From 21c1ad1005e3415f8448a21f17ed3ba3be9715b8 Mon Sep 17 00:00:00 2001 From: louis Date: Fri, 23 Aug 2013 17:11:38 +0200 Subject: Initial Push Version 0.0.1 --- COPYING | 340 ++++++++++++++++++ HISTORY | 6 + Makefile | 128 +++++++ README | 85 +++++ config.c | 57 +++ config.h | 24 ++ imageserver.c | 205 +++++++++++ imageserver.h | 20 ++ po/de_DE.po | 36 ++ services.h | 56 +++ setup.c | 111 ++++++ setup.h | 30 ++ themoviedbscraper/moviedbactors.c | 83 +++++ themoviedbscraper/moviedbactors.h | 37 ++ themoviedbscraper/moviedbmovie.c | 140 ++++++++ themoviedbscraper/moviedbmovie.h | 37 ++ themoviedbscraper/themoviedbscraper.c | 151 ++++++++ themoviedbscraper/themoviedbscraper.h | 32 ++ thetvdbscraper/thetvdbscraper.c | 129 +++++++ thetvdbscraper/thetvdbscraper.h | 28 ++ thetvdbscraper/tvdbactors.c | 100 ++++++ thetvdbscraper/tvdbactors.h | 37 ++ thetvdbscraper/tvdbmedia.c | 149 ++++++++ thetvdbscraper/tvdbmedia.h | 45 +++ thetvdbscraper/tvdbmirrors.c | 97 ++++++ thetvdbscraper/tvdbmirrors.h | 27 ++ thetvdbscraper/tvdbseries.c | 92 +++++ thetvdbscraper/tvdbseries.h | 29 ++ tools/curlfuncs.cpp | 235 +++++++++++++ tools/curlfuncs.h | 45 +++ tools/filesystem.c | 44 +++ tools/fuzzy.c | 70 ++++ tools/splitstring.c | 32 ++ tvscraper.c | 223 ++++++++++++ tvscraperdb.c | 628 ++++++++++++++++++++++++++++++++++ tvscraperdb.h | 50 +++ worker.c | 233 +++++++++++++ worker.h | 47 +++ 38 files changed, 3918 insertions(+) create mode 100644 COPYING create mode 100644 HISTORY create mode 100644 Makefile create mode 100644 README create mode 100644 config.c create mode 100644 config.h create mode 100644 imageserver.c create mode 100644 imageserver.h create mode 100644 po/de_DE.po create mode 100644 services.h create mode 100644 setup.c create mode 100644 setup.h create mode 100644 themoviedbscraper/moviedbactors.c create mode 100644 themoviedbscraper/moviedbactors.h create mode 100644 themoviedbscraper/moviedbmovie.c create mode 100644 themoviedbscraper/moviedbmovie.h create mode 100644 themoviedbscraper/themoviedbscraper.c create mode 100644 themoviedbscraper/themoviedbscraper.h create mode 100644 thetvdbscraper/thetvdbscraper.c create mode 100644 thetvdbscraper/thetvdbscraper.h create mode 100644 thetvdbscraper/tvdbactors.c create mode 100644 thetvdbscraper/tvdbactors.h create mode 100644 thetvdbscraper/tvdbmedia.c create mode 100644 thetvdbscraper/tvdbmedia.h create mode 100644 thetvdbscraper/tvdbmirrors.c create mode 100644 thetvdbscraper/tvdbmirrors.h create mode 100644 thetvdbscraper/tvdbseries.c create mode 100644 thetvdbscraper/tvdbseries.h create mode 100644 tools/curlfuncs.cpp create mode 100644 tools/curlfuncs.h create mode 100644 tools/filesystem.c create mode 100644 tools/fuzzy.c create mode 100644 tools/splitstring.c create mode 100644 tvscraper.c create mode 100644 tvscraperdb.c create mode 100644 tvscraperdb.h create mode 100644 worker.c create mode 100644 worker.h diff --git a/COPYING b/COPYING new file mode 100644 index 0000000..f90922e --- /dev/null +++ b/COPYING @@ -0,0 +1,340 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc. + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/HISTORY b/HISTORY new file mode 100644 index 0000000..f816225 --- /dev/null +++ b/HISTORY @@ -0,0 +1,6 @@ +VDR Plugin 'tvscraper' Revision History +---------------------------------------- + +2013-07-26: Version 0.0.1 + +- Initial revision. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..52439ba --- /dev/null +++ b/Makefile @@ -0,0 +1,128 @@ +# +# Makefile for a Video Disk Recorder plugin +# +# $Id$ + +# The official name of this plugin. +# This name will be used in the '-P...' option of VDR to load the plugin. +# By default the main source file also carries this name. + +PLUGIN = tvscraper + +### The version number of this plugin (taken from the main source file): + +VERSION = $(shell grep 'static const char \*VERSION *=' $(PLUGIN).c | awk '{ print $$6 }' | sed -e 's/[";]//g') + +### The directory environment: + +# Use package data if installed...otherwise assume we're under the VDR source directory: +PKGCFG = $(if $(VDRDIR),$(shell pkg-config --variable=$(1) $(VDRDIR)/vdr.pc),$(shell pkg-config --variable=$(1) vdr || pkg-config --variable=$(1) ../../../vdr.pc)) +LIBDIR = $(call PKGCFG,libdir) +LOCDIR = $(call PKGCFG,locdir) +PLGCFG = $(call PKGCFG,plgcfg) +# +TMPDIR ?= /tmp + +### The compiler options: + +export CFLAGS = $(call PKGCFG,cflags) +export CXXFLAGS = $(call PKGCFG,cxxflags) + +### The version number of VDR's plugin API: + +APIVERSION = $(call PKGCFG,apiversion) + +### Allow user defined options to overwrite defaults: + +-include $(PLGCFG) + +### The name of the distribution archive: + +ARCHIVE = $(PLUGIN)-$(VERSION) +PACKAGE = vdr-$(ARCHIVE) + +### The name of the shared object file: + +SOFILE = libvdr-$(PLUGIN).so + +### Includes and Defines (add further entries here): + +INCLUDES += + +DEFINES += -DPLUGIN_NAME_I18N='"$(PLUGIN)"' +DEFINES += $(shell xml2-config --cflags) + +LIBS += $(shell xml2-config --libs) +LIBS += $(shell pkg-config --libs libcurl) +LIBS += $(shell pkg-config --libs sqlite3) +LIBS += $(shell pkg-config --cflags --libs jansson) + +### The object files (add further files here): + +OBJS = $(PLUGIN).o + +### The main target: + +all: $(SOFILE) i18n + +### Implicit rules: + +%.o: %.c + $(CXX) $(CXXFLAGS) -c $(DEFINES) $(INCLUDES) -o $@ $< + +### Dependencies: + +MAKEDEP = $(CXX) -MM -MG +DEPFILE = .dependencies +$(DEPFILE): Makefile + @$(MAKEDEP) $(CXXFLAGS) $(DEFINES) $(INCLUDES) $(OBJS:%.o=%.c) > $@ + +-include $(DEPFILE) + +### Internationalization (I18N): + +PODIR = po +I18Npo = $(wildcard $(PODIR)/*.po) +I18Nmo = $(addsuffix .mo, $(foreach file, $(I18Npo), $(basename $(file)))) +I18Nmsgs = $(addprefix $(DESTDIR)$(LOCDIR)/, $(addsuffix /LC_MESSAGES/vdr-$(PLUGIN).mo, $(notdir $(foreach file, $(I18Npo), $(basename $(file)))))) +I18Npot = $(PODIR)/$(PLUGIN).pot + +%.mo: %.po + msgfmt -c -o $@ $< + +$(I18Npot): $(wildcard *.c) + xgettext -C -cTRANSLATORS --no-wrap --no-location -k -ktr -ktrNOOP --package-name=vdr-$(PLUGIN) --package-version=$(VERSION) --msgid-bugs-address='' -o $@ `ls $^` + +%.po: $(I18Npot) + msgmerge -U --no-wrap --no-location --backup=none -q -N $@ $< + @touch $@ + +$(I18Nmsgs): $(DESTDIR)$(LOCDIR)/%/LC_MESSAGES/vdr-$(PLUGIN).mo: $(PODIR)/%.mo + install -D -m644 $< $@ + +.PHONY: i18n +i18n: $(I18Nmo) $(I18Npot) + +install-i18n: $(I18Nmsgs) + +### Targets: + +$(SOFILE): $(OBJS) + $(CXX) $(CXXFLAGS) $(LDFLAGS) -shared $(OBJS) $(LIBS) -o $@ + +install-lib: $(SOFILE) + install -D $^ $(DESTDIR)$(LIBDIR)/$^.$(APIVERSION) + +install: install-lib install-i18n + +dist: $(I18Npo) clean + @-rm -rf $(TMPDIR)/$(ARCHIVE) + @mkdir $(TMPDIR)/$(ARCHIVE) + @cp -a * $(TMPDIR)/$(ARCHIVE) + @tar czf $(PACKAGE).tgz -C $(TMPDIR) $(ARCHIVE) + @-rm -rf $(TMPDIR)/$(ARCHIVE) + @echo Distribution package created as $(PACKAGE).tgz + +clean: + @-rm -f $(PODIR)/*.mo $(PODIR)/*.pot + @-rm -f $(OBJS) $(DEPFILE) *.so *.tgz core* *~ diff --git a/README b/README new file mode 100644 index 0000000..86ea416 --- /dev/null +++ b/README @@ -0,0 +1,85 @@ +This is a "plugin" for the Video Disk Recorder (VDR). + +Written by: Stefan Braun + +Project's homepage: +http://projects.vdr-developer.org/projects/plg-tvscraper + +Latest version available at: +http://projects.vdr-developer.org/projects/plg-tvscraper/files + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. +See the file COPYING for more information. + +Description +----------- + +TVScraper runs in the background and collects metadata (posters, +banners, fanart, actor thumbs and roles, descriptions) for all +available EPG events on selectable channels and for recordings. +Additionally the plugin provides the collected metadata via the VDR +service interface to other plugins which deal with EPG information. + +TVScraper uses the thetvdb.com API for collecting series metadata and +themoviedb.org API for movies. Check the websites of both services for +the terms of use. + +Important: To avoid unnecessary traffic, only activate these channels +to be scrapped which are reasonable. After plugin installation all +channels are deactivated by default, so please consider this point when +you activate the channels you are interested in ;) + +Additionally you are invited to contribute to the used web services with +providing missing data for your favorite movies and series. + +Requirements +------------ + +To run the plugin the following libaries have to be installed: +- libsqlite3 +- libcurl +- libXML2 +- libjansson + +Installation and configuration +------------------------------ + +Just install the plugin depending on your used distribution. During VDR +startup the plugin base directory can be set with the following option: + +-d , --dir= Set directory where database and images + are stored + +If no directory is provided, the plugin uses VDRCACHEDIR as default. +Please care about that the user who runs VDR has full read/write access +to this directory, otherwise the plugin will not start. + +As already mentioned, after first installations no channels are activated +to be scrapped. Please configure these channels in the plugin setup menu. +Additionally you can trigger that your already existing recordings are +scrapped, so that also for this recordings metadata is available. + +The plugins uses a sqlite3 database to store the necessary information. +If /dev/shm/ is available, the database is kept in memory during runtime +which improves performance. In the configured plugin basedir only a +persistant backup of the database is stored then. If /dev/shm/ is not +available, only the database file in the plugin base directory is used. + +Usage +----- + +After the initial configuration the plugin runs completely independent in +the background, you don't have to care about anything. The plugins checks +at least every 24 hours for new EPG events and collects the metadata for +these events automatically. + +Before each run the plugin performs a cleanup, all images for movies which +are not available in the current EPG are deleted. Series and actors thumbs +are kept to avoid unnecessary traffic for the web services, because the +propability that this data is needed in the future again is rather high. + +If a running recording is detected, the plugin marks the corresponding movie +meta data so that the information for this movie will be kept permanentely. \ No newline at end of file diff --git a/config.c b/config.c new file mode 100644 index 0000000..4110bd3 --- /dev/null +++ b/config.c @@ -0,0 +1,57 @@ +#include "config.h" + +using namespace std; + +cTVScraperConfig::cTVScraperConfig() { + enableDebug = 0; +} + +cTVScraperConfig::~cTVScraperConfig() { +} + +void cTVScraperConfig::ClearChannels(void) { + channels.clear(); +} + +void cTVScraperConfig::AddChannel(string channelID) { + channels.push_back(channelID); +} + +bool cTVScraperConfig::ChannelActive(int channelNum) { + cChannel *channel = Channels.GetByNumber(channelNum); + if (channel) { + string channelID = ""; + channelID = *(channel->GetChannelID().ToString()); + int numChannels = channels.size(); + for (int i=0; i flds = s.split(';'); + int numChannels = flds.size(); + for (int i=0; i channels; + public: + cTVScraperConfig(); + ~cTVScraperConfig(); + int enableDebug; + void SetBaseDir(string dir) { baseDir = dir; }; + string GetBaseDir(void) { return baseDir; }; + void ClearChannels(void); + void AddChannel(string channelID); + bool ChannelActive(int channelNum); + bool SetupParse(const char *Name, const char *Value); + vector GetChannels(void) { return channels; }; + void PrintChannels(void); +}; + +#endif //__TVSCRAPER_CONFIG_H \ No newline at end of file diff --git a/imageserver.c b/imageserver.c new file mode 100644 index 0000000..975d417 --- /dev/null +++ b/imageserver.c @@ -0,0 +1,205 @@ +#include "imageserver.h" + +using namespace std; + +cImageServer::cImageServer(cTVScraperDB *db) { + this->db = db; +} + +cImageServer::~cImageServer() { +} + +scrapType cImageServer::GetScrapType(const cEvent *event) { + scrapType type = scrapNone; + int duration = event->Duration() / 60; + if ((duration > 19) && (duration < 65)) { + type = scrapSeries; + } else if (duration > 80) { + type = scrapMovie; + } + return type; +} + +int cImageServer::GetID(int eventID, scrapType type, bool isRecording) { + int id = 0; + if (type == scrapSeries) { + id = db->GetSeriesID(eventID, isRecording); + } else if (type == scrapMovie) { + id = db->GetMovieID(eventID, isRecording); + } + return id; +} + +tvMedia cImageServer::GetPosterOrBanner(int id, scrapType type) { + tvMedia media; + media.path = ""; + media.width = 0; + media.height = 0; + if (type == scrapSeries) { + stringstream path; + path << config.GetBaseDir() << "/series/" << id << "/banner.jpg"; + media.path = path.str(); + media.width = 758; + media.height = 140; + } else if (type == scrapMovie) { + stringstream path; + path << config.GetBaseDir() << "/movies/" << id << "_poster.jpg"; + media.path = path.str(); + media.width = 500; + media.height = 750; + } + return media; +} + +tvMedia cImageServer::GetPoster(int id, scrapType type) { + tvMedia media; + media.path = ""; + media.width = 0; + media.height = 0; + if (type == scrapSeries) { + stringstream path; + path << config.GetBaseDir() << "/series/" << id << "/poster_0.jpg"; + string filePoster = path.str(); + if (FileExists(filePoster)) { + media.path = filePoster; + media.width = 680; + media.height = 1000; + } + } else if (type == scrapMovie) { + stringstream path; + path << config.GetBaseDir() << "/movies/" << id << "_poster.jpg"; + string filePoster = path.str(); + if (FileExists(filePoster)) { + media.path = path.str(); + media.width = 500; + media.height = 750; + } + } + return media; +} + +tvMedia cImageServer::GetBanner(int id) { + tvMedia media; + stringstream path; + path << config.GetBaseDir() << "/series/" << id << "/banner.jpg"; + media.path = path.str(); + media.width = 758; + media.height = 140; + return media; +} + +vector cImageServer::GetPosters(int id, scrapType type) { + vector posters; + if (type == scrapSeries) { + for (int i=0; i<3; i++) { + stringstream path; + path << config.GetBaseDir() << "/series/" << id << "/poster_" << i << ".jpg"; + string filePoster = path.str(); + if (FileExists(filePoster)) { + tvMedia media; + media.path = filePoster; + media.width = 680; + media.height = 1000; + posters.push_back(media); + } else + break; + } + } else if (type == scrapMovie) { + stringstream path; + path << config.GetBaseDir() << "/movies/" << id << "_poster.jpg"; + string filePoster = path.str(); + if (FileExists(filePoster)) { + tvMedia media; + media.path = path.str(); + media.width = 500; + media.height = 750; + posters.push_back(media); + } + } + return posters; +} + +vector cImageServer::GetFanart(int id, scrapType type) { + vector fanart; + if (type == scrapSeries) { + for (int i=0; i<3; i++) { + stringstream path; + path << config.GetBaseDir() << "/series/" << id << "/fanart_" << i << ".jpg"; + string fileFanart = path.str(); + if (FileExists(fileFanart)) { + tvMedia media; + media.path = fileFanart; + media.width = 1920; + media.height = 1080; + fanart.push_back(media); + } else + break; + } + } else if (type == scrapMovie) { + stringstream path; + path << config.GetBaseDir() << "/movies/" << id << "_backdrop.jpg"; + string fileFanart = path.str(); + if (FileExists(fileFanart)) { + tvMedia media; + media.path = fileFanart; + media.width = 1280; + media.height = 720; + fanart.push_back(media); + } + } + return fanart; +} + +vector cImageServer::GetActors(int id, scrapType type) { + vector actors; + if (type == scrapSeries) { + vector > actorsDB = db->GetActorsSeries(id); + int numActors = actorsDB.size(); + for (int i=0; i < numActors; i++) { + vector row = actorsDB[i]; + if (row.size() == 3) { + tvActor actor; + actor.name = row[0]; + actor.role = row[1]; + tvMedia thumb; + stringstream thumbPath; + thumbPath << config.GetBaseDir() << "/series/" << id << "/" << row[2]; + thumb.path = thumbPath.str(); + thumb.width = 300; + thumb.height = 450; + actor.thumb = thumb; + actors.push_back(actor); + } + } + } else if (type == scrapMovie) { + vector > actorsDB = db->GetActorsMovie(id); + int numActors = actorsDB.size(); + for (int i=0; i < numActors; i++) { + vector row = actorsDB[i]; + if (row.size() == 3) { + tvActor actor; + actor.name = row[1]; + actor.role = row[2]; + stringstream thumbPath; + thumbPath << config.GetBaseDir() << "/movies/actors/actor_" << row[0] << ".jpg"; + tvMedia thumb; + thumb.path = thumbPath.str(); + thumb.width = 421; + thumb.height = 632; + actor.thumb = thumb; + actors.push_back(actor); + } + } + } + return actors; +} + +string cImageServer::GetDescription(int id, scrapType type) { + string description; + if (type == scrapSeries) { + description = db->GetDescriptionSeries(id); + } else if (type == scrapMovie) { + description = db->GetDescriptionMovie(id); + } + return description; +} diff --git a/imageserver.h b/imageserver.h new file mode 100644 index 0000000..a8bfd8b --- /dev/null +++ b/imageserver.h @@ -0,0 +1,20 @@ +using namespace std; + +// --- cImageServer -------------------------------------------------------- + +class cImageServer { +private: + cTVScraperDB *db; +public: + cImageServer(cTVScraperDB *db); + virtual ~cImageServer(void); + scrapType GetScrapType(const cEvent *event); + int GetID(int eventID, scrapType type, bool isRecording); + tvMedia GetPosterOrBanner(int id, scrapType type); + tvMedia GetPoster(int id, scrapType type); + tvMedia GetBanner(int id); + vector GetPosters(int id, scrapType type); + vector GetFanart(int id, scrapType type); + vector GetActors(int id, scrapType type); + string GetDescription(int id, scrapType type); +}; \ No newline at end of file diff --git a/po/de_DE.po b/po/de_DE.po new file mode 100644 index 0000000..f2ab8eb --- /dev/null +++ b/po/de_DE.po @@ -0,0 +1,36 @@ +msgid "" +msgstr "" +"Project-Id-Version: vdr-tvscrapper 0.0.1\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2013-08-23 15:28+0200\n" +"PO-Revision-Date: 2013-08-20 08:27+0200\n" +"Last-Translator: Stefan Braun \n" +"Language-Team:\n" +"Language: de \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" + +msgid "Configure channels to be scrapped" +msgstr "Kanäle zum scrapen festlegen" + +msgid "Trigger scrapping Video Directory" +msgstr "" + +msgid "Trigger EPG scrapping" +msgstr "" + +msgid "Enable Debug Logging" +msgstr "Debug Logging aktivieren" + +msgid "don't scrap" +msgstr "nicht scrapen" + +msgid "scrap" +msgstr "scrapen" + +#~ msgid "Trigger scraping Video Directory" +#~ msgstr "Scrapen des Video Verzeichnisses starten" + +#~ msgid "Trigger EPG scraping" +#~ msgstr "EPG Scraping starten" diff --git a/services.h b/services.h new file mode 100644 index 0000000..fc278ad --- /dev/null +++ b/services.h @@ -0,0 +1,56 @@ +enum tvMediaType { + typeSeries, + typeMovie, + typeNone, +}; + +struct tvMedia { + std::string path; + int width; + int height; +}; + +struct tvActor { + std::string name; + std::string role; + tvMedia thumb; +}; + +// Data structure for service "TVScraper-GetPosterOrBanner" +struct TVScraperGetPosterOrBanner +{ +// in + const cEvent *event; // search image for this event +//out + tvMediaType type; //typeSeries or typeMovie + tvMedia media; //banner or poster +}; + +// Data structure for service "TVScraper-GetPoster" +struct TVScraperGetPoster +{ +// in + const cEvent *event; // search image for this event + bool isRecording; // search in current EPG or recordings +//out + tvMedia media; //poster +}; + + +/* Data structure for service "TVScraper-GetFullEPGInformation" +if type == typeMovie a poster and a fanart image is delivered +if type == typeSeries a banner and up to three posters and fanarts are delivered +*/ +struct TVScraperGetFullInformation +{ +// in + const cEvent *event; // search all media for this event + bool isRecording; // search in current EPG or recordings +//out + tvMediaType type; + tvMedia banner; + std::vector posters; + std::vector fanart; + std::vector actors; + std::string description; +}; \ No newline at end of file diff --git a/setup.c b/setup.c new file mode 100644 index 0000000..0415d31 --- /dev/null +++ b/setup.c @@ -0,0 +1,111 @@ +#include "setup.h" + +using namespace std; + +/* cTVScraperSetup */ + +cTVScraperSetup::cTVScraperSetup(cTVScraperWorker *workerThread) { + worker = workerThread; + int numChannels = Channels.Count(); + for (int i=0; iText(); + if (strcmp(ItemText, tr("Configure channels to be scrapped")) == 0) + state = AddSubMenu(new cTVScraperChannelSetup(&channelsScrap)); + else if (strcmp(ItemText, tr("Trigger scrapping Video Directory")) == 0) { + Skins.Message(mtInfo, "Scrapping Video Directory started"); + worker->InitVideoDirScan(); + state = osContinue; + } else if (strcmp(ItemText, tr("Trigger EPG scrapping")) == 0) { + Skins.Message(mtInfo, "EPG Scrapping started"); + worker->InitManualScan(); + state = osContinue; + } + } + return state; +} + +void cTVScraperSetup::Store(void) { + config.ClearChannels(); + stringstream channelsToScrap; + int numChannels = channelsScrap.size(); + for (int i=0; iGetChannelID().ToString()); + channelsToScrap << channelID << ";"; + config.AddChannel(channelID); + } + } + } + SetupStore("ScrapChannels", channelsToScrap.str().c_str()); + SetupStore("enableDebug", config.enableDebug); +} + + +/* cTVScraperChannelSetup */ + +cTVScraperChannelSetup ::cTVScraperChannelSetup (vector *channelsScrap) : cOsdMenu(tr("Configure channels to be scrapped"), 30) { + this->channelsScrap = channelsScrap; + SetMenuCategory(mcSetupPlugins); + Setup(); +} + +cTVScraperChannelSetup ::~cTVScraperChannelSetup () { +} + + +void cTVScraperChannelSetup ::Setup(void) { + int currentItem = Current(); + Clear(); + int i=0; + for (cChannel *channel = Channels.First(); channel; channel = Channels.Next(channel)) { + if (!channel->GroupSep()) { + Add(new cMenuEditBoolItem(channel->Name(), &channelsScrap->at(i), tr("don't scrap"), tr("scrap"))); + i++; + } + } + SetCurrent(Get(currentItem)); + Display(); +} + +eOSState cTVScraperChannelSetup ::ProcessKey(eKeys Key) { + eOSState state = cOsdMenu::ProcessKey(Key); + switch (Key) { + case kOk: + return osBack; + default: + break; + } + return state; +} \ No newline at end of file diff --git a/setup.h b/setup.h new file mode 100644 index 0000000..a0f2410 --- /dev/null +++ b/setup.h @@ -0,0 +1,30 @@ +#ifndef __TVSCRAPER_SETUP_H +#define __TVSCRAPER_SETUP_H + +using namespace std; + +class cTVScraperSetup : public cMenuSetupPage { + public: + cTVScraperSetup(cTVScraperWorker *workerThread); + virtual ~cTVScraperSetup(); + private: + vector channelsScrap; + cTVScraperWorker *worker; + void Setup(void); + protected: + virtual eOSState ProcessKey(eKeys Key); + virtual void Store(void); +}; + +class cTVScraperChannelSetup : public cOsdMenu { + public: + cTVScraperChannelSetup(vector *channelsScrap); + virtual ~cTVScraperChannelSetup(); + private: + vector *channelsScrap; + void Setup(void); + protected: + virtual eOSState ProcessKey(eKeys Key); +}; + +#endif //__TVSCRAPER_SETUP_H \ No newline at end of file diff --git a/themoviedbscraper/moviedbactors.c b/themoviedbscraper/moviedbactors.c new file mode 100644 index 0000000..b3880f3 --- /dev/null +++ b/themoviedbscraper/moviedbactors.c @@ -0,0 +1,83 @@ +#include +#include +#include +#include +#include +#include "moviedbactors.h" + +using namespace std; + +cMovieDbActors::cMovieDbActors(string json) { + this->json = json; +} + +cMovieDbActors::~cMovieDbActors() { + actors.clear(); +} + +void cMovieDbActors::ParseJSON(void) { + json_t *jActors; + json_error_t error; + jActors = json_loads(json.c_str(), 0, &error); + if (!jActors) { + return; + } + if(!json_is_object(jActors)) { + return; + } + json_t *cast = json_object_get(jActors, "cast"); + if(!json_is_array(cast)) { + return; + } + size_t numActors = json_array_size(cast); + for (size_t i = 0; i < numActors; i++) { + json_t *jActor = json_array_get(cast, i); + if (!json_is_object(jActor)) { + return; + } + json_t *jId = json_object_get(jActor, "id"); + json_t *jName = json_object_get(jActor, "name"); + json_t *jRole = json_object_get(jActor, "character"); + json_t *jPath = json_object_get(jActor, "profile_path"); + if (!json_is_integer(jId) || !json_is_string(jName) || !json_is_string(jRole) || !json_is_string(jPath)) + return; + cMovieDBActor *actor = new cMovieDBActor(); + actor->id = json_integer_value(jId); + actor->name = json_string_value(jName); + actor->role = json_string_value(jRole); + actor->path = json_string_value(jPath); + actors.push_back(actor); + } +} + +void cMovieDbActors::StoreDB(cTVScraperDB *db, int movieID) { + int numActors = actors.size(); + for (int i=0; iInsertMovieActor(movieID, actors[i]->id, actors[i]->name, actors[i]->role); + } +} + +void cMovieDbActors::Store(string baseUrl, string destDir) { + int size = actors.size(); + string path; + string url; + for (int i=0; ipath; + url = strUrl.str(); + stringstream fullPath; + fullPath << destDir << "/actor_" << actors[i]->id << ".jpg"; + path = fullPath.str(); + if (!FileExists(path)) { + CurlGetUrlFile(url.c_str(), path.c_str()); + } + } +} + +void cMovieDbActors::Dump(void) { + int numActors = actors.size(); + esyslog("tvscraper: %d Actors:", numActors); + for (int i=0; iid, actors[i]->name.c_str(), actors[i]->role.c_str(), actors[i]->path.c_str()); + } +} \ No newline at end of file diff --git a/themoviedbscraper/moviedbactors.h b/themoviedbscraper/moviedbactors.h new file mode 100644 index 0000000..1169b85 --- /dev/null +++ b/themoviedbscraper/moviedbactors.h @@ -0,0 +1,37 @@ +#ifndef __TVSCRAPER_MOVIEDBACTORS_H +#define __TVSCRAPER_MOVIEDBACTORS_H + +using namespace std; + +// --- cMovieDBActor ------------------------------------------------------------- +class cMovieDBActor { +public: + cMovieDBActor(void) { + id = 0; + path = ""; + name = ""; + role = ""; + }; + int id; + string path; + string name; + string role; +}; + +// --- cMovieDBActors ------------------------------------------------------------- + +class cMovieDbActors { +private: + string json; + vector actors; +public: + cMovieDbActors(string json); + virtual ~cMovieDbActors(void); + void ParseJSON(void); + void StoreDB(cTVScraperDB *db, int movieID); + void Store(string baseUrl, string destDir); + void Dump(void); +}; + + +#endif //__TVSCRAPER_MOVIEDBACTORS_H diff --git a/themoviedbscraper/moviedbmovie.c b/themoviedbscraper/moviedbmovie.c new file mode 100644 index 0000000..26aee7f --- /dev/null +++ b/themoviedbscraper/moviedbmovie.c @@ -0,0 +1,140 @@ +#include +#include +#include +#include +#include +#include "moviedbmovie.h" + +using namespace std; + +cMovieDbMovie::cMovieDbMovie(string json) { + this->json = json; + title = ""; + originalTitle = ""; + overview = ""; + backdropPath = ""; + posterPath = ""; +} + +cMovieDbMovie::~cMovieDbMovie() { + +} + +void cMovieDbMovie::ParseJSON(void) { + json_t *movie; + json_error_t error; + movie = json_loads(json.c_str(), 0, &error); + if (!movie) { + return; + } + if(!json_is_object(movie)) { + return; + } + json_t *jTitle = json_object_get(movie, "title"); + if(json_is_string(jTitle)) { + title = json_string_value(jTitle);; + } + json_t *jOriginalTitle = json_object_get(movie, "original_title"); + if(json_is_string(jOriginalTitle)) { + originalTitle = json_string_value(jOriginalTitle); + } + json_t *jOverview = json_object_get(movie, "overview"); + if(json_is_string(jOverview)) { + overview = json_string_value(jOverview); + } + json_t *jBackdrop = json_object_get(movie, "backdrop_path"); + if(json_is_string(jBackdrop)) { + backdropPath = json_string_value(jBackdrop); + } + json_t *jPoster = json_object_get(movie, "poster_path"); + if(json_is_string(jPoster)) { + posterPath = json_string_value(jPoster); + } +} + +int cMovieDbMovie::ParseJSONForMovieId(string movieSearchString) { + //convert searchstring to lower case + transform(movieSearchString.begin(), movieSearchString.end(), movieSearchString.begin(), ::tolower); + json_t *root; + json_error_t error; + root = json_loads(json.c_str(), 0, &error); + if (!root) { + return -1; + } + if(!json_is_object(root)) { + return -1; + } + json_t *results = json_object_get(root, "results"); + if(!json_is_array(results)) { + return -1; + } + size_t numResults = json_array_size(results); + for (size_t res = 0; res < numResults; res++) { + json_t *result = json_array_get(results, res); + if (!json_is_object(result)) { + return -1; + } + json_t *title = json_object_get(result, "title"); + if (!json_is_string(title)) { + return -1; + } + string resultTitle = json_string_value(title); + //convert result to lower case + transform(resultTitle.begin(), resultTitle.end(), resultTitle.begin(), ::tolower); + json_t *jId = json_object_get(result, "id"); + if (json_is_integer(jId)) { + int id = (int)json_integer_value(jId); + searchResult sRes; + sRes.id = id; + sRes.distance = sentence_distance(resultTitle, movieSearchString); + resultSet.push_back(sRes); + } + } + return FindBestResult(); +} + +int cMovieDbMovie::FindBestResult(void) { + int resID = -1; + int bestMatch = -1; + int numResults = resultSet.size(); + for (int i=0; iInsertMovie(id, title, originalTitle, overview); +} + +void cMovieDbMovie::Dump(void) { + esyslog("tvscraper: -------------- MOVIE DUMP ---------------"); + esyslog("tvscraper: title %s", title.c_str()); + esyslog("tvscraper: originalTitle %s", originalTitle.c_str()); + esyslog("tvscraper: overview %s", overview.c_str()); + esyslog("tvscraper: backdropPath %s", backdropPath.c_str()); + esyslog("tvscraper: posterPath %s", posterPath.c_str()); +} \ No newline at end of file diff --git a/themoviedbscraper/moviedbmovie.h b/themoviedbscraper/moviedbmovie.h new file mode 100644 index 0000000..e1bd6d9 --- /dev/null +++ b/themoviedbscraper/moviedbmovie.h @@ -0,0 +1,37 @@ +#ifndef __TVSCRAPER_MOVIEDBMOVIE_H +#define __TVSCRAPER_MOVIEDBMOVIE_H + +using namespace std; + +struct searchResult { + int id; + int distance; +}; + +// --- cMovieDbMovie ------------------------------------------------------------- + +class cMovieDbMovie { +private: + int id; + string json; + string title; + string originalTitle; + string overview; + string backdropPath; + string posterPath; + vector resultSet; + int FindBestResult(void); +public: + cMovieDbMovie(string json); + virtual ~cMovieDbMovie(void); + int ParseJSONForMovieId(string movieSearchString); + void ParseJSON(void); + void SetID(int movieID) { id = movieID; }; + int ID(void) { return id; }; + void StoreDB(cTVScraperDB *db); + void StoreMedia(string posterBaseUrl, string backdropBaseUrl, string destDir); + void Dump(); +}; + + +#endif //__TVSCRAPER_TVDBSERIES_H diff --git a/themoviedbscraper/themoviedbscraper.c b/themoviedbscraper/themoviedbscraper.c new file mode 100644 index 0000000..2f9e7e9 --- /dev/null +++ b/themoviedbscraper/themoviedbscraper.c @@ -0,0 +1,151 @@ +#include +#include +#include +#include +#include "themoviedbscraper.h" + +using namespace std; + +cMovieDBScraper::cMovieDBScraper(string baseDir, cTVScraperDB *db, string language) { + apiKey = "abb01b5a277b9c2c60ec0302d83c5ee9"; + //TODO: use language from VDR settings + this->language = language; + baseURL = "api.themoviedb.org/3"; + this->baseDir = baseDir; + this->db = db; + posterSize = "w500"; + backdropSize = "w1280"; + actorthumbSize = "h632"; +} + +cMovieDBScraper::~cMovieDBScraper() { +} + +void cMovieDBScraper::Scrap(const cEvent *event, int recordingID) { + string movieName = event->Title(); + int eventID = (int)event->EventID(); + if (config.enableDebug) + esyslog("tvscraper: scrapping movie \"%s\"", movieName.c_str()); + int movieID = SearchMovie(movieName); + if (movieID < 1) { + if (config.enableDebug) + esyslog("tvscraper: nothing found for \"%s\"", movieName.c_str()); + return; + } + if (!recordingID) { + time_t validTill = event->EndTime(); + db->InsertEventMovie(eventID, validTill, movieID); + } else { + db->InsertRecording(recordingID, 0, movieID); + } + if (db->MovieExists(movieID)) { + return; + } + cMovieDbMovie *movie = ReadMovie(movieID); + if (!movie) + return; + movie->StoreDB(db); + cMovieDbActors *actors = ReadActors(movieID); + if (!actors) { + delete movie; + return; + } + actors->StoreDB(db, movieID); + StoreMedia(movie, actors); + delete movie; + delete actors; + if (config.enableDebug) + esyslog("tvscraper: \"%s\" successfully scrapped, id %d", movieName.c_str(), movieID); +} + +bool cMovieDBScraper::Connect(void) { + stringstream url; + url << baseURL << "/configuration?api_key=" << apiKey; + string configJSON; + if (CurlGetUrl(url.str().c_str(), &configJSON)) { + return parseJSON(configJSON); + } + return false; +} + +bool cMovieDBScraper::parseJSON(string jsonString) { + json_t *root; + json_error_t error; + + root = json_loads(jsonString.c_str(), 0, &error); + if (!root) { + return false; + } + if(!json_is_object(root)) { + return false; + } + json_t *images; + images = json_object_get(root, "images"); + if(!json_is_object(images)) { + return false; + } + + json_t *imgUrl; + imgUrl = json_object_get(images, "base_url"); + if(!json_is_string(imgUrl)) { + return false; + } + imageUrl = json_string_value(imgUrl); + return true; +} + +int cMovieDBScraper::SearchMovie(string movieName) { + stringstream movieEscaped; + movieEscaped << "\"" << movieName << "\""; + stringstream url; + url << baseURL << "/search/movie?api_key=" << apiKey << "&query=" << CurlEscape(movieEscaped.str().c_str()) << "&language=" << language.c_str(); + string movieJSON; + int movieID = -1; + if (CurlGetUrl(url.str().c_str(), &movieJSON)) { + cMovieDbMovie *movie = new cMovieDbMovie(movieJSON); + movieID = movie->ParseJSONForMovieId(movieName); + delete movie; + } + return movieID; +} + +cMovieDbMovie *cMovieDBScraper::ReadMovie(int movieID) { + stringstream url; + url << baseURL << "/movie/" << movieID << "?api_key=" << apiKey << "&language=" << language.c_str(); + string movieJSON; + cMovieDbMovie *movie = NULL; + if (CurlGetUrl(url.str().c_str(), &movieJSON)) { + movie = new cMovieDbMovie(movieJSON); + movie->SetID(movieID); + movie->ParseJSON(); + } + return movie; +} + +cMovieDbActors *cMovieDBScraper::ReadActors(int movieID) { + stringstream url; + url << baseURL << "/movie/" << movieID << "/casts?api_key=" << apiKey; + string actorsJSON; + cMovieDbActors *actors = NULL; + if (CurlGetUrl(url.str().c_str(), &actorsJSON)) { + actors = new cMovieDbActors(actorsJSON); + actors->ParseJSON(); + } + return actors; +} + +void cMovieDBScraper::StoreMedia(cMovieDbMovie *movie, cMovieDbActors *actors) { + stringstream posterUrl; + posterUrl << imageUrl << posterSize; + stringstream backdropUrl; + backdropUrl << imageUrl << backdropSize; + stringstream destDir; + destDir << baseDir << "/"; + movie->StoreMedia(posterUrl.str(), backdropUrl.str(), destDir.str()); + stringstream actorsUrl; + actorsUrl << imageUrl << actorthumbSize; + stringstream actorsDestDir; + actorsDestDir << baseDir << "/actors"; + CreateDirectory(actorsDestDir.str()); + actors->Store(actorsUrl.str(), actorsDestDir.str()); +} diff --git a/themoviedbscraper/themoviedbscraper.h b/themoviedbscraper/themoviedbscraper.h new file mode 100644 index 0000000..54d463b --- /dev/null +++ b/themoviedbscraper/themoviedbscraper.h @@ -0,0 +1,32 @@ +#ifndef __TVSCRAPER_MOVIEDBSCRAPER_H +#define __TVSCRAPER_MOVIEDBSCRAPER_H + +using namespace std; + +// --- cMovieDBScraper ------------------------------------------------------------- + +class cMovieDBScraper { +private: + string apiKey; + string language; + string baseURL; + string baseDir; + string imageUrl; + string posterSize; + string backdropSize; + string actorthumbSize; + cTVScraperDB *db; + bool parseJSON(string jsonString); + int SearchMovie(string movieName); + cMovieDbMovie *ReadMovie(int movieID); + cMovieDbActors *ReadActors(int movieID); + void StoreMedia(cMovieDbMovie *movie, cMovieDbActors *actors); +public: + cMovieDBScraper(string baseDir, cTVScraperDB *db, string language); + virtual ~cMovieDBScraper(void); + bool Connect(void); + void Scrap(const cEvent *event, int recordingID = 0); +}; + + +#endif //__TVSCRAPER_MOVIEDBSCRAPER_H diff --git a/thetvdbscraper/thetvdbscraper.c b/thetvdbscraper/thetvdbscraper.c new file mode 100644 index 0000000..5695ba0 --- /dev/null +++ b/thetvdbscraper/thetvdbscraper.c @@ -0,0 +1,129 @@ +#include +#include +#include +#include +#include +#include "thetvdbscraper.h" + +using namespace std; + +cTVDBScraper::cTVDBScraper(string baseDir, cTVScraperDB *db, string language) { + apiKey = "E9DBB94CA50832ED"; + baseURL = "thetvdb.com"; + this->baseDir = baseDir; + this->language = language; + this->db = db; + mirrors = NULL; +} + +cTVDBScraper::~cTVDBScraper() { + if (mirrors) + delete mirrors; +} + +void cTVDBScraper::Scrap(const cEvent *event, int recordingID) { + string seriesName = event->Title(); + if (config.enableDebug) + esyslog("tvscraper: scrapping series \"%s\"", seriesName.c_str()); + int eventID = (int)event->EventID(); + cTVDBSeries *series = ReadSeries(seriesName); + cTVDBSeriesMedia *media = NULL; + cTVDBActors *actors = NULL; + if (!series) + return; + if (series->ID() < 1) { + if (config.enableDebug) + esyslog("tvscraper: nothing found for \"%s\"", seriesName.c_str()); + return; + } + if (!db->SeriesExists(series->ID())) { + series->StoreDB(db); + media = ReadSeriesMedia(series->ID()); + actors = ReadSeriesActors(series->ID()); + if (actors) + actors->StoreDB(db, series->ID()); + StoreMedia(series, media, actors); + } + if (!recordingID) { + time_t validTill = event->EndTime(); + db->InsertEventSeries(eventID, validTill, series->ID()); + } else { + db->InsertRecording(recordingID, series->ID(), 0); + } + delete series; + if (media) + delete media; + if (actors) + delete actors; + if (config.enableDebug) + esyslog("tvscraper: \"%s\" successfully scrapped, id %d", seriesName.c_str(), series->ID()); +} + + +bool cTVDBScraper::Connect(void) { + stringstream url; + url << baseURL << "/api/" << apiKey << "/mirrors.xml"; + string mirrorsXML; + if (CurlGetUrl(url.str().c_str(), &mirrorsXML)) { + mirrors = new cTVDBMirrors(mirrorsXML); + return mirrors->ParseXML(); + } + return false; +} + +cTVDBSeries *cTVDBScraper::ReadSeries(string seriesName) { + stringstream seriesEscape; + seriesEscape << "\"" << seriesName << "\""; + stringstream url; + url << mirrors->GetMirrorXML() << "/api/GetSeries.php?seriesname=" << CurlEscape(seriesEscape.str().c_str()) << "&language=" << language.c_str(); + string seriesXML; + cTVDBSeries *series = NULL; + if (CurlGetUrl(url.str().c_str(), &seriesXML)) { + series = new cTVDBSeries(seriesXML); + series->ParseXML(); + } + return series; +} + +cTVDBSeriesMedia *cTVDBScraper::ReadSeriesMedia(int seriesID) { + stringstream url; + url << mirrors->GetMirrorXML() << "/api/" << apiKey << "/series/" << seriesID << "/banners.xml"; + string bannersXML; + cTVDBSeriesMedia *media = NULL; + if (CurlGetUrl(url.str().c_str(), &bannersXML)) { + media = new cTVDBSeriesMedia(bannersXML, language); + media->ParseXML(); + } + return media; +} + +cTVDBActors *cTVDBScraper::ReadSeriesActors(int seriesID) { + stringstream url; + url << mirrors->GetMirrorXML() << "/api/" << apiKey << "/series/" << seriesID << "/actors.xml"; + string actorsXML; + cTVDBActors *actors = NULL; + if (CurlGetUrl(url.str().c_str(), &actorsXML)) { + actors = new cTVDBActors(actorsXML, language); + actors->ParseXML(); + } + return actors; +} + +void cTVDBScraper::StoreMedia(cTVDBSeries *series, cTVDBSeriesMedia *media, cTVDBActors *actors) { + stringstream baseUrl; + baseUrl << mirrors->GetMirrorBanner() << "/banners/"; + stringstream destDir; + destDir << baseDir << "/" << series->ID() << "/"; + bool ok = CreateDirectory(destDir.str().c_str()); + if (!ok) + return; + if (series) { + series->StoreBanner(baseUrl.str(), destDir.str()); + } + if (media) { + media->Store(baseUrl.str(), destDir.str()); + } + if (actors) { + actors->Store(baseUrl.str(), destDir.str()); + } +} diff --git a/thetvdbscraper/thetvdbscraper.h b/thetvdbscraper/thetvdbscraper.h new file mode 100644 index 0000000..2659bdc --- /dev/null +++ b/thetvdbscraper/thetvdbscraper.h @@ -0,0 +1,28 @@ +#ifndef __TVSCRAPER_TVDBSCRAPER_H +#define __TVSCRAPER_TVDBSCRAPER_H + +using namespace std; + +// --- cTVDBScraper ------------------------------------------------------------- + +class cTVDBScraper { +private: + string apiKey; + string baseURL; + string baseDir; + string language; + cTVScraperDB *db; + cTVDBMirrors *mirrors; + cTVDBSeries *ReadSeries(string seriesName); + cTVDBSeriesMedia *ReadSeriesMedia(int seriesID); + cTVDBActors *ReadSeriesActors(int seriesID); + void StoreMedia(cTVDBSeries *series, cTVDBSeriesMedia *media, cTVDBActors *actors); +public: + cTVDBScraper(string baseDir, cTVScraperDB *db, string language); + virtual ~cTVDBScraper(void); + bool Connect(void); + void Scrap(const cEvent *event, int recordingID = 0); +}; + + +#endif //__TVSCRAPER_TVDBSCRAPER_H diff --git a/thetvdbscraper/tvdbactors.c b/thetvdbscraper/tvdbactors.c new file mode 100644 index 0000000..404cd38 --- /dev/null +++ b/thetvdbscraper/tvdbactors.c @@ -0,0 +1,100 @@ +#include +#include +#include +#include +#include +#include "tvdbactors.h" + +using namespace std; + +cTVDBActors::cTVDBActors(string xml, string language) { + doc = NULL; + SetXMLDoc(xml); + this->language = language; +} + +cTVDBActors::~cTVDBActors() { + xmlFreeDoc(doc); + actors.clear(); +} + +void cTVDBActors::SetXMLDoc(string xml) { + xmlInitParser(); + doc = xmlReadMemory(xml.c_str(), strlen(xml.c_str()), "noname.xml", NULL, 0); +} + +void cTVDBActors::ParseXML(void) { + if (doc == NULL) + return; + //Root Element has to be + xmlNode *node = NULL; + node = xmlDocGetRootElement(doc); + if (!(node && !xmlStrcmp(node->name, (const xmlChar *)"Actors"))) + return; + //Looping through actors + node = node->children; + xmlNode *cur_node = NULL; + for (cur_node = node; cur_node; cur_node = cur_node->next) { + if ((cur_node->type == XML_ELEMENT_NODE) && !xmlStrcmp(cur_node->name, (const xmlChar *)"Actor")) { + ReadEntry(cur_node->children); + } + } +} + +void cTVDBActors::ReadEntry(xmlNode *node) { + xmlNode *cur_node = NULL; + xmlChar *node_content; + cTVDBActor *actor = new cTVDBActor(); + for (cur_node = node; cur_node; cur_node = cur_node->next) { + if (cur_node->type == XML_ELEMENT_NODE) { + node_content = xmlNodeListGetString(doc, cur_node->xmlChildrenNode, 1); + if (!node_content) + continue; + if (!xmlStrcmp(cur_node->name, (const xmlChar *)"Image")) { + actor->path = (const char *)node_content; + } else if (!xmlStrcmp(cur_node->name, (const xmlChar *)"Name")) { + actor->name = (const char *)node_content; + } else if (!xmlStrcmp(cur_node->name, (const xmlChar *)"Role")) { + actor->role = (const char *)node_content; + } + xmlFree(node_content); + } + } + actors.push_back(actor); +} + +void cTVDBActors::StoreDB(cTVScraperDB *db, int series_id) { + int size = actors.size(); + for (int i=0; iInsertActor(series_id, actors[i]->name, actors[i]->role, strThumb.str()); + } +} + +void cTVDBActors::Store(string baseUrl, string destDir) { + int size = actors.size(); + string path; + string url; + for (int i=0; ipath; + url = strUrl.str(); + stringstream fullPath; + fullPath << destDir << "actor_" << i << ".jpg"; + path = fullPath.str(); + if (actors[i]->path.size() > 0) { + CurlGetUrlFile(url.c_str(), path.c_str()); + } + } +} + +void cTVDBActors::Dump(bool verbose) { + int size = actors.size(); + esyslog("tvscraper: read %d actor entries", size); + if (!verbose) + return; + for (int i=0; ipath.c_str(), actors[i]->name.c_str(), actors[i]->role.c_str()); + } +} \ No newline at end of file diff --git a/thetvdbscraper/tvdbactors.h b/thetvdbscraper/tvdbactors.h new file mode 100644 index 0000000..819d1d4 --- /dev/null +++ b/thetvdbscraper/tvdbactors.h @@ -0,0 +1,37 @@ +#ifndef __TVSCRAPER_TVDBACTORS_H +#define __TVSCRAPER_TVDBACTORS_H + +using namespace std; + +// --- cTVDBActor ------------------------------------------------------------- +class cTVDBActor { +public: + cTVDBActor(void) { + path = ""; + name = ""; + role = ""; + }; + string path; + string name; + string role; +}; + +// --- cTVDBActors -------------------------------------------------------- + +class cTVDBActors { +private: + xmlDoc *doc; + string language; + vector actors; + void SetXMLDoc(string xml); + void ReadEntry(xmlNode *node); +public: + cTVDBActors(string xml, string language); + virtual ~cTVDBActors(void); + void ParseXML(void); + void StoreDB(cTVScraperDB *db, int series_id); + void Store(string baseUrl, string destDir); + void Dump(bool verbose); +}; + +#endif //__TVSCRAPER_TVDBACTORS_H diff --git a/thetvdbscraper/tvdbmedia.c b/thetvdbscraper/tvdbmedia.c new file mode 100644 index 0000000..f9672b6 --- /dev/null +++ b/thetvdbscraper/tvdbmedia.c @@ -0,0 +1,149 @@ +#include +#include +#include +#include +#include +#include "tvdbmedia.h" + +using namespace std; + +cTVDBSeriesMedia::cTVDBSeriesMedia(string xml, string language) { + doc = NULL; + SetXMLDoc(xml); + this->language = language; +} + +cTVDBSeriesMedia::~cTVDBSeriesMedia() { + xmlFreeDoc(doc); + medias.clear(); +} + +void cTVDBSeriesMedia::SetXMLDoc(string xml) { + xmlInitParser(); + doc = xmlReadMemory(xml.c_str(), strlen(xml.c_str()), "noname.xml", NULL, 0); +} + +void cTVDBSeriesMedia::ParseXML(void) { + if (doc == NULL) + return; + //Root Element has to be + xmlNode *node = NULL; + node = xmlDocGetRootElement(doc); + if (!(node && !xmlStrcmp(node->name, (const xmlChar *)"Banners"))) + return; + //Looping through banners + node = node->children; + xmlNode *cur_node = NULL; + for (cur_node = node; cur_node; cur_node = cur_node->next) { + if ((cur_node->type == XML_ELEMENT_NODE) && !xmlStrcmp(cur_node->name, (const xmlChar *)"Banner")) { + ReadEntry(cur_node->children); + } + } +} + +void cTVDBSeriesMedia::ReadEntry(xmlNode *node) { + xmlNode *cur_node = NULL; + xmlChar *node_content; + cTVDBMedia *media = new cTVDBMedia(); + for (cur_node = node; cur_node; cur_node = cur_node->next) { + if (cur_node->type == XML_ELEMENT_NODE) { + node_content = xmlNodeListGetString(doc, cur_node->xmlChildrenNode, 1); + if (!node_content) + continue; + if (!xmlStrcmp(cur_node->name, (const xmlChar *)"BannerPath")) { + media->path = (const char *)node_content; + } else if (!xmlStrcmp(cur_node->name, (const xmlChar *)"BannerType")) { + if (!xmlStrcmp(node_content, (const xmlChar *)"poster")) + media->type = mediaPoster; + else if (!xmlStrcmp(node_content, (const xmlChar *)"fanart")) + media->type = mediaFanart; + else if (!xmlStrcmp(node_content, (const xmlChar *)"banner")) + media->type = mediaBanner; + else if (!xmlStrcmp(node_content, (const xmlChar *)"season")) + media->type = mediaSeason; + else + media->type = mediaUnknown; + } else if (!xmlStrcmp(cur_node->name, (const xmlChar *)"Season")) { + media->season = atoi((const char *)node_content); + } else if (!xmlStrcmp(cur_node->name, (const xmlChar *)"Language")) { + media->language = (const char *)node_content; + } + xmlFree(node_content); + } + } + if (media->type == mediaUnknown) + delete media; + else if ((media->type == mediaSeason) && (media->language).compare(language)) + delete media; + else + medias.push_back(media); +} + +void cTVDBSeriesMedia::Store(string baseUrl, string destDir) { + int size = medias.size(); + string path; + string url; + int maxPix = 3; + int numFanart = 0; + int numPoster = 0; + int numSeason = 0; + int numBanner = 0; + for (int i=0; ipath.size() == 0) + continue; + stringstream strUrl; + strUrl << baseUrl << medias[i]->path; + url = strUrl.str(); + switch (medias[i]->type) { + case mediaFanart: { + if (numFanart >= maxPix) + continue; + stringstream fullPath; + fullPath << destDir << "fanart_" << numFanart << ".jpg"; + path = fullPath.str(); + numFanart++; + break; } + case mediaPoster: { + if (numPoster >= maxPix) + continue; + stringstream fullPath; + fullPath << destDir << "poster_" << numPoster << ".jpg"; + path = fullPath.str(); + numPoster++; + break; } + case mediaSeason: { + stringstream fullPath; + if (medias[i]->season > 0) { + fullPath << destDir << "season_poster_" << medias[i]->season << ".jpg"; + } else { + if (numSeason >= maxPix) + continue; + fullPath << destDir << "season_" << numSeason << ".jpg"; + numSeason++; + } + path = fullPath.str(); + break; } + case mediaBanner: { + if (numBanner > 9) + continue; + stringstream fullPath; + fullPath << destDir << "banner_" << numBanner << ".jpg"; + path = fullPath.str(); + numBanner++; + break; } + default: + break; + } + CurlGetUrlFile(url.c_str(), path.c_str()); + } +} + +void cTVDBSeriesMedia::Dump(bool verbose) { + int size = medias.size(); + esyslog("tvscraper: read %d banner entries", size); + if (!verbose) + return; + for (int i=0; itype, medias[i]->path.c_str(), medias[i]->language.c_str(), medias[i]->season); + } +} \ No newline at end of file diff --git a/thetvdbscraper/tvdbmedia.h b/thetvdbscraper/tvdbmedia.h new file mode 100644 index 0000000..b9e9b0c --- /dev/null +++ b/thetvdbscraper/tvdbmedia.h @@ -0,0 +1,45 @@ +#ifndef __TVSCRAPER_TVDBMEDIA_H +#define __TVSCRAPER_TVDBMEDIA_H + +using namespace std; + +enum mediaType { + mediaUnknown, + mediaPoster, + mediaFanart, + mediaSeason, + mediaBanner, +}; + +// --- cTVDBMedia ------------------------------------------------------------- +class cTVDBMedia { +public: + cTVDBMedia(void) { + path = ""; + language = ""; + season = 0; + }; + mediaType type; + string path; + string language; + int season; +}; + +// --- cTVDBSeriesMedia -------------------------------------------------------- + +class cTVDBSeriesMedia { +private: + xmlDoc *doc; + string language; + vector medias; + void SetXMLDoc(string xml); + void ReadEntry(xmlNode *node); +public: + cTVDBSeriesMedia(string xml, string language); + virtual ~cTVDBSeriesMedia(void); + void ParseXML(void); + void Store(string baseUrl, string destDir); + void Dump(bool verbose); +}; + +#endif //__TVSCRAPER_TVDBMEDIA_H diff --git a/thetvdbscraper/tvdbmirrors.c b/thetvdbscraper/tvdbmirrors.c new file mode 100644 index 0000000..d0849c2 --- /dev/null +++ b/thetvdbscraper/tvdbmirrors.c @@ -0,0 +1,97 @@ +#include +#include +#include +#include +#include +#include "tvdbmirrors.h" + +using namespace std; + +cTVDBMirrors::cTVDBMirrors(string xml) { + doc = NULL; + SetXMLDoc(xml); +} + +cTVDBMirrors::~cTVDBMirrors() { + xmlFreeDoc(doc); +} + +void cTVDBMirrors::SetXMLDoc(string xml) { + xmlInitParser(); + doc = xmlReadMemory(xml.c_str(), strlen(xml.c_str()), "noname.xml", NULL, 0); +} + +string cTVDBMirrors::GetMirrorXML(void) { + if (xmlmirrors.size() == 0) + return ""; + int randMirror = rand() % xmlmirrors.size(); + return xmlmirrors[randMirror]; +} +string cTVDBMirrors::GetMirrorBanner(void) { + if (bannermirrors.size() == 0) + return ""; + int randMirror = rand() % bannermirrors.size(); + return bannermirrors[randMirror]; +} + +string cTVDBMirrors::GetMirrorZip(void) { + if (zipmirrors.size() == 0) + return ""; + int randMirror = rand() % zipmirrors.size(); + return zipmirrors[randMirror]; +} + +bool cTVDBMirrors::ParseXML(void) { + if (doc == NULL) + return false; + //Root Element has to be + xmlNode *node = NULL; + node = xmlDocGetRootElement(doc); + if (!(node && !xmlStrcmp(node->name, (const xmlChar *)"Mirrors"))) + return false; + //Loop through + node = node->children; + xmlNode *cur_node = NULL; + bool ok = false; + for (cur_node = node; cur_node; cur_node = cur_node->next) { + if ((cur_node->type == XML_ELEMENT_NODE) && !xmlStrcmp(cur_node->name, (const xmlChar *)"Mirror")) { + ok = ReadEntry(cur_node->children); + } + } + return ok; +} + +bool cTVDBMirrors::ReadEntry(xmlNode *node) { + xmlNode *cur_node = NULL; + xmlChar *node_content; + string path = ""; + int typemask = 0; + for (cur_node = node; cur_node; cur_node = cur_node->next) { + if (cur_node->type == XML_ELEMENT_NODE) { + node_content = xmlNodeListGetString(doc, cur_node->xmlChildrenNode, 1); + if (!node_content) + continue; + if (!xmlStrcmp(cur_node->name, (const xmlChar *)"mirrorpath")) { + path = (const char *)node_content; + } else if (!xmlStrcmp(cur_node->name, (const xmlChar *)"typemask")) { + typemask = atoi((const char *)node_content); + } + xmlFree(node_content); + } + } + return CreateMirror(path, typemask); +} + +bool cTVDBMirrors::CreateMirror(string path, int typemask) { + if (path.size() < 1) + return false; + if (typemask < 1 || typemask > 7) + return false; + if (typemask & 1) + xmlmirrors.push_back(path); + if (typemask & 2) + bannermirrors.push_back(path); + if (typemask & 4) + zipmirrors.push_back(path); + return true; +} \ No newline at end of file diff --git a/thetvdbscraper/tvdbmirrors.h b/thetvdbscraper/tvdbmirrors.h new file mode 100644 index 0000000..67ad92c --- /dev/null +++ b/thetvdbscraper/tvdbmirrors.h @@ -0,0 +1,27 @@ +#ifndef __TVSCRAPER_TVDBMIRRORS_H +#define __TVSCRAPER_TVDBMIRRORS_H + +using namespace std; + +// --- cTVDBMirrors ------------------------------------------------------------- + +class cTVDBMirrors { +private: + xmlDoc *doc; + vector xmlmirrors; + vector bannermirrors; + vector zipmirrors; + void SetXMLDoc(string xml); + bool ReadEntry(xmlNode *node); + bool CreateMirror(string path, int typemask); +public: + cTVDBMirrors(string xml); + virtual ~cTVDBMirrors(void); + bool ParseXML(void); + string GetMirrorXML(void); + string GetMirrorBanner(void); + string GetMirrorZip(void); +}; + + +#endif //__TVSCRAPER_TVDBMIRRORS_H diff --git a/thetvdbscraper/tvdbseries.c b/thetvdbscraper/tvdbseries.c new file mode 100644 index 0000000..b679e5e --- /dev/null +++ b/thetvdbscraper/tvdbseries.c @@ -0,0 +1,92 @@ +#include +#include +#include +#include +#include +#include "tvdbseries.h" + +using namespace std; + +cTVDBSeries::cTVDBSeries(string xml) { + doc = NULL; + SetXMLDoc(xml); + seriesID = 0; + name = ""; + banner = ""; + overview = ""; + imbdid = ""; +} + +cTVDBSeries::~cTVDBSeries() { + xmlFreeDoc(doc); +} + +void cTVDBSeries::SetXMLDoc(string xml) { + xmlInitParser(); + doc = xmlReadMemory(xml.c_str(), strlen(xml.c_str()), "noname.xml", NULL, 0); +} + +void cTVDBSeries::ParseXML(void) { + if (doc == NULL) + return; + //Root Element has to be + xmlNode *node = NULL; + node = xmlDocGetRootElement(doc); + if (!(node && !xmlStrcmp(node->name, (const xmlChar *)"Data"))) + return; + //Searching for + node = node->children; + xmlNode *cur_node = NULL; + for (cur_node = node; cur_node; cur_node = cur_node->next) { + if ((cur_node->type == XML_ELEMENT_NODE) && !xmlStrcmp(cur_node->name, (const xmlChar *)"Series")) { + node = cur_node; + break; + } else { + node = NULL; + } + } + if (!node) + return; + //now read the first series + node = node->children; + xmlChar *node_content; + for (cur_node = node; cur_node; cur_node = cur_node->next) { + if (cur_node->type == XML_ELEMENT_NODE) { + node_content = xmlNodeListGetString(doc, cur_node->xmlChildrenNode, 1); + if (!node_content) + continue; + if (!xmlStrcmp(cur_node->name, (const xmlChar *)"seriesid")) { + seriesID = atoi((const char *)node_content); + } else if (!xmlStrcmp(cur_node->name, (const xmlChar *)"SeriesName")) { + name = (const char *)node_content; + } else if (!xmlStrcmp(cur_node->name, (const xmlChar *)"Overview")) { + overview = (const char *)node_content; + } else if (!xmlStrcmp(cur_node->name, (const xmlChar *)"banner")) { + banner = (const char *)node_content; + } else if (!xmlStrcmp(cur_node->name, (const xmlChar *)"IMDB_ID")) { + imbdid = (const char *)node_content; + } + xmlFree(node_content); + } + } +} + +void cTVDBSeries::StoreDB(cTVScraperDB *db) { + db->InsertSeries(seriesID, name, overview); +} + +void cTVDBSeries::StoreBanner(string baseUrl, string destDir) { + if (banner.size() == 0) + return; + stringstream strUrl; + strUrl << baseUrl << banner; + string url = strUrl.str(); + stringstream fullPath; + fullPath << destDir << "banner.jpg"; + string path = fullPath.str(); + CurlGetUrlFile(url.c_str(), path.c_str()); +} + +void cTVDBSeries::Dump() { + esyslog("tvscraper: series %s, id: %d, overview %s, imdb %s", name.c_str(), seriesID, overview.c_str(), imbdid.c_str()); +} diff --git a/thetvdbscraper/tvdbseries.h b/thetvdbscraper/tvdbseries.h new file mode 100644 index 0000000..98fb75d --- /dev/null +++ b/thetvdbscraper/tvdbseries.h @@ -0,0 +1,29 @@ +#ifndef __TVSCRAPER_TVDBSERIES_H +#define __TVSCRAPER_TVDBSERIES_H + +using namespace std; + +// --- cTVDBSeries ------------------------------------------------------------- + +class cTVDBSeries { +private: + xmlDoc *doc; + int seriesID; + string name; + string banner; + string overview; + string imbdid; + void SetXMLDoc(string xml); +public: + cTVDBSeries(string xml); + virtual ~cTVDBSeries(void); + void ParseXML(void); + int ID(void) { return seriesID; }; + const char *Name(void) { return name.c_str(); }; + void StoreDB(cTVScraperDB *db); + void StoreBanner(string baseUrl, string destDir); + void Dump(); +}; + + +#endif //__TVSCRAPER_TVDBSERIES_H diff --git a/tools/curlfuncs.cpp b/tools/curlfuncs.cpp new file mode 100644 index 0000000..de4c72f --- /dev/null +++ b/tools/curlfuncs.cpp @@ -0,0 +1,235 @@ +/* +Copyright (c) 2002, Mayukh Bose +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +* Neither the name of Mayukh Bose nor the names of other +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +/* + Change History: + 11/23/2004 - Removed the #include line because I didn't + need it. Wonder why I had it there in the first place :). + 10/20/2004 - Publicly released this code. +*/ +#include +#include +#include +#include +#include "curlfuncs.h" + +#ifndef TRUE +#define TRUE 1 +#endif + +using namespace std; + +// Local function prototypes +int CurlDoPost(const char *url, string *sOutput, const string &sReferer, + struct curl_httppost *formpost, struct curl_slist *headerlist); + +namespace curlfuncs { + string sBuf; + bool bInitialized = false; + CURL *curl = NULL; +} + +size_t collect_data(void *ptr, size_t size, size_t nmemb, void *stream) +{ + string sTmp; + register size_t actualsize = size * nmemb; + if ((FILE *)stream == NULL) { + sTmp.assign((char *)ptr, actualsize); + curlfuncs::sBuf += sTmp; + } + else { + size_t xxx = fwrite(ptr, size, nmemb, (FILE *)stream); + } + return actualsize; +} + +inline void InitCurlLibraryIfNeeded() +{ + if (!curlfuncs::bInitialized) { + curl_global_init(CURL_GLOBAL_ALL); + curlfuncs::curl = curl_easy_init(); + if (!curlfuncs::curl) + throw string("Could not create new curl instance"); + curl_easy_setopt(curlfuncs::curl, CURLOPT_NOPROGRESS, 1); // Do not show progress + curl_easy_setopt(curlfuncs::curl, CURLOPT_WRITEFUNCTION, collect_data); + curl_easy_setopt(curlfuncs::curl, CURLOPT_WRITEDATA, 0); // Set option to write to string + curl_easy_setopt(curlfuncs::curl, CURLOPT_FOLLOWLOCATION, TRUE); + curl_easy_setopt(curlfuncs::curl, CURLOPT_USERAGENT, "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; Mayukh's libcurl wrapper http://www.mayukhbose.com/)"); + curlfuncs::bInitialized = true; + } +} + +int CurlGetUrl(const char *url, string *sOutput, const string &sReferer) +{ + InitCurlLibraryIfNeeded(); + + curl_easy_setopt(curlfuncs::curl, CURLOPT_URL, url); // Set the URL to get + if (sReferer != "") + curl_easy_setopt(curlfuncs::curl, CURLOPT_REFERER, sReferer.c_str()); + curl_easy_setopt(curlfuncs::curl, CURLOPT_HTTPGET, TRUE); + curl_easy_setopt(curlfuncs::curl, CURLOPT_WRITEDATA, 0); // Set option to write to string + curlfuncs::sBuf = ""; + if (curl_easy_perform(curlfuncs::curl) == 0) + *sOutput = curlfuncs::sBuf; + else { + // We have an error here mate! + *sOutput = ""; + return 0; + } + + return 1; +} + +int CurlGetUrlFile(const char *url, const char *filename, const string &sReferer) +{ + int nRet = 0; + InitCurlLibraryIfNeeded(); + + // Point the output to a file + FILE *fp; + if ((fp = fopen(filename, "w")) == NULL) + return 0; + + curl_easy_setopt(curlfuncs::curl, CURLOPT_WRITEDATA, fp); // Set option to write to file + curl_easy_setopt(curlfuncs::curl, CURLOPT_URL, url); // Set the URL to get + if (sReferer != "") + curl_easy_setopt(curlfuncs::curl, CURLOPT_REFERER, sReferer.c_str()); + curl_easy_setopt(curlfuncs::curl, CURLOPT_HTTPGET, TRUE); + if (curl_easy_perform(curlfuncs::curl) == 0) + nRet = 1; + else + nRet = 0; + + curl_easy_setopt(curlfuncs::curl, CURLOPT_WRITEDATA, NULL); // Set option back to default (string) + fclose(fp); + return nRet; +} + +int CurlPostUrl(const char *url, const string &sPost, string *sOutput, const string &sReferer) +{ + InitCurlLibraryIfNeeded(); + + int retval = 1; + string::size_type nStart = 0, nEnd, nPos; + string sTmp, sName, sValue; + struct curl_httppost *formpost=NULL; + struct curl_httppost *lastptr=NULL; + struct curl_slist *headerlist=NULL; + + // Add the POST variables here + while ((nEnd = sPost.find("##", nStart)) != string::npos) { + sTmp = sPost.substr(nStart, nEnd - nStart); + if ((nPos = sTmp.find("=")) == string::npos) + return 0; + sName = sTmp.substr(0, nPos); + sValue = sTmp.substr(nPos+1); + curl_formadd(&formpost, &lastptr, CURLFORM_COPYNAME, sName.c_str(), CURLFORM_COPYCONTENTS, sValue.c_str(), CURLFORM_END); + nStart = nEnd + 2; + } + sTmp = sPost.substr(nStart); + if ((nPos = sTmp.find("=")) == string::npos) + return 0; + sName = sTmp.substr(0, nPos); + sValue = sTmp.substr(nPos+1); + curl_formadd(&formpost, &lastptr, CURLFORM_COPYNAME, sName.c_str(), CURLFORM_COPYCONTENTS, sValue.c_str(), CURLFORM_END); + + retval = CurlDoPost(url, sOutput, sReferer, formpost, headerlist); + + curl_formfree(formpost); + curl_slist_free_all(headerlist); + return retval; +} + +int CurlPostRaw(const char *url, const string &sPost, string *sOutput, const string &sReferer) +{ + InitCurlLibraryIfNeeded(); + + int retval; + struct curl_httppost *formpost=NULL; + struct curl_slist *headerlist=NULL; + + curl_easy_setopt(curlfuncs::curl, CURLOPT_POSTFIELDS, sPost.c_str()); + curl_easy_setopt(curlfuncs::curl, CURLOPT_POSTFIELDSIZE, 0); //FIXME: Should this be the size instead, in case this is binary string? + + retval = CurlDoPost(url, sOutput, sReferer, formpost, headerlist); + + curl_formfree(formpost); + curl_slist_free_all(headerlist); + return retval; +} + +int CurlDoPost(const char *url, string *sOutput, const string &sReferer, + struct curl_httppost *formpost, struct curl_slist *headerlist) +{ + headerlist = curl_slist_append(headerlist, "Expect:"); + + // Now do the form post + curl_easy_setopt(curlfuncs::curl, CURLOPT_URL, url); + if (sReferer != "") + curl_easy_setopt(curlfuncs::curl, CURLOPT_REFERER, sReferer.c_str()); + curl_easy_setopt(curlfuncs::curl, CURLOPT_HTTPPOST, formpost); + + curl_easy_setopt(curlfuncs::curl, CURLOPT_WRITEDATA, 0); // Set option to write to string + curlfuncs::sBuf = ""; + if (curl_easy_perform(curlfuncs::curl) == 0) { + *sOutput = curlfuncs::sBuf; + return 1; + } + else { + // We have an error here mate! + *sOutput = ""; + return 0; + } +} + +void FreeCurlLibrary(void) +{ + if (curlfuncs::curl) + curl_easy_cleanup(curlfuncs::curl); + curl_global_cleanup(); + curlfuncs::bInitialized = false; +} + +int CurlSetCookieFile(char *filename) +{ + InitCurlLibraryIfNeeded(); + if (curl_easy_setopt(curlfuncs::curl, CURLOPT_COOKIEFILE, filename) != 0) + return 0; + if (curl_easy_setopt(curlfuncs::curl, CURLOPT_COOKIEJAR, filename) != 0) + return 0; + return 1; +} + +char *CurlEscape(const char *url) { + InitCurlLibraryIfNeeded(); + return curl_escape(url , strlen(url)); +} diff --git a/tools/curlfuncs.h b/tools/curlfuncs.h new file mode 100644 index 0000000..e0f76cf --- /dev/null +++ b/tools/curlfuncs.h @@ -0,0 +1,45 @@ +/* +Copyright (c) 2002, Mayukh Bose +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +* Neither the name of Mayukh Bose nor the names of other +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef __CURLFUNCS_H_20020513__ +#define __CURLFUNCS_H_20020513__ +#include + using namespace std; + +int CurlGetUrl(const char *url, string *sOutput, const string &sReferer=""); +int CurlGetUrlFile(const char *url, const char *filename, const string &sReferer=""); +void FreeCurlLibrary(void); +int CurlSetCookieFile(char *filename); +int CurlPostUrl(const char *url, const string &sPost, string *sOutput, const string &sReferer = ""); +int CurlPostRaw(const char *url, const string &sPost, string *sOutput, const string &sReferer = ""); +char *CurlEscape(const char *url); +#endif diff --git a/tools/filesystem.c b/tools/filesystem.c new file mode 100644 index 0000000..724b28c --- /dev/null +++ b/tools/filesystem.c @@ -0,0 +1,44 @@ +#include +#include +#include +#include + +using namespace std; + +bool CreateDirectory(string dir) { + mkdir(dir.c_str(), 0775); + //check if successfull + DIR *pDir; + bool exists = false; + pDir = opendir(dir.c_str()); + if (pDir != NULL) { + exists = true; + closedir(pDir); + } + return exists; +} + +bool FileExists(string filename) { + ifstream ifile(filename.c_str()); + if (ifile) { + //a valid image should be larger then 500 bytes + ifile.seekg (0, ifile.end); + int length = ifile.tellg(); + if (length > 500) + return true; + } + return false; +} + +bool CheckDirExists(const char* dirName) { + struct statfs statfsbuf; + if (statfs(dirName,&statfsbuf)==-1) return false; + if ((statfsbuf.f_type!=0x01021994) && (statfsbuf.f_type!=0x28cd3d45)) return false; + if (access(dirName,R_OK|W_OK)==-1) return false; + return true; + +} + +void DeleteFile(string filename) { + remove(filename.c_str()); +} \ No newline at end of file diff --git a/tools/fuzzy.c b/tools/fuzzy.c new file mode 100644 index 0000000..188f017 --- /dev/null +++ b/tools/fuzzy.c @@ -0,0 +1,70 @@ +#include +#include +#include +#include +#include +#include + +template +size_t +seq_distance(const T& seq1, const T& seq2, const C& cost, + const typename T::value_type& empty = typename T::value_type()) { + const size_t size1 = seq1.size(); + const size_t size2 = seq2.size(); + + std::vector curr_col(size2 + 1); + std::vector prev_col(size2 + 1); + + // Prime the previous column for use in the following loop: + prev_col[0] = 0; + for (size_t idx2 = 0; idx2 < size2; ++idx2) { + prev_col[idx2 + 1] = prev_col[idx2] + cost(empty, seq2[idx2]); + } + + for (size_t idx1 = 0; idx1 < size1; ++idx1) { + curr_col[0] = curr_col[0] + cost(seq1[idx1], empty); + + for (size_t idx2 = 0; idx2 < size2; ++idx2) { + curr_col[idx2 + 1] = std::min(std::min( + curr_col[idx2] + cost(empty, seq2[idx2]), + prev_col[idx2 + 1] + cost(seq1[idx1], empty)), + prev_col[idx2] + cost(seq1[idx1], seq2[idx2])); + } + + curr_col.swap(prev_col); + curr_col[0] = prev_col[0]; + } + + return prev_col[size2]; +} + +size_t +letter_distance(char letter1, char letter2) { + return letter1 != letter2 ? 1 : 0; +} + +size_t +word_distance(const std::string& word1, const std::string& word2) { + return seq_distance(word1, word2, &letter_distance); +} + +size_t +sentence_distance(const std::string& sentence1, const std::string& sentence2) { + std::vector words1; + std::vector words2; + std::istringstream iss1(sentence1); + std::istringstream iss2(sentence2); + for(std::istream_iterator it(iss1), end; ; ) { + words1.push_back(*it); + if(++it == end) + break; + words1.push_back(" "); + } + for(std::istream_iterator it(iss2), end; ; ) { + words2.push_back(*it); + if(++it == end) + break; + words2.push_back(" "); + } + return seq_distance(words1, words2, &word_distance); +} \ No newline at end of file diff --git a/tools/splitstring.c b/tools/splitstring.c new file mode 100644 index 0000000..14af577 --- /dev/null +++ b/tools/splitstring.c @@ -0,0 +1,32 @@ +using namespace std; + +class splitstring : public string { + vector flds; +public: + splitstring(const char *s) : string(s) { }; + vector& split(char delim, int rep=0); +}; + +// split: receives a char delimiter; returns a vector of strings +// By default ignores repeated delimiters, unless argument rep == 1. +vector& splitstring::split(char delim, int rep) { + if (!flds.empty()) flds.clear(); // empty vector if necessary + string work = data(); + string buf = ""; + int i = 0; + while (i < work.length()) { + if (work[i] != delim) + buf += work[i]; + else if (rep == 1) { + flds.push_back(buf); + buf = ""; + } else if (buf.length() > 0) { + flds.push_back(buf); + buf = ""; + } + i++; + } + if (!buf.empty()) + flds.push_back(buf); + return flds; +} diff --git a/tvscraper.c b/tvscraper.c new file mode 100644 index 0000000..616af73 --- /dev/null +++ b/tvscraper.c @@ -0,0 +1,223 @@ +/* + * tvscraper.c: A plugin for the Video Disk Recorder + * + * See the README file for copyright information and how to reach the author. + * + * $Id$ + */ +#include +#include +#include "tools/curlfuncs.cpp" +#include "tools/filesystem.c" +#include "tools/fuzzy.c" +#include "tools/splitstring.c" +#include "config.c" +cTVScraperConfig config; +#include "tvscraperdb.c" +#include "thetvdbscraper/tvdbmirrors.c" +#include "thetvdbscraper/tvdbseries.c" +#include "thetvdbscraper/tvdbmedia.c" +#include "thetvdbscraper/tvdbactors.c" +#include "thetvdbscraper/thetvdbscraper.c" +#include "themoviedbscraper/moviedbmovie.c" +#include "themoviedbscraper/moviedbactors.c" +#include "themoviedbscraper/themoviedbscraper.c" +#include "worker.c" +#include "services.h" +#include "imageserver.c" +#include "setup.c" + +static const char *VERSION = "0.0.1"; +static const char *DESCRIPTION = "Scraping movie and series info"; +static const char *MAINMENUENTRY = "TV Scraper"; + +class cPluginTvscraper : public cPlugin { +private: + bool cacheDirSet; + cTVScraperDB *db; + cTVScraperWorker *workerThread; + cImageServer *imageServer; +public: + cPluginTvscraper(void); + virtual ~cPluginTvscraper(); + virtual const char *Version(void) { return VERSION; } + virtual const char *Description(void) { return DESCRIPTION; } + virtual const char *CommandLineHelp(void); + virtual bool ProcessArgs(int argc, char *argv[]); + virtual bool Initialize(void); + virtual bool Start(void); + virtual void Stop(void); + virtual void Housekeeping(void); + virtual void MainThreadHook(void); + virtual cString Active(void); + virtual time_t WakeupTime(void); + virtual const char *MainMenuEntry(void) { return NULL; } + virtual cOsdObject *MainMenuAction(void); + virtual cMenuSetupPage *SetupMenu(void); + virtual bool SetupParse(const char *Name, const char *Value); + virtual bool Service(const char *Id, void *Data = NULL); + virtual const char **SVDRPHelpPages(void); + virtual cString SVDRPCommand(const char *Command, const char *Option, int &ReplyCode); +}; + +cPluginTvscraper::cPluginTvscraper(void) { +} + +cPluginTvscraper::~cPluginTvscraper() { +} + +const char *cPluginTvscraper::CommandLineHelp(void) { + return " -d , --dir= Set directory where database and images are stored\n"; +} + +bool cPluginTvscraper::ProcessArgs(int argc, char *argv[]) { + static const struct option long_options[] = { + { "dir", required_argument, NULL, 'd' }, + { 0, 0, 0, 0 } + }; + + int c; + cacheDirSet = false; + while ((c = getopt_long(argc, argv, "d:", long_options, NULL)) != -1) { + switch (c) { + case 'd': + cacheDirSet = true; + config.SetBaseDir(optarg); + break; + default: + return false; + } + } + return true; +} + +bool cPluginTvscraper::Initialize(void) { + return true; +} + +bool cPluginTvscraper::Start(void) { + if (!cacheDirSet) { + config.SetBaseDir(cPlugin::CacheDirectory(PLUGIN_NAME_I18N)); + } + db = new cTVScraperDB(); + if (!db->Connect()) { + esyslog("tvscraper: could not connect to Database. Aborting!"); + return false; + }; + imageServer = new cImageServer(db); + workerThread = new cTVScraperWorker(db); + workerThread->SetDirectories(); + workerThread->SetLanguage(); + workerThread->Start(); + return true; +} + +void cPluginTvscraper::Stop(void) { + while (workerThread->Active()) { + workerThread->Stop(); + } + delete workerThread; + delete imageServer; + delete db; +} + +void cPluginTvscraper::Housekeeping(void) { +} + +void cPluginTvscraper::MainThreadHook(void) { +} + +cString cPluginTvscraper::Active(void) { + return NULL; +} + +time_t cPluginTvscraper::WakeupTime(void) { + return 0; +} + +cOsdObject *cPluginTvscraper::MainMenuAction(void) { + return NULL; +} + +cMenuSetupPage *cPluginTvscraper::SetupMenu(void) { + return new cTVScraperSetup(workerThread); +} + +bool cPluginTvscraper::SetupParse(const char *Name, const char *Value) { + return config.SetupParse(Name, Value); +} + +bool cPluginTvscraper::Service(const char *Id, void *Data) { + if (strcmp(Id, "TVScraperGetPosterOrBanner") == 0) { + if (Data == NULL) + return false; + TVScraperGetPosterOrBanner* call = (TVScraperGetPosterOrBanner*) Data; + if (!call->event) + return false; + scrapType type = imageServer->GetScrapType(call->event); + if (type == scrapSeries) + call->type = typeSeries; + else if (type == scrapMovie) + call->type = typeMovie; + else + call->type = typeNone; + int id = imageServer->GetID(call->event->EventID(), type, false); + if (id > 0) { + call->media = imageServer->GetPosterOrBanner(id, type); + return true; + } + return false; + } + if (strcmp(Id, "TVScraperGetPoster") == 0) { + if (Data == NULL) + return false; + TVScraperGetPoster* call = (TVScraperGetPoster*) Data; + if (!call->event) + return false; + scrapType type = imageServer->GetScrapType(call->event); + int id = imageServer->GetID(call->event->EventID(), type, call->isRecording); + if (id > 0) { + call->media = imageServer->GetPoster(id, type); + return true; + } + return false; + } + if (strcmp(Id, "TVScraperGetFullInformation") == 0) { + if (Data == NULL) + return false; + TVScraperGetFullInformation* call = (TVScraperGetFullInformation*) Data; + if (!call->event) + return false; + + scrapType type = imageServer->GetScrapType(call->event); + int id = imageServer->GetID(call->event->EventID(), type, call->isRecording); + + if (id == 0) + return false; + + if (type == scrapSeries) { + call->type = typeSeries; + call->banner = imageServer->GetBanner(id); + } else if (type == scrapMovie) { + call->type = typeMovie; + } else { + call->type = typeNone; + } + call->posters = imageServer->GetPosters(id, type); + call->fanart = imageServer->GetFanart(id, type); + call->actors = imageServer->GetActors(id, type); + call->description = imageServer->GetDescription(id, type); + return true; + } + return false; +} + +const char **cPluginTvscraper::SVDRPHelpPages(void) { + return NULL; +} + +cString cPluginTvscraper::SVDRPCommand(const char *Command, const char *Option, int &ReplyCode) { + return NULL; +} + +VDRPLUGINCREATOR(cPluginTvscraper); diff --git a/tvscraperdb.c b/tvscraperdb.c new file mode 100644 index 0000000..e89be33 --- /dev/null +++ b/tvscraperdb.c @@ -0,0 +1,628 @@ +#include +#include +#include +#include +#include "tvscraperdb.h" + +using namespace std; + +cTVScraperDB::cTVScraperDB(void) { + db = NULL; + string memHD = "/dev/shm/"; + inMem = CheckDirExists(memHD.c_str()); + if (inMem) { + stringstream sstrDbFileMem; + sstrDbFileMem << memHD << "tvscraper.db"; + dbPathMem = sstrDbFileMem.str(); + } + stringstream sstrDbFile; + sstrDbFile << config.GetBaseDir() << "/tvscraper.db"; + dbPathPhys = sstrDbFile.str(); +} + +cTVScraperDB::~cTVScraperDB() { + sqlite3_close(db); +} + +vector > cTVScraperDB::Query(string query) { + sqlite3_stmt *statement; + vector > results; + if(sqlite3_prepare_v2(db, query.c_str(), -1, &statement, 0) == SQLITE_OK) { + int cols = sqlite3_column_count(statement); + int result = 0; + while(true) { + result = sqlite3_step(statement); + if(result == SQLITE_ROW) { + vector values; + for(int col = 0; col < cols; col++) { + values.push_back((char*)sqlite3_column_text(statement, col)); + } + results.push_back(values); + } else { + break; + } + } + sqlite3_finalize(statement); + } + string error = sqlite3_errmsg(db); + if(error != "not an error") { + esyslog("tvscraper: query failed: %s , error: %s", query.c_str(), error.c_str()); + } + return results; +} + +bool cTVScraperDB::Connect(void) { + if (inMem) { + if (sqlite3_open(dbPathMem.c_str(),&db)!=SQLITE_OK) { + esyslog("tvscraper: failed to open or create %s", dbPathMem.c_str()); + return false; + } + esyslog("tvscraper: connecting to db %s", dbPathMem.c_str()); + int rc = LoadOrSaveDb(db, dbPathPhys.c_str(), false); + if (rc != SQLITE_OK) { + esyslog("tvscraper: error while loading data from %s, errorcode %d", dbPathPhys.c_str(), rc); + return false; + } + } else { + if (sqlite3_open(dbPathPhys.c_str(),&db)!=SQLITE_OK) { + esyslog("tvscraper: failed to open or create %s", dbPathPhys.c_str()); + return false; + } + esyslog("tvscraper: connecting to db %s", dbPathPhys.c_str()); + } + CreateTables(); + return true; +} + +void cTVScraperDB::BackupToDisc(void) { + if (inMem) { + LoadOrSaveDb(db, dbPathPhys.c_str(), true); + } +} + +int cTVScraperDB::LoadOrSaveDb(sqlite3 *pInMemory, const char *zFilename, int isSave) { + int rc; /* Function return code */ + sqlite3 *pFile; /* Database connection opened on zFilename */ + sqlite3_backup *pBackup; /* Backup object used to copy data */ + sqlite3 *pTo; /* Database to copy to (pFile or pInMemory) */ + sqlite3 *pFrom; /* Database to copy from (pFile or pInMemory) */ + + rc = sqlite3_open(zFilename, &pFile); + if( rc==SQLITE_OK ){ + pFrom = (isSave ? pInMemory : pFile); + pTo = (isSave ? pFile : pInMemory); + pBackup = sqlite3_backup_init(pTo, "main", pFrom, "main"); + if( pBackup ){ + (void)sqlite3_backup_step(pBackup, -1); + (void)sqlite3_backup_finish(pBackup); + } + rc = sqlite3_errcode(pTo); + } + + (void)sqlite3_close(pFile); + return rc; +} + +bool cTVScraperDB::CreateTables(void) { + stringstream sql; + sql << "CREATE TABLE IF NOT EXISTS series ("; + sql << "series_id integer primary key, "; + sql << "series_name nvarchar(255), "; + sql << "series_overview text"; + sql << ");"; + + sql << "CREATE TABLE IF NOT EXISTS series_actors ("; + sql << "actor_series_id integer, "; + sql << "actor_name nvarchar(255), "; + sql << "actor_role nvarchar(255), "; + sql << "actor_thumbnail nvarchar(255)"; + sql << ");"; + sql << "CREATE INDEX IF NOT EXISTS idx1 on series_actors (actor_series_id); "; + + sql << "CREATE TABLE IF NOT EXISTS movies ("; + sql << "movie_id integer primary key, "; + sql << "movie_title nvarchar(255), "; + sql << "movie_original_title nvarchar(255), "; + sql << "movie_overview text"; + sql << ");"; + + sql << "CREATE TABLE IF NOT EXISTS actors ("; + sql << "actor_id integer primary key, "; + sql << "actor_name nvarchar(255)"; + sql << ");"; + + sql << "CREATE TABLE IF NOT EXISTS actor_movie ("; + sql << "actor_id integer, "; + sql << "movie_id integer, "; + sql << "actor_role nvarchar(255)"; + sql << ");"; + sql << "CREATE UNIQUE INDEX IF NOT EXISTS idx2 on actor_movie (actor_id, movie_id); "; + + sql << "CREATE TABLE IF NOT EXISTS event_movie ("; + sql << "event_id integer, "; + sql << "valid_till integer, "; + sql << "movie_id integer "; + sql << ");"; + sql << "CREATE UNIQUE INDEX IF NOT EXISTS idx3 on event_movie (event_id, movie_id); "; + + sql << "CREATE TABLE IF NOT EXISTS event_series ("; + sql << "event_id integer, "; + sql << "valid_till integer, "; + sql << "series_id integer "; + sql << ");"; + sql << "CREATE UNIQUE INDEX IF NOT EXISTS idx4 on event_series (event_id, series_id); "; + + sql << "CREATE TABLE IF NOT EXISTS recordings ("; + sql << "rec_event_id integer, "; + sql << "series_id integer, "; + sql << "movie_id integer "; + sql << ");"; + sql << "CREATE UNIQUE INDEX IF NOT EXISTS idx5 on recordings (rec_event_id, series_id, movie_id); "; + + sql << "CREATE TABLE IF NOT EXISTS scrap_history ("; + sql << "channel_id nvarchar(255), "; + sql << "newest_scrapped integer "; + sql << ");"; + + sql << "CREATE TABLE IF NOT EXISTS scrap_checker ("; + sql << "last_scrapped integer "; + sql << ");"; + + char *errmsg; + if (sqlite3_exec(db,sql.str().c_str(),NULL,NULL,&errmsg)!=SQLITE_OK) { + esyslog("tvscraper: createdb: %s", errmsg); + sqlite3_free(errmsg); + sqlite3_close(db); + return false; + } + return true; +} + +void cTVScraperDB::ClearOutdated(string movieDir) { + //first check which movie images can be deleted + time_t now = time(0); + stringstream sql; + sql << "select movie_id from event_movie where valid_till < " << now; + vector > result = Query(sql.str()); + int numOutdated = result.size(); + if (numOutdated > 0) { + for (int i=0; i < numOutdated; i++) { + bool keepMovie = false; + vector row = result[i]; + if (row.size() > 0) { + int movieID = atoi(row[0].c_str()); + if (movieID > 0) { + //are there other still valid events pointing to that movie? + keepMovie = CheckMovieOutdatedEvents(movieID); + if (!keepMovie) { + //are there recordings pointing to that movie? + keepMovie = CheckMovieOutdatedRecordings(movieID); + } + if (!keepMovie) { + DeleteMovie(movieID, movieDir); + } + } + } + } + } + //delete all invalid events pointing to movies + stringstream sql2; + sql2 << "delete from event_movie where valid_till < " << now; + sqlite3_stmt *stmt; + sqlite3_prepare_v2(db, sql2.str().c_str(), -1, &stmt, NULL); + int ret = sqlite3_step(stmt); + //delete all invalid events pointing to series, series will all be kept for later use + stringstream sql3; + sql3 << "delete from event_series where valid_till < " << now; + sqlite3_stmt *stmt2; + sqlite3_prepare_v2(db, sql3.str().c_str(), -1, &stmt2, NULL); + int ret2 = sqlite3_step(stmt2); + esyslog("tvscraper: Cleanup Done"); +} + +void cTVScraperDB::DeleteMovie(int movieID, string movieDir) { + //delete images + stringstream backdrop; + backdrop << movieDir << "/" << movieID << "_backdrop.jpg"; + stringstream poster; + poster << movieDir << "/" << movieID << "_poster.jpg"; + DeleteFile(backdrop.str()); + DeleteFile(poster.str()); + //delete cast of this movie + stringstream sql; + sql << "DELETE FROM actor_movie WHERE movie_id = " << movieID; + sqlite3_stmt *stmt; + sqlite3_prepare_v2(db, sql.str().c_str(), -1, &stmt, NULL); + int ret = sqlite3_step(stmt); +} + +bool cTVScraperDB::CheckMovieOutdatedEvents(int movieID) { + time_t now = time(0); + stringstream sql; + sql << "select event_id from event_movie where movie_id = " << movieID << " and valid_till > " << now; + int eventID = 0; + vector > result = Query(sql.str()); + if (result.size() > 0) { + vector >::iterator it = result.begin(); + vector row = *it; + if (row.size() > 0) { + eventID = atoi(row[0].c_str()); + } + } + if (eventID > 0) + return true; + return false; +} + +bool cTVScraperDB::CheckMovieOutdatedRecordings(int movieID) { + stringstream sql; + sql << "select rec_event_id from recordings where movie_id = " << movieID; + int eventID = 0; + vector > result = Query(sql.str()); + if (result.size() > 0) { + vector >::iterator it = result.begin(); + vector row = *it; + if (row.size() > 0) { + eventID = atoi(row[0].c_str()); + } + } + if (eventID > 0) + return true; + return false; +} + +void cTVScraperDB::InsertSeries(int seriesID, string name, string overview) { + stringstream sql; + sql << "INSERT INTO series (series_id, series_name, series_overview) "; + sql << "VALUES ("; + sql << seriesID << ", "; + sql << "?, ? "; + sql << ");"; + sqlite3_stmt *stmt; + sqlite3_prepare_v2(db, sql.str().c_str(), -1, &stmt, NULL); + sqlite3_bind_text(stmt, 1, name.c_str(), -1, SQLITE_TRANSIENT); + sqlite3_bind_text(stmt, 2, overview.c_str(), -1, SQLITE_TRANSIENT); + int ret = sqlite3_step(stmt); +} + +void cTVScraperDB::InsertEventSeries(int eventID, time_t validTill, int seriesID) { + stringstream sql; + sql << "INSERT INTO event_series (event_id, valid_till, series_id) "; + sql << "VALUES ("; + sql << eventID << ", " << validTill << ", " << seriesID; + sql << ");"; + sqlite3_stmt *stmt; + sqlite3_prepare_v2(db, sql.str().c_str(), -1, &stmt, NULL); + int ret = sqlite3_step(stmt); +} + +void cTVScraperDB::InsertActor(int seriesID, string name, string role, string thumb) { + stringstream sql; + sql << "INSERT INTO series_actors (actor_series_id, actor_name, actor_role, actor_thumbnail) "; + sql << "VALUES ("; + sql << seriesID << ", "; + sql << "?, ? ,"; + sql << "'" << thumb << "'"; + sql << ");"; + sqlite3_stmt *stmt; + sqlite3_prepare_v2(db, sql.str().c_str(), -1, &stmt, NULL); + sqlite3_bind_text(stmt, 1, name.c_str(), -1, SQLITE_TRANSIENT); + sqlite3_bind_text(stmt, 2, role.c_str(), -1, SQLITE_TRANSIENT); + int ret = sqlite3_step(stmt); +} + +void cTVScraperDB::InsertMovie(int movieID, string title, string originalTitle, string overview) { + stringstream sql; + sql << "INSERT INTO movies (movie_id, movie_title, movie_original_title, movie_overview) "; + sql << "VALUES ("; + sql << movieID << ", "; + sql << "?, ?, ?"; + sql << ");"; + sqlite3_stmt *stmt; + sqlite3_prepare_v2(db, sql.str().c_str(), -1, &stmt, NULL); + sqlite3_bind_text(stmt, 1, title.c_str(), -1, SQLITE_TRANSIENT); + sqlite3_bind_text(stmt, 2, originalTitle.c_str(), -1, SQLITE_TRANSIENT); + sqlite3_bind_text(stmt, 3, overview.c_str(), -1, SQLITE_TRANSIENT); + int ret = sqlite3_step(stmt); +} + +void cTVScraperDB::InsertEventMovie(int eventID, time_t validTill, int movieID) { + stringstream sql; + sql << "INSERT INTO event_movie (event_id, valid_till, movie_id) "; + sql << "VALUES ("; + sql << eventID << ", " << validTill << ", " << movieID; + sql << ");"; + sqlite3_stmt *stmt; + sqlite3_prepare_v2(db, sql.str().c_str(), -1, &stmt, NULL); + int ret = sqlite3_step(stmt); +} + + +void cTVScraperDB::InsertMovieActor(int movieID, int actorID, string name, string role) { + stringstream sql; + sql << "INSERT INTO actors (actor_id, actor_name) "; + sql << "VALUES ("; + sql << actorID << ", ?"; + sql << ");"; + sqlite3_stmt *stmt; + sqlite3_prepare_v2(db, sql.str().c_str(), -1, &stmt, NULL); + sqlite3_bind_text(stmt, 1, name.c_str(), -1, SQLITE_TRANSIENT); + int ret = sqlite3_step(stmt); + + stringstream sql2; + sql2 << "INSERT INTO actor_movie (actor_id, movie_id, actor_role) "; + sql2 << "VALUES ("; + sql2 << actorID << ", " << movieID << ", ?"; + sql2 << ");"; + sqlite3_stmt *stmt2; + sqlite3_prepare_v2(db, sql2.str().c_str(), -1, &stmt2, NULL); + sqlite3_bind_text(stmt2, 1, role.c_str(), -1, SQLITE_TRANSIENT); + ret = sqlite3_step(stmt2); +} + +bool cTVScraperDB::MovieExists(int movieID) { + stringstream sql; + sql << "select count(movies.movie_id) as found from movies where movies.movie_id = " << movieID; + vector > result = Query(sql.str()); + int found = 0; + if (result.size() > 0) { + vector >::iterator it = result.begin(); + vector row = *it; + if (row.size() > 0) { + found = atoi(row[0].c_str()); + } + } + if (found == 1) + return true; + return false; +} + +bool cTVScraperDB::SeriesExists(int seriesID) { + stringstream sql; + sql << "select count(series.series_id) as found from series where series.series_id = " << seriesID; + vector > result = Query(sql.str()); + int found = 0; + if (result.size() > 0) { + vector >::iterator it = result.begin(); + vector row = *it; + if (row.size() > 0) { + found = atoi(row[0].c_str()); + } + } + if (found == 1) + return true; + return false; +} + +int cTVScraperDB::SearchMovie(string movieTitle) { + stringstream sql; + sql << "select movie_id from movies where movie_title='" << movieTitle.c_str() << "'"; + vector > result = Query(sql.str()); + int movieID = 0; + if (result.size() > 0) { + vector >::iterator it = result.begin(); + vector row = *it; + if (row.size() > 0) { + movieID = atoi(row[0].c_str()); + } + } + return movieID; +} + +int cTVScraperDB::SearchSeries(string seriesTitle) { + stringstream sql; + sql << "select series_id from series where series_name='" << seriesTitle.c_str() << "'"; + vector > result = Query(sql.str()); + int seriesID = 0; + if (result.size() > 0) { + vector >::iterator it = result.begin(); + vector row = *it; + if (row.size() > 0) { + seriesID = atoi(row[0].c_str()); + } + } + return seriesID; +} + +void cTVScraperDB::InsertRecording(int recEventID, int seriesID, int movieID) { + stringstream sql; + sql << "INSERT INTO recordings (rec_event_id, series_id, movie_id) "; + sql << "VALUES ("; + sql << recEventID << ", " << seriesID << ", " << movieID; + sql << ");"; + sqlite3_stmt *stmt; + sqlite3_prepare_v2(db, sql.str().c_str(), -1, &stmt, NULL); + int ret = sqlite3_step(stmt); +} + +void cTVScraperDB::SetRecordingSeries(int eventID) { + stringstream sql; + sql << "select series_id from event_series where event_id = " << eventID; + vector > result = Query(sql.str()); + if (result.size() > 0) { + vector >::iterator it = result.begin(); + vector row = *it; + if (row.size() > 0) { + int seriesID = atoi(row[0].c_str()); + InsertRecording(eventID, seriesID, 0); + } + } +} + +void cTVScraperDB::SetRecordingMovie(int eventID) { + stringstream sql; + sql << "select movie_id from event_movie where event_id =" << eventID; + vector > result = Query(sql.str()); + if (result.size() > 0) { + vector >::iterator it = result.begin(); + vector row = *it; + if (row.size() > 0) { + int movieID = atoi(row[0].c_str()); + InsertRecording(eventID, 0, movieID); + } + } +} + +void cTVScraperDB::ClearRecordings(void) { + stringstream sql; + sql << "DELETE FROM recordings where 0 = 0"; + sqlite3_stmt *stmt; + sqlite3_prepare_v2(db, sql.str().c_str(), -1, &stmt, NULL); + int ret = sqlite3_step(stmt); +} + +bool cTVScraperDB::CheckScrap(time_t timeStamp, string channelID) { + bool doScrap = false; + stringstream sql; + sql << "select newest_scrapped from scrap_history where channel_id = '" << channelID.c_str() << "'"; + vector > result = Query(sql.str()); + if (result.size() > 0) { + vector >::iterator it = result.begin(); + vector row = *it; + if (row.size() > 0) { + char *ep; + time_t newestScrapped = strtoul(row[0].c_str(), &ep, 10); + if (newestScrapped < timeStamp) { + doScrap = true; + stringstream sql2; + sql2 << "UPDATE scrap_history set newest_scrapped = " << timeStamp << " where channel_id='" << channelID.c_str() << "'"; + sqlite3_stmt *stmt; + sqlite3_prepare_v2(db, sql2.str().c_str(), -1, &stmt, NULL); + sqlite3_step(stmt); + } + } + } else { + doScrap = true; + stringstream sql2; + sql2 << "INSERT INTO scrap_history (channel_id, newest_scrapped) VALUES ('" << channelID.c_str() << "', " << timeStamp << ")"; + sqlite3_stmt *stmt; + sqlite3_prepare_v2(db, sql2.str().c_str(), -1, &stmt, NULL); + sqlite3_step(stmt); + } + return doScrap; +} + +bool cTVScraperDB::CheckStartScrapping(int minimumDistance) { + bool startScrapping = false; + time_t now = time(0); + stringstream sql; + sql << "select last_scrapped from scrap_checker"; + vector > result = Query(sql.str()); + if (result.size() > 0) { + vector >::iterator it = result.begin(); + vector row = *it; + if (row.size() > 0) { + char *ep; + time_t last_scrapped = strtoul(row[0].c_str(), &ep, 10); + int difference = (int)(now - last_scrapped); + if (difference > minimumDistance) { + startScrapping = true; + stringstream sql2; + sql2 << "delete from scrap_checker"; + sqlite3_stmt *stmt; + sqlite3_prepare_v2(db, sql2.str().c_str(), -1, &stmt, NULL); + sqlite3_step(stmt); + stringstream sql3; + sql3 << "INSERT INTO scrap_checker (last_scrapped) VALUES (" << now << ")"; + sqlite3_stmt *stmt2; + sqlite3_prepare_v2(db, sql3.str().c_str(), -1, &stmt2, NULL); + sqlite3_step(stmt2); + } + } + } else { + startScrapping = true; + stringstream sql2; + sql2 << "INSERT INTO scrap_checker (last_scrapped) VALUES (" << now << ")"; + sqlite3_stmt *stmt; + sqlite3_prepare_v2(db, sql2.str().c_str(), -1, &stmt, NULL); + sqlite3_step(stmt); + } + return startScrapping; +} + +int cTVScraperDB::GetSeriesID(int eventID, bool isRecording) { + stringstream sql; + if (!isRecording) + sql << "select series_id from event_series where event_id = " << eventID; + else + sql << "select series_id from recordings where rec_event_id = " << eventID; + vector > result = Query(sql.str()); + int seriesID = 0; + if (result.size() > 0) { + vector >::iterator it = result.begin(); + vector row = *it; + if (row.size() > 0) { + seriesID = atoi(row[0].c_str()); + } + } + return seriesID; +} + +int cTVScraperDB::GetMovieID(int eventID, bool isRecording) { + stringstream sql; + if (!isRecording) + sql << "select movie_id from event_movie where event_id = " << eventID; + else + sql << "select movie_id from recordings where rec_event_id = " << eventID; + vector > result = Query(sql.str()); + int movieID = 0; + if (result.size() > 0) { + vector >::iterator it = result.begin(); + vector row = *it; + if (row.size() > 0) { + movieID = atoi(row[0].c_str()); + } + } + return movieID; +} + +vector > cTVScraperDB::GetActorsSeries(int seriesID) { + stringstream sql; + sql << "select actor_name, actor_role, actor_thumbnail "; + sql << "from series_actors "; + sql << "where actor_series_id = " << seriesID; + vector > result = Query(sql.str()); + return result; +} + +vector > cTVScraperDB::GetActorsMovie(int movieID) { + stringstream sql; + sql << "select actors.actor_id, actor_name, actor_role "; + sql << "from actors, actor_movie "; + sql << "where actor_movie.actor_id = actors.actor_id "; + sql << "and actor_movie.movie_id = " << movieID; + vector > result = Query(sql.str()); + return result; +} + +string cTVScraperDB::GetDescriptionSeries(int seriesID) { + string description = ""; + stringstream sql; + sql << "select series_overview from series "; + sql << "where series_id = " << seriesID; + vector > result = Query(sql.str()); + if (result.size() > 0) { + vector >::iterator it = result.begin(); + vector row = *it; + if (row.size() > 0) { + description= row[0]; + } + } + return description; +} + +string cTVScraperDB::GetDescriptionMovie(int movieID) { + string description = ""; + stringstream sql; + sql << "select movie_overview from movies "; + sql << "where movie_id = " << movieID; + vector > result = Query(sql.str()); + if (result.size() > 0) { + vector >::iterator it = result.begin(); + vector row = *it; + if (row.size() > 0) { + description= row[0]; + } + } + return description; +} diff --git a/tvscraperdb.h b/tvscraperdb.h new file mode 100644 index 0000000..a8f06ee --- /dev/null +++ b/tvscraperdb.h @@ -0,0 +1,50 @@ +#ifndef __TVSCRAPER_TVSCRAPERDB_H +#define __TVSCRAPER_TVSCRAPERDB_H + +using namespace std; + +// --- cTVScraperDB -------------------------------------------------------- + +class cTVScraperDB { +private: + sqlite3 *db; + string dbPathPhys; + string dbPathMem; + bool inMem; + vector > Query(string query); + int LoadOrSaveDb(sqlite3 *pInMemory, const char *zFilename, int isSave); + bool CreateTables(void); +public: + cTVScraperDB(void); + virtual ~cTVScraperDB(void); + bool Connect(void); + void BackupToDisc(void); + void ClearOutdated(string movieDir); + bool CheckMovieOutdatedEvents(int movieID); + bool CheckMovieOutdatedRecordings(int movieID); + void DeleteMovie(int movieID, string movieDir); + void InsertSeries(int seriesID, string name, string overview); + void InsertEventSeries(int eventID, time_t validTill, int seriesID); + void InsertActor(int seriesID, string name, string role, string thumb); + void InsertMovie(int movieID, string title, string originalTitle, string overview); + void InsertEventMovie(int eventID, time_t validTill, int movieID); + void InsertMovieActor(int movieID, int actorID, string name, string role); + bool MovieExists(int movieID); + bool SeriesExists(int seriesID); + int SearchMovie(string movieTitle); + int SearchSeries(string seriesTitle); + void InsertRecording(int recEventID, int seriesID, int movieID); + void SetRecordingSeries(int eventID); + void SetRecordingMovie(int eventID); + void ClearRecordings(void); + bool CheckScrap(time_t timeStamp, string channelID); + bool CheckStartScrapping(int minimumDistance); + int GetSeriesID(int eventID, bool isRecording); + int GetMovieID(int eventID, bool isRecording); + vector > GetActorsSeries(int seriesID); + vector > GetActorsMovie(int movieID); + string GetDescriptionSeries(int seriesID); + string GetDescriptionMovie(int movieID); +}; + +#endif //__TVSCRAPER_TVSCRAPPERDB_H diff --git a/worker.c b/worker.c new file mode 100644 index 0000000..09b5752 --- /dev/null +++ b/worker.c @@ -0,0 +1,233 @@ +#include +#include "worker.h" + +using namespace std; + +cTVScraperWorker::cTVScraperWorker(cTVScraperDB *db) : cThread("tvscraper", true) { + startLoop = true; + scanVideoDir = false; + manualScan = false; + this->db = db; + moviedbScraper = NULL; + tvdbScraper = NULL; + initSleep = 2 * 60 * 1000; + loopSleep = 5 * 60 * 1000; + language = ""; +} + +cTVScraperWorker::~cTVScraperWorker() { + if (moviedbScraper) + delete moviedbScraper; + if (tvdbScraper) + delete tvdbScraper; +} + +void cTVScraperWorker::SetLanguage(void) { + string loc = setlocale(LC_NAME, NULL); + size_t index = loc.find_first_of("_"); + string langISO = ""; + if (index > 0) { + langISO = loc.substr(0, index); + } + if (langISO.size() == 2) { + language = langISO.c_str(); + dsyslog("tvscraper: using language %s", language.c_str()); + return; + } + language = "en"; + dsyslog("tvscraper: using fallback language %s", language.c_str()); +} + +void cTVScraperWorker::Stop(void) { + waitCondition.Broadcast(); // wakeup the thread + Cancel(5); // wait up to 5 seconds for thread was stopping + db->BackupToDisc(); +} + +void cTVScraperWorker::InitVideoDirScan(void) { + scanVideoDir = true; + waitCondition.Broadcast(); +} + +void cTVScraperWorker::InitManualScan(void) { + manualScan = true; + waitCondition.Broadcast(); +} + +void cTVScraperWorker::SetDirectories(void) { + plgBaseDir = config.GetBaseDir(); + stringstream strSeriesDir; + strSeriesDir << plgBaseDir << "/series"; + seriesDir = strSeriesDir.str(); + stringstream strMovieDir; + strMovieDir << plgBaseDir << "/movies"; + movieDir = strMovieDir.str(); + bool ok = false; + ok = CreateDirectory(plgBaseDir); + if (ok) + ok = CreateDirectory(seriesDir); + if (ok) + ok = CreateDirectory(movieDir); + if (!ok) { + esyslog("tvscraper: ERROR: check %s for write permissions", plgBaseDir.c_str()); + startLoop = false; + } else { + dsyslog("tvscraper: using base directory %s", plgBaseDir.c_str()); + } +} + +scrapType cTVScraperWorker::GetScrapType(const cEvent *event) { + scrapType type = scrapNone; + int duration = event->Duration() / 60; + if ((duration > 19) && (duration < 65)) { + type = scrapSeries; + } else if (duration > 80) { + type = scrapMovie; + } + return type; +} + +bool cTVScraperWorker::ConnectScrapers(void) { + if (!moviedbScraper) { + moviedbScraper = new cMovieDBScraper(movieDir, db, language); + if (!moviedbScraper->Connect()) { + esyslog("tvscraper: ERROR, connection to TheMovieDB failed"); + return false; + } + } + if (!tvdbScraper) { + tvdbScraper = new cTVDBScraper(seriesDir, db, language); + if (!tvdbScraper->Connect()) { + esyslog("tvscraper: ERROR, connection to TheTVDB failed"); + return false; + } + } + return true; +} + +void cTVScraperWorker::DisconnectScrapers(void) { + if (moviedbScraper) { + delete moviedbScraper; + moviedbScraper = NULL; + } + if (tvdbScraper) { + delete tvdbScraper; + tvdbScraper = NULL; + } +} + +void cTVScraperWorker::ScrapEPG(void) { + vector channels = config.GetChannels(); + int numChannels = channels.size(); + for (int i=0; iName(), channelID.c_str()); + cSchedulesLock schedulesLock; + const cSchedules *schedules = cSchedules::Schedules(schedulesLock); + const cSchedule *Schedule = schedules->GetSchedule(channel); + if (Schedule) { + const cEvent *event = NULL; + for (event = Schedule->Events()->First(); event; event = Schedule->Events()->Next(event)) { + if (!Running()) + return; + scrapType type = GetScrapType(event); + if (type != scrapNone) { + if (!db->CheckScrap(event->StartTime(), channelID)) + continue; + if (type == scrapSeries) { + tvdbScraper->Scrap(event); + } else if (type == scrapMovie) { + moviedbScraper->Scrap(event); + } + waitCondition.TimedWait(mutex, 100); + } + } + } + } +} + +void cTVScraperWorker::ScrapRecordings(void) { + db->ClearRecordings(); + for (cRecording *rec = Recordings.First(); rec; rec = Recordings.Next(rec)) { + const cRecordingInfo *recInfo = rec->Info(); + const cEvent *recEvent = recInfo->GetEvent(); + if (recEvent) { + tEventID recEventID = recEvent->EventID(); + string recTitle = recInfo->Title(); + scrapType type = GetScrapType(recEvent); + if (type == scrapSeries) { + int seriesID = db->SearchSeries(recEvent->Title()); + if (seriesID) { + db->InsertRecording((int)recEventID, seriesID, 0); + } else { + tvdbScraper->Scrap(recEvent, (int)recEventID); + } + } else if (type == scrapMovie) { + int movieID = db->SearchMovie(recEvent->Title()); + if (movieID) { + db->InsertRecording((int)recEventID, 0, movieID); + } else { + moviedbScraper->Scrap(recEvent, (int)recEventID); + } + } + } + } +} + +void cTVScraperWorker::CheckRunningTimers(void) { + for (cTimer *timer = Timers.First(); timer; timer = Timers.Next(timer)) { + if (timer->Recording()) { + const cEvent *event = timer->Event(); + scrapType type = GetScrapType(event); + if (type == scrapSeries) { + db->SetRecordingSeries((int)event->EventID()); + } else if (type == scrapMovie) { + db->SetRecordingMovie((int)event->EventID()); + } + } + } +} + +bool cTVScraperWorker::StartScrapping(void) { + if (manualScan) { + manualScan = false; + return true; + } + //wait at least one day from last scrapping to scrap again + int minTime = 24 * 60 * 60; + return db->CheckStartScrapping(minTime); +} + + +void cTVScraperWorker::Action(void) { + if (!startLoop) + return; + mutex.Lock(); + dsyslog("tvscraper: waiting %d minutes to start main loop", initSleep / 1000 / 60); + waitCondition.TimedWait(mutex, initSleep); + + while (Running()) { + if (scanVideoDir) { + scanVideoDir = false; + dsyslog("tvscraper: scanning video dir"); + if (ConnectScrapers()) { + ScrapRecordings(); + } + DisconnectScrapers(); + continue; + } + CheckRunningTimers(); + if (StartScrapping()) { + dsyslog("tvscraper: start scrapping epg"); + db->ClearOutdated(movieDir); + if (ConnectScrapers()) { + ScrapEPG(); + } + DisconnectScrapers(); + db->BackupToDisc(); + dsyslog("tvscraper: epg scrapping done"); + } + waitCondition.TimedWait(mutex, loopSleep); + } +} diff --git a/worker.h b/worker.h new file mode 100644 index 0000000..d89ab8b --- /dev/null +++ b/worker.h @@ -0,0 +1,47 @@ +#ifndef __TVSCRAPER_WORKER_H +#define __TVSCRAPER_WORKER_H + +// --- cTVScraperWorker ------------------------------------------------------------- + +enum scrapType { + scrapSeries, + scrapMovie, + scrapNone +}; + +class cTVScraperWorker : public cThread { +private: + bool startLoop; + bool scanVideoDir; + bool manualScan; + string language; + string plgBaseDir; + string seriesDir; + string movieDir; + int initSleep; + int loopSleep; + cCondVar waitCondition; + cMutex mutex; + cTVScraperDB *db; + cMovieDBScraper *moviedbScraper; + cTVDBScraper *tvdbScraper; + scrapType GetScrapType(const cEvent *event); + bool ConnectScrapers(void); + void DisconnectScrapers(void); + void CheckRunningTimers(void); + void ScrapEPG(void); + void ScrapRecordings(void); + bool StartScrapping(void); +public: + cTVScraperWorker(cTVScraperDB *db); + virtual ~cTVScraperWorker(void); + void SetLanguage(void); + void SetDirectories(void); + void InitVideoDirScan(void); + void InitManualScan(void); + virtual void Action(void); + void Stop(void); +}; + + +#endif //__TVSCRAPER_WORKER_H -- cgit v1.2.3