diff options
author | louis <louis.braun@gmx.de> | 2013-08-23 17:11:38 +0200 |
---|---|---|
committer | louis <louis.braun@gmx.de> | 2013-08-23 17:11:38 +0200 |
commit | 21c1ad1005e3415f8448a21f17ed3ba3be9715b8 (patch) | |
tree | e8cae7353b6047d1d1bd104ad54e5b15d0c85260 | |
download | vdr-plugin-tvscraper-21c1ad1005e3415f8448a21f17ed3ba3be9715b8.tar.gz vdr-plugin-tvscraper-21c1ad1005e3415f8448a21f17ed3ba3be9715b8.tar.bz2 |
Initial Push Version 0.0.1
38 files changed, 3918 insertions, 0 deletions
@@ -0,0 +1,340 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc. + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + <one line to give the program's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + <signature of Ty Coon>, 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. @@ -0,0 +1,6 @@ +VDR Plugin 'tvscraper' Revision History +---------------------------------------- + +2013-07-26: Version 0.0.1 + +- Initial revision. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..52439ba --- /dev/null +++ b/Makefile @@ -0,0 +1,128 @@ +# +# Makefile for a Video Disk Recorder plugin +# +# $Id$ + +# The official name of this plugin. +# This name will be used in the '-P...' option of VDR to load the plugin. +# By default the main source file also carries this name. + +PLUGIN = tvscraper + +### The version number of this plugin (taken from the main source file): + +VERSION = $(shell grep 'static const char \*VERSION *=' $(PLUGIN).c | awk '{ print $$6 }' | sed -e 's/[";]//g') + +### The directory environment: + +# Use package data if installed...otherwise assume we're under the VDR source directory: +PKGCFG = $(if $(VDRDIR),$(shell pkg-config --variable=$(1) $(VDRDIR)/vdr.pc),$(shell pkg-config --variable=$(1) vdr || pkg-config --variable=$(1) ../../../vdr.pc)) +LIBDIR = $(call PKGCFG,libdir) +LOCDIR = $(call PKGCFG,locdir) +PLGCFG = $(call PKGCFG,plgcfg) +# +TMPDIR ?= /tmp + +### The compiler options: + +export CFLAGS = $(call PKGCFG,cflags) +export CXXFLAGS = $(call PKGCFG,cxxflags) + +### The version number of VDR's plugin API: + +APIVERSION = $(call PKGCFG,apiversion) + +### Allow user defined options to overwrite defaults: + +-include $(PLGCFG) + +### The name of the distribution archive: + +ARCHIVE = $(PLUGIN)-$(VERSION) +PACKAGE = vdr-$(ARCHIVE) + +### The name of the shared object file: + +SOFILE = libvdr-$(PLUGIN).so + +### Includes and Defines (add further entries here): + +INCLUDES += + +DEFINES += -DPLUGIN_NAME_I18N='"$(PLUGIN)"' +DEFINES += $(shell xml2-config --cflags) + +LIBS += $(shell xml2-config --libs) +LIBS += $(shell pkg-config --libs libcurl) +LIBS += $(shell pkg-config --libs sqlite3) +LIBS += $(shell pkg-config --cflags --libs jansson) + +### The object files (add further files here): + +OBJS = $(PLUGIN).o + +### The main target: + +all: $(SOFILE) i18n + +### Implicit rules: + +%.o: %.c + $(CXX) $(CXXFLAGS) -c $(DEFINES) $(INCLUDES) -o $@ $< + +### Dependencies: + +MAKEDEP = $(CXX) -MM -MG +DEPFILE = .dependencies +$(DEPFILE): Makefile + @$(MAKEDEP) $(CXXFLAGS) $(DEFINES) $(INCLUDES) $(OBJS:%.o=%.c) > $@ + +-include $(DEPFILE) + +### Internationalization (I18N): + +PODIR = po +I18Npo = $(wildcard $(PODIR)/*.po) +I18Nmo = $(addsuffix .mo, $(foreach file, $(I18Npo), $(basename $(file)))) +I18Nmsgs = $(addprefix $(DESTDIR)$(LOCDIR)/, $(addsuffix /LC_MESSAGES/vdr-$(PLUGIN).mo, $(notdir $(foreach file, $(I18Npo), $(basename $(file)))))) +I18Npot = $(PODIR)/$(PLUGIN).pot + +%.mo: %.po + msgfmt -c -o $@ $< + +$(I18Npot): $(wildcard *.c) + xgettext -C -cTRANSLATORS --no-wrap --no-location -k -ktr -ktrNOOP --package-name=vdr-$(PLUGIN) --package-version=$(VERSION) --msgid-bugs-address='<see README>' -o $@ `ls $^` + +%.po: $(I18Npot) + msgmerge -U --no-wrap --no-location --backup=none -q -N $@ $< + @touch $@ + +$(I18Nmsgs): $(DESTDIR)$(LOCDIR)/%/LC_MESSAGES/vdr-$(PLUGIN).mo: $(PODIR)/%.mo + install -D -m644 $< $@ + +.PHONY: i18n +i18n: $(I18Nmo) $(I18Npot) + +install-i18n: $(I18Nmsgs) + +### Targets: + +$(SOFILE): $(OBJS) + $(CXX) $(CXXFLAGS) $(LDFLAGS) -shared $(OBJS) $(LIBS) -o $@ + +install-lib: $(SOFILE) + install -D $^ $(DESTDIR)$(LIBDIR)/$^.$(APIVERSION) + +install: install-lib install-i18n + +dist: $(I18Npo) clean + @-rm -rf $(TMPDIR)/$(ARCHIVE) + @mkdir $(TMPDIR)/$(ARCHIVE) + @cp -a * $(TMPDIR)/$(ARCHIVE) + @tar czf $(PACKAGE).tgz -C $(TMPDIR) $(ARCHIVE) + @-rm -rf $(TMPDIR)/$(ARCHIVE) + @echo Distribution package created as $(PACKAGE).tgz + +clean: + @-rm -f $(PODIR)/*.mo $(PODIR)/*.pot + @-rm -f $(OBJS) $(DEPFILE) *.so *.tgz core* *~ @@ -0,0 +1,85 @@ +This is a "plugin" for the Video Disk Recorder (VDR). + +Written by: Stefan Braun <louis.braun@gmx.de> + +Project's homepage: +http://projects.vdr-developer.org/projects/plg-tvscraper + +Latest version available at: +http://projects.vdr-developer.org/projects/plg-tvscraper/files + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. +See the file COPYING for more information. + +Description +----------- + +TVScraper runs in the background and collects metadata (posters, +banners, fanart, actor thumbs and roles, descriptions) for all +available EPG events on selectable channels and for recordings. +Additionally the plugin provides the collected metadata via the VDR +service interface to other plugins which deal with EPG information. + +TVScraper uses the thetvdb.com API for collecting series metadata and +themoviedb.org API for movies. Check the websites of both services for +the terms of use. + +Important: To avoid unnecessary traffic, only activate these channels +to be scrapped which are reasonable. After plugin installation all +channels are deactivated by default, so please consider this point when +you activate the channels you are interested in ;) + +Additionally you are invited to contribute to the used web services with +providing missing data for your favorite movies and series. + +Requirements +------------ + +To run the plugin the following libaries have to be installed: +- libsqlite3 +- libcurl +- libXML2 +- libjansson + +Installation and configuration +------------------------------ + +Just install the plugin depending on your used distribution. During VDR +startup the plugin base directory can be set with the following option: + +-d <CACHEDIR>, --dir=<CACHEDIR> Set directory where database and images + are stored + +If no directory is provided, the plugin uses VDRCACHEDIR as default. +Please care about that the user who runs VDR has full read/write access +to this directory, otherwise the plugin will not start. + +As already mentioned, after first installations no channels are activated +to be scrapped. Please configure these channels in the plugin setup menu. +Additionally you can trigger that your already existing recordings are +scrapped, so that also for this recordings metadata is available. + +The plugins uses a sqlite3 database to store the necessary information. +If /dev/shm/ is available, the database is kept in memory during runtime +which improves performance. In the configured plugin basedir only a +persistant backup of the database is stored then. If /dev/shm/ is not +available, only the database file in the plugin base directory is used. + +Usage +----- + +After the initial configuration the plugin runs completely independent in +the background, you don't have to care about anything. The plugins checks +at least every 24 hours for new EPG events and collects the metadata for +these events automatically. + +Before each run the plugin performs a cleanup, all images for movies which +are not available in the current EPG are deleted. Series and actors thumbs +are kept to avoid unnecessary traffic for the web services, because the +propability that this data is needed in the future again is rather high. + +If a running recording is detected, the plugin marks the corresponding movie +meta data so that the information for this movie will be kept permanentely.
\ No newline at end of file diff --git a/config.c b/config.c new file mode 100644 index 0000000..4110bd3 --- /dev/null +++ b/config.c @@ -0,0 +1,57 @@ +#include "config.h" + +using namespace std; + +cTVScraperConfig::cTVScraperConfig() { + enableDebug = 0; +} + +cTVScraperConfig::~cTVScraperConfig() { +} + +void cTVScraperConfig::ClearChannels(void) { + channels.clear(); +} + +void cTVScraperConfig::AddChannel(string channelID) { + channels.push_back(channelID); +} + +bool cTVScraperConfig::ChannelActive(int channelNum) { + cChannel *channel = Channels.GetByNumber(channelNum); + if (channel) { + string channelID = ""; + channelID = *(channel->GetChannelID().ToString()); + int numChannels = channels.size(); + for (int i=0; i<numChannels; i++) { + if (!channels[i].compare(channelID)) { + return true; + } + } + } + return false; +} + +bool cTVScraperConfig::SetupParse(const char *Name, const char *Value) { + if (strcmp(Name, "ScrapChannels") == 0) { + splitstring s(Value); + vector<string> flds = s.split(';'); + int numChannels = flds.size(); + for (int i=0; i<numChannels; i++) { + channels.push_back(flds[i]); + } + return true; + } else if (strcmp(Name, "enableDebug") == 0) { + enableDebug = atoi(Value); + return true; + } + return false; +} + +void cTVScraperConfig::PrintChannels(void) { + int numChannels = channels.size(); + esyslog("tvscraper: %d channel to be scrapped", numChannels); + for (int i=0; i<numChannels; i++) { + esyslog("tvscraper: channel to be scrapped: %s", channels[i].c_str()); + } +} diff --git a/config.h b/config.h new file mode 100644 index 0000000..88cce72 --- /dev/null +++ b/config.h @@ -0,0 +1,24 @@ +#ifndef __TVSCRAPER_CONFIG_H +#define __TVSCRAPER_CONFIG_H + +using namespace std; + +class cTVScraperConfig { + private: + string baseDir; + vector<string> channels; + public: + cTVScraperConfig(); + ~cTVScraperConfig(); + int enableDebug; + void SetBaseDir(string dir) { baseDir = dir; }; + string GetBaseDir(void) { return baseDir; }; + void ClearChannels(void); + void AddChannel(string channelID); + bool ChannelActive(int channelNum); + bool SetupParse(const char *Name, const char *Value); + vector<string> GetChannels(void) { return channels; }; + void PrintChannels(void); +}; + +#endif //__TVSCRAPER_CONFIG_H
\ No newline at end of file diff --git a/imageserver.c b/imageserver.c new file mode 100644 index 0000000..975d417 --- /dev/null +++ b/imageserver.c @@ -0,0 +1,205 @@ +#include "imageserver.h" + +using namespace std; + +cImageServer::cImageServer(cTVScraperDB *db) { + this->db = db; +} + +cImageServer::~cImageServer() { +} + +scrapType cImageServer::GetScrapType(const cEvent *event) { + scrapType type = scrapNone; + int duration = event->Duration() / 60; + if ((duration > 19) && (duration < 65)) { + type = scrapSeries; + } else if (duration > 80) { + type = scrapMovie; + } + return type; +} + +int cImageServer::GetID(int eventID, scrapType type, bool isRecording) { + int id = 0; + if (type == scrapSeries) { + id = db->GetSeriesID(eventID, isRecording); + } else if (type == scrapMovie) { + id = db->GetMovieID(eventID, isRecording); + } + return id; +} + +tvMedia cImageServer::GetPosterOrBanner(int id, scrapType type) { + tvMedia media; + media.path = ""; + media.width = 0; + media.height = 0; + if (type == scrapSeries) { + stringstream path; + path << config.GetBaseDir() << "/series/" << id << "/banner.jpg"; + media.path = path.str(); + media.width = 758; + media.height = 140; + } else if (type == scrapMovie) { + stringstream path; + path << config.GetBaseDir() << "/movies/" << id << "_poster.jpg"; + media.path = path.str(); + media.width = 500; + media.height = 750; + } + return media; +} + +tvMedia cImageServer::GetPoster(int id, scrapType type) { + tvMedia media; + media.path = ""; + media.width = 0; + media.height = 0; + if (type == scrapSeries) { + stringstream path; + path << config.GetBaseDir() << "/series/" << id << "/poster_0.jpg"; + string filePoster = path.str(); + if (FileExists(filePoster)) { + media.path = filePoster; + media.width = 680; + media.height = 1000; + } + } else if (type == scrapMovie) { + stringstream path; + path << config.GetBaseDir() << "/movies/" << id << "_poster.jpg"; + string filePoster = path.str(); + if (FileExists(filePoster)) { + media.path = path.str(); + media.width = 500; + media.height = 750; + } + } + return media; +} + +tvMedia cImageServer::GetBanner(int id) { + tvMedia media; + stringstream path; + path << config.GetBaseDir() << "/series/" << id << "/banner.jpg"; + media.path = path.str(); + media.width = 758; + media.height = 140; + return media; +} + +vector<tvMedia> cImageServer::GetPosters(int id, scrapType type) { + vector<tvMedia> posters; + if (type == scrapSeries) { + for (int i=0; i<3; i++) { + stringstream path; + path << config.GetBaseDir() << "/series/" << id << "/poster_" << i << ".jpg"; + string filePoster = path.str(); + if (FileExists(filePoster)) { + tvMedia media; + media.path = filePoster; + media.width = 680; + media.height = 1000; + posters.push_back(media); + } else + break; + } + } else if (type == scrapMovie) { + stringstream path; + path << config.GetBaseDir() << "/movies/" << id << "_poster.jpg"; + string filePoster = path.str(); + if (FileExists(filePoster)) { + tvMedia media; + media.path = path.str(); + media.width = 500; + media.height = 750; + posters.push_back(media); + } + } + return posters; +} + +vector<tvMedia> cImageServer::GetFanart(int id, scrapType type) { + vector<tvMedia> fanart; + if (type == scrapSeries) { + for (int i=0; i<3; i++) { + stringstream path; + path << config.GetBaseDir() << "/series/" << id << "/fanart_" << i << ".jpg"; + string fileFanart = path.str(); + if (FileExists(fileFanart)) { + tvMedia media; + media.path = fileFanart; + media.width = 1920; + media.height = 1080; + fanart.push_back(media); + } else + break; + } + } else if (type == scrapMovie) { + stringstream path; + path << config.GetBaseDir() << "/movies/" << id << "_backdrop.jpg"; + string fileFanart = path.str(); + if (FileExists(fileFanart)) { + tvMedia media; + media.path = fileFanart; + media.width = 1280; + media.height = 720; + fanart.push_back(media); + } + } + return fanart; +} + +vector<tvActor> cImageServer::GetActors(int id, scrapType type) { + vector<tvActor> actors; + if (type == scrapSeries) { + vector<vector<string> > actorsDB = db->GetActorsSeries(id); + int numActors = actorsDB.size(); + for (int i=0; i < numActors; i++) { + vector<string> row = actorsDB[i]; + if (row.size() == 3) { + tvActor actor; + actor.name = row[0]; + actor.role = row[1]; + tvMedia thumb; + stringstream thumbPath; + thumbPath << config.GetBaseDir() << "/series/" << id << "/" << row[2]; + thumb.path = thumbPath.str(); + thumb.width = 300; + thumb.height = 450; + actor.thumb = thumb; + actors.push_back(actor); + } + } + } else if (type == scrapMovie) { + vector<vector<string> > actorsDB = db->GetActorsMovie(id); + int numActors = actorsDB.size(); + for (int i=0; i < numActors; i++) { + vector<string> row = actorsDB[i]; + if (row.size() == 3) { + tvActor actor; + actor.name = row[1]; + actor.role = row[2]; + stringstream thumbPath; + thumbPath << config.GetBaseDir() << "/movies/actors/actor_" << row[0] << ".jpg"; + tvMedia thumb; + thumb.path = thumbPath.str(); + thumb.width = 421; + thumb.height = 632; + actor.thumb = thumb; + actors.push_back(actor); + } + } + } + return actors; +} + +string cImageServer::GetDescription(int id, scrapType type) { + string description; + if (type == scrapSeries) { + description = db->GetDescriptionSeries(id); + } else if (type == scrapMovie) { + description = db->GetDescriptionMovie(id); + } + return description; +} diff --git a/imageserver.h b/imageserver.h new file mode 100644 index 0000000..a8bfd8b --- /dev/null +++ b/imageserver.h @@ -0,0 +1,20 @@ +using namespace std; + +// --- cImageServer -------------------------------------------------------- + +class cImageServer { +private: + cTVScraperDB *db; +public: + cImageServer(cTVScraperDB *db); + virtual ~cImageServer(void); + scrapType GetScrapType(const cEvent *event); + int GetID(int eventID, scrapType type, bool isRecording); + tvMedia GetPosterOrBanner(int id, scrapType type); + tvMedia GetPoster(int id, scrapType type); + tvMedia GetBanner(int id); + vector<tvMedia> GetPosters(int id, scrapType type); + vector<tvMedia> GetFanart(int id, scrapType type); + vector<tvActor> GetActors(int id, scrapType type); + string GetDescription(int id, scrapType type); +};
\ No newline at end of file diff --git a/po/de_DE.po b/po/de_DE.po new file mode 100644 index 0000000..f2ab8eb --- /dev/null +++ b/po/de_DE.po @@ -0,0 +1,36 @@ +msgid "" +msgstr "" +"Project-Id-Version: vdr-tvscrapper 0.0.1\n" +"Report-Msgid-Bugs-To: <see README>\n" +"POT-Creation-Date: 2013-08-23 15:28+0200\n" +"PO-Revision-Date: 2013-08-20 08:27+0200\n" +"Last-Translator: Stefan Braun <louis.braun@gmx.de>\n" +"Language-Team:\n" +"Language: de \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" + +msgid "Configure channels to be scrapped" +msgstr "Kanäle zum scrapen festlegen" + +msgid "Trigger scrapping Video Directory" +msgstr "" + +msgid "Trigger EPG scrapping" +msgstr "" + +msgid "Enable Debug Logging" +msgstr "Debug Logging aktivieren" + +msgid "don't scrap" +msgstr "nicht scrapen" + +msgid "scrap" +msgstr "scrapen" + +#~ msgid "Trigger scraping Video Directory" +#~ msgstr "Scrapen des Video Verzeichnisses starten" + +#~ msgid "Trigger EPG scraping" +#~ msgstr "EPG Scraping starten" diff --git a/services.h b/services.h new file mode 100644 index 0000000..fc278ad --- /dev/null +++ b/services.h @@ -0,0 +1,56 @@ +enum tvMediaType { + typeSeries, + typeMovie, + typeNone, +}; + +struct tvMedia { + std::string path; + int width; + int height; +}; + +struct tvActor { + std::string name; + std::string role; + tvMedia thumb; +}; + +// Data structure for service "TVScraper-GetPosterOrBanner" +struct TVScraperGetPosterOrBanner +{ +// in + const cEvent *event; // search image for this event +//out + tvMediaType type; //typeSeries or typeMovie + tvMedia media; //banner or poster +}; + +// Data structure for service "TVScraper-GetPoster" +struct TVScraperGetPoster +{ +// in + const cEvent *event; // search image for this event + bool isRecording; // search in current EPG or recordings +//out + tvMedia media; //poster +}; + + +/* Data structure for service "TVScraper-GetFullEPGInformation" +if type == typeMovie a poster and a fanart image is delivered +if type == typeSeries a banner and up to three posters and fanarts are delivered +*/ +struct TVScraperGetFullInformation +{ +// in + const cEvent *event; // search all media for this event + bool isRecording; // search in current EPG or recordings +//out + tvMediaType type; + tvMedia banner; + std::vector<tvMedia> posters; + std::vector<tvMedia> fanart; + std::vector<tvActor> actors; + std::string description; +};
\ No newline at end of file @@ -0,0 +1,111 @@ +#include "setup.h" + +using namespace std; + +/* cTVScraperSetup */ + +cTVScraperSetup::cTVScraperSetup(cTVScraperWorker *workerThread) { + worker = workerThread; + int numChannels = Channels.Count(); + for (int i=0; i<numChannels; i++) { + int akt = 0; + if (config.ChannelActive(i+1)) + akt = 1; + channelsScrap.push_back(akt); + } + Setup(); +} + +cTVScraperSetup::~cTVScraperSetup() { +} + + +void cTVScraperSetup::Setup(void) { + int currentItem = Current(); + Clear(); + Add(new cOsdItem(tr("Configure channels to be scrapped"))); + Add(new cOsdItem(tr("Trigger scrapping Video Directory"))); + Add(new cOsdItem(tr("Trigger EPG scrapping"))); + Add(new cMenuEditBoolItem(tr("Enable Debug Logging"), &config.enableDebug)); + + SetCurrent(Get(currentItem)); + Display(); +} + +eOSState cTVScraperSetup::ProcessKey(eKeys Key) { + bool hadSubMenu = HasSubMenu(); + if (hadSubMenu && Key == kOk) + Store(); + eOSState state = cMenuSetupPage::ProcessKey(Key); + if (!hadSubMenu && (Key == kOk)) { + const char* ItemText = Get(Current())->Text(); + if (strcmp(ItemText, tr("Configure channels to be scrapped")) == 0) + state = AddSubMenu(new cTVScraperChannelSetup(&channelsScrap)); + else if (strcmp(ItemText, tr("Trigger scrapping Video Directory")) == 0) { + Skins.Message(mtInfo, "Scrapping Video Directory started"); + worker->InitVideoDirScan(); + state = osContinue; + } else if (strcmp(ItemText, tr("Trigger EPG scrapping")) == 0) { + Skins.Message(mtInfo, "EPG Scrapping started"); + worker->InitManualScan(); + state = osContinue; + } + } + return state; +} + +void cTVScraperSetup::Store(void) { + config.ClearChannels(); + stringstream channelsToScrap; + int numChannels = channelsScrap.size(); + for (int i=0; i<numChannels; i++) { + if (channelsScrap[i] == 1) { + cChannel *channel = Channels.GetByNumber(i+1); + if (channel) { + string channelID = *(channel->GetChannelID().ToString()); + channelsToScrap << channelID << ";"; + config.AddChannel(channelID); + } + } + } + SetupStore("ScrapChannels", channelsToScrap.str().c_str()); + SetupStore("enableDebug", config.enableDebug); +} + + +/* cTVScraperChannelSetup */ + +cTVScraperChannelSetup ::cTVScraperChannelSetup (vector<int> *channelsScrap) : cOsdMenu(tr("Configure channels to be scrapped"), 30) { + this->channelsScrap = channelsScrap; + SetMenuCategory(mcSetupPlugins); + Setup(); +} + +cTVScraperChannelSetup ::~cTVScraperChannelSetup () { +} + + +void cTVScraperChannelSetup ::Setup(void) { + int currentItem = Current(); + Clear(); + int i=0; + for (cChannel *channel = Channels.First(); channel; channel = Channels.Next(channel)) { + if (!channel->GroupSep()) { + Add(new cMenuEditBoolItem(channel->Name(), &channelsScrap->at(i), tr("don't scrap"), tr("scrap"))); + i++; + } + } + SetCurrent(Get(currentItem)); + Display(); +} + +eOSState cTVScraperChannelSetup ::ProcessKey(eKeys Key) { + eOSState state = cOsdMenu::ProcessKey(Key); + switch (Key) { + case kOk: + return osBack; + default: + break; + } + return state; +}
\ No newline at end of file @@ -0,0 +1,30 @@ +#ifndef __TVSCRAPER_SETUP_H +#define __TVSCRAPER_SETUP_H + +using namespace std; + +class cTVScraperSetup : public cMenuSetupPage { + public: + cTVScraperSetup(cTVScraperWorker *workerThread); + virtual ~cTVScraperSetup(); + private: + vector<int> channelsScrap; + cTVScraperWorker *worker; + void Setup(void); + protected: + virtual eOSState ProcessKey(eKeys Key); + virtual void Store(void); +}; + +class cTVScraperChannelSetup : public cOsdMenu { + public: + cTVScraperChannelSetup(vector<int> *channelsScrap); + virtual ~cTVScraperChannelSetup(); + private: + vector<int> *channelsScrap; + void Setup(void); + protected: + virtual eOSState ProcessKey(eKeys Key); +}; + +#endif //__TVSCRAPER_SETUP_H
\ No newline at end of file diff --git a/themoviedbscraper/moviedbactors.c b/themoviedbscraper/moviedbactors.c new file mode 100644 index 0000000..b3880f3 --- /dev/null +++ b/themoviedbscraper/moviedbactors.c @@ -0,0 +1,83 @@ +#include <string>
+#include <sstream>
+#include <vector>
+#include <algorithm>
+#include <jansson.h>
+#include "moviedbactors.h"
+
+using namespace std;
+
+cMovieDbActors::cMovieDbActors(string json) {
+ this->json = json;
+}
+
+cMovieDbActors::~cMovieDbActors() {
+ actors.clear();
+}
+
+void cMovieDbActors::ParseJSON(void) {
+ json_t *jActors;
+ json_error_t error;
+ jActors = json_loads(json.c_str(), 0, &error);
+ if (!jActors) {
+ return;
+ }
+ if(!json_is_object(jActors)) {
+ return;
+ }
+ json_t *cast = json_object_get(jActors, "cast");
+ if(!json_is_array(cast)) {
+ return;
+ }
+ size_t numActors = json_array_size(cast);
+ for (size_t i = 0; i < numActors; i++) {
+ json_t *jActor = json_array_get(cast, i);
+ if (!json_is_object(jActor)) {
+ return;
+ }
+ json_t *jId = json_object_get(jActor, "id");
+ json_t *jName = json_object_get(jActor, "name");
+ json_t *jRole = json_object_get(jActor, "character");
+ json_t *jPath = json_object_get(jActor, "profile_path");
+ if (!json_is_integer(jId) || !json_is_string(jName) || !json_is_string(jRole) || !json_is_string(jPath))
+ return;
+ cMovieDBActor *actor = new cMovieDBActor();
+ actor->id = json_integer_value(jId);
+ actor->name = json_string_value(jName);
+ actor->role = json_string_value(jRole);
+ actor->path = json_string_value(jPath);
+ actors.push_back(actor);
+ }
+}
+
+void cMovieDbActors::StoreDB(cTVScraperDB *db, int movieID) {
+ int numActors = actors.size();
+ for (int i=0; i<numActors; i++) {
+ db->InsertMovieActor(movieID, actors[i]->id, actors[i]->name, actors[i]->role);
+ }
+}
+
+void cMovieDbActors::Store(string baseUrl, string destDir) {
+ int size = actors.size();
+ string path;
+ string url;
+ for (int i=0; i<size; i++) {
+ stringstream strUrl;
+ strUrl << baseUrl << actors[i]->path;
+ url = strUrl.str();
+ stringstream fullPath;
+ fullPath << destDir << "/actor_" << actors[i]->id << ".jpg";
+ path = fullPath.str();
+ if (!FileExists(path)) {
+ CurlGetUrlFile(url.c_str(), path.c_str());
+ }
+ }
+}
+
+void cMovieDbActors::Dump(void) {
+ int numActors = actors.size();
+ esyslog("tvscraper: %d Actors:", numActors);
+ for (int i=0; i<numActors; i++) {
+ esyslog("tvscraper: id %d, name %s, role %s, path %s", actors[i]->id, actors[i]->name.c_str(), actors[i]->role.c_str(), actors[i]->path.c_str());
+ }
+}
\ No newline at end of file diff --git a/themoviedbscraper/moviedbactors.h b/themoviedbscraper/moviedbactors.h new file mode 100644 index 0000000..1169b85 --- /dev/null +++ b/themoviedbscraper/moviedbactors.h @@ -0,0 +1,37 @@ +#ifndef __TVSCRAPER_MOVIEDBACTORS_H
+#define __TVSCRAPER_MOVIEDBACTORS_H
+
+using namespace std;
+
+// --- cMovieDBActor -------------------------------------------------------------
+class cMovieDBActor {
+public:
+ cMovieDBActor(void) {
+ id = 0;
+ path = "";
+ name = "";
+ role = "";
+ };
+ int id;
+ string path;
+ string name;
+ string role;
+};
+
+// --- cMovieDBActors -------------------------------------------------------------
+
+class cMovieDbActors {
+private:
+ string json;
+ vector<cMovieDBActor*> actors;
+public:
+ cMovieDbActors(string json);
+ virtual ~cMovieDbActors(void);
+ void ParseJSON(void);
+ void StoreDB(cTVScraperDB *db, int movieID);
+ void Store(string baseUrl, string destDir);
+ void Dump(void);
+};
+
+
+#endif //__TVSCRAPER_MOVIEDBACTORS_H
diff --git a/themoviedbscraper/moviedbmovie.c b/themoviedbscraper/moviedbmovie.c new file mode 100644 index 0000000..26aee7f --- /dev/null +++ b/themoviedbscraper/moviedbmovie.c @@ -0,0 +1,140 @@ +#include <string>
+#include <sstream>
+#include <vector>
+#include <algorithm>
+#include <jansson.h>
+#include "moviedbmovie.h"
+
+using namespace std;
+
+cMovieDbMovie::cMovieDbMovie(string json) {
+ this->json = json;
+ title = "";
+ originalTitle = "";
+ overview = "";
+ backdropPath = "";
+ posterPath = "";
+}
+
+cMovieDbMovie::~cMovieDbMovie() {
+
+}
+
+void cMovieDbMovie::ParseJSON(void) {
+ json_t *movie;
+ json_error_t error;
+ movie = json_loads(json.c_str(), 0, &error);
+ if (!movie) {
+ return;
+ }
+ if(!json_is_object(movie)) {
+ return;
+ }
+ json_t *jTitle = json_object_get(movie, "title");
+ if(json_is_string(jTitle)) {
+ title = json_string_value(jTitle);;
+ }
+ json_t *jOriginalTitle = json_object_get(movie, "original_title");
+ if(json_is_string(jOriginalTitle)) {
+ originalTitle = json_string_value(jOriginalTitle);
+ }
+ json_t *jOverview = json_object_get(movie, "overview");
+ if(json_is_string(jOverview)) {
+ overview = json_string_value(jOverview);
+ }
+ json_t *jBackdrop = json_object_get(movie, "backdrop_path");
+ if(json_is_string(jBackdrop)) {
+ backdropPath = json_string_value(jBackdrop);
+ }
+ json_t *jPoster = json_object_get(movie, "poster_path");
+ if(json_is_string(jPoster)) {
+ posterPath = json_string_value(jPoster);
+ }
+}
+
+int cMovieDbMovie::ParseJSONForMovieId(string movieSearchString) {
+ //convert searchstring to lower case
+ transform(movieSearchString.begin(), movieSearchString.end(), movieSearchString.begin(), ::tolower);
+ json_t *root;
+ json_error_t error;
+ root = json_loads(json.c_str(), 0, &error);
+ if (!root) {
+ return -1;
+ }
+ if(!json_is_object(root)) {
+ return -1;
+ }
+ json_t *results = json_object_get(root, "results");
+ if(!json_is_array(results)) {
+ return -1;
+ }
+ size_t numResults = json_array_size(results);
+ for (size_t res = 0; res < numResults; res++) {
+ json_t *result = json_array_get(results, res);
+ if (!json_is_object(result)) {
+ return -1;
+ }
+ json_t *title = json_object_get(result, "title");
+ if (!json_is_string(title)) {
+ return -1;
+ }
+ string resultTitle = json_string_value(title);
+ //convert result to lower case
+ transform(resultTitle.begin(), resultTitle.end(), resultTitle.begin(), ::tolower);
+ json_t *jId = json_object_get(result, "id");
+ if (json_is_integer(jId)) {
+ int id = (int)json_integer_value(jId);
+ searchResult sRes;
+ sRes.id = id;
+ sRes.distance = sentence_distance(resultTitle, movieSearchString);
+ resultSet.push_back(sRes);
+ }
+ }
+ return FindBestResult();
+}
+
+int cMovieDbMovie::FindBestResult(void) {
+ int resID = -1;
+ int bestMatch = -1;
+ int numResults = resultSet.size();
+ for (int i=0; i<numResults; i++) {
+ if (i == 0) {
+ bestMatch = resultSet[i].distance;
+ resID = resultSet[i].id;
+ } else if (resultSet[i].distance < bestMatch) {
+ bestMatch = resultSet[i].distance;
+ resID = resultSet[i].id;
+ }
+ }
+ return resID;
+}
+
+void cMovieDbMovie::StoreMedia(string posterBaseUrl, string backdropBaseUrl, string destDir) {
+ stringstream pathPoster;
+ pathPoster << destDir << id << "_poster.jpg";
+ stringstream posterUrl;
+ posterUrl << posterBaseUrl << posterPath;
+ if (!FileExists(pathPoster.str())) {
+ CurlGetUrlFile(posterUrl.str().c_str(), pathPoster.str().c_str());
+ }
+ stringstream pathBackdrop;
+ pathBackdrop << destDir << id << "_backdrop.jpg";
+ stringstream backdropUrl;
+ backdropUrl << backdropBaseUrl << backdropPath;
+ if (!FileExists(pathBackdrop.str())) {
+ CurlGetUrlFile(backdropUrl.str().c_str(), pathBackdrop.str().c_str());
+ }
+}
+
+void cMovieDbMovie::StoreDB(cTVScraperDB *db) {
+ db->InsertMovie(id, title, originalTitle, overview);
+}
+
+void cMovieDbMovie::Dump(void) {
+ esyslog("tvscraper: -------------- MOVIE DUMP ---------------");
+ esyslog("tvscraper: title %s", title.c_str());
+ esyslog("tvscraper: originalTitle %s", originalTitle.c_str());
+ esyslog("tvscraper: overview %s", overview.c_str());
+ esyslog("tvscraper: backdropPath %s", backdropPath.c_str());
+ esyslog("tvscraper: posterPath %s", posterPath.c_str());
+}
\ No newline at end of file diff --git a/themoviedbscraper/moviedbmovie.h b/themoviedbscraper/moviedbmovie.h new file mode 100644 index 0000000..e1bd6d9 --- /dev/null +++ b/themoviedbscraper/moviedbmovie.h @@ -0,0 +1,37 @@ +#ifndef __TVSCRAPER_MOVIEDBMOVIE_H
+#define __TVSCRAPER_MOVIEDBMOVIE_H
+
+using namespace std;
+
+struct searchResult {
+ int id;
+ int distance;
+};
+
+// --- cMovieDbMovie -------------------------------------------------------------
+
+class cMovieDbMovie {
+private:
+ int id;
+ string json;
+ string title;
+ string originalTitle;
+ string overview;
+ string backdropPath;
+ string posterPath;
+ vector<searchResult> resultSet;
+ int FindBestResult(void);
+public:
+ cMovieDbMovie(string json);
+ virtual ~cMovieDbMovie(void);
+ int ParseJSONForMovieId(string movieSearchString);
+ void ParseJSON(void);
+ void SetID(int movieID) { id = movieID; };
+ int ID(void) { return id; };
+ void StoreDB(cTVScraperDB *db);
+ void StoreMedia(string posterBaseUrl, string backdropBaseUrl, string destDir);
+ void Dump();
+};
+
+
+#endif //__TVSCRAPER_TVDBSERIES_H
diff --git a/themoviedbscraper/themoviedbscraper.c b/themoviedbscraper/themoviedbscraper.c new file mode 100644 index 0000000..2f9e7e9 --- /dev/null +++ b/themoviedbscraper/themoviedbscraper.c @@ -0,0 +1,151 @@ +#include <string>
+#include <sstream>
+#include <vector>
+#include <jansson.h>
+#include "themoviedbscraper.h"
+
+using namespace std;
+
+cMovieDBScraper::cMovieDBScraper(string baseDir, cTVScraperDB *db, string language) {
+ apiKey = "abb01b5a277b9c2c60ec0302d83c5ee9";
+ //TODO: use language from VDR settings
+ this->language = language;
+ baseURL = "api.themoviedb.org/3";
+ this->baseDir = baseDir;
+ this->db = db;
+ posterSize = "w500";
+ backdropSize = "w1280";
+ actorthumbSize = "h632";
+}
+
+cMovieDBScraper::~cMovieDBScraper() {
+}
+
+void cMovieDBScraper::Scrap(const cEvent *event, int recordingID) {
+ string movieName = event->Title();
+ int eventID = (int)event->EventID();
+ if (config.enableDebug)
+ esyslog("tvscraper: scrapping movie \"%s\"", movieName.c_str());
+ int movieID = SearchMovie(movieName);
+ if (movieID < 1) {
+ if (config.enableDebug)
+ esyslog("tvscraper: nothing found for \"%s\"", movieName.c_str());
+ return;
+ }
+ if (!recordingID) {
+ time_t validTill = event->EndTime();
+ db->InsertEventMovie(eventID, validTill, movieID);
+ } else {
+ db->InsertRecording(recordingID, 0, movieID);
+ }
+ if (db->MovieExists(movieID)) {
+ return;
+ }
+ cMovieDbMovie *movie = ReadMovie(movieID);
+ if (!movie)
+ return;
+ movie->StoreDB(db);
+ cMovieDbActors *actors = ReadActors(movieID);
+ if (!actors) {
+ delete movie;
+ return;
+ }
+ actors->StoreDB(db, movieID);
+ StoreMedia(movie, actors);
+ delete movie;
+ delete actors;
+ if (config.enableDebug)
+ esyslog("tvscraper: \"%s\" successfully scrapped, id %d", movieName.c_str(), movieID);
+}
+
+bool cMovieDBScraper::Connect(void) {
+ stringstream url;
+ url << baseURL << "/configuration?api_key=" << apiKey;
+ string configJSON;
+ if (CurlGetUrl(url.str().c_str(), &configJSON)) {
+ return parseJSON(configJSON);
+ }
+ return false;
+}
+
+bool cMovieDBScraper::parseJSON(string jsonString) {
+ json_t *root;
+ json_error_t error;
+
+ root = json_loads(jsonString.c_str(), 0, &error);
+ if (!root) {
+ return false;
+ }
+ if(!json_is_object(root)) {
+ return false;
+ }
+ json_t *images;
+ images = json_object_get(root, "images");
+ if(!json_is_object(images)) {
+ return false;
+ }
+
+ json_t *imgUrl;
+ imgUrl = json_object_get(images, "base_url");
+ if(!json_is_string(imgUrl)) {
+ return false;
+ }
+ imageUrl = json_string_value(imgUrl);
+ return true;
+}
+
+int cMovieDBScraper::SearchMovie(string movieName) {
+ stringstream movieEscaped;
+ movieEscaped << "\"" << movieName << "\"";
+ stringstream url;
+ url << baseURL << "/search/movie?api_key=" << apiKey << "&query=" << CurlEscape(movieEscaped.str().c_str()) << "&language=" << language.c_str();
+ string movieJSON;
+ int movieID = -1;
+ if (CurlGetUrl(url.str().c_str(), &movieJSON)) {
+ cMovieDbMovie *movie = new cMovieDbMovie(movieJSON);
+ movieID = movie->ParseJSONForMovieId(movieName);
+ delete movie;
+ }
+ return movieID;
+}
+
+cMovieDbMovie *cMovieDBScraper::ReadMovie(int movieID) {
+ stringstream url;
+ url << baseURL << "/movie/" << movieID << "?api_key=" << apiKey << "&language=" << language.c_str();
+ string movieJSON;
+ cMovieDbMovie *movie = NULL;
+ if (CurlGetUrl(url.str().c_str(), &movieJSON)) {
+ movie = new cMovieDbMovie(movieJSON);
+ movie->SetID(movieID);
+ movie->ParseJSON();
+ }
+ return movie;
+}
+
+cMovieDbActors *cMovieDBScraper::ReadActors(int movieID) {
+ stringstream url;
+ url << baseURL << "/movie/" << movieID << "/casts?api_key=" << apiKey;
+ string actorsJSON;
+ cMovieDbActors *actors = NULL;
+ if (CurlGetUrl(url.str().c_str(), &actorsJSON)) {
+ actors = new cMovieDbActors(actorsJSON);
+ actors->ParseJSON();
+ }
+ return actors;
+}
+
+void cMovieDBScraper::StoreMedia(cMovieDbMovie *movie, cMovieDbActors *actors) {
+ stringstream posterUrl;
+ posterUrl << imageUrl << posterSize;
+ stringstream backdropUrl;
+ backdropUrl << imageUrl << backdropSize;
+ stringstream destDir;
+ destDir << baseDir << "/";
+ movie->StoreMedia(posterUrl.str(), backdropUrl.str(), destDir.str());
+ stringstream actorsUrl;
+ actorsUrl << imageUrl << actorthumbSize;
+ stringstream actorsDestDir;
+ actorsDestDir << baseDir << "/actors";
+ CreateDirectory(actorsDestDir.str());
+ actors->Store(actorsUrl.str(), actorsDestDir.str());
+}
diff --git a/themoviedbscraper/themoviedbscraper.h b/themoviedbscraper/themoviedbscraper.h new file mode 100644 index 0000000..54d463b --- /dev/null +++ b/themoviedbscraper/themoviedbscraper.h @@ -0,0 +1,32 @@ +#ifndef __TVSCRAPER_MOVIEDBSCRAPER_H
+#define __TVSCRAPER_MOVIEDBSCRAPER_H
+
+using namespace std;
+
+// --- cMovieDBScraper -------------------------------------------------------------
+
+class cMovieDBScraper {
+private:
+ string apiKey;
+ string language;
+ string baseURL;
+ string baseDir;
+ string imageUrl;
+ string posterSize;
+ string backdropSize;
+ string actorthumbSize;
+ cTVScraperDB *db;
+ bool parseJSON(string jsonString);
+ int SearchMovie(string movieName);
+ cMovieDbMovie *ReadMovie(int movieID);
+ cMovieDbActors *ReadActors(int movieID);
+ void StoreMedia(cMovieDbMovie *movie, cMovieDbActors *actors);
+public:
+ cMovieDBScraper(string baseDir, cTVScraperDB *db, string language);
+ virtual ~cMovieDBScraper(void);
+ bool Connect(void);
+ void Scrap(const cEvent *event, int recordingID = 0);
+};
+
+
+#endif //__TVSCRAPER_MOVIEDBSCRAPER_H
diff --git a/thetvdbscraper/thetvdbscraper.c b/thetvdbscraper/thetvdbscraper.c new file mode 100644 index 0000000..5695ba0 --- /dev/null +++ b/thetvdbscraper/thetvdbscraper.c @@ -0,0 +1,129 @@ +#include <string>
+#include <sstream>
+#include <vector>
+#include <libxml/parser.h>
+#include <libxml/tree.h>
+#include "thetvdbscraper.h"
+
+using namespace std;
+
+cTVDBScraper::cTVDBScraper(string baseDir, cTVScraperDB *db, string language) {
+ apiKey = "E9DBB94CA50832ED";
+ baseURL = "thetvdb.com";
+ this->baseDir = baseDir;
+ this->language = language;
+ this->db = db;
+ mirrors = NULL;
+}
+
+cTVDBScraper::~cTVDBScraper() {
+ if (mirrors)
+ delete mirrors;
+}
+
+void cTVDBScraper::Scrap(const cEvent *event, int recordingID) {
+ string seriesName = event->Title();
+ if (config.enableDebug)
+ esyslog("tvscraper: scrapping series \"%s\"", seriesName.c_str());
+ int eventID = (int)event->EventID();
+ cTVDBSeries *series = ReadSeries(seriesName);
+ cTVDBSeriesMedia *media = NULL;
+ cTVDBActors *actors = NULL;
+ if (!series)
+ return;
+ if (series->ID() < 1) {
+ if (config.enableDebug)
+ esyslog("tvscraper: nothing found for \"%s\"", seriesName.c_str());
+ return;
+ }
+ if (!db->SeriesExists(series->ID())) {
+ series->StoreDB(db);
+ media = ReadSeriesMedia(series->ID());
+ actors = ReadSeriesActors(series->ID());
+ if (actors)
+ actors->StoreDB(db, series->ID());
+ StoreMedia(series, media, actors);
+ }
+ if (!recordingID) {
+ time_t validTill = event->EndTime();
+ db->InsertEventSeries(eventID, validTill, series->ID());
+ } else {
+ db->InsertRecording(recordingID, series->ID(), 0);
+ }
+ delete series;
+ if (media)
+ delete media;
+ if (actors)
+ delete actors;
+ if (config.enableDebug)
+ esyslog("tvscraper: \"%s\" successfully scrapped, id %d", seriesName.c_str(), series->ID());
+}
+
+
+bool cTVDBScraper::Connect(void) {
+ stringstream url;
+ url << baseURL << "/api/" << apiKey << "/mirrors.xml";
+ string mirrorsXML;
+ if (CurlGetUrl(url.str().c_str(), &mirrorsXML)) {
+ mirrors = new cTVDBMirrors(mirrorsXML);
+ return mirrors->ParseXML();
+ }
+ return false;
+}
+
+cTVDBSeries *cTVDBScraper::ReadSeries(string seriesName) {
+ stringstream seriesEscape;
+ seriesEscape << "\"" << seriesName << "\"";
+ stringstream url;
+ url << mirrors->GetMirrorXML() << "/api/GetSeries.php?seriesname=" << CurlEscape(seriesEscape.str().c_str()) << "&language=" << language.c_str();
+ string seriesXML;
+ cTVDBSeries *series = NULL;
+ if (CurlGetUrl(url.str().c_str(), &seriesXML)) {
+ series = new cTVDBSeries(seriesXML);
+ series->ParseXML();
+ }
+ return series;
+}
+
+cTVDBSeriesMedia *cTVDBScraper::ReadSeriesMedia(int seriesID) {
+ stringstream url;
+ url << mirrors->GetMirrorXML() << "/api/" << apiKey << "/series/" << seriesID << "/banners.xml";
+ string bannersXML;
+ cTVDBSeriesMedia *media = NULL;
+ if (CurlGetUrl(url.str().c_str(), &bannersXML)) {
+ media = new cTVDBSeriesMedia(bannersXML, language);
+ media->ParseXML();
+ }
+ return media;
+}
+
+cTVDBActors *cTVDBScraper::ReadSeriesActors(int seriesID) {
+ stringstream url;
+ url << mirrors->GetMirrorXML() << "/api/" << apiKey << "/series/" << seriesID << "/actors.xml";
+ string actorsXML;
+ cTVDBActors *actors = NULL;
+ if (CurlGetUrl(url.str().c_str(), &actorsXML)) {
+ actors = new cTVDBActors(actorsXML, language);
+ actors->ParseXML();
+ }
+ return actors;
+}
+
+void cTVDBScraper::StoreMedia(cTVDBSeries *series, cTVDBSeriesMedia *media, cTVDBActors *actors) {
+ stringstream baseUrl;
+ baseUrl << mirrors->GetMirrorBanner() << "/banners/";
+ stringstream destDir;
+ destDir << baseDir << "/" << series->ID() << "/";
+ bool ok = CreateDirectory(destDir.str().c_str());
+ if (!ok)
+ return;
+ if (series) {
+ series->StoreBanner(baseUrl.str(), destDir.str());
+ }
+ if (media) {
+ media->Store(baseUrl.str(), destDir.str());
+ }
+ if (actors) {
+ actors->Store(baseUrl.str(), destDir.str());
+ }
+}
diff --git a/thetvdbscraper/thetvdbscraper.h b/thetvdbscraper/thetvdbscraper.h new file mode 100644 index 0000000..2659bdc --- /dev/null +++ b/thetvdbscraper/thetvdbscraper.h @@ -0,0 +1,28 @@ +#ifndef __TVSCRAPER_TVDBSCRAPER_H
+#define __TVSCRAPER_TVDBSCRAPER_H
+
+using namespace std;
+
+// --- cTVDBScraper -------------------------------------------------------------
+
+class cTVDBScraper {
+private:
+ string apiKey;
+ string baseURL;
+ string baseDir;
+ string language;
+ cTVScraperDB *db;
+ cTVDBMirrors *mirrors;
+ cTVDBSeries *ReadSeries(string seriesName);
+ cTVDBSeriesMedia *ReadSeriesMedia(int seriesID);
+ cTVDBActors *ReadSeriesActors(int seriesID);
+ void StoreMedia(cTVDBSeries *series, cTVDBSeriesMedia *media, cTVDBActors *actors);
+public:
+ cTVDBScraper(string baseDir, cTVScraperDB *db, string language);
+ virtual ~cTVDBScraper(void);
+ bool Connect(void);
+ void Scrap(const cEvent *event, int recordingID = 0);
+};
+
+
+#endif //__TVSCRAPER_TVDBSCRAPER_H
diff --git a/thetvdbscraper/tvdbactors.c b/thetvdbscraper/tvdbactors.c new file mode 100644 index 0000000..404cd38 --- /dev/null +++ b/thetvdbscraper/tvdbactors.c @@ -0,0 +1,100 @@ +#include <string>
+#include <sstream>
+#include <vector>
+#include <libxml/parser.h>
+#include <libxml/tree.h>
+#include "tvdbactors.h"
+
+using namespace std;
+
+cTVDBActors::cTVDBActors(string xml, string language) {
+ doc = NULL;
+ SetXMLDoc(xml);
+ this->language = language;
+}
+
+cTVDBActors::~cTVDBActors() {
+ xmlFreeDoc(doc);
+ actors.clear();
+}
+
+void cTVDBActors::SetXMLDoc(string xml) {
+ xmlInitParser();
+ doc = xmlReadMemory(xml.c_str(), strlen(xml.c_str()), "noname.xml", NULL, 0);
+}
+
+void cTVDBActors::ParseXML(void) {
+ if (doc == NULL)
+ return;
+ //Root Element has to be <Actors>
+ xmlNode *node = NULL;
+ node = xmlDocGetRootElement(doc);
+ if (!(node && !xmlStrcmp(node->name, (const xmlChar *)"Actors")))
+ return;
+ //Looping through actors
+ node = node->children;
+ xmlNode *cur_node = NULL;
+ for (cur_node = node; cur_node; cur_node = cur_node->next) {
+ if ((cur_node->type == XML_ELEMENT_NODE) && !xmlStrcmp(cur_node->name, (const xmlChar *)"Actor")) {
+ ReadEntry(cur_node->children);
+ }
+ }
+}
+
+void cTVDBActors::ReadEntry(xmlNode *node) {
+ xmlNode *cur_node = NULL;
+ xmlChar *node_content;
+ cTVDBActor *actor = new cTVDBActor();
+ for (cur_node = node; cur_node; cur_node = cur_node->next) {
+ if (cur_node->type == XML_ELEMENT_NODE) {
+ node_content = xmlNodeListGetString(doc, cur_node->xmlChildrenNode, 1);
+ if (!node_content)
+ continue;
+ if (!xmlStrcmp(cur_node->name, (const xmlChar *)"Image")) {
+ actor->path = (const char *)node_content;
+ } else if (!xmlStrcmp(cur_node->name, (const xmlChar *)"Name")) {
+ actor->name = (const char *)node_content;
+ } else if (!xmlStrcmp(cur_node->name, (const xmlChar *)"Role")) {
+ actor->role = (const char *)node_content;
+ }
+ xmlFree(node_content);
+ }
+ }
+ actors.push_back(actor);
+}
+
+void cTVDBActors::StoreDB(cTVScraperDB *db, int series_id) {
+ int size = actors.size();
+ for (int i=0; i<size; i++) {
+ stringstream strThumb;
+ strThumb << "actor_" << i << ".jpg";
+ db->InsertActor(series_id, actors[i]->name, actors[i]->role, strThumb.str());
+ }
+}
+
+void cTVDBActors::Store(string baseUrl, string destDir) {
+ int size = actors.size();
+ string path;
+ string url;
+ for (int i=0; i<size; i++) {
+ stringstream strUrl;
+ strUrl << baseUrl << actors[i]->path;
+ url = strUrl.str();
+ stringstream fullPath;
+ fullPath << destDir << "actor_" << i << ".jpg";
+ path = fullPath.str();
+ if (actors[i]->path.size() > 0) {
+ CurlGetUrlFile(url.c_str(), path.c_str());
+ }
+ }
+}
+
+void cTVDBActors::Dump(bool verbose) {
+ int size = actors.size();
+ esyslog("tvscraper: read %d actor entries", size);
+ if (!verbose)
+ return;
+ for (int i=0; i<size; i++) {
+ esyslog("tvscraper: path %s, name: %s, role %s", actors[i]->path.c_str(), actors[i]->name.c_str(), actors[i]->role.c_str());
+ }
+}
\ No newline at end of file diff --git a/thetvdbscraper/tvdbactors.h b/thetvdbscraper/tvdbactors.h new file mode 100644 index 0000000..819d1d4 --- /dev/null +++ b/thetvdbscraper/tvdbactors.h @@ -0,0 +1,37 @@ +#ifndef __TVSCRAPER_TVDBACTORS_H
+#define __TVSCRAPER_TVDBACTORS_H
+
+using namespace std;
+
+// --- cTVDBActor -------------------------------------------------------------
+class cTVDBActor {
+public:
+ cTVDBActor(void) {
+ path = "";
+ name = "";
+ role = "";
+ };
+ string path;
+ string name;
+ string role;
+};
+
+// --- cTVDBActors --------------------------------------------------------
+
+class cTVDBActors {
+private:
+ xmlDoc *doc;
+ string language;
+ vector<cTVDBActor*> actors;
+ void SetXMLDoc(string xml);
+ void ReadEntry(xmlNode *node);
+public:
+ cTVDBActors(string xml, string language);
+ virtual ~cTVDBActors(void);
+ void ParseXML(void);
+ void StoreDB(cTVScraperDB *db, int series_id);
+ void Store(string baseUrl, string destDir);
+ void Dump(bool verbose);
+};
+
+#endif //__TVSCRAPER_TVDBACTORS_H
diff --git a/thetvdbscraper/tvdbmedia.c b/thetvdbscraper/tvdbmedia.c new file mode 100644 index 0000000..f9672b6 --- /dev/null +++ b/thetvdbscraper/tvdbmedia.c @@ -0,0 +1,149 @@ +#include <string>
+#include <sstream>
+#include <vector>
+#include <libxml/parser.h>
+#include <libxml/tree.h>
+#include "tvdbmedia.h"
+
+using namespace std;
+
+cTVDBSeriesMedia::cTVDBSeriesMedia(string xml, string language) {
+ doc = NULL;
+ SetXMLDoc(xml);
+ this->language = language;
+}
+
+cTVDBSeriesMedia::~cTVDBSeriesMedia() {
+ xmlFreeDoc(doc);
+ medias.clear();
+}
+
+void cTVDBSeriesMedia::SetXMLDoc(string xml) {
+ xmlInitParser();
+ doc = xmlReadMemory(xml.c_str(), strlen(xml.c_str()), "noname.xml", NULL, 0);
+}
+
+void cTVDBSeriesMedia::ParseXML(void) {
+ if (doc == NULL)
+ return;
+ //Root Element has to be <Banners>
+ xmlNode *node = NULL;
+ node = xmlDocGetRootElement(doc);
+ if (!(node && !xmlStrcmp(node->name, (const xmlChar *)"Banners")))
+ return;
+ //Looping through banners
+ node = node->children;
+ xmlNode *cur_node = NULL;
+ for (cur_node = node; cur_node; cur_node = cur_node->next) {
+ if ((cur_node->type == XML_ELEMENT_NODE) && !xmlStrcmp(cur_node->name, (const xmlChar *)"Banner")) {
+ ReadEntry(cur_node->children);
+ }
+ }
+}
+
+void cTVDBSeriesMedia::ReadEntry(xmlNode *node) {
+ xmlNode *cur_node = NULL;
+ xmlChar *node_content;
+ cTVDBMedia *media = new cTVDBMedia();
+ for (cur_node = node; cur_node; cur_node = cur_node->next) {
+ if (cur_node->type == XML_ELEMENT_NODE) {
+ node_content = xmlNodeListGetString(doc, cur_node->xmlChildrenNode, 1);
+ if (!node_content)
+ continue;
+ if (!xmlStrcmp(cur_node->name, (const xmlChar *)"BannerPath")) {
+ media->path = (const char *)node_content;
+ } else if (!xmlStrcmp(cur_node->name, (const xmlChar *)"BannerType")) {
+ if (!xmlStrcmp(node_content, (const xmlChar *)"poster"))
+ media->type = mediaPoster;
+ else if (!xmlStrcmp(node_content, (const xmlChar *)"fanart"))
+ media->type = mediaFanart;
+ else if (!xmlStrcmp(node_content, (const xmlChar *)"banner"))
+ media->type = mediaBanner;
+ else if (!xmlStrcmp(node_content, (const xmlChar *)"season"))
+ media->type = mediaSeason;
+ else
+ media->type = mediaUnknown;
+ } else if (!xmlStrcmp(cur_node->name, (const xmlChar *)"Season")) {
+ media->season = atoi((const char *)node_content);
+ } else if (!xmlStrcmp(cur_node->name, (const xmlChar *)"Language")) {
+ media->language = (const char *)node_content;
+ }
+ xmlFree(node_content);
+ }
+ }
+ if (media->type == mediaUnknown)
+ delete media;
+ else if ((media->type == mediaSeason) && (media->language).compare(language))
+ delete media;
+ else
+ medias.push_back(media);
+}
+
+void cTVDBSeriesMedia::Store(string baseUrl, string destDir) {
+ int size = medias.size();
+ string path;
+ string url;
+ int maxPix = 3;
+ int numFanart = 0;
+ int numPoster = 0;
+ int numSeason = 0;
+ int numBanner = 0;
+ for (int i=0; i<size; i++) {
+ if (medias[i]->path.size() == 0)
+ continue;
+ stringstream strUrl;
+ strUrl << baseUrl << medias[i]->path;
+ url = strUrl.str();
+ switch (medias[i]->type) {
+ case mediaFanart: {
+ if (numFanart >= maxPix)
+ continue;
+ stringstream fullPath;
+ fullPath << destDir << "fanart_" << numFanart << ".jpg";
+ path = fullPath.str();
+ numFanart++;
+ break; }
+ case mediaPoster: {
+ if (numPoster >= maxPix)
+ continue;
+ stringstream fullPath;
+ fullPath << destDir << "poster_" << numPoster << ".jpg";
+ path = fullPath.str();
+ numPoster++;
+ break; }
+ case mediaSeason: {
+ stringstream fullPath;
+ if (medias[i]->season > 0) {
+ fullPath << destDir << "season_poster_" << medias[i]->season << ".jpg";
+ } else {
+ if (numSeason >= maxPix)
+ continue;
+ fullPath << destDir << "season_" << numSeason << ".jpg";
+ numSeason++;
+ }
+ path = fullPath.str();
+ break; }
+ case mediaBanner: {
+ if (numBanner > 9)
+ continue;
+ stringstream fullPath;
+ fullPath << destDir << "banner_" << numBanner << ".jpg";
+ path = fullPath.str();
+ numBanner++;
+ break; }
+ default:
+ break;
+ }
+ CurlGetUrlFile(url.c_str(), path.c_str());
+ }
+}
+
+void cTVDBSeriesMedia::Dump(bool verbose) {
+ int size = medias.size();
+ esyslog("tvscraper: read %d banner entries", size);
+ if (!verbose)
+ return;
+ for (int i=0; i<size; i++) {
+ esyslog("tvscraper: type %d, path %s, lang: %s, season %d", medias[i]->type, medias[i]->path.c_str(), medias[i]->language.c_str(), medias[i]->season);
+ }
+}
\ No newline at end of file diff --git a/thetvdbscraper/tvdbmedia.h b/thetvdbscraper/tvdbmedia.h new file mode 100644 index 0000000..b9e9b0c --- /dev/null +++ b/thetvdbscraper/tvdbmedia.h @@ -0,0 +1,45 @@ +#ifndef __TVSCRAPER_TVDBMEDIA_H
+#define __TVSCRAPER_TVDBMEDIA_H
+
+using namespace std;
+
+enum mediaType {
+ mediaUnknown,
+ mediaPoster,
+ mediaFanart,
+ mediaSeason,
+ mediaBanner,
+};
+
+// --- cTVDBMedia -------------------------------------------------------------
+class cTVDBMedia {
+public:
+ cTVDBMedia(void) {
+ path = "";
+ language = "";
+ season = 0;
+ };
+ mediaType type;
+ string path;
+ string language;
+ int season;
+};
+
+// --- cTVDBSeriesMedia --------------------------------------------------------
+
+class cTVDBSeriesMedia {
+private:
+ xmlDoc *doc;
+ string language;
+ vector<cTVDBMedia*> medias;
+ void SetXMLDoc(string xml);
+ void ReadEntry(xmlNode *node);
+public:
+ cTVDBSeriesMedia(string xml, string language);
+ virtual ~cTVDBSeriesMedia(void);
+ void ParseXML(void);
+ void Store(string baseUrl, string destDir);
+ void Dump(bool verbose);
+};
+
+#endif //__TVSCRAPER_TVDBMEDIA_H
diff --git a/thetvdbscraper/tvdbmirrors.c b/thetvdbscraper/tvdbmirrors.c new file mode 100644 index 0000000..d0849c2 --- /dev/null +++ b/thetvdbscraper/tvdbmirrors.c @@ -0,0 +1,97 @@ +#include <string>
+#include <sstream>
+#include <vector>
+#include <libxml/parser.h>
+#include <libxml/tree.h>
+#include "tvdbmirrors.h"
+
+using namespace std;
+
+cTVDBMirrors::cTVDBMirrors(string xml) {
+ doc = NULL;
+ SetXMLDoc(xml);
+}
+
+cTVDBMirrors::~cTVDBMirrors() {
+ xmlFreeDoc(doc);
+}
+
+void cTVDBMirrors::SetXMLDoc(string xml) {
+ xmlInitParser();
+ doc = xmlReadMemory(xml.c_str(), strlen(xml.c_str()), "noname.xml", NULL, 0);
+}
+
+string cTVDBMirrors::GetMirrorXML(void) {
+ if (xmlmirrors.size() == 0)
+ return "";
+ int randMirror = rand() % xmlmirrors.size();
+ return xmlmirrors[randMirror];
+}
+string cTVDBMirrors::GetMirrorBanner(void) {
+ if (bannermirrors.size() == 0)
+ return "";
+ int randMirror = rand() % bannermirrors.size();
+ return bannermirrors[randMirror];
+}
+
+string cTVDBMirrors::GetMirrorZip(void) {
+ if (zipmirrors.size() == 0)
+ return "";
+ int randMirror = rand() % zipmirrors.size();
+ return zipmirrors[randMirror];
+}
+
+bool cTVDBMirrors::ParseXML(void) {
+ if (doc == NULL)
+ return false;
+ //Root Element has to be <Mirrors>
+ xmlNode *node = NULL;
+ node = xmlDocGetRootElement(doc);
+ if (!(node && !xmlStrcmp(node->name, (const xmlChar *)"Mirrors")))
+ return false;
+ //Loop through <Mirror>
+ node = node->children;
+ xmlNode *cur_node = NULL;
+ bool ok = false;
+ for (cur_node = node; cur_node; cur_node = cur_node->next) {
+ if ((cur_node->type == XML_ELEMENT_NODE) && !xmlStrcmp(cur_node->name, (const xmlChar *)"Mirror")) {
+ ok = ReadEntry(cur_node->children);
+ }
+ }
+ return ok;
+}
+
+bool cTVDBMirrors::ReadEntry(xmlNode *node) {
+ xmlNode *cur_node = NULL;
+ xmlChar *node_content;
+ string path = "";
+ int typemask = 0;
+ for (cur_node = node; cur_node; cur_node = cur_node->next) {
+ if (cur_node->type == XML_ELEMENT_NODE) {
+ node_content = xmlNodeListGetString(doc, cur_node->xmlChildrenNode, 1);
+ if (!node_content)
+ continue;
+ if (!xmlStrcmp(cur_node->name, (const xmlChar *)"mirrorpath")) {
+ path = (const char *)node_content;
+ } else if (!xmlStrcmp(cur_node->name, (const xmlChar *)"typemask")) {
+ typemask = atoi((const char *)node_content);
+ }
+ xmlFree(node_content);
+ }
+ }
+ return CreateMirror(path, typemask);
+}
+
+bool cTVDBMirrors::CreateMirror(string path, int typemask) {
+ if (path.size() < 1)
+ return false;
+ if (typemask < 1 || typemask > 7)
+ return false;
+ if (typemask & 1)
+ xmlmirrors.push_back(path);
+ if (typemask & 2)
+ bannermirrors.push_back(path);
+ if (typemask & 4)
+ zipmirrors.push_back(path);
+ return true;
+}
\ No newline at end of file diff --git a/thetvdbscraper/tvdbmirrors.h b/thetvdbscraper/tvdbmirrors.h new file mode 100644 index 0000000..67ad92c --- /dev/null +++ b/thetvdbscraper/tvdbmirrors.h @@ -0,0 +1,27 @@ +#ifndef __TVSCRAPER_TVDBMIRRORS_H
+#define __TVSCRAPER_TVDBMIRRORS_H
+
+using namespace std;
+
+// --- cTVDBMirrors -------------------------------------------------------------
+
+class cTVDBMirrors {
+private:
+ xmlDoc *doc;
+ vector<string> xmlmirrors;
+ vector<string> bannermirrors;
+ vector<string> zipmirrors;
+ void SetXMLDoc(string xml);
+ bool ReadEntry(xmlNode *node);
+ bool CreateMirror(string path, int typemask);
+public:
+ cTVDBMirrors(string xml);
+ virtual ~cTVDBMirrors(void);
+ bool ParseXML(void);
+ string GetMirrorXML(void);
+ string GetMirrorBanner(void);
+ string GetMirrorZip(void);
+};
+
+
+#endif //__TVSCRAPER_TVDBMIRRORS_H
diff --git a/thetvdbscraper/tvdbseries.c b/thetvdbscraper/tvdbseries.c new file mode 100644 index 0000000..b679e5e --- /dev/null +++ b/thetvdbscraper/tvdbseries.c @@ -0,0 +1,92 @@ +#include <string>
+#include <sstream>
+#include <vector>
+#include <libxml/parser.h>
+#include <libxml/tree.h>
+#include "tvdbseries.h"
+
+using namespace std;
+
+cTVDBSeries::cTVDBSeries(string xml) {
+ doc = NULL;
+ SetXMLDoc(xml);
+ seriesID = 0;
+ name = "";
+ banner = "";
+ overview = "";
+ imbdid = "";
+}
+
+cTVDBSeries::~cTVDBSeries() {
+ xmlFreeDoc(doc);
+}
+
+void cTVDBSeries::SetXMLDoc(string xml) {
+ xmlInitParser();
+ doc = xmlReadMemory(xml.c_str(), strlen(xml.c_str()), "noname.xml", NULL, 0);
+}
+
+void cTVDBSeries::ParseXML(void) {
+ if (doc == NULL)
+ return;
+ //Root Element has to be <Data>
+ xmlNode *node = NULL;
+ node = xmlDocGetRootElement(doc);
+ if (!(node && !xmlStrcmp(node->name, (const xmlChar *)"Data")))
+ return;
+ //Searching for <Series>
+ node = node->children;
+ xmlNode *cur_node = NULL;
+ for (cur_node = node; cur_node; cur_node = cur_node->next) {
+ if ((cur_node->type == XML_ELEMENT_NODE) && !xmlStrcmp(cur_node->name, (const xmlChar *)"Series")) {
+ node = cur_node;
+ break;
+ } else {
+ node = NULL;
+ }
+ }
+ if (!node)
+ return;
+ //now read the first series
+ node = node->children;
+ xmlChar *node_content;
+ for (cur_node = node; cur_node; cur_node = cur_node->next) {
+ if (cur_node->type == XML_ELEMENT_NODE) {
+ node_content = xmlNodeListGetString(doc, cur_node->xmlChildrenNode, 1);
+ if (!node_content)
+ continue;
+ if (!xmlStrcmp(cur_node->name, (const xmlChar *)"seriesid")) {
+ seriesID = atoi((const char *)node_content);
+ } else if (!xmlStrcmp(cur_node->name, (const xmlChar *)"SeriesName")) {
+ name = (const char *)node_content;
+ } else if (!xmlStrcmp(cur_node->name, (const xmlChar *)"Overview")) {
+ overview = (const char *)node_content;
+ } else if (!xmlStrcmp(cur_node->name, (const xmlChar *)"banner")) {
+ banner = (const char *)node_content;
+ } else if (!xmlStrcmp(cur_node->name, (const xmlChar *)"IMDB_ID")) {
+ imbdid = (const char *)node_content;
+ }
+ xmlFree(node_content);
+ }
+ }
+}
+
+void cTVDBSeries::StoreDB(cTVScraperDB *db) {
+ db->InsertSeries(seriesID, name, overview);
+}
+
+void cTVDBSeries::StoreBanner(string baseUrl, string destDir) {
+ if (banner.size() == 0)
+ return;
+ stringstream strUrl;
+ strUrl << baseUrl << banner;
+ string url = strUrl.str();
+ stringstream fullPath;
+ fullPath << destDir << "banner.jpg";
+ string path = fullPath.str();
+ CurlGetUrlFile(url.c_str(), path.c_str());
+}
+
+void cTVDBSeries::Dump() {
+ esyslog("tvscraper: series %s, id: %d, overview %s, imdb %s", name.c_str(), seriesID, overview.c_str(), imbdid.c_str());
+}
diff --git a/thetvdbscraper/tvdbseries.h b/thetvdbscraper/tvdbseries.h new file mode 100644 index 0000000..98fb75d --- /dev/null +++ b/thetvdbscraper/tvdbseries.h @@ -0,0 +1,29 @@ +#ifndef __TVSCRAPER_TVDBSERIES_H
+#define __TVSCRAPER_TVDBSERIES_H
+
+using namespace std;
+
+// --- cTVDBSeries -------------------------------------------------------------
+
+class cTVDBSeries {
+private:
+ xmlDoc *doc;
+ int seriesID;
+ string name;
+ string banner;
+ string overview;
+ string imbdid;
+ void SetXMLDoc(string xml);
+public:
+ cTVDBSeries(string xml);
+ virtual ~cTVDBSeries(void);
+ void ParseXML(void);
+ int ID(void) { return seriesID; };
+ const char *Name(void) { return name.c_str(); };
+ void StoreDB(cTVScraperDB *db);
+ void StoreBanner(string baseUrl, string destDir);
+ void Dump();
+};
+
+
+#endif //__TVSCRAPER_TVDBSERIES_H
diff --git a/tools/curlfuncs.cpp b/tools/curlfuncs.cpp new file mode 100644 index 0000000..de4c72f --- /dev/null +++ b/tools/curlfuncs.cpp @@ -0,0 +1,235 @@ +/* +Copyright (c) 2002, Mayukh Bose +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +* Neither the name of Mayukh Bose nor the names of other +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +/* + Change History: + 11/23/2004 - Removed the #include <unistd.h> line because I didn't + need it. Wonder why I had it there in the first place :). + 10/20/2004 - Publicly released this code. +*/ +#include <string> +#include <cstdio> +#include <curl/curl.h> +#include <curl/easy.h> +#include "curlfuncs.h" + +#ifndef TRUE +#define TRUE 1 +#endif + +using namespace std; + +// Local function prototypes +int CurlDoPost(const char *url, string *sOutput, const string &sReferer, + struct curl_httppost *formpost, struct curl_slist *headerlist); + +namespace curlfuncs { + string sBuf; + bool bInitialized = false; + CURL *curl = NULL; +} + +size_t collect_data(void *ptr, size_t size, size_t nmemb, void *stream) +{ + string sTmp; + register size_t actualsize = size * nmemb; + if ((FILE *)stream == NULL) { + sTmp.assign((char *)ptr, actualsize); + curlfuncs::sBuf += sTmp; + } + else { + size_t xxx = fwrite(ptr, size, nmemb, (FILE *)stream); + } + return actualsize; +} + +inline void InitCurlLibraryIfNeeded() +{ + if (!curlfuncs::bInitialized) { + curl_global_init(CURL_GLOBAL_ALL); + curlfuncs::curl = curl_easy_init(); + if (!curlfuncs::curl) + throw string("Could not create new curl instance"); + curl_easy_setopt(curlfuncs::curl, CURLOPT_NOPROGRESS, 1); // Do not show progress + curl_easy_setopt(curlfuncs::curl, CURLOPT_WRITEFUNCTION, collect_data); + curl_easy_setopt(curlfuncs::curl, CURLOPT_WRITEDATA, 0); // Set option to write to string + curl_easy_setopt(curlfuncs::curl, CURLOPT_FOLLOWLOCATION, TRUE); + curl_easy_setopt(curlfuncs::curl, CURLOPT_USERAGENT, "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; Mayukh's libcurl wrapper http://www.mayukhbose.com/)"); + curlfuncs::bInitialized = true; + } +} + +int CurlGetUrl(const char *url, string *sOutput, const string &sReferer) +{ + InitCurlLibraryIfNeeded(); + + curl_easy_setopt(curlfuncs::curl, CURLOPT_URL, url); // Set the URL to get + if (sReferer != "") + curl_easy_setopt(curlfuncs::curl, CURLOPT_REFERER, sReferer.c_str()); + curl_easy_setopt(curlfuncs::curl, CURLOPT_HTTPGET, TRUE); + curl_easy_setopt(curlfuncs::curl, CURLOPT_WRITEDATA, 0); // Set option to write to string + curlfuncs::sBuf = ""; + if (curl_easy_perform(curlfuncs::curl) == 0) + *sOutput = curlfuncs::sBuf; + else { + // We have an error here mate! + *sOutput = ""; + return 0; + } + + return 1; +} + +int CurlGetUrlFile(const char *url, const char *filename, const string &sReferer) +{ + int nRet = 0; + InitCurlLibraryIfNeeded(); + + // Point the output to a file + FILE *fp; + if ((fp = fopen(filename, "w")) == NULL) + return 0; + + curl_easy_setopt(curlfuncs::curl, CURLOPT_WRITEDATA, fp); // Set option to write to file + curl_easy_setopt(curlfuncs::curl, CURLOPT_URL, url); // Set the URL to get + if (sReferer != "") + curl_easy_setopt(curlfuncs::curl, CURLOPT_REFERER, sReferer.c_str()); + curl_easy_setopt(curlfuncs::curl, CURLOPT_HTTPGET, TRUE); + if (curl_easy_perform(curlfuncs::curl) == 0) + nRet = 1; + else + nRet = 0; + + curl_easy_setopt(curlfuncs::curl, CURLOPT_WRITEDATA, NULL); // Set option back to default (string) + fclose(fp); + return nRet; +} + +int CurlPostUrl(const char *url, const string &sPost, string *sOutput, const string &sReferer) +{ + InitCurlLibraryIfNeeded(); + + int retval = 1; + string::size_type nStart = 0, nEnd, nPos; + string sTmp, sName, sValue; + struct curl_httppost *formpost=NULL; + struct curl_httppost *lastptr=NULL; + struct curl_slist *headerlist=NULL; + + // Add the POST variables here + while ((nEnd = sPost.find("##", nStart)) != string::npos) { + sTmp = sPost.substr(nStart, nEnd - nStart); + if ((nPos = sTmp.find("=")) == string::npos) + return 0; + sName = sTmp.substr(0, nPos); + sValue = sTmp.substr(nPos+1); + curl_formadd(&formpost, &lastptr, CURLFORM_COPYNAME, sName.c_str(), CURLFORM_COPYCONTENTS, sValue.c_str(), CURLFORM_END); + nStart = nEnd + 2; + } + sTmp = sPost.substr(nStart); + if ((nPos = sTmp.find("=")) == string::npos) + return 0; + sName = sTmp.substr(0, nPos); + sValue = sTmp.substr(nPos+1); + curl_formadd(&formpost, &lastptr, CURLFORM_COPYNAME, sName.c_str(), CURLFORM_COPYCONTENTS, sValue.c_str(), CURLFORM_END); + + retval = CurlDoPost(url, sOutput, sReferer, formpost, headerlist); + + curl_formfree(formpost); + curl_slist_free_all(headerlist); + return retval; +} + +int CurlPostRaw(const char *url, const string &sPost, string *sOutput, const string &sReferer) +{ + InitCurlLibraryIfNeeded(); + + int retval; + struct curl_httppost *formpost=NULL; + struct curl_slist *headerlist=NULL; + + curl_easy_setopt(curlfuncs::curl, CURLOPT_POSTFIELDS, sPost.c_str()); + curl_easy_setopt(curlfuncs::curl, CURLOPT_POSTFIELDSIZE, 0); //FIXME: Should this be the size instead, in case this is binary string? + + retval = CurlDoPost(url, sOutput, sReferer, formpost, headerlist); + + curl_formfree(formpost); + curl_slist_free_all(headerlist); + return retval; +} + +int CurlDoPost(const char *url, string *sOutput, const string &sReferer, + struct curl_httppost *formpost, struct curl_slist *headerlist) +{ + headerlist = curl_slist_append(headerlist, "Expect:"); + + // Now do the form post + curl_easy_setopt(curlfuncs::curl, CURLOPT_URL, url); + if (sReferer != "") + curl_easy_setopt(curlfuncs::curl, CURLOPT_REFERER, sReferer.c_str()); + curl_easy_setopt(curlfuncs::curl, CURLOPT_HTTPPOST, formpost); + + curl_easy_setopt(curlfuncs::curl, CURLOPT_WRITEDATA, 0); // Set option to write to string + curlfuncs::sBuf = ""; + if (curl_easy_perform(curlfuncs::curl) == 0) { + *sOutput = curlfuncs::sBuf; + return 1; + } + else { + // We have an error here mate! + *sOutput = ""; + return 0; + } +} + +void FreeCurlLibrary(void) +{ + if (curlfuncs::curl) + curl_easy_cleanup(curlfuncs::curl); + curl_global_cleanup(); + curlfuncs::bInitialized = false; +} + +int CurlSetCookieFile(char *filename) +{ + InitCurlLibraryIfNeeded(); + if (curl_easy_setopt(curlfuncs::curl, CURLOPT_COOKIEFILE, filename) != 0) + return 0; + if (curl_easy_setopt(curlfuncs::curl, CURLOPT_COOKIEJAR, filename) != 0) + return 0; + return 1; +} + +char *CurlEscape(const char *url) { + InitCurlLibraryIfNeeded(); + return curl_escape(url , strlen(url)); +} diff --git a/tools/curlfuncs.h b/tools/curlfuncs.h new file mode 100644 index 0000000..e0f76cf --- /dev/null +++ b/tools/curlfuncs.h @@ -0,0 +1,45 @@ +/* +Copyright (c) 2002, Mayukh Bose +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +* Neither the name of Mayukh Bose nor the names of other +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef __CURLFUNCS_H_20020513__ +#define __CURLFUNCS_H_20020513__ +#include <string> + using namespace std; + +int CurlGetUrl(const char *url, string *sOutput, const string &sReferer=""); +int CurlGetUrlFile(const char *url, const char *filename, const string &sReferer=""); +void FreeCurlLibrary(void); +int CurlSetCookieFile(char *filename); +int CurlPostUrl(const char *url, const string &sPost, string *sOutput, const string &sReferer = ""); +int CurlPostRaw(const char *url, const string &sPost, string *sOutput, const string &sReferer = ""); +char *CurlEscape(const char *url); +#endif diff --git a/tools/filesystem.c b/tools/filesystem.c new file mode 100644 index 0000000..724b28c --- /dev/null +++ b/tools/filesystem.c @@ -0,0 +1,44 @@ +#include <sys/stat.h>
+#include <sys/types.h>
+#include <sys/vfs.h>
+#include <fstream>
+
+using namespace std;
+
+bool CreateDirectory(string dir) {
+ mkdir(dir.c_str(), 0775);
+ //check if successfull
+ DIR *pDir;
+ bool exists = false;
+ pDir = opendir(dir.c_str());
+ if (pDir != NULL) {
+ exists = true;
+ closedir(pDir);
+ }
+ return exists;
+}
+
+bool FileExists(string filename) {
+ ifstream ifile(filename.c_str());
+ if (ifile) {
+ //a valid image should be larger then 500 bytes
+ ifile.seekg (0, ifile.end);
+ int length = ifile.tellg();
+ if (length > 500)
+ return true;
+ }
+ return false;
+}
+
+bool CheckDirExists(const char* dirName) {
+ struct statfs statfsbuf;
+ if (statfs(dirName,&statfsbuf)==-1) return false;
+ if ((statfsbuf.f_type!=0x01021994) && (statfsbuf.f_type!=0x28cd3d45)) return false;
+ if (access(dirName,R_OK|W_OK)==-1) return false;
+ return true;
+
+}
+
+void DeleteFile(string filename) {
+ remove(filename.c_str());
+}
\ No newline at end of file diff --git a/tools/fuzzy.c b/tools/fuzzy.c new file mode 100644 index 0000000..188f017 --- /dev/null +++ b/tools/fuzzy.c @@ -0,0 +1,70 @@ +#include <iterator> +#include <algorithm> +#include <vector> +#include <string> +#include <sstream> +#include <iostream> + +template<typename T, typename C> +size_t +seq_distance(const T& seq1, const T& seq2, const C& cost, + const typename T::value_type& empty = typename T::value_type()) { + const size_t size1 = seq1.size(); + const size_t size2 = seq2.size(); + + std::vector<size_t> curr_col(size2 + 1); + std::vector<size_t> prev_col(size2 + 1); + + // Prime the previous column for use in the following loop: + prev_col[0] = 0; + for (size_t idx2 = 0; idx2 < size2; ++idx2) { + prev_col[idx2 + 1] = prev_col[idx2] + cost(empty, seq2[idx2]); + } + + for (size_t idx1 = 0; idx1 < size1; ++idx1) { + curr_col[0] = curr_col[0] + cost(seq1[idx1], empty); + + for (size_t idx2 = 0; idx2 < size2; ++idx2) { + curr_col[idx2 + 1] = std::min(std::min( + curr_col[idx2] + cost(empty, seq2[idx2]), + prev_col[idx2 + 1] + cost(seq1[idx1], empty)), + prev_col[idx2] + cost(seq1[idx1], seq2[idx2])); + } + + curr_col.swap(prev_col); + curr_col[0] = prev_col[0]; + } + + return prev_col[size2]; +} + +size_t +letter_distance(char letter1, char letter2) { + return letter1 != letter2 ? 1 : 0; +} + +size_t +word_distance(const std::string& word1, const std::string& word2) { + return seq_distance(word1, word2, &letter_distance); +} + +size_t +sentence_distance(const std::string& sentence1, const std::string& sentence2) { + std::vector<std::string> words1; + std::vector<std::string> words2; + std::istringstream iss1(sentence1); + std::istringstream iss2(sentence2); + for(std::istream_iterator<std::string> it(iss1), end; ; ) { + words1.push_back(*it); + if(++it == end) + break; + words1.push_back(" "); + } + for(std::istream_iterator<std::string> it(iss2), end; ; ) { + words2.push_back(*it); + if(++it == end) + break; + words2.push_back(" "); + } + return seq_distance(words1, words2, &word_distance); +}
\ No newline at end of file diff --git a/tools/splitstring.c b/tools/splitstring.c new file mode 100644 index 0000000..14af577 --- /dev/null +++ b/tools/splitstring.c @@ -0,0 +1,32 @@ +using namespace std; + +class splitstring : public string { + vector<string> flds; +public: + splitstring(const char *s) : string(s) { }; + vector<string>& split(char delim, int rep=0); +}; + +// split: receives a char delimiter; returns a vector of strings +// By default ignores repeated delimiters, unless argument rep == 1. +vector<string>& splitstring::split(char delim, int rep) { + if (!flds.empty()) flds.clear(); // empty vector if necessary + string work = data(); + string buf = ""; + int i = 0; + while (i < work.length()) { + if (work[i] != delim) + buf += work[i]; + else if (rep == 1) { + flds.push_back(buf); + buf = ""; + } else if (buf.length() > 0) { + flds.push_back(buf); + buf = ""; + } + i++; + } + if (!buf.empty()) + flds.push_back(buf); + return flds; +} diff --git a/tvscraper.c b/tvscraper.c new file mode 100644 index 0000000..616af73 --- /dev/null +++ b/tvscraper.c @@ -0,0 +1,223 @@ +/* + * tvscraper.c: A plugin for the Video Disk Recorder + * + * See the README file for copyright information and how to reach the author. + * + * $Id$ + */ +#include <getopt.h> +#include <vdr/plugin.h> +#include "tools/curlfuncs.cpp" +#include "tools/filesystem.c" +#include "tools/fuzzy.c" +#include "tools/splitstring.c" +#include "config.c" +cTVScraperConfig config; +#include "tvscraperdb.c" +#include "thetvdbscraper/tvdbmirrors.c" +#include "thetvdbscraper/tvdbseries.c" +#include "thetvdbscraper/tvdbmedia.c" +#include "thetvdbscraper/tvdbactors.c" +#include "thetvdbscraper/thetvdbscraper.c" +#include "themoviedbscraper/moviedbmovie.c" +#include "themoviedbscraper/moviedbactors.c" +#include "themoviedbscraper/themoviedbscraper.c" +#include "worker.c" +#include "services.h" +#include "imageserver.c" +#include "setup.c" + +static const char *VERSION = "0.0.1"; +static const char *DESCRIPTION = "Scraping movie and series info"; +static const char *MAINMENUENTRY = "TV Scraper"; + +class cPluginTvscraper : public cPlugin { +private: + bool cacheDirSet; + cTVScraperDB *db; + cTVScraperWorker *workerThread; + cImageServer *imageServer; +public: + cPluginTvscraper(void); + virtual ~cPluginTvscraper(); + virtual const char *Version(void) { return VERSION; } + virtual const char *Description(void) { return DESCRIPTION; } + virtual const char *CommandLineHelp(void); + virtual bool ProcessArgs(int argc, char *argv[]); + virtual bool Initialize(void); + virtual bool Start(void); + virtual void Stop(void); + virtual void Housekeeping(void); + virtual void MainThreadHook(void); + virtual cString Active(void); + virtual time_t WakeupTime(void); + virtual const char *MainMenuEntry(void) { return NULL; } + virtual cOsdObject *MainMenuAction(void); + virtual cMenuSetupPage *SetupMenu(void); + virtual bool SetupParse(const char *Name, const char *Value); + virtual bool Service(const char *Id, void *Data = NULL); + virtual const char **SVDRPHelpPages(void); + virtual cString SVDRPCommand(const char *Command, const char *Option, int &ReplyCode); +}; + +cPluginTvscraper::cPluginTvscraper(void) { +} + +cPluginTvscraper::~cPluginTvscraper() { +} + +const char *cPluginTvscraper::CommandLineHelp(void) { + return " -d <CACHEDIR>, --dir=<CACHEDIR> Set directory where database and images are stored\n"; +} + +bool cPluginTvscraper::ProcessArgs(int argc, char *argv[]) { + static const struct option long_options[] = { + { "dir", required_argument, NULL, 'd' }, + { 0, 0, 0, 0 } + }; + + int c; + cacheDirSet = false; + while ((c = getopt_long(argc, argv, "d:", long_options, NULL)) != -1) { + switch (c) { + case 'd': + cacheDirSet = true; + config.SetBaseDir(optarg); + break; + default: + return false; + } + } + return true; +} + +bool cPluginTvscraper::Initialize(void) { + return true; +} + +bool cPluginTvscraper::Start(void) { + if (!cacheDirSet) { + config.SetBaseDir(cPlugin::CacheDirectory(PLUGIN_NAME_I18N)); + } + db = new cTVScraperDB(); + if (!db->Connect()) { + esyslog("tvscraper: could not connect to Database. Aborting!"); + return false; + }; + imageServer = new cImageServer(db); + workerThread = new cTVScraperWorker(db); + workerThread->SetDirectories(); + workerThread->SetLanguage(); + workerThread->Start(); + return true; +} + +void cPluginTvscraper::Stop(void) { + while (workerThread->Active()) { + workerThread->Stop(); + } + delete workerThread; + delete imageServer; + delete db; +} + +void cPluginTvscraper::Housekeeping(void) { +} + +void cPluginTvscraper::MainThreadHook(void) { +} + +cString cPluginTvscraper::Active(void) { + return NULL; +} + +time_t cPluginTvscraper::WakeupTime(void) { + return 0; +} + +cOsdObject *cPluginTvscraper::MainMenuAction(void) { + return NULL; +} + +cMenuSetupPage *cPluginTvscraper::SetupMenu(void) { + return new cTVScraperSetup(workerThread); +} + +bool cPluginTvscraper::SetupParse(const char *Name, const char *Value) { + return config.SetupParse(Name, Value); +} + +bool cPluginTvscraper::Service(const char *Id, void *Data) { + if (strcmp(Id, "TVScraperGetPosterOrBanner") == 0) { + if (Data == NULL) + return false; + TVScraperGetPosterOrBanner* call = (TVScraperGetPosterOrBanner*) Data; + if (!call->event) + return false; + scrapType type = imageServer->GetScrapType(call->event); + if (type == scrapSeries) + call->type = typeSeries; + else if (type == scrapMovie) + call->type = typeMovie; + else + call->type = typeNone; + int id = imageServer->GetID(call->event->EventID(), type, false); + if (id > 0) { + call->media = imageServer->GetPosterOrBanner(id, type); + return true; + } + return false; + } + if (strcmp(Id, "TVScraperGetPoster") == 0) { + if (Data == NULL) + return false; + TVScraperGetPoster* call = (TVScraperGetPoster*) Data; + if (!call->event) + return false; + scrapType type = imageServer->GetScrapType(call->event); + int id = imageServer->GetID(call->event->EventID(), type, call->isRecording); + if (id > 0) { + call->media = imageServer->GetPoster(id, type); + return true; + } + return false; + } + if (strcmp(Id, "TVScraperGetFullInformation") == 0) { + if (Data == NULL) + return false; + TVScraperGetFullInformation* call = (TVScraperGetFullInformation*) Data; + if (!call->event) + return false; + + scrapType type = imageServer->GetScrapType(call->event); + int id = imageServer->GetID(call->event->EventID(), type, call->isRecording); + + if (id == 0) + return false; + + if (type == scrapSeries) { + call->type = typeSeries; + call->banner = imageServer->GetBanner(id); + } else if (type == scrapMovie) { + call->type = typeMovie; + } else { + call->type = typeNone; + } + call->posters = imageServer->GetPosters(id, type); + call->fanart = imageServer->GetFanart(id, type); + call->actors = imageServer->GetActors(id, type); + call->description = imageServer->GetDescription(id, type); + return true; + } + return false; +} + +const char **cPluginTvscraper::SVDRPHelpPages(void) { + return NULL; +} + +cString cPluginTvscraper::SVDRPCommand(const char *Command, const char *Option, int &ReplyCode) { + return NULL; +} + +VDRPLUGINCREATOR(cPluginTvscraper); diff --git a/tvscraperdb.c b/tvscraperdb.c new file mode 100644 index 0000000..e89be33 --- /dev/null +++ b/tvscraperdb.c @@ -0,0 +1,628 @@ +#include <string>
+#include <sstream>
+#include <vector>
+#include <sqlite3.h>
+#include "tvscraperdb.h"
+
+using namespace std;
+
+cTVScraperDB::cTVScraperDB(void) {
+ db = NULL;
+ string memHD = "/dev/shm/";
+ inMem = CheckDirExists(memHD.c_str());
+ if (inMem) {
+ stringstream sstrDbFileMem;
+ sstrDbFileMem << memHD << "tvscraper.db";
+ dbPathMem = sstrDbFileMem.str();
+ }
+ stringstream sstrDbFile;
+ sstrDbFile << config.GetBaseDir() << "/tvscraper.db";
+ dbPathPhys = sstrDbFile.str();
+}
+
+cTVScraperDB::~cTVScraperDB() {
+ sqlite3_close(db);
+}
+
+vector<vector<string> > cTVScraperDB::Query(string query) {
+ sqlite3_stmt *statement;
+ vector<vector<string> > results;
+ if(sqlite3_prepare_v2(db, query.c_str(), -1, &statement, 0) == SQLITE_OK) {
+ int cols = sqlite3_column_count(statement);
+ int result = 0;
+ while(true) {
+ result = sqlite3_step(statement);
+ if(result == SQLITE_ROW) {
+ vector<string> values;
+ for(int col = 0; col < cols; col++) {
+ values.push_back((char*)sqlite3_column_text(statement, col));
+ }
+ results.push_back(values);
+ } else {
+ break;
+ }
+ }
+ sqlite3_finalize(statement);
+ }
+ string error = sqlite3_errmsg(db);
+ if(error != "not an error") {
+ esyslog("tvscraper: query failed: %s , error: %s", query.c_str(), error.c_str());
+ }
+ return results;
+}
+
+bool cTVScraperDB::Connect(void) {
+ if (inMem) {
+ if (sqlite3_open(dbPathMem.c_str(),&db)!=SQLITE_OK) {
+ esyslog("tvscraper: failed to open or create %s", dbPathMem.c_str());
+ return false;
+ }
+ esyslog("tvscraper: connecting to db %s", dbPathMem.c_str());
+ int rc = LoadOrSaveDb(db, dbPathPhys.c_str(), false);
+ if (rc != SQLITE_OK) {
+ esyslog("tvscraper: error while loading data from %s, errorcode %d", dbPathPhys.c_str(), rc);
+ return false;
+ }
+ } else {
+ if (sqlite3_open(dbPathPhys.c_str(),&db)!=SQLITE_OK) {
+ esyslog("tvscraper: failed to open or create %s", dbPathPhys.c_str());
+ return false;
+ }
+ esyslog("tvscraper: connecting to db %s", dbPathPhys.c_str());
+ }
+ CreateTables();
+ return true;
+}
+
+void cTVScraperDB::BackupToDisc(void) {
+ if (inMem) {
+ LoadOrSaveDb(db, dbPathPhys.c_str(), true);
+ }
+}
+
+int cTVScraperDB::LoadOrSaveDb(sqlite3 *pInMemory, const char *zFilename, int isSave) {
+ int rc; /* Function return code */
+ sqlite3 *pFile; /* Database connection opened on zFilename */
+ sqlite3_backup *pBackup; /* Backup object used to copy data */
+ sqlite3 *pTo; /* Database to copy to (pFile or pInMemory) */
+ sqlite3 *pFrom; /* Database to copy from (pFile or pInMemory) */
+
+ rc = sqlite3_open(zFilename, &pFile);
+ if( rc==SQLITE_OK ){
+ pFrom = (isSave ? pInMemory : pFile);
+ pTo = (isSave ? pFile : pInMemory);
+ pBackup = sqlite3_backup_init(pTo, "main", pFrom, "main");
+ if( pBackup ){
+ (void)sqlite3_backup_step(pBackup, -1);
+ (void)sqlite3_backup_finish(pBackup);
+ }
+ rc = sqlite3_errcode(pTo);
+ }
+
+ (void)sqlite3_close(pFile);
+ return rc;
+}
+
+bool cTVScraperDB::CreateTables(void) {
+ stringstream sql;
+ sql << "CREATE TABLE IF NOT EXISTS series (";
+ sql << "series_id integer primary key, ";
+ sql << "series_name nvarchar(255), ";
+ sql << "series_overview text";
+ sql << ");";
+
+ sql << "CREATE TABLE IF NOT EXISTS series_actors (";
+ sql << "actor_series_id integer, ";
+ sql << "actor_name nvarchar(255), ";
+ sql << "actor_role nvarchar(255), ";
+ sql << "actor_thumbnail nvarchar(255)";
+ sql << ");";
+ sql << "CREATE INDEX IF NOT EXISTS idx1 on series_actors (actor_series_id); ";
+
+ sql << "CREATE TABLE IF NOT EXISTS movies (";
+ sql << "movie_id integer primary key, ";
+ sql << "movie_title nvarchar(255), ";
+ sql << "movie_original_title nvarchar(255), ";
+ sql << "movie_overview text";
+ sql << ");";
+
+ sql << "CREATE TABLE IF NOT EXISTS actors (";
+ sql << "actor_id integer primary key, ";
+ sql << "actor_name nvarchar(255)";
+ sql << ");";
+
+ sql << "CREATE TABLE IF NOT EXISTS actor_movie (";
+ sql << "actor_id integer, ";
+ sql << "movie_id integer, ";
+ sql << "actor_role nvarchar(255)";
+ sql << ");";
+ sql << "CREATE UNIQUE INDEX IF NOT EXISTS idx2 on actor_movie (actor_id, movie_id); ";
+
+ sql << "CREATE TABLE IF NOT EXISTS event_movie (";
+ sql << "event_id integer, ";
+ sql << "valid_till integer, ";
+ sql << "movie_id integer ";
+ sql << ");";
+ sql << "CREATE UNIQUE INDEX IF NOT EXISTS idx3 on event_movie (event_id, movie_id); ";
+
+ sql << "CREATE TABLE IF NOT EXISTS event_series (";
+ sql << "event_id integer, ";
+ sql << "valid_till integer, ";
+ sql << "series_id integer ";
+ sql << ");";
+ sql << "CREATE UNIQUE INDEX IF NOT EXISTS idx4 on event_series (event_id, series_id); ";
+
+ sql << "CREATE TABLE IF NOT EXISTS recordings (";
+ sql << "rec_event_id integer, ";
+ sql << "series_id integer, ";
+ sql << "movie_id integer ";
+ sql << ");";
+ sql << "CREATE UNIQUE INDEX IF NOT EXISTS idx5 on recordings (rec_event_id, series_id, movie_id); ";
+
+ sql << "CREATE TABLE IF NOT EXISTS scrap_history (";
+ sql << "channel_id nvarchar(255), ";
+ sql << "newest_scrapped integer ";
+ sql << ");";
+
+ sql << "CREATE TABLE IF NOT EXISTS scrap_checker (";
+ sql << "last_scrapped integer ";
+ sql << ");";
+
+ char *errmsg;
+ if (sqlite3_exec(db,sql.str().c_str(),NULL,NULL,&errmsg)!=SQLITE_OK) {
+ esyslog("tvscraper: createdb: %s", errmsg);
+ sqlite3_free(errmsg);
+ sqlite3_close(db);
+ return false;
+ }
+ return true;
+}
+
+void cTVScraperDB::ClearOutdated(string movieDir) {
+ //first check which movie images can be deleted
+ time_t now = time(0);
+ stringstream sql;
+ sql << "select movie_id from event_movie where valid_till < " << now;
+ vector<vector<string> > result = Query(sql.str());
+ int numOutdated = result.size();
+ if (numOutdated > 0) {
+ for (int i=0; i < numOutdated; i++) {
+ bool keepMovie = false;
+ vector<string> row = result[i];
+ if (row.size() > 0) {
+ int movieID = atoi(row[0].c_str());
+ if (movieID > 0) {
+ //are there other still valid events pointing to that movie?
+ keepMovie = CheckMovieOutdatedEvents(movieID);
+ if (!keepMovie) {
+ //are there recordings pointing to that movie?
+ keepMovie = CheckMovieOutdatedRecordings(movieID);
+ }
+ if (!keepMovie) {
+ DeleteMovie(movieID, movieDir);
+ }
+ }
+ }
+ }
+ }
+ //delete all invalid events pointing to movies
+ stringstream sql2;
+ sql2 << "delete from event_movie where valid_till < " << now;
+ sqlite3_stmt *stmt;
+ sqlite3_prepare_v2(db, sql2.str().c_str(), -1, &stmt, NULL);
+ int ret = sqlite3_step(stmt);
+ //delete all invalid events pointing to series, series will all be kept for later use
+ stringstream sql3;
+ sql3 << "delete from event_series where valid_till < " << now;
+ sqlite3_stmt *stmt2;
+ sqlite3_prepare_v2(db, sql3.str().c_str(), -1, &stmt2, NULL);
+ int ret2 = sqlite3_step(stmt2);
+ esyslog("tvscraper: Cleanup Done");
+}
+
+void cTVScraperDB::DeleteMovie(int movieID, string movieDir) {
+ //delete images
+ stringstream backdrop;
+ backdrop << movieDir << "/" << movieID << "_backdrop.jpg";
+ stringstream poster;
+ poster << movieDir << "/" << movieID << "_poster.jpg";
+ DeleteFile(backdrop.str());
+ DeleteFile(poster.str());
+ //delete cast of this movie
+ stringstream sql;
+ sql << "DELETE FROM actor_movie WHERE movie_id = " << movieID;
+ sqlite3_stmt *stmt;
+ sqlite3_prepare_v2(db, sql.str().c_str(), -1, &stmt, NULL);
+ int ret = sqlite3_step(stmt);
+}
+
+bool cTVScraperDB::CheckMovieOutdatedEvents(int movieID) {
+ time_t now = time(0);
+ stringstream sql;
+ sql << "select event_id from event_movie where movie_id = " << movieID << " and valid_till > " << now;
+ int eventID = 0;
+ vector<vector<string> > result = Query(sql.str());
+ if (result.size() > 0) {
+ vector<vector<string> >::iterator it = result.begin();
+ vector<string> row = *it;
+ if (row.size() > 0) {
+ eventID = atoi(row[0].c_str());
+ }
+ }
+ if (eventID > 0)
+ return true;
+ return false;
+}
+
+bool cTVScraperDB::CheckMovieOutdatedRecordings(int movieID) {
+ stringstream sql;
+ sql << "select rec_event_id from recordings where movie_id = " << movieID;
+ int eventID = 0;
+ vector<vector<string> > result = Query(sql.str());
+ if (result.size() > 0) {
+ vector<vector<string> >::iterator it = result.begin();
+ vector<string> row = *it;
+ if (row.size() > 0) {
+ eventID = atoi(row[0].c_str());
+ }
+ }
+ if (eventID > 0)
+ return true;
+ return false;
+}
+
+void cTVScraperDB::InsertSeries(int seriesID, string name, string overview) {
+ stringstream sql;
+ sql << "INSERT INTO series (series_id, series_name, series_overview) ";
+ sql << "VALUES (";
+ sql << seriesID << ", ";
+ sql << "?, ? ";
+ sql << ");";
+ sqlite3_stmt *stmt;
+ sqlite3_prepare_v2(db, sql.str().c_str(), -1, &stmt, NULL);
+ sqlite3_bind_text(stmt, 1, name.c_str(), -1, SQLITE_TRANSIENT);
+ sqlite3_bind_text(stmt, 2, overview.c_str(), -1, SQLITE_TRANSIENT);
+ int ret = sqlite3_step(stmt);
+}
+
+void cTVScraperDB::InsertEventSeries(int eventID, time_t validTill, int seriesID) {
+ stringstream sql;
+ sql << "INSERT INTO event_series (event_id, valid_till, series_id) ";
+ sql << "VALUES (";
+ sql << eventID << ", " << validTill << ", " << seriesID;
+ sql << ");";
+ sqlite3_stmt *stmt;
+ sqlite3_prepare_v2(db, sql.str().c_str(), -1, &stmt, NULL);
+ int ret = sqlite3_step(stmt);
+}
+
+void cTVScraperDB::InsertActor(int seriesID, string name, string role, string thumb) {
+ stringstream sql;
+ sql << "INSERT INTO series_actors (actor_series_id, actor_name, actor_role, actor_thumbnail) ";
+ sql << "VALUES (";
+ sql << seriesID << ", ";
+ sql << "?, ? ,";
+ sql << "'" << thumb << "'";
+ sql << ");";
+ sqlite3_stmt *stmt;
+ sqlite3_prepare_v2(db, sql.str().c_str(), -1, &stmt, NULL);
+ sqlite3_bind_text(stmt, 1, name.c_str(), -1, SQLITE_TRANSIENT);
+ sqlite3_bind_text(stmt, 2, role.c_str(), -1, SQLITE_TRANSIENT);
+ int ret = sqlite3_step(stmt);
+}
+
+void cTVScraperDB::InsertMovie(int movieID, string title, string originalTitle, string overview) {
+ stringstream sql;
+ sql << "INSERT INTO movies (movie_id, movie_title, movie_original_title, movie_overview) ";
+ sql << "VALUES (";
+ sql << movieID << ", ";
+ sql << "?, ?, ?";
+ sql << ");";
+ sqlite3_stmt *stmt;
+ sqlite3_prepare_v2(db, sql.str().c_str(), -1, &stmt, NULL);
+ sqlite3_bind_text(stmt, 1, title.c_str(), -1, SQLITE_TRANSIENT);
+ sqlite3_bind_text(stmt, 2, originalTitle.c_str(), -1, SQLITE_TRANSIENT);
+ sqlite3_bind_text(stmt, 3, overview.c_str(), -1, SQLITE_TRANSIENT);
+ int ret = sqlite3_step(stmt);
+}
+
+void cTVScraperDB::InsertEventMovie(int eventID, time_t validTill, int movieID) {
+ stringstream sql;
+ sql << "INSERT INTO event_movie (event_id, valid_till, movie_id) ";
+ sql << "VALUES (";
+ sql << eventID << ", " << validTill << ", " << movieID;
+ sql << ");";
+ sqlite3_stmt *stmt;
+ sqlite3_prepare_v2(db, sql.str().c_str(), -1, &stmt, NULL);
+ int ret = sqlite3_step(stmt);
+}
+
+
+void cTVScraperDB::InsertMovieActor(int movieID, int actorID, string name, string role) {
+ stringstream sql;
+ sql << "INSERT INTO actors (actor_id, actor_name) ";
+ sql << "VALUES (";
+ sql << actorID << ", ?";
+ sql << ");";
+ sqlite3_stmt *stmt;
+ sqlite3_prepare_v2(db, sql.str().c_str(), -1, &stmt, NULL);
+ sqlite3_bind_text(stmt, 1, name.c_str(), -1, SQLITE_TRANSIENT);
+ int ret = sqlite3_step(stmt);
+
+ stringstream sql2;
+ sql2 << "INSERT INTO actor_movie (actor_id, movie_id, actor_role) ";
+ sql2 << "VALUES (";
+ sql2 << actorID << ", " << movieID << ", ?";
+ sql2 << ");";
+ sqlite3_stmt *stmt2;
+ sqlite3_prepare_v2(db, sql2.str().c_str(), -1, &stmt2, NULL);
+ sqlite3_bind_text(stmt2, 1, role.c_str(), -1, SQLITE_TRANSIENT);
+ ret = sqlite3_step(stmt2);
+}
+
+bool cTVScraperDB::MovieExists(int movieID) {
+ stringstream sql;
+ sql << "select count(movies.movie_id) as found from movies where movies.movie_id = " << movieID;
+ vector<vector<string> > result = Query(sql.str());
+ int found = 0;
+ if (result.size() > 0) {
+ vector<vector<string> >::iterator it = result.begin();
+ vector<string> row = *it;
+ if (row.size() > 0) {
+ found = atoi(row[0].c_str());
+ }
+ }
+ if (found == 1)
+ return true;
+ return false;
+}
+
+bool cTVScraperDB::SeriesExists(int seriesID) {
+ stringstream sql;
+ sql << "select count(series.series_id) as found from series where series.series_id = " << seriesID;
+ vector<vector<string> > result = Query(sql.str());
+ int found = 0;
+ if (result.size() > 0) {
+ vector<vector<string> >::iterator it = result.begin();
+ vector<string> row = *it;
+ if (row.size() > 0) {
+ found = atoi(row[0].c_str());
+ }
+ }
+ if (found == 1)
+ return true;
+ return false;
+}
+
+int cTVScraperDB::SearchMovie(string movieTitle) {
+ stringstream sql;
+ sql << "select movie_id from movies where movie_title='" << movieTitle.c_str() << "'";
+ vector<vector<string> > result = Query(sql.str());
+ int movieID = 0;
+ if (result.size() > 0) {
+ vector<vector<string> >::iterator it = result.begin();
+ vector<string> row = *it;
+ if (row.size() > 0) {
+ movieID = atoi(row[0].c_str());
+ }
+ }
+ return movieID;
+}
+
+int cTVScraperDB::SearchSeries(string seriesTitle) {
+ stringstream sql;
+ sql << "select series_id from series where series_name='" << seriesTitle.c_str() << "'";
+ vector<vector<string> > result = Query(sql.str());
+ int seriesID = 0;
+ if (result.size() > 0) {
+ vector<vector<string> >::iterator it = result.begin();
+ vector<string> row = *it;
+ if (row.size() > 0) {
+ seriesID = atoi(row[0].c_str());
+ }
+ }
+ return seriesID;
+}
+
+void cTVScraperDB::InsertRecording(int recEventID, int seriesID, int movieID) {
+ stringstream sql;
+ sql << "INSERT INTO recordings (rec_event_id, series_id, movie_id) ";
+ sql << "VALUES (";
+ sql << recEventID << ", " << seriesID << ", " << movieID;
+ sql << ");";
+ sqlite3_stmt *stmt;
+ sqlite3_prepare_v2(db, sql.str().c_str(), -1, &stmt, NULL);
+ int ret = sqlite3_step(stmt);
+}
+
+void cTVScraperDB::SetRecordingSeries(int eventID) {
+ stringstream sql;
+ sql << "select series_id from event_series where event_id = " << eventID;
+ vector<vector<string> > result = Query(sql.str());
+ if (result.size() > 0) {
+ vector<vector<string> >::iterator it = result.begin();
+ vector<string> row = *it;
+ if (row.size() > 0) {
+ int seriesID = atoi(row[0].c_str());
+ InsertRecording(eventID, seriesID, 0);
+ }
+ }
+}
+
+void cTVScraperDB::SetRecordingMovie(int eventID) {
+ stringstream sql;
+ sql << "select movie_id from event_movie where event_id =" << eventID;
+ vector<vector<string> > result = Query(sql.str());
+ if (result.size() > 0) {
+ vector<vector<string> >::iterator it = result.begin();
+ vector<string> row = *it;
+ if (row.size() > 0) {
+ int movieID = atoi(row[0].c_str());
+ InsertRecording(eventID, 0, movieID);
+ }
+ }
+}
+
+void cTVScraperDB::ClearRecordings(void) {
+ stringstream sql;
+ sql << "DELETE FROM recordings where 0 = 0";
+ sqlite3_stmt *stmt;
+ sqlite3_prepare_v2(db, sql.str().c_str(), -1, &stmt, NULL);
+ int ret = sqlite3_step(stmt);
+}
+
+bool cTVScraperDB::CheckScrap(time_t timeStamp, string channelID) {
+ bool doScrap = false;
+ stringstream sql;
+ sql << "select newest_scrapped from scrap_history where channel_id = '" << channelID.c_str() << "'";
+ vector<vector<string> > result = Query(sql.str());
+ if (result.size() > 0) {
+ vector<vector<string> >::iterator it = result.begin();
+ vector<string> row = *it;
+ if (row.size() > 0) {
+ char *ep;
+ time_t newestScrapped = strtoul(row[0].c_str(), &ep, 10);
+ if (newestScrapped < timeStamp) {
+ doScrap = true;
+ stringstream sql2;
+ sql2 << "UPDATE scrap_history set newest_scrapped = " << timeStamp << " where channel_id='" << channelID.c_str() << "'";
+ sqlite3_stmt *stmt;
+ sqlite3_prepare_v2(db, sql2.str().c_str(), -1, &stmt, NULL);
+ sqlite3_step(stmt);
+ }
+ }
+ } else {
+ doScrap = true;
+ stringstream sql2;
+ sql2 << "INSERT INTO scrap_history (channel_id, newest_scrapped) VALUES ('" << channelID.c_str() << "', " << timeStamp << ")";
+ sqlite3_stmt *stmt;
+ sqlite3_prepare_v2(db, sql2.str().c_str(), -1, &stmt, NULL);
+ sqlite3_step(stmt);
+ }
+ return doScrap;
+}
+
+bool cTVScraperDB::CheckStartScrapping(int minimumDistance) {
+ bool startScrapping = false;
+ time_t now = time(0);
+ stringstream sql;
+ sql << "select last_scrapped from scrap_checker";
+ vector<vector<string> > result = Query(sql.str());
+ if (result.size() > 0) {
+ vector<vector<string> >::iterator it = result.begin();
+ vector<string> row = *it;
+ if (row.size() > 0) {
+ char *ep;
+ time_t last_scrapped = strtoul(row[0].c_str(), &ep, 10);
+ int difference = (int)(now - last_scrapped);
+ if (difference > minimumDistance) {
+ startScrapping = true;
+ stringstream sql2;
+ sql2 << "delete from scrap_checker";
+ sqlite3_stmt *stmt;
+ sqlite3_prepare_v2(db, sql2.str().c_str(), -1, &stmt, NULL);
+ sqlite3_step(stmt);
+ stringstream sql3;
+ sql3 << "INSERT INTO scrap_checker (last_scrapped) VALUES (" << now << ")";
+ sqlite3_stmt *stmt2;
+ sqlite3_prepare_v2(db, sql3.str().c_str(), -1, &stmt2, NULL);
+ sqlite3_step(stmt2);
+ }
+ }
+ } else {
+ startScrapping = true;
+ stringstream sql2;
+ sql2 << "INSERT INTO scrap_checker (last_scrapped) VALUES (" << now << ")";
+ sqlite3_stmt *stmt;
+ sqlite3_prepare_v2(db, sql2.str().c_str(), -1, &stmt, NULL);
+ sqlite3_step(stmt);
+ }
+ return startScrapping;
+}
+
+int cTVScraperDB::GetSeriesID(int eventID, bool isRecording) {
+ stringstream sql;
+ if (!isRecording)
+ sql << "select series_id from event_series where event_id = " << eventID;
+ else
+ sql << "select series_id from recordings where rec_event_id = " << eventID;
+ vector<vector<string> > result = Query(sql.str());
+ int seriesID = 0;
+ if (result.size() > 0) {
+ vector<vector<string> >::iterator it = result.begin();
+ vector<string> row = *it;
+ if (row.size() > 0) {
+ seriesID = atoi(row[0].c_str());
+ }
+ }
+ return seriesID;
+}
+
+int cTVScraperDB::GetMovieID(int eventID, bool isRecording) {
+ stringstream sql;
+ if (!isRecording)
+ sql << "select movie_id from event_movie where event_id = " << eventID;
+ else
+ sql << "select movie_id from recordings where rec_event_id = " << eventID;
+ vector<vector<string> > result = Query(sql.str());
+ int movieID = 0;
+ if (result.size() > 0) {
+ vector<vector<string> >::iterator it = result.begin();
+ vector<string> row = *it;
+ if (row.size() > 0) {
+ movieID = atoi(row[0].c_str());
+ }
+ }
+ return movieID;
+}
+
+vector<vector<string> > cTVScraperDB::GetActorsSeries(int seriesID) {
+ stringstream sql;
+ sql << "select actor_name, actor_role, actor_thumbnail ";
+ sql << "from series_actors ";
+ sql << "where actor_series_id = " << seriesID;
+ vector<vector<string> > result = Query(sql.str());
+ return result;
+}
+
+vector<vector<string> > cTVScraperDB::GetActorsMovie(int movieID) {
+ stringstream sql;
+ sql << "select actors.actor_id, actor_name, actor_role ";
+ sql << "from actors, actor_movie ";
+ sql << "where actor_movie.actor_id = actors.actor_id ";
+ sql << "and actor_movie.movie_id = " << movieID;
+ vector<vector<string> > result = Query(sql.str());
+ return result;
+}
+
+string cTVScraperDB::GetDescriptionSeries(int seriesID) {
+ string description = "";
+ stringstream sql;
+ sql << "select series_overview from series ";
+ sql << "where series_id = " << seriesID;
+ vector<vector<string> > result = Query(sql.str());
+ if (result.size() > 0) {
+ vector<vector<string> >::iterator it = result.begin();
+ vector<string> row = *it;
+ if (row.size() > 0) {
+ description= row[0];
+ }
+ }
+ return description;
+}
+
+string cTVScraperDB::GetDescriptionMovie(int movieID) {
+ string description = "";
+ stringstream sql;
+ sql << "select movie_overview from movies ";
+ sql << "where movie_id = " << movieID;
+ vector<vector<string> > result = Query(sql.str());
+ if (result.size() > 0) {
+ vector<vector<string> >::iterator it = result.begin();
+ vector<string> row = *it;
+ if (row.size() > 0) {
+ description= row[0];
+ }
+ }
+ return description;
+}
diff --git a/tvscraperdb.h b/tvscraperdb.h new file mode 100644 index 0000000..a8f06ee --- /dev/null +++ b/tvscraperdb.h @@ -0,0 +1,50 @@ +#ifndef __TVSCRAPER_TVSCRAPERDB_H
+#define __TVSCRAPER_TVSCRAPERDB_H
+
+using namespace std;
+
+// --- cTVScraperDB --------------------------------------------------------
+
+class cTVScraperDB {
+private:
+ sqlite3 *db;
+ string dbPathPhys;
+ string dbPathMem;
+ bool inMem;
+ vector<vector<string> > Query(string query);
+ int LoadOrSaveDb(sqlite3 *pInMemory, const char *zFilename, int isSave);
+ bool CreateTables(void);
+public:
+ cTVScraperDB(void);
+ virtual ~cTVScraperDB(void);
+ bool Connect(void);
+ void BackupToDisc(void);
+ void ClearOutdated(string movieDir);
+ bool CheckMovieOutdatedEvents(int movieID);
+ bool CheckMovieOutdatedRecordings(int movieID);
+ void DeleteMovie(int movieID, string movieDir);
+ void InsertSeries(int seriesID, string name, string overview);
+ void InsertEventSeries(int eventID, time_t validTill, int seriesID);
+ void InsertActor(int seriesID, string name, string role, string thumb);
+ void InsertMovie(int movieID, string title, string originalTitle, string overview);
+ void InsertEventMovie(int eventID, time_t validTill, int movieID);
+ void InsertMovieActor(int movieID, int actorID, string name, string role);
+ bool MovieExists(int movieID);
+ bool SeriesExists(int seriesID);
+ int SearchMovie(string movieTitle);
+ int SearchSeries(string seriesTitle);
+ void InsertRecording(int recEventID, int seriesID, int movieID);
+ void SetRecordingSeries(int eventID);
+ void SetRecordingMovie(int eventID);
+ void ClearRecordings(void);
+ bool CheckScrap(time_t timeStamp, string channelID);
+ bool CheckStartScrapping(int minimumDistance);
+ int GetSeriesID(int eventID, bool isRecording);
+ int GetMovieID(int eventID, bool isRecording);
+ vector<vector<string> > GetActorsSeries(int seriesID);
+ vector<vector<string> > GetActorsMovie(int movieID);
+ string GetDescriptionSeries(int seriesID);
+ string GetDescriptionMovie(int movieID);
+};
+
+#endif //__TVSCRAPER_TVSCRAPPERDB_H
diff --git a/worker.c b/worker.c new file mode 100644 index 0000000..09b5752 --- /dev/null +++ b/worker.c @@ -0,0 +1,233 @@ +#include <locale.h> +#include "worker.h" + +using namespace std; + +cTVScraperWorker::cTVScraperWorker(cTVScraperDB *db) : cThread("tvscraper", true) { + startLoop = true; + scanVideoDir = false; + manualScan = false; + this->db = db; + moviedbScraper = NULL; + tvdbScraper = NULL; + initSleep = 2 * 60 * 1000; + loopSleep = 5 * 60 * 1000; + language = ""; +} + +cTVScraperWorker::~cTVScraperWorker() { + if (moviedbScraper) + delete moviedbScraper; + if (tvdbScraper) + delete tvdbScraper; +} + +void cTVScraperWorker::SetLanguage(void) { + string loc = setlocale(LC_NAME, NULL); + size_t index = loc.find_first_of("_"); + string langISO = ""; + if (index > 0) { + langISO = loc.substr(0, index); + } + if (langISO.size() == 2) { + language = langISO.c_str(); + dsyslog("tvscraper: using language %s", language.c_str()); + return; + } + language = "en"; + dsyslog("tvscraper: using fallback language %s", language.c_str()); +} + +void cTVScraperWorker::Stop(void) { + waitCondition.Broadcast(); // wakeup the thread + Cancel(5); // wait up to 5 seconds for thread was stopping + db->BackupToDisc(); +} + +void cTVScraperWorker::InitVideoDirScan(void) { + scanVideoDir = true; + waitCondition.Broadcast(); +} + +void cTVScraperWorker::InitManualScan(void) { + manualScan = true; + waitCondition.Broadcast(); +} + +void cTVScraperWorker::SetDirectories(void) { + plgBaseDir = config.GetBaseDir(); + stringstream strSeriesDir; + strSeriesDir << plgBaseDir << "/series"; + seriesDir = strSeriesDir.str(); + stringstream strMovieDir; + strMovieDir << plgBaseDir << "/movies"; + movieDir = strMovieDir.str(); + bool ok = false; + ok = CreateDirectory(plgBaseDir); + if (ok) + ok = CreateDirectory(seriesDir); + if (ok) + ok = CreateDirectory(movieDir); + if (!ok) { + esyslog("tvscraper: ERROR: check %s for write permissions", plgBaseDir.c_str()); + startLoop = false; + } else { + dsyslog("tvscraper: using base directory %s", plgBaseDir.c_str()); + } +} + +scrapType cTVScraperWorker::GetScrapType(const cEvent *event) { + scrapType type = scrapNone; + int duration = event->Duration() / 60; + if ((duration > 19) && (duration < 65)) { + type = scrapSeries; + } else if (duration > 80) { + type = scrapMovie; + } + return type; +} + +bool cTVScraperWorker::ConnectScrapers(void) { + if (!moviedbScraper) { + moviedbScraper = new cMovieDBScraper(movieDir, db, language); + if (!moviedbScraper->Connect()) { + esyslog("tvscraper: ERROR, connection to TheMovieDB failed"); + return false; + } + } + if (!tvdbScraper) { + tvdbScraper = new cTVDBScraper(seriesDir, db, language); + if (!tvdbScraper->Connect()) { + esyslog("tvscraper: ERROR, connection to TheTVDB failed"); + return false; + } + } + return true; +} + +void cTVScraperWorker::DisconnectScrapers(void) { + if (moviedbScraper) { + delete moviedbScraper; + moviedbScraper = NULL; + } + if (tvdbScraper) { + delete tvdbScraper; + tvdbScraper = NULL; + } +} + +void cTVScraperWorker::ScrapEPG(void) { + vector<string> channels = config.GetChannels(); + int numChannels = channels.size(); + for (int i=0; i<numChannels; i++) { + string channelID = channels[i]; + const cChannel *channel = Channels.GetByChannelID(tChannelID::FromString(channelID.c_str())); + dsyslog("tvscraper: scrapping Channel %s %s", channel->Name(), channelID.c_str()); + cSchedulesLock schedulesLock; + const cSchedules *schedules = cSchedules::Schedules(schedulesLock); + const cSchedule *Schedule = schedules->GetSchedule(channel); + if (Schedule) { + const cEvent *event = NULL; + for (event = Schedule->Events()->First(); event; event = Schedule->Events()->Next(event)) { + if (!Running()) + return; + scrapType type = GetScrapType(event); + if (type != scrapNone) { + if (!db->CheckScrap(event->StartTime(), channelID)) + continue; + if (type == scrapSeries) { + tvdbScraper->Scrap(event); + } else if (type == scrapMovie) { + moviedbScraper->Scrap(event); + } + waitCondition.TimedWait(mutex, 100); + } + } + } + } +} + +void cTVScraperWorker::ScrapRecordings(void) { + db->ClearRecordings(); + for (cRecording *rec = Recordings.First(); rec; rec = Recordings.Next(rec)) { + const cRecordingInfo *recInfo = rec->Info(); + const cEvent *recEvent = recInfo->GetEvent(); + if (recEvent) { + tEventID recEventID = recEvent->EventID(); + string recTitle = recInfo->Title(); + scrapType type = GetScrapType(recEvent); + if (type == scrapSeries) { + int seriesID = db->SearchSeries(recEvent->Title()); + if (seriesID) { + db->InsertRecording((int)recEventID, seriesID, 0); + } else { + tvdbScraper->Scrap(recEvent, (int)recEventID); + } + } else if (type == scrapMovie) { + int movieID = db->SearchMovie(recEvent->Title()); + if (movieID) { + db->InsertRecording((int)recEventID, 0, movieID); + } else { + moviedbScraper->Scrap(recEvent, (int)recEventID); + } + } + } + } +} + +void cTVScraperWorker::CheckRunningTimers(void) { + for (cTimer *timer = Timers.First(); timer; timer = Timers.Next(timer)) { + if (timer->Recording()) { + const cEvent *event = timer->Event(); + scrapType type = GetScrapType(event); + if (type == scrapSeries) { + db->SetRecordingSeries((int)event->EventID()); + } else if (type == scrapMovie) { + db->SetRecordingMovie((int)event->EventID()); + } + } + } +} + +bool cTVScraperWorker::StartScrapping(void) { + if (manualScan) { + manualScan = false; + return true; + } + //wait at least one day from last scrapping to scrap again + int minTime = 24 * 60 * 60; + return db->CheckStartScrapping(minTime); +} + + +void cTVScraperWorker::Action(void) { + if (!startLoop) + return; + mutex.Lock(); + dsyslog("tvscraper: waiting %d minutes to start main loop", initSleep / 1000 / 60); + waitCondition.TimedWait(mutex, initSleep); + + while (Running()) { + if (scanVideoDir) { + scanVideoDir = false; + dsyslog("tvscraper: scanning video dir"); + if (ConnectScrapers()) { + ScrapRecordings(); + } + DisconnectScrapers(); + continue; + } + CheckRunningTimers(); + if (StartScrapping()) { + dsyslog("tvscraper: start scrapping epg"); + db->ClearOutdated(movieDir); + if (ConnectScrapers()) { + ScrapEPG(); + } + DisconnectScrapers(); + db->BackupToDisc(); + dsyslog("tvscraper: epg scrapping done"); + } + waitCondition.TimedWait(mutex, loopSleep); + } +} diff --git a/worker.h b/worker.h new file mode 100644 index 0000000..d89ab8b --- /dev/null +++ b/worker.h @@ -0,0 +1,47 @@ +#ifndef __TVSCRAPER_WORKER_H +#define __TVSCRAPER_WORKER_H + +// --- cTVScraperWorker ------------------------------------------------------------- + +enum scrapType { + scrapSeries, + scrapMovie, + scrapNone +}; + +class cTVScraperWorker : public cThread { +private: + bool startLoop; + bool scanVideoDir; + bool manualScan; + string language; + string plgBaseDir; + string seriesDir; + string movieDir; + int initSleep; + int loopSleep; + cCondVar waitCondition; + cMutex mutex; + cTVScraperDB *db; + cMovieDBScraper *moviedbScraper; + cTVDBScraper *tvdbScraper; + scrapType GetScrapType(const cEvent *event); + bool ConnectScrapers(void); + void DisconnectScrapers(void); + void CheckRunningTimers(void); + void ScrapEPG(void); + void ScrapRecordings(void); + bool StartScrapping(void); +public: + cTVScraperWorker(cTVScraperDB *db); + virtual ~cTVScraperWorker(void); + void SetLanguage(void); + void SetDirectories(void); + void InitVideoDirScan(void); + void InitManualScan(void); + virtual void Action(void); + void Stop(void); +}; + + +#endif //__TVSCRAPER_WORKER_H |