From d0ac75a3ce3224d1071c771fa33244cf81c95a86 Mon Sep 17 00:00:00 2001 From: horchi Date: Sat, 18 Mar 2017 08:04:30 +0100 Subject: 2017-03-18: version 1.1.111 (horchi)\n - change: modified repeaing timer check statement to handle '' values like null values again\n - added: new view eventsview-perlbo.sql\n - change: fixed some typos\n\n --- HISTORY.h | 9 +++++++-- configs/epgd.conf | 2 +- lib/searchtimer.c | 4 ++-- scripts/epgd-showmerge | 2 +- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/HISTORY.h b/HISTORY.h index 4719880..e778153 100644 --- a/HISTORY.h +++ b/HISTORY.h @@ -4,8 +4,8 @@ * ----------------------------------- */ -#define _VERSION "1.1.110" -#define VERSION_DATE "16.03.2017" +#define _VERSION "1.1.111" +#define VERSION_DATE "18.03.2017" #define DB_API 4 #ifdef GIT_REV @@ -17,6 +17,11 @@ /* * ------------------------------------ +2017-03-18: version 1.1.111 (horchi) + - change: modified repeaing timer check statement to handle '' values like null values again + - added: new view eventsview-perlbo.sql + - change: fixed some typos + 2017-03-16: version 1.1.110 (horchi) - added: Congig option for MovieDb api-key diff --git a/configs/epgd.conf b/configs/epgd.conf index 019c43f..7fbddc7 100644 --- a/configs/epgd.conf +++ b/configs/epgd.conf @@ -32,7 +32,7 @@ MaxImagesPerEvent = 1 SeriesEnabled = 1 SeriesPort = 2006 SeriesStoreToFs = 0 -SeriesUrl = eplists.constabel.net +SeriesUrl = www.eplists.de ScrapEpg = 1 ScrapRecordings = 1 diff --git a/lib/searchtimer.c b/lib/searchtimer.c index b516359..e9081fb 100644 --- a/lib/searchtimer.c +++ b/lib/searchtimer.c @@ -1224,8 +1224,8 @@ int cSearchTimer::isAlreadyDone(int repeatfields, json_t* obj, int silent) if (repeatfields & sfFolge) { - selectDoneTimer->build(" and (field('%s',ifnull(compshorttext,''),ifnull(episodecomppartname,'')) > 0" - " or field('%s',ifnull(compshorttext,''),ifnull(episodecomppartname,'')) > 0)", + selectDoneTimer->build(" and (field('%s',ifnull(compshorttext,'NoShortnameAvailable'),ifnull(episodecomppartname,'NoShortnameAvailable')) > 0" + " or field('%s',ifnull(compshorttext,''),ifnull(episodecomppartname,'NoShortnameAvailable')) > 0)", useeventsDb->getStrValue("COMPSHORTTEXT"), useeventsDb->getStrValue("EPISODECOMPPARTNAME")); } diff --git a/scripts/epgd-showmerge b/scripts/epgd-showmerge index 1dc022a..5c4fee2 100755 --- a/scripts/epgd-showmerge +++ b/scripts/epgd-showmerge @@ -10,7 +10,7 @@ if [ "$1" == "-h" ]; then exit 0 fi -mysql -u epg2vdr -Depg2vdr -e "select from_unixtime(lastmerge) as 'las merge was at' from vdrs where uuid = 'epgd';" +mysql -u epg2vdr -Depg2vdr -e "select from_unixtime(lastmerge) as 'last merge was at' from vdrs where uuid = 'epgd';" mysql -u epg2vdr -Depg2vdr -e "select count(1) as 'events pending for merge' from events where source = 'vdr' and updsp > (select lastmerge from vdrs where uuid = 'epgd');" mysql -u epg2vdr -Depg2vdr -e "select count(source),source from events group by source;" -- cgit v1.2.3 From 543cb573da71ff4a1ac54f79509a4d08b004e274 Mon Sep 17 00:00:00 2001 From: horchi Date: Sat, 18 Mar 2017 08:05:05 +0100 Subject: configs/eventsview-perlbo.sql --- configs/eventsview-perlbo.sql | 64 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 configs/eventsview-perlbo.sql diff --git a/configs/eventsview-perlbo.sql b/configs/eventsview-perlbo.sql new file mode 100644 index 0000000..03d6a2c --- /dev/null +++ b/configs/eventsview-perlbo.sql @@ -0,0 +1,64 @@ +CREATE VIEW eventsview as select cnt_useid useid, cnt_eventid eventid, cnt_channelid channelid, cnt_source source, all_updsp updsp, cnt_updflg updflg, cnt_delflg delflg, cnt_fileref fileref, cnt_tableid tableid, cnt_version version, sub_title title, +case + when sub_shorttext is null then + case when sub_genre is null then '' else concat(sub_genre,' (', sub_country, ' ',sub_year, ')') end + else + case when epi_season is null then + sub_shorttext + else + concat('(S', lpad(format(epi_season, 0), 2, '0'), 'E', lpad(format(epi_part, 0), 2, '0'), ') ', epi_partname) + end +end shorttext, +cnt_starttime starttime, cnt_duration duration, cnt_parentalrating parentalrating, cnt_vps vps, cnt_contents contents, replace( +concat( + TRIM(LEADING '|' FROM + concat( + case when sub_genre is Null then '' else concat('|','Genre: ',sub_genre) end, + case when sub_category is Null then '' else concat('|','Kategorie: ',sub_category) end, + case when sub_country is Null then '' else concat('|','Land: ',sub_country) end, + case when sub_year is Null then '' else concat('|','Jahr: ',substring(sub_year,1,4)) end + ) + ), + concat( + case when sub_shortdescription is Null then '' else concat('||',sub_shortdescription) end, + case when sub_txtrating is Null and sub_shortreview is null then '' else '||' end, + case when sub_txtrating is null then '' else sub_txtrating end, + case when sub_txtrating is Null or sub_shortreview is null then '' else ', ' end, + case when sub_shortreview is Null then '' else sub_shortreview end, + case when sub_tipp is Null and sub_rating is Null then '' else '||' end, + case when sub_tipp is Null then '' else concat('|',sub_tipp) end, + case when sub_rating is Null then '' else concat('|',sub_rating) end, + case when sub_topic is Null then '' else concat('||','Thema: ',sub_topic) end, + case when sub_longdescription is Null then '' else concat('||',sub_longdescription) end, + case when sub_moderator is Null then '' else concat('||','Moderator: ',sub_moderator) end, + case when sub_commentator is Null then '' else concat('||','Kommentar: ',sub_commentator) end, + case when sub_guest is Null then '' else concat('|','Gäste: ',sub_guest) end, + case when cnt_parentalrating is Null or cnt_parentalrating = 0 then '' else concat('||','Altersempfehlung: ab ',cnt_parentalrating) end, + case when sub_actor is Null and sub_producer is Null and sub_other is Null then '' else '|' end, + case when sub_actor is Null then '' else concat('|','Darsteller: ',sub_actor) end, + case when sub_producer is Null then '' else concat('|','Produzent: ',sub_producer) end, + case when sub_other is Null then '' else concat('|','Sonstige: ',sub_other) end, + case when sub_director is Null and sub_screenplay is Null and sub_camera is Null and sub_music is Null and sub_audio is Null and sub_flags is Null then '' else '|' end, + case when sub_director is Null then '' else concat('|','Regie: ',sub_director) end, + case when sub_screenplay is Null then '' else concat('|','Drehbuch: ',sub_screenplay) end, + case when sub_camera is Null then '' else concat('|','Kamera: ',sub_camera) end, + case when sub_music is Null then '' else concat('|','Musik: ',sub_music) end, + case when sub_audio is Null then '' else concat('|','Audio: ',sub_audio) end, + case when sub_flags is Null then '' else concat('|','Flags: ',sub_flags) end, + case when epi_episodename is Null then '' else concat('||','Serie: ',epi_episodename) end, + case when epi_shortname is Null then '' else concat('|','Kurzname: ',epi_shortname) end, + case when epi_partname is Null then '' else concat('|','Episode: ',epi_partname) end, + case when epi_extracol1 is Null then '' else concat('|',epi_extracol1) end, + case when epi_extracol2 is Null then '' else concat('|',epi_extracol2) end, + case when epi_extracol3 is Null then '' else concat('|',epi_extracol3) end, + case when epi_season is Null then '' else concat('|','Staffel: ',cast(epi_season as char)) end, + case when epi_part is Null then '' else concat('|','Staffelfolge: ',cast(epi_part as char)) end, + case when epi_part is Null then '' else concat('|','Staffelfolgen: ',cast(epi_parts as char)) end, + case when epi_number is Null then '' else concat('|','Folge: ',cast(epi_number as char)) end, + case when cnt_source <> sub_source then concat('||','Quelle: ',upper(replace(cnt_source,'vdr','dvb')),'/',upper(sub_source)) else concat('||','Quelle: ',upper(replace(cnt_source,'vdr','dvb'))) end + ) +) +,'|', ' +') as description +from + useevents; -- cgit v1.2.3 From 7d30c12632dd4df671594767798a876ccf3e5ef6 Mon Sep 17 00:00:00 2001 From: horchi Date: Sat, 18 Mar 2017 09:02:45 +0100 Subject: reset curl header option --- lib/curl.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/curl.c b/lib/curl.c index c8dda08..6fc5d9d 100644 --- a/lib/curl.c +++ b/lib/curl.c @@ -108,6 +108,7 @@ int cCurl::init(const char* httpproxy) curl_easy_setopt(handle, CURLOPT_PROXY, httpproxy); // Specify HTTP proxy } + curl_easy_setopt(handle, CURLOPT_HTTPHEADER, 0); curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, collect_data); curl_easy_setopt(handle, CURLOPT_WRITEDATA, 0); // Set option to write to string curl_easy_setopt(handle, CURLOPT_FOLLOWLOCATION, yes); @@ -439,7 +440,7 @@ int cCurl::downloadFile(const char* url, int& size, MemoryStruct* data, int time curl_easy_setopt(handle, CURLOPT_ACCEPT_ENCODING, "gzip"); // if (headerlist) - curl_easy_setopt(handle, CURLOPT_HTTPHEADER, headerlist); + curl_easy_setopt(handle, CURLOPT_HTTPHEADER, headerlist); // perform http-get -- cgit v1.2.3 From 48392aab9ed36c389db18fb95e230d3eed08f18c Mon Sep 17 00:00:00 2001 From: horchi Date: Sat, 18 Mar 2017 09:22:29 +0100 Subject: 2017-03-18: version 1.1.112 (horchi)\n - bugfix: fixed imagecount value in events table\n\n --- HISTORY.h | 5 ++++- update.c | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/HISTORY.h b/HISTORY.h index e778153..1620dee 100644 --- a/HISTORY.h +++ b/HISTORY.h @@ -4,7 +4,7 @@ * ----------------------------------- */ -#define _VERSION "1.1.111" +#define _VERSION "1.1.112" #define VERSION_DATE "18.03.2017" #define DB_API 4 @@ -17,6 +17,9 @@ /* * ------------------------------------ +2017-03-18: version 1.1.112 (horchi) + - bugfix: fixed imagecount value in events table + 2017-03-18: version 1.1.111 (horchi) - change: modified repeaing timer check statement to handle '' values like null values again - added: new view eventsview-perlbo.sql diff --git a/update.c b/update.c index f79ab68..b0d61be 100644 --- a/update.c +++ b/update.c @@ -1808,7 +1808,7 @@ int cEpgd::parseEvent(cDbRow* event, xmlNode* node) event->getBigintValue("EVENTID"), event->getStrValue("FILEREF")); - event->setValue("IMAGECOUNT", imgCnt); + event->setValue("IMAGECOUNT", min(imgCnt, EpgdConfig.maximagesperevent)); free(images); free(imagetype); @@ -1829,6 +1829,8 @@ int cEpgd::storeImageRefs(tEventId evtId, const char* source, const char* images char* imagesCsv = strdup(images); int count = 0; + // #TODO limit here to EpgdConfig.maximagesperevent + for (char* p = imagesCsv; p && *p; p = next, lfn++) { if ((next = strchr(p, ','))) -- cgit v1.2.3 From 3d410346bfd642e152453c898c8d98b4b7a4800d Mon Sep 17 00:00:00 2001 From: horchi Date: Sat, 18 Mar 2017 11:56:16 +0100 Subject: added dynamic pause hadling for scraper movie download --- lib/searchtimer.c | 2 +- update.c | 20 +++++++++++++++----- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/lib/searchtimer.c b/lib/searchtimer.c index e9081fb..50861f3 100644 --- a/lib/searchtimer.c +++ b/lib/searchtimer.c @@ -690,7 +690,7 @@ int cSearchTimer::checkTimers() { int count = 0; - tell(1, "Checking timers against actual epg and serarchtimer settings"); + tell(1, "Checking timers against actual epg and searchtimer settings"); if (checkConnection() != success) return 0; diff --git a/update.c b/update.c index b0d61be..139f7c0 100644 --- a/update.c +++ b/update.c @@ -2316,9 +2316,6 @@ int cEpgd::scrapNewEvents() if (seriesCur % 10 == 0) tell(0, "series episode %d / %d scraped...continuing scraping", seriesCur, seriesTotal); - if (seriesCur % 50 == 0) - sleep(1); - tvdbManager->ProcessSeries(*it); if (doShutDown()) @@ -2356,6 +2353,8 @@ int cEpgd::scrapNewEvents() tell(0, "%d new movies to scrap in db", moviesTotal); + time_t sectionStartAt = time(0); // split download in parts of 40 + for (vector::iterator it = moviesToScrap.begin(); it != moviesToScrap.end(); ++it) { movieCur++; @@ -2363,8 +2362,19 @@ int cEpgd::scrapNewEvents() if (movieCur % 10 == 0) tell(0, "movie %d / %d scraped...continuing scraping", movieCur, moviesTotal); - if (movieCur % 50 == 0) - sleep(1); + if (movieCur % 40 == 0) + { + int duration = time(0) - sectionStartAt; + + if (duration < 10) + { + duration = 10 - duration; + tell(0, "Waiting %d seconds..", duration); + sleep(duration); + } + + sectionStartAt = time(0); + } movieDbManager->ProcessMovie(*it); -- cgit v1.2.3 From 4c2906c0ee6947cd8c04e36820e62e2e00fbc868 Mon Sep 17 00:00:00 2001 From: horchi Date: Sun, 19 Mar 2017 08:34:22 +0100 Subject: minor change --- configs/epg.dat | 1 + 1 file changed, 1 insertion(+) diff --git a/configs/epg.dat b/configs/epg.dat index 0959093..b33e4a4 100644 --- a/configs/epg.dat +++ b/configs/epg.dat @@ -400,6 +400,7 @@ Table useevents MASTERID "" cnt_masterid UInt 0 Data|Meta, USEID "" cnt_useid UInt 0 Data, +// SOURCE "" source Ascii 25 Data SUBSOURCE "" sub_source Ascii 10 Data|Meta, SUBEVENTID "" sub_eventid UBigInt 0 Data|Meta, -- cgit v1.2.3 From 4ab12d84b2738fbfc9aafed6b4710180aa2d37e4 Mon Sep 17 00:00:00 2001 From: horchi Date: Sun, 19 Mar 2017 15:26:57 +0100 Subject: minor imrovement of db api --- lib/db.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/lib/db.c b/lib/db.c index 7b50bcd..2dbca80 100644 --- a/lib/db.c +++ b/lib/db.c @@ -734,15 +734,18 @@ int cDbTable::init(int allowAlter) // check/create table ... - if (exist() && allowAlter) - validateStructure(allowAlter); + if (allowAlter) + { + if (exist()) + validateStructure(allowAlter); - if (createTable() != success) - return fail; + if (!exist() && createTable() != success) + return fail; - // check/create indices + // check/create indices - createIndices(); + createIndices(); + } // ------------------------------ // prepare BASIC statements -- cgit v1.2.3