epgdata2xmltv now handles wrong charset and encoding (only sort of!)v0.0.2

added eplist support

13 files changed, 254 insertions, 31 deletions

diff --git a/Makefile b/Makefile
index 6e30fef..63a45ec 100644
--- a/Makefile
+++ b/Makefile
@@ -114,7 +114,9 @@ install:
 dist: clean
 	@-rm -rf $(TMPDIR)/$(ARCHIVE)
 	@mkdir $(TMPDIR)/$(ARCHIVE)
-	@cp -a *.cpp *.h HISTORY Makefile README po $(TMPDIR)/$(ARCHIVE)
+	@cp -a *.cpp *.h HISTORY COPYING Makefile README po $(TMPDIR)/$(ARCHIVE)
+	@mkdir -p $(TMPDIR)/$(ARCHIVE)/dist/epgdata2xmltv
+	@cp -a dist/epgdata2xmltv/*.cpp dist/epgdata2xmltv/*.h dist/epgdata2xmltv/Makefile dist/epgdata2xmltv/INSTALL dist/epgdata2xmltv/COPYING dist/epgdata2xmltv/epgdata2xmltv.dist $(TMPDIR)/$(ARCHIVE)/dist/epgdata2xmltv 
 	@tar czf $(PACKAGE).tgz -C $(TMPDIR) --exclude debian --exclude CVS --exclude .svn $(ARCHIVE)
 	@-rm -rf $(TMPDIR)/$(ARCHIVE)
 	@echo Distribution package created as $(PACKAGE).tgz
diff --git a/dist/epgdata2xmltv/Makefile b/dist/epgdata2xmltv/Makefile
index 2ff7f4e..e13b84d 100644
--- a/dist/epgdata2xmltv/Makefile
+++ b/dist/epgdata2xmltv/Makefile
@@ -11,8 +11,8 @@ STRIP ?= strip
 
 ### Includes and Defines (add further entries here):
 
-PKG-LIBS += libxml-2.0 libxslt libexslt libcurl libzip
-PKG-INCLUDES += libxml-2.0 libxslt libexslt libcurl libzip
+PKG-LIBS += libxml-2.0 libxslt libexslt libcurl libzip libpcrecpp enca
+PKG-INCLUDES += libxml-2.0 libxslt libexslt libcurl libzip libpcrecpp enca
 
 DEFINES += -D_GNU_SOURCE 
 DEFINES += -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE -D_LARGEFILE64_SOURCE
diff --git a/dist/epgdata2xmltv/epgdata2xmltv.cpp b/dist/epgdata2xmltv/epgdata2xmltv.cpp
index f432c07..7515cb0 100644
--- a/dist/epgdata2xmltv/epgdata2xmltv.cpp
+++ b/dist/epgdata2xmltv/epgdata2xmltv.cpp
@@ -8,6 +8,8 @@
 #include <string.h>
 #include <locale.h>
 #include <zip.h>
+#include <pcrecpp.h>
+#include <enca.h>
 #include <libxml/parserInternals.h>
 #include "epgdata2xmltv.h"
 #include "epgdata2xmltv_xsl.h"
@@ -381,7 +383,6 @@ int cepgdata2xmltv::Process(int argc, char *argv[])
                 break;
             }
             dtdmem[size]=0;
-            dtdmem=strreplace(dtdmem,"ISO-8859-1","Windows-1252");
             zip_fclose(zfile);
 
             int entries=zip_get_num_files(zip);
@@ -484,11 +485,38 @@ int cepgdata2xmltv::Process(int argc, char *argv[])
 
         xmlDocPtr pxmlDoc;
         if (!pxsltStylesheet) LoadXSLT();
-        if ((pxmlDoc=xmlParseMemory(xmlmem,strlen(xmlmem)))==NULL)
+        int xmlsize=strlen(xmlmem);
+        if ((pxmlDoc=xmlParseMemory(xmlmem,xmlsize))==NULL)
         {
-            esyslog("failed parsing xml");
-            free(xmlmem);
-            continue;
+            EncaAnalyser analyser=enca_analyser_alloc("__");
+            if (analyser) {
+                EncaEncoding encoding=enca_analyse_const(analyser, (unsigned char *) xmlmem,xmlsize);
+                const char *cs=enca_charset_name(encoding.charset, ENCA_NAME_STYLE_ICONV);
+                if (cs) {
+                    if (!strcmp(cs,"UTF-8")) {
+                        xmlmem=strreplace(xmlmem,"Windows-1252","UTF-8");
+                    } else {
+                        esyslog("enca returned %s, please report!",cs);
+                    }
+                }
+                enca_analyser_free(analyser);
+            }
+
+            string s = xmlmem;
+            int reps=pcrecpp::RE("&(?![a-zA-Z]{1,8};)").GlobalReplace("%amp;",&s);
+            if (reps) {
+                xmlmem = (char *)realloc(xmlmem, s.size()+1);
+                xmlsize = s.size();
+                strcpy(xmlmem,s.c_str());
+            }
+
+            if ((pxmlDoc=xmlParseMemory(xmlmem,xmlsize))==NULL)
+            {
+                esyslog("failed parsing xml");
+                free(xmlmem);
+                xmlmem=NULL;
+                continue;
+            }
         }
 
         for (;;)
diff --git a/dist/epgdata2xmltv/epgdata2xmltv.h b/dist/epgdata2xmltv/epgdata2xmltv.h
index c49794d..818e883 100644
--- a/dist/epgdata2xmltv/epgdata2xmltv.h
+++ b/dist/epgdata2xmltv/epgdata2xmltv.h
@@ -49,9 +49,6 @@ private:
     int  DownloadData(const char *url);
     bool Translate(xmlDocPtr pxmlDoc, const char **params);
     void LoadXSLT();
-/*    xmlParserInputPtr xmlMyExternalEntityLoader(const char *URL,
-            const char *ID,
-            xmlParserCtxtPtr ctxt);        */
   public:
     cepgdata2xmltv();
     ~cepgdata2xmltv();
diff --git a/maps.h b/maps.h
index 6f9fefe..71458ef 100644
--- a/maps.h
+++ b/maps.h
@@ -30,6 +30,8 @@
 #define USE_VIDEO              0x100
 #define USE_AUDIO              0x200
 
+#define USE_SEASON             0x400
+
 #define CREDITS_ACTORS         0x100000
 #define CREDITS_DIRECTORS      0x200000
 #define CREDITS_OTHERS         0x400000
diff --git a/parse.cpp b/parse.cpp
index 19812e8..7a2de45 100644
--- a/parse.cpp
+++ b/parse.cpp
@@ -14,6 +14,7 @@
 #include <locale.h>
 #include <langinfo.h>
 #include <time.h>
+#include <pwd.h>
 
 #include "xmltv2vdr.h"
 #include "parse.h"
@@ -179,6 +180,8 @@ void cXMLTVEvent::Clear()
     eventid=0;
     credits.Clear();
     categories.Clear();
+    season=0;
+    episode=0;
 }
 
 cXMLTVEvent::cXMLTVEvent()
@@ -350,12 +353,19 @@ cEvent *cParse::SearchEvent(cSchedule* schedule, cXMLTVEvent *xevent)
     if (f) return f;
     // 2nd with StartTime
     f=(cEvent *) schedule->GetEvent((tEventID) 0,start);
-    if (f) return f;
+    if (f)
+    {
+        if (!strcmp(f->Title(),xevent->Title()))
+        {
+            return f;
+        }
+    }
     // 3rd with StartTime +/- WaitTime
     int maxdiff=INT_MAX;
     int eventTimeDiff=0;
     if (xevent->Duration()) eventTimeDiff=xevent->Duration()/4;
-    if (eventTimeDiff<600) eventTimeDiff=600;
+    if (eventTimeDiff<780) eventTimeDiff=780;
+
     for (cEvent *p = schedule->Events()->First(); p; p = schedule->Events()->Next(p))
     {
         int diff=abs((int) difftime(p->StartTime(),start));
@@ -372,6 +382,7 @@ cEvent *cParse::SearchEvent(cSchedule* schedule, cXMLTVEvent *xevent)
             }
             else
             {
+                if (f) continue; // we already have an event!
                 // cut both titles into pieces and check
                 // if we have at least one match with
                 // minimum length of 4 characters
@@ -406,9 +417,10 @@ cEvent *cParse::SearchEvent(cSchedule* schedule, cXMLTVEvent *xevent)
                             }
                         }
                     }
-                    free(s1);
-                    free(s2);
                 }
+                if (s1) free(s1);
+                if (s2) free(s2);
+
                 if (wfound)
                 {
                     if (diff<=maxdiff)
@@ -443,6 +455,56 @@ cEvent *cParse::GetEventBefore(cSchedule* schedule, time_t start)
     return NULL;
 }
 
+void cParse::FetchSeasonEpisode(cEvent *event)
+{
+    if (!epdir) return;
+    if (!event) return;
+    if (!event->ShortText()) return;
+    if (!event->Title()) return;
+    char *epfile=NULL;
+    if (asprintf(&epfile,"%s/.eplists/lists/%s.episodes",epdir,event->Title())==-1) return;
+
+    struct stat statbuf;
+    if (stat(epfile,&statbuf)==-1)
+    {
+        free(epfile);
+        return;
+    }
+
+    FILE *f=fopen(epfile,"r");
+    if (!f)
+    {
+        free(epfile);
+        return;
+    }
+
+    char *line=NULL;
+    size_t length;
+    while (getline(&line,&length,f)!=-1)
+    {
+        if (line[0]=='#') continue;
+        char title[256]="";
+        int season;
+        int episode;
+        if (sscanf(line,"%i\t%i\t%*i\t%255c",&season,&episode,title)==3)
+        {
+            char *lf=strchr(title,'\n');
+            if (lf) *lf=0;
+            if (!strcmp(event->ShortText(),title))
+            {
+                xevent.SetSeason(season);
+                xevent.SetEpisode(episode);
+                break;
+            }
+        }
+    }
+    if (line) free(line);
+    fclose(f);
+
+    free(epfile);
+    return;
+}
+
 bool cParse::PutEvent(cSchedule* schedule, cEvent *event, cXMLTVEvent *xevent, cEPGMapping *map)
 {
     if (!schedule) return false;
@@ -576,17 +638,7 @@ bool cParse::PutEvent(cSchedule* schedule, cEvent *event, cXMLTVEvent *xevent, c
             return true;
         }
     }
-    else
-    {
-        if (event->TableID()==0) return true;
-        start=event->StartTime();
-        end=event->EndTime();
-        localtime_r(&start,&tm);
-        strftime(from,sizeof(from)-1,"%b %d %H:%M",&tm);
-        localtime_r(&end,&tm);
-        strftime(till,sizeof(till)-1,"%b %d %H:%M",&tm);
-        source->Dlog("changing '%s'@%s-%s",event->Title(),from,till);
-    }
+
     if ((map->Flags() & USE_SHORTTEXT)==USE_SHORTTEXT)
     {
         if (xevent->ShortText() && (strlen(xevent->ShortText())>0))
@@ -602,6 +654,13 @@ bool cParse::PutEvent(cSchedule* schedule, cEvent *event, cXMLTVEvent *xevent, c
             }
         }
     }
+
+    if (event->ShortText() && (strlen(event->ShortText())>0) && ((map->Flags() & USE_SEASON)==USE_SEASON))
+    {
+        // Try to fetch season and episode from eplist
+        FetchSeasonEpisode(event);
+    }
+
     if ((map->Flags() & USE_LONGTEXT)==USE_LONGTEXT)
     {
         if (xevent->Description() && (strlen(xevent->Description())>0))
@@ -697,6 +756,21 @@ bool cParse::PutEvent(cSchedule* schedule, cEvent *event, cXMLTVEvent *xevent, c
             if (text) addExt=xevent->Add2Description(text->Value(),xevent->Year());
         }
     }
+    if ((map->Flags() & USE_SEASON)==USE_SEASON)
+    {
+        if (xevent->Season())
+        {
+            cTEXTMapping *text=TEXTMapping("season");
+            if (text) addExt=xevent->Add2Description(text->Value(),xevent->Season());
+        }
+
+        if (xevent->Episode())
+        {
+            cTEXTMapping *text=TEXTMapping("episode");
+            if (text) addExt=xevent->Add2Description(text->Value(),xevent->Episode());
+        }
+    }
+
     if (((map->Flags() & USE_ORIGTITLE)==USE_ORIGTITLE) && (xevent->OrigTitle()))
     {
         cTEXTMapping *text=TEXTMapping("originaltitle");
@@ -726,6 +800,17 @@ bool cParse::PutEvent(cSchedule* schedule, cEvent *event, cXMLTVEvent *xevent, c
         cTEXTMapping *text=TEXTMapping("review");
         if (text) addExt=xevent->Add2Description(text->Value(),xevent->Review());
     }
+    if (event->TableID()==0) return true;
+    if ((map->Flags() & OPT_APPEND)!=OPT_APPEND)
+    {
+        start=event->StartTime();
+        end=event->EndTime();
+        localtime_r(&start,&tm);
+        strftime(from,sizeof(from)-1,"%b %d %H:%M",&tm);
+        localtime_r(&end,&tm);
+        strftime(till,sizeof(till)-1,"%b %d %H:%M",&tm);
+        source->Dlog("changing '%s'@%s-%s",event->Title(),from,till);
+    }
     if (addExt) event->SetDescription(xevent->Description());
     event->SetTableID(0); // prevent EIT EPG to update this event
     return true;
@@ -1212,9 +1297,20 @@ cParse::cParse(cEPGSource *Source, cEPGMappings *Maps, cTEXTMappings *Texts)
     }
     source->Dlog("vdr codeset is '%s'",CodeSet ? CodeSet : "US-ASCII//TRANSLIT");
     conv = new cCharSetConv("UTF-8",CodeSet ? CodeSet : "US-ASCII//TRANSLIT");
+
+    struct passwd *pw=getpwuid(getuid());
+    if (pw)
+    {
+        epdir=strdup(pw->pw_dir);
+    }
+    else
+    {
+        epdir=NULL;
+    }
 }
 
 cParse::~cParse()
 {
+    if (epdir) free(epdir);
     delete conv;
 }
diff --git a/parse.h b/parse.h
index 3d4f1e0..96b8f7b 100644
--- a/parse.h
+++ b/parse.h
@@ -33,6 +33,8 @@ private:
     time_t starttime;
     int duration;
     time_t vps;
+    int season;
+    int episode;
     tEventID eventid;
     cStringList credits;
     cStringList categories;
@@ -57,6 +59,14 @@ public:
     void SetCountry(const char *Country);
     void SetReview(const char *Review);
     void SetRating(const char *System, const char *Rating);
+    void SetSeason(int Season)
+    {
+        season=Season;
+    }
+    void SetEpisode(int Episode)
+    {
+        episode=Episode;
+    }
     void SetYear(int Year)
     {
         year=Year;
@@ -137,6 +147,14 @@ public:
     {
         return &categories;
     }
+    int Season(void)
+    {
+        return season;
+    }
+    int Episode(void)
+    {
+        return episode;
+    }
 };
 
 class cParse
@@ -164,6 +182,7 @@ private:
     cTEXTMappings *texts;
     cXMLTVEvent xevent;
     cCharSetConv *conv;
+    char *epdir;
     char cbuf[80];
     char *RemoveNonASCII(const char *src);
     struct split split(char *in, char delim);
@@ -171,6 +190,7 @@ private:
     cEvent *SearchEvent(cSchedule* schedule, cXMLTVEvent *xevent);
     time_t ConvertXMLTVTime2UnixTime(char *xmltvtime);
     bool FetchEvent(xmlNodePtr node);
+    void FetchSeasonEpisode(cEvent *event);
     cEPGMapping *EPGMapping(const char *ChannelName);
     cTEXTMapping *TEXTMapping(const char *Name);
     bool PutEvent(cSchedule* schedule,cEvent *event,cXMLTVEvent *xevent, cEPGMapping *map);
diff --git a/po/de_DE.po b/po/de_DE.po
index 6e9f045..1d894b8 100644
--- a/po/de_DE.po
+++ b/po/de_DE.po
@@ -4,7 +4,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: vdr\n"
 "Report-Msgid-Bugs-To: <see README>\n"
-"POT-Creation-Date: 2011-08-07 18:55+0200\n"
+"POT-Creation-Date: 2011-11-25 19:04+0100\n"
 "PO-Revision-Date: 2010-12-23 23:59+0100\n"
 "Last-Translator: Jochen Dolze <vdr@dolze.de>\n"
 "Language-Team: <vdr@linuxtv.org>\n"
@@ -106,6 +106,12 @@ msgstr "Kritik"
 msgid "category"
 msgstr "Kategorie"
 
+msgid "season"
+msgstr "Staffel"
+
+msgid "episode"
+msgstr "Episode"
+
 msgid "country and date"
 msgstr "Ort und Jahr"
 
@@ -115,6 +121,9 @@ msgstr "Originaltitel"
 msgid "credits"
 msgstr "Mitwirkende"
 
+msgid "season and episode"
+msgstr "Staffel und Episode"
+
 msgid "overview"
 msgstr "Übersicht"
 
@@ -231,4 +240,3 @@ msgstr "xmltv2vdr plugin ist noch aktiv"
 
 msgid "Imports xmltv epg into vdr"
 msgstr "Importiert xmltv epg in den VDR"
-
diff --git a/po/it_IT.po b/po/it_IT.po
index deb796c..63df24b 100644
--- a/po/it_IT.po
+++ b/po/it_IT.po
@@ -4,7 +4,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: vdr\n"
 "Report-Msgid-Bugs-To: <see README>\n"
-"POT-Creation-Date: 2011-08-07 18:55+0200\n"
+"POT-Creation-Date: 2011-11-25 19:04+0100\n"
 "PO-Revision-Date: 2011-03-05 15:45+0100\n"
 "Last-Translator: Diego Pierotto <vdr-italian@tiscali.it>\n"
 "Language-Team:  <vdr@linuxtv.org>\n"
@@ -109,6 +109,12 @@ msgstr "Anteprima"
 msgid "category"
 msgstr "Categoria"
 
+msgid "season"
+msgstr ""
+
+msgid "episode"
+msgstr ""
+
 msgid "country and date"
 msgstr "Paese e data"
 
@@ -118,6 +124,9 @@ msgstr "Titolo originale"
 msgid "credits"
 msgstr "Crediti"
 
+msgid "season and episode"
+msgstr ""
+
 msgid "overview"
 msgstr ""
 
@@ -234,4 +243,3 @@ msgstr "Plugin XMLTV2VDR ancora in esecuzione"
 
 msgid "Imports xmltv epg into vdr"
 msgstr "Importa EPG di XMLTV in VDR"
-
diff --git a/setup.cpp b/setup.cpp
index 82d5b6f..b9fe697 100644
--- a/setup.cpp
+++ b/setup.cpp
@@ -9,6 +9,7 @@
 
 #include <vdr/osdbase.h>
 #include <time.h>
+#include <pwd.h>
 
 #define CHNUMWIDTH (numdigits(Channels.MaxNumber())+1)
 
@@ -431,6 +432,26 @@ cMenuSetupXmltv2vdrTextMap::cMenuSetupXmltv2vdrTextMap(cPluginXmltv2vdr *Plugin)
         strcpy(category,tr("category"));
     }
 
+    textmap=baseplugin->TEXTMapping("season");
+    if (textmap)
+    {
+        strn0cpy(season,textmap->Value(),sizeof(season)-1);
+    }
+    else
+    {
+        strcpy(season,tr("season"));
+    }
+
+    textmap=baseplugin->TEXTMapping("episode");
+    if (textmap)
+    {
+        strn0cpy(episode,textmap->Value(),sizeof(episode)-1);
+    }
+    else
+    {
+        strcpy(episode,tr("episode"));
+    }
+
     Add(NewTitle(tr("country and date")));
     Add(new cMenuEditStrItem("country",country,sizeof(country)));
     Add(new cMenuEditStrItem("date",date,sizeof(date)));
@@ -455,6 +476,11 @@ cMenuSetupXmltv2vdrTextMap::cMenuSetupXmltv2vdrTextMap(cPluginXmltv2vdr *Plugin)
 
     Add(NewTitle(tr("review")));
     Add(new cMenuEditStrItem("review",review,sizeof(review)));
+
+    Add(NewTitle(tr("season and episode")));
+    Add(new cMenuEditStrItem("season",season,sizeof(season)));
+    Add(new cMenuEditStrItem("episode",episode,sizeof(episode)));
+
 }
 
 void cMenuSetupXmltv2vdrTextMap::Store()
@@ -596,6 +622,24 @@ void cMenuSetupXmltv2vdrTextMap::Store()
     {
         baseplugin->TEXTMappingAdd(new cTEXTMapping("review",review));
     }
+    textmap=baseplugin->TEXTMapping("season");
+    if (textmap)
+    {
+        textmap->ChangeValue(season);
+    }
+    else
+    {
+        baseplugin->TEXTMappingAdd(new cTEXTMapping("season",season));
+    }
+    textmap=baseplugin->TEXTMapping("episode");
+    if (textmap)
+    {
+        textmap->ChangeValue(episode);
+    }
+    else
+    {
+        baseplugin->TEXTMappingAdd(new cTEXTMapping("episode",episode));
+    }
 
     SetupStore("textmap.country",country);
     SetupStore("textmap.date",date);
@@ -612,6 +656,8 @@ void cMenuSetupXmltv2vdrTextMap::Store()
     SetupStore("textmap.producer",producer);
     SetupStore("textmap.writer",writer);
     SetupStore("textmap.review",review);
+    SetupStore("textmap.season",season);
+    SetupStore("textmap.episode",episode);
 }
 
 // --------------------------------------------------------------------------------------------------------
@@ -1047,6 +1093,18 @@ void cMenuSetupXmltv2vdrChannelMap::output(void)
     Add(new cMyMenuEditBitItem(tr("audio"),&flags,USE_AUDIO),true);
     Add(new cMyMenuEditBitItem(tr("vps"),&flags,OPT_VPS),true);
 
+    struct passwd *pw=getpwuid(getuid());
+    if (pw)
+    {
+        char *path=NULL;
+        if (asprintf(&path,"%s/.eplists/lists",pw->pw_dir)!=-1)
+        {
+            if (!access(path,R_OK))
+                Add(new cMyMenuEditBitItem(tr("season and episode"),&flags,USE_SEASON),true);
+            free(path);
+        }
+    }
+
     hasmaps=false;
     Add(NewTitle(tr("epg source channel mappings")),true);
     for (int i=0; i<map->NumChannelIDs(); i++)
diff --git a/setup.h b/setup.h
index 4475f90..1cd0b60 100644
--- a/setup.h
+++ b/setup.h
@@ -84,6 +84,8 @@ private:
     char guest[255];
     char review[255];
     char category[255];
+    char season[255];
+    char episode[255];
 public:
     cMenuSetupXmltv2vdrTextMap(cPluginXmltv2vdr *Plugin);
 };
diff --git a/xmltv2vdr.cpp b/xmltv2vdr.cpp
index cf68805..52aa814 100644
--- a/xmltv2vdr.cpp
+++ b/xmltv2vdr.cpp
@@ -831,6 +831,8 @@ cPluginXmltv2vdr::cPluginXmltv2vdr(void) : epgexecutor(&epgsources)
     TEXTMappingAdd(new cTEXTMapping("producer",tr("producer")));
     TEXTMappingAdd(new cTEXTMapping("writer",tr("writer")));
     TEXTMappingAdd(new cTEXTMapping("review",tr("review")));
+    TEXTMappingAdd(new cTEXTMapping("season",tr("season")));
+    TEXTMappingAdd(new cTEXTMapping("episode",tr("episode")));
 }
 
 cPluginXmltv2vdr::~cPluginXmltv2vdr()
diff --git a/xmltv2vdr.h b/xmltv2vdr.h
index 1067494..8acfac2 100644
--- a/xmltv2vdr.h
+++ b/xmltv2vdr.h
@@ -14,7 +14,7 @@
 #include "maps.h"
 #include "parse.h"
 
-static const char *VERSION        = "0.0.2pre";
+static const char *VERSION        = "0.0.2";
 static const char *DESCRIPTION    = trNOOP("Imports xmltv epg into vdr");
 
 class cEPGChannel : public cListObject