path: root/PLUGINS/epgdata/epgdata.c

/*
 * epgdata.c
 *
 * See the README file for copyright information
 *
 */

#include <dirent.h>
#include <unistd.h>

#include "epgdata.h"

//***************************************************************************
// Epgdata
//***************************************************************************

Epgdata::Epgdata()
   : Plugin()
{
   pxsltStylesheet = 0;
   stmtByFileRef = 0;
   stmtCleanDouble = 0;
   stmtMarkOldEvents = 0;
   selectByDate = 0;
   selectId = 0;

   pin = 0;
   timeout = 3 * tmeSecondsPerMinute;
   baseurl = strdup("http://www.epgdata.com");
}

Epgdata::~Epgdata()
{
   if (pxsltStylesheet)
      xsltFreeStylesheet(pxsltStylesheet);

   delete stmtByFileRef;
   delete selectByDate;
   delete stmtCleanDouble;
   delete stmtMarkOldEvents;

   free(baseurl);
   free(pin);
}

int Epgdata::init(cEpgd* aObject, int aUtf8)
{
   Plugin::init(aObject, aUtf8);

   pxsltStylesheet = loadXSLT(getSource(), confDir, utf8);

   return done;
}

int Epgdata::initDb()
{
   int status = success;

   // --------
   // by fileref (for pictures)
   // select name from fileref 
   //     where source = ? and fileref = ?

   stmtByFileRef = new cDbStatement(obj->fileDb);

   stmtByFileRef->build("select ");
   stmtByFileRef->bind("NAME", cDBS::bndOut);
   stmtByFileRef->build(" from %s where ", obj->fileDb->TableName());
   stmtByFileRef->bind("SOURCE", cDBS::bndIn | cDBS::bndSet);
   stmtByFileRef->bind("FILEREF", cDBS::bndIn | cDBS::bndSet, " and ");
   
   status += stmtByFileRef->prepare();

   // ---------------
   // (for cleanup double)
   // select name from fileref 
   //     where source = ? order by name desc

   stmtCleanDouble = new cDbStatement(obj->fileDb);

   stmtCleanDouble->build("select ");
   stmtCleanDouble->bind("NAME", cDBS::bndOut);
   stmtCleanDouble->build(" from %s where ", obj->fileDb->TableName());
   stmtCleanDouble->bind("SOURCE", cDBS::bndIn | cDBS::bndSet);
   stmtCleanDouble->build(" order by name desc");
   
   status += stmtCleanDouble->prepare();

   // ---------
   // select channelid, merge, mergesp from channelmap 
   //     where source = ? and extid = ?

   selectId = new cDbStatement(obj->mapDb);

   selectId->build("select ");
   selectId->bind("CHANNELID", cDBS::bndOut);
   selectId->bind("MERGESP", cDBS::bndOut, ", ");
   selectId->bind("MERGE", cDBS::bndOut, ", ");
   selectId->build(" from %s where ", obj->mapDb->TableName());
   selectId->bind("SOURCE", cDBS::bndIn | cDBS::bndSet);
   selectId->bind("EXTERNALID", cDBS::bndIn | cDBS::bndSet, " and ");
   
   status += selectId->prepare();

   // ---------
   // select name, tag from filerf
   //  where source = 'epgdata' 
   //    and name like ?

   valueName.setField(obj->fileDb->getField("NAME"));
   valueNameLike.setField(obj->fileDb->getField("NAME"));

   selectByDate = new cDbStatement(obj->fileDb);

   selectByDate->build("select ");
   selectByDate->bind(&valueName, cDBS::bndOut);
   selectByDate->bind("FILEREF", cDBS::bndOut, ", ");
   selectByDate->build(" from %s where source = '%s' and ", 
                       obj->fileDb->TableName(), getSource());
   selectByDate->bindCmp(0, &valueNameLike, "like");

   status += selectByDate->prepare();

//    // --------
//    // update events set delflg = ?, updflg = ?, fileref = ?, updsp = ?
//    //    where fileref = ? 
//    //      and source = ?;
//    //      and updflg in (....)

//    valueFileRef.setField(obj->eventsDb->getField("FileRef"));
//    stmtSetDelByFileref = new cDbStatement(obj->eventsDb);
   
//    stmtSetDelByFileref->build("update %s set ", obj->eventsDb->TableName());
//    stmtSetDelByFileref->bind("DelFlg", cDbService::bndIn |cDbService:: bndSet);
//    stmtSetDelByFileref->bind("UpdFlg", cDbService::bndIn |cDbService:: bndSet, ", ");
//    stmtSetDelByFileref->bind("FileRef", cDbService::bndIn | cDbService::bndSet, ", ");
//    stmtSetDelByFileref->bind("UpdSp", cDbService::bndIn | cDbService::bndSet, ", ");
//    stmtSetDelByFileref->build( " where ");
//    stmtSetDelByFileref->bind(&valueFileRef, cDbService::bndIn |cDbService:: bndSet);
//    stmtSetDelByFileref->bind("Source", cDbService::bndIn | cDbService::bndSet, " and ");
//    stmtSetDelByFileref->build(" and updflg in (%s)", Us::getDeletable());

//    status += stmtSetDelByFileref->prepare();

   // ----------
   // update events 
   //   set updflg = case when updflg in (...) then 'D' else updflg end, 
   //       delflg = 'Y',
   //       updsp = unix_timestamp()
   //   where source = '...'
   //     and (source, fileref) not in (select source,fileref from fileref)

   stmtMarkOldEvents = new cDbStatement(obj->eventsDb);

   stmtMarkOldEvents->build("update %s set ", obj->eventsDb->TableName());
   stmtMarkOldEvents->build("updflg = case when updflg in (%s) then 'D' else updflg end, ", cEventState::getDeletable());
   stmtMarkOldEvents->build("delflg = 'Y', updsp = unix_timestamp()");
   stmtMarkOldEvents->build(" where source = '%s'", getSource());
   stmtMarkOldEvents->build(" and  (source, fileref) not in (select source,fileref from fileref)");

   status += stmtMarkOldEvents->prepare();  

   // ----------
   // if no epgdata entry in fileref read files from FS to table

   int count = 0;
   obj->fileDb->countWhere("source = 'epgdata'", count);

   if (!count)
   {
      char* path = 0;
      DIR* dir;
      dirent* dp;

      asprintf(&path, "%s/%s", EpgdConfig.cachePath, getSource());
      chkDir(path);

      if (!(dir = opendir(path)))
      {
         tell(0, "Error: Opening cache path '%s' failed, %s", path, strerror(errno));
         free(path);
         return fail;
      }

      while ((dp = readdir(dir)))
      {
         char* fileRef = 0;
         char* file = 0;
         char* tag = 0;
         struct stat sb;

         if (!strstr(dp->d_name, "_de_qy.zip"))
            continue;

         asprintf(&file, "%s/%s", path, dp->d_name);
         stat(file, &sb);
         free(file);

         asprintf(&tag, "%ld", sb.st_size);
         asprintf(&fileRef, "%s-%s", dp->d_name, tag);
         
         // store file and let tag NULL to indicate that processing is needed

         obj->fileDb->clear();
         obj->fileDb->setValue("NAME", dp->d_name);
         obj->fileDb->setValue("SOURCE", getSource());
         obj->fileDb->setValue("EXTERNALID", "0");
         obj->fileDb->setValue("FILEREF", fileRef);
         obj->fileDb->store();
         
         tell(1, "Added '%s' to table fileref", dp->d_name);
         free(fileRef);
         free(tag);
      }

      free(path);
      closedir(dir);
   }

   return success;
}

int Epgdata::exitDb()
{
   delete stmtByFileRef;       stmtByFileRef = 0;
   delete stmtCleanDouble;     stmtCleanDouble = 0;
   delete selectByDate;        selectByDate = 0;
   delete selectId;            selectId = 0;
   delete stmtMarkOldEvents;   stmtMarkOldEvents = 0;

   return success;
}

//***************************************************************************
// At Config Item
//***************************************************************************

int Epgdata::atConfigItem(const char* Name, const char* Value)
{
   if      (!strcasecmp(Name, "Url"))      { free(baseurl); baseurl = strdup(Value); }
   else if (!strcasecmp(Name, "Pin"))      { free(pin); pin = strdup(Value); }
   else if (!strcasecmp(Name, "Timeout"))  { timeout = atoi(Value); }

   else return fail;

   return success;
}

//***************************************************************************
// Ready
//***************************************************************************

int Epgdata::ready() 
{ 
   static int count = na;

   if (isEmpty(pin))
      return no;

   if (count == na)
   {
      char* where;

      asprintf(&where, "source = '%s'", getSource());

      if (obj->mapDb->countWhere(where, count) != success)
         count = na;

      free(where);
   }

   return count > 0;
}

//***************************************************************************
// Process Day
//***************************************************************************

int Epgdata::processDay(int day, int fullupdate, Statistic* statistic)
{
   char* directory = 0;
   char* fspath = 0;
   char* path = 0;           // name of the zip file including the path
   int haveOneForThisDay = no;
   MemoryStruct data;
   int fileSize = 0;
   char* fileRef = 0;
   char* url = 0;
   char* logurl = 0;
   int load = yes;
   char* like = 0;
   int bSize = 0;
   char entryName[200+TB];
   MemoryStruct uzdata;
   // char* oldFileRef = 0;

   int status;

   // path to zip files, url, ..

   asprintf(&directory, "%s/%s", EpgdConfig.cachePath, getSource());
   chkDir(directory);
   asprintf(&url, "%s/index.php?action=sendPackage&iOEM=VDR&pin=%s&dayOffset=%d&dataType=xml", baseurl, pin, day);
   asprintf(&logurl, "%s/index.php?action=sendPackage&iOEM=VDR&pin=%s&dayOffset=%d&dataType=xml", baseurl, "insert-your-pin-here", day);

   // first get the http header

   data.clear();
   data.headerOnly = yes;

   status = obj->downloadFile(url, fileSize, &data);

   if (status != success || isEmpty(data.name))
   {
      tell(1, "Download header for day (%d) at '%s' failed, aborting, got name '%s', status was %d", 
           day, logurl, data.name ? data.name : "", status);
      status = fail;
      goto Exit;
   }

   tell(2, "Got info for day (%d), file '%s' with tag '%s'", day, data.name, data.tag);

   asprintf(&fileRef, "%s-%s", data.name, data.tag);
   asprintf(&path, "%s/%s", directory, data.name);

   // lookup file
   
   obj->fileDb->clear();
   obj->fileDb->setValue("NAME", data.name);
   obj->fileDb->setValue("SOURCE", getSource());

   // 1:1 match ?

   obj->fileDb->find();

   asprintf(&like, "%.8s_%%", data.name);
   valueNameLike.setValue(like);
   free(like);
   
   // check for update 

   if (selectByDate->find())
   {
      haveOneForThisDay = yes;
      // oldFileRef = strdup(obj->fileDb->getStrValue("FileRef"));
   }

   if (haveOneForThisDay && day >= EpgdConfig.upddays)
   {
      // don't check for update of existing files more than 'upddays' in the future
      
      tell(2, "Skipping update check of file '%s' for day %d", data.name, day);

      statistic->nonUpdates++;
      status = success;
      load = no;
   }
   else if (haveOneForThisDay && obj->fileDb->hasValue("FileRef", fileRef))
   {
      tell(2, "Skipping download of day (%d) due to non-update", day);
      statistic->nonUpdates++;
      status = success;
      load = no;
   }

   if (!load && !obj->fileDb->getRow()->getValue("Tag")->isNull())
      goto Exit;

   // not exist, update or not processed -> work

   // first check if we have it already on fs

   asprintf(&fspath, "%s/%s", directory, valueName.getStrValue());

   if (!load && fileExists(fspath))
   {
      tell(1, "File '%s' exist, loading from filesystem", fspath);
      
      obj->loadFromFs(&data, valueName.getStrValue(), getSource());

      free(fileRef);
      free(path);
      path = strdup(fspath);
      asprintf(&fileRef, "%s-%s", valueName.getStrValue(), data.tag);

      obj->fileDb->clear();
      obj->fileDb->setValue("NAME", valueName.getStrValue());
      obj->fileDb->setValue("SOURCE", getSource());
   }

   free(fspath);

   if (load)
   {
      tell(0, "Download file: '%s' to '%s", logurl, data.name);
      
      data.clear();
      
      if (obj->downloadFile(url, fileSize, &data, timeout) != success)
      {
         tell(0, "Download of day (%d) from '%s' failed", day, logurl);
         status = fail;
         goto Exit;
      }

      statistic->bytes += data.size;
      statistic->files++;

      // store zip to FS
      
      obj->storeToFs(&data, data.name, getSource());   
   }

   if (data.isEmpty())
      goto Exit;

   // unzip ...
   
   uzdata.clear();
   
   if (unzip(path, /*filter*/ ".xml", uzdata.memory, bSize, entryName) == success)
   {
      tell(0, "Processing file '%s' for day %d (%d/%d)", 
           fileRef, day, haveOneForThisDay, load);

      uzdata.size = bSize;
             
      // store ?
      
      if (EpgdConfig.storeXmlToFs)
         obj->storeToFs(&uzdata, entryName, getSource());
      
      // process file ..
      
      obj->connection->startTransaction();
      
      if ((status = processFile(uzdata.memory, uzdata.size, fileRef)) != success)
         statistic->rejected++;
      
      if (!obj->dbConnected())
      {
         status = fail;
         goto Exit;
      }

// we can use this code instead of "stmtMarkOldEvents" !!
// but we have to complete it like tvm plugin!
//       if (haveOneForThisDay && load && strcmp(oldFileRef, fileRef) != 0)
//       {
//          // mark 'old' entrys in events table as deleted
//          // and 'fake' fileref to new to avoid deletion at cleanup
         
//          tell(0, "Removing events of fileref '%s' for day %d", oldFileRef, day);
         
//          obj->eventsDb->clear();
//          obj->eventsDb->setValue("DelFlg", "Y");
//          obj->eventsDb->setValue("UpdFlg", "D");
//          obj->eventsDb->setValue("FileRef", fileRef);     // fake to new fileref
//          obj->eventsDb->setValue("UpdSp", time(0));
//          obj->eventsDb->setValue("Source", getSource());
//          valueFileRef.setValue(oldFileRef);                             // old fileref
//          stmtSetDelByFileref->execute();
//       }
      
      // Confirm processing of file
      
      obj->fileDb->setValue("EXTERNALID", "0");
      obj->fileDb->setValue("TAG", data.tag);
      obj->fileDb->setValue("FILEREF", fileRef);
      obj->fileDb->store();
      
      obj->connection->commit();
   }

  Exit:
   
   // free(oldFileRef);
   obj->fileDb->reset();
   selectByDate->freeResult();

   uzdata.clear();
   free(url);
   free(logurl);
   free(fileRef);
   free(directory);
   free(path);

   return status;
}

//***************************************************************************
// Process File
//***************************************************************************

int Epgdata::processFile(const char* data, int size, const char* fileRef)
{
   xmlDocPtr transformedDoc;
   xmlNodePtr xmlRoot;
   int count = 0;

   if ((transformedDoc = obj->transformXml(data, size, pxsltStylesheet, fileRef)) == 0)
   {
      tell(0, "XSLT transformation for '%s' failed, ignoring", fileRef);
      return fail;
   }

   if (!(xmlRoot = xmlDocGetRootElement(transformedDoc)))
   {
      tell(0, "Invalid xml document returned from xslt for '%s', ignoring", fileRef);
      return fail;
   }
   
   // DEBUG: xmlSaveFile("/tmp/test.xml", transformedDoc);

   for (xmlNodePtr node = xmlRoot->xmlChildrenNode; node && obj->dbConnected(); node = node->next)
   {
      char* prop = 0;
      tEventId eventid;
      char* extid = 0;
      
      // skip all unexpected elements
      
      if (node->type != XML_ELEMENT_NODE || strcmp((char*)node->name, "event") != 0)
         continue;
      
      // get/check eventid
      
      if (!(prop = (char*)xmlGetProp(node, (xmlChar*)"id")) || !*prop || !(eventid = atoll(prop)))
      {
         xmlFree(prop);
         tell(0, "Missing event id, ignoring!");
         continue;
      }
      
      xmlFree(prop);

      // get/check provider id
      
      if (!(prop = (char*)xmlGetProp(node, (xmlChar*)"provid")) || !*prop || !atoi(prop))
      {
         xmlFree(prop);
         tell(0, "Missing provider id, ignoring!");
         continue;
      }

      extid = strdup(prop);
      xmlFree(prop);

      obj->mapDb->clear();
      obj->mapDb->setValue("EXTERNALID", extid);
      obj->mapDb->setValue("SOURCE", getSource());
      free(extid);

      for (int f = selectId->find(); f; f = selectId->fetch())
      {
         int insert;
         const char* channelId = obj->mapDb->getStrValue("CHANNELID");
         
         // create event ..
         
         obj->eventsDb->clear();
         obj->eventsDb->setBigintValue("EVENTID", eventid);
         obj->eventsDb->setValue("CHANNELID", channelId);

         insert = !obj->eventsDb->find();

         obj->eventsDb->setValue("SOURCE", getSource());
         obj->eventsDb->setValue("FILEREF", fileRef);

         // auto parse and set other fields

         obj->parseEvent(obj->eventsDb->getRow(), node);

         // ...

         time_t mergesp = obj->mapDb->getIntValue("MERGESP");
         long starttime = obj->eventsDb->getIntValue("STARTTIME");
         int merge = obj->mapDb->getIntValue("MERGE");

         // store ..
         
         if (insert)
         {
            // handle insert

            obj->eventsDb->setValue("VERSION", 0xFF);
            obj->eventsDb->setValue("TABLEID", 0L);
            obj->eventsDb->setValue("USEID", 0L);

            if (starttime <= mergesp)
               obj->eventsDb->setCharValue("UPDFLG", cEventState::usInactive);
            else
               obj->eventsDb->setCharValue("UPDFLG", merge > 1 ? cEventState::usMergeSpare : cEventState::usActive);

            obj->eventsDb->insert();
         }
         else
         {
            if (obj->eventsDb->hasValue("DELFLG", "Y"))
               obj->eventsDb->setValue("DELFLG", "N");

            if (obj->eventsDb->hasValue("UPDFLG", "D"))
            {
               if (starttime <= mergesp)
                  obj->eventsDb->setCharValue("UPDFLG", cEventState::usInactive);
               else
                  obj->eventsDb->setCharValue("UPDFLG", merge > 1 ? cEventState::usMergeSpare : cEventState::usActive);
            }

            obj->eventsDb->update();
         }

         obj->eventsDb->reset();
         count++;
      }
   }

   selectId->freeResult();

   xmlFreeDoc(transformedDoc);

   tell(2, "XML File '%s' processed, updated %d events", fileRef, count);

   return success;
}

//***************************************************************************
// Get Picture
//***************************************************************************

int Epgdata::getPicture(const char* imagename, const char* fileRef, MemoryStruct* data)
{
   int fileSize = 0;
   char* path = 0;
   char entryName[200+TB];

   data->clear();

   // lookup file information

   obj->fileDb->clear();
   obj->fileDb->setValue("FILEREF", fileRef);
   obj->fileDb->setValue("SOURCE", getSource());

   if (stmtByFileRef->find())
      asprintf(&path, "%s/epgdata/%s", EpgdConfig.cachePath, 
               obj->fileDb->getStrValue("Name"));

   stmtByFileRef->freeResult();

   if (!path)
   {
      tell(0, "Error: No entry with fileref '%s' to lookup image '%s' found",
           fileRef, imagename);
      return 0;
   }

   if (unzip(path, imagename, data->memory, fileSize, entryName) == success)
   {
      data->size = fileSize;
      tell(2, "Unzip of image '%s' succeeded", imagename);
   }

   free(path);

   return fileSize;
}

int Epgdata::cleanupAfter()
{
   const char* ext = ".zip";
   struct dirent* dirent;   
   DIR* dir;
   char* pdir;
   int count = 0;
   char* last = 0;

   // cleanup *.zip in FS cache ... 

   // remove old versions for each day
 
   obj->fileDb->clear();
   obj->fileDb->setValue("SOURCE", getSource());

   for (int f = stmtCleanDouble->find(); f; f = stmtCleanDouble->fetch())
   {
      const char* name = obj->fileDb->getStrValue("NAME");

      if (last && strncmp(name, last, 8) == 0)
      {
         char* where;
         tell(1, "Remove old epgdata file '%s' from table", name);
         asprintf(&where, "name = '%s'", name);
         obj->fileDb->deleteWhere("%s", where);
         free(where);
      }
       
      free(last);
      last = strdup(name);
   }
   
   free(last);
   stmtCleanDouble->freeResult();

   // mark wasted events (delflg, ...)

   stmtMarkOldEvents->execute();   

   // cleanup filesystem, remove files which not referenced in table

   asprintf(&pdir, "%s/%s", EpgdConfig.cachePath, getSource());

   if (!(dir = opendir(pdir)))
   {
      tell(1, "Can't open directory '%s', '%s'", pdir, strerror(errno));
      free(pdir);

      return done;
   }

   tell(1, "Starting cleanup of epgdata zip's in '%s'", pdir);

   free(pdir);
 
   while ((dirent = readdir(dir)))
   {
      // check extension

      if (strncmp(dirent->d_name + strlen(dirent->d_name) - strlen(ext), ext, strlen(ext)) != 0)
         continue;

      // lookup file
   
      obj->fileDb->clear();
      obj->fileDb->setValue("NAME", dirent->d_name);
      obj->fileDb->setValue("SOURCE", getSource());

      if (!obj->fileDb->find())
      {
         asprintf(&pdir, "%s/%s/%s", EpgdConfig.cachePath, getSource(), dirent->d_name);

         if (!removeFile(pdir))
            count++;
         
         free(pdir);
      }

      obj->fileDb->reset();
   }

   closedir(dir);

   tell(1, "Cleanup finished, removed (%d) epgdata files", count);

   return success;
}

//***************************************************************************

extern "C" void* EPGPluginCreator() { return new Epgdata(); }