Version 0.1.2

9 files changed, 716 insertions, 58 deletions

diff --git a/lib/Makefile b/lib/Makefile
index f27782b..0716ea6 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -1,7 +1,14 @@
 
+BASELIBS = -lrt -lz -lmysqlclient -lcurl  -luuid
+BASELIBS += $(shell mysql_config --libs)
+
+CFLAGS   += $(shell mysql_config --include)
 
 all: 
-	g++ -ggdb -DPLGDIR='"."' test.c common.c config.c db.c tabledef.c -lrt -lz -lmysqlclient -o t
+	g++ -ggdb -DPLGDIR='"."' $(CFLAGS) test.c dbdict.c common.c config.c db.c tabledef.c $(BASELIBS) -o t
+
+demo: demo.c
+	g++ -ggdb -DUSEUUID -DPLGDIR='"."' $(CFLAGS) demo.c common.c db.c tabledef.c config.c $(BASELIBS) -o demo
 
 clean:
-	rm -f *.o *.a *~ core
+	rm -f *.o *.a *~ core demo
diff --git a/lib/common.c b/lib/common.c
index f3a0436..b5130a7 100644
--- a/lib/common.c
+++ b/lib/common.c
@@ -6,6 +6,7 @@
  */
 
 #include <sys/stat.h>
+#include <sys/time.h>
 
 #ifdef USEUUID
 # include <uuid/uuid.h>
@@ -17,6 +18,7 @@
 #include <syslog.h>
 #include <unistd.h>
 #include <zlib.h>
+#include <errno.h>
 
 #ifdef USELIBARCHIVE
 # include <archive.h>
@@ -54,7 +56,7 @@ void tell(int eloquence, const char* format, ...)
    va_start(ap, format);
 
 #ifdef VDR_PLUGIN
-   snprintf(t, sizeBuffer, "scraper2vdr: ");
+   snprintf(t, sizeBuffer, LOG_PREFIX);
 #endif
 
    vsnprintf(t+strlen(t), sizeBuffer-strlen(t), format, ap);
@@ -62,10 +64,18 @@ void tell(int eloquence, const char* format, ...)
    if (EPG2VDRConfig.logstdout)
    {
       char buf[50+TB];
-      time_t now;
-      time(&now);
-      strftime(buf, 50, "%y.%m.%d %H:%M:%S", localtime(&now));
+      timeval tp;
+
+      gettimeofday(&tp, 0);
+      
+      tm* tm = localtime(&tp.tv_sec);
+      
+      sprintf(buf,"%2.2d:%2.2d:%2.2d,%3.3ld ",
+              tm->tm_hour, tm->tm_min, tm->tm_sec, 
+              tp.tv_usec / 1000);
+
       printf("%s %s\n", buf, t);
+      
    }
    else
       syslog(LOG_ERR, "%s", t);
@@ -74,6 +84,36 @@ void tell(int eloquence, const char* format, ...)
 }
 
 //***************************************************************************
+// Save Realloc
+//***************************************************************************
+
+char* srealloc(void* ptr, size_t size)
+{
+   void* n = realloc(ptr, size);
+
+   if (!n)
+   {
+      free(ptr);
+      ptr = 0;
+   }
+
+   return (char*)n;
+}
+
+//***************************************************************************
+// us now
+//***************************************************************************
+
+double usNow()
+{
+   struct timeval tp;
+
+   gettimeofday(&tp, 0); 
+
+   return tp.tv_sec * 1000000.0 + tp.tv_usec;
+}
+
+//***************************************************************************
 // Host ID
 //***************************************************************************
 
@@ -121,6 +161,38 @@ void toUpper(std::string& str)
    free(dest);
 }
 
+//***************************************************************************
+// To Case (UTF-8 save)
+//***************************************************************************
+
+const char* toCase(Case cs, char* str)
+{
+   char* s = str;
+   int lenSrc = strlen(str);
+
+   int csSrc;  // size of character
+
+   for (int ps = 0; ps < lenSrc; ps += csSrc)
+   {
+      csSrc = max(mblen(&s[ps], lenSrc-ps), 1);
+      
+      if (csSrc == 1)
+         s[ps] = cs == cUpper ? toupper(s[ps]) : tolower(s[ps]);
+      else if (csSrc == 2 && s[ps] == (char)0xc3 && s[ps+1] >= (char)0xa0)
+      {
+         s[ps] = s[ps];
+         s[ps+1] = cs == cUpper ? toupper(s[ps+1]) : tolower(s[ps+1]);
+      }
+      else
+      {
+         for (int i = 0; i < csSrc; i++)
+            s[ps+i] = s[ps+i];
+      }
+   }
+
+   return str;
+}
+
 void removeChars(std::string& str, const char* ignore)
 {
    const char* s = str.c_str();
@@ -330,6 +402,86 @@ const char* c2s(char c, char* buf)
 }
 
 //***************************************************************************
+// Store To File
+//***************************************************************************
+
+int storeToFile(const char* filename, const char* data, int size)
+{
+   FILE* fout;
+
+   if ((fout = fopen(filename, "w+")))
+   {
+      fwrite(data, sizeof(char), size, fout);
+      fclose(fout);
+   }
+   else
+   {
+      tell(0, "Error, can't store '%s' to filesystem '%s'", filename, strerror(errno));
+      return fail;
+   }
+
+   return success;
+}
+
+//***************************************************************************
+// Load From File
+//***************************************************************************
+
+int loadFromFile(const char* infile, MemoryStruct* data)
+{
+   FILE* fin;
+   struct stat sb;
+
+   data->clear();
+
+   if (!fileExists(infile))
+   {
+      tell(0, "File '%s' not found'", infile);
+      return fail;
+   }
+
+   if (stat(infile, &sb) < 0)
+   {
+      tell(0, "Can't get info of '%s', error was '%s'", infile, strerror(errno));
+      return fail;
+   }
+
+   if ((fin = fopen(infile, "r")))
+   {
+      const char* sfx = suffixOf(infile);
+
+      data->size = sb.st_size;
+      data->modTime = sb.st_mtime;
+      data->memory = (char*)malloc(data->size);
+      fread(data->memory, sizeof(char), data->size, fin);
+      fclose(fin);
+      sprintf(data->tag, "%ld", data->size);
+
+      if (strcmp(sfx, "gz") == 0)
+         sprintf(data->contentEncoding, "gzip");
+      
+      if (strcmp(sfx, "js") == 0)
+         sprintf(data->contentType, "application/javascript");
+
+      else if (strcmp(sfx, "png") == 0 || strcmp(sfx, "jpg") == 0 || strcmp(sfx, "gif") == 0)
+         sprintf(data->contentType, "image/%s", sfx);
+
+      else if (strcmp(sfx, "ico") == 0)
+         strcpy(data->contentType, "image/x-icon");
+
+      else
+         sprintf(data->contentType, "text/%s", sfx);
+   }
+   else
+   {
+      tell(0, "Error, can't open '%s' for reading, error was '%s'", infile, strerror(errno));
+      return fail;
+   }
+
+   return success;
+}
+
+//***************************************************************************
 // TOOLS
 //***************************************************************************
 
@@ -338,6 +490,33 @@ int isEmpty(const char* str)
    return !str || !*str;
 }
 
+const char* notNull(const char* str)
+{
+   if (!str)
+      return "<null>";
+
+   return str;
+}
+
+int isZero(const char* str)
+{
+   const char* p = str;
+
+   while (p && *p)
+   {
+      if (*p != '0')
+         return no;
+
+      p++;
+   }
+
+   return yes;
+}
+
+//***************************************************************************
+// 
+//***************************************************************************
+
 char* sstrcpy(char* dest, const char* src, int max)
 {
    if (!dest || !src)
@@ -356,11 +535,21 @@ int isLink(const char* path)
    if (lstat(path, &sb) == 0)
       return S_ISLNK(sb.st_mode);
 
-   tell(0, "Error: Detecting state for '%s' failed, error was '%m'", path);
+   tell(0, "Error: Detecting state for '%s' failed, error was '%s'", path, strerror(errno));
 
    return false;
 }
 
+const char* suffixOf(const char* path)
+{
+   const char* p;
+
+   if (path && (p = strrchr(path, '.')))
+      return p+1;
+
+   return "";
+}
+
 int fileSize(const char* path)
 {
    struct stat sb;
@@ -368,9 +557,21 @@ int fileSize(const char* path)
    if (lstat(path, &sb) == 0)
       return sb.st_size;
 
-   tell(0, "Error: Detecting state for '%s' failed, error was '%m'", path);
+   tell(0, "Error: Detecting state for '%s' failed, error was '%s'", path, strerror(errno));
 
-   return false;
+   return 0;
+}
+
+time_t fileModTime(const char* path)
+{
+   struct stat sb;
+
+   if (lstat(path, &sb) == 0)
+      return sb.st_mtime;
+
+   tell(0, "Error: Detecting state for '%s' failed, error was '%s'", path, strerror(errno));
+
+   return 0;
 }
 
 
@@ -390,7 +591,7 @@ int createLink(const char* link, const char* dest, int force)
       
       if (symlink(dest, link) != 0)
       {
-         tell(0, "Failed to create symlink '%s', error was '%m'", link);
+         tell(0, "Failed to create symlink '%s', error was '%s'", link, strerror(errno));
          return fail;
       }
    }
@@ -408,7 +609,7 @@ int removeFile(const char* filename)
 
    if (unlink(filename) != 0)
    {
-      tell(0, "Can't remove file '%s', '%m'", filename);
+      tell(0, "Can't remove file '%s', '%s'", filename, strerror(errno));
       
       return 1;
    }
@@ -432,7 +633,7 @@ int chkDir(const char* path)
       
       if (mkdir(path, ACCESSPERMS) == -1)
       {
-         tell(0, "Can't create directory '%m'");
+         tell(0, "Can't create directory '%s'", strerror(errno));
          return fail;
       }
    }
@@ -541,6 +742,63 @@ int gunzip(MemoryStruct* zippedData, MemoryStruct* unzippedData)
    return success;
 }
 
+//***************************************************************************
+// gzip
+//***************************************************************************
+
+int _gzip(Bytef* dest, uLongf* destLen, const Bytef* source, uLong sourceLen)
+{
+    z_stream stream;
+    int res;
+
+    stream.next_in = (Bytef *)source;
+    stream.avail_in = (uInt)sourceLen;
+    stream.next_out = dest;
+    stream.avail_out = (uInt)*destLen;
+    if ((uLong)stream.avail_out != *destLen) return Z_BUF_ERROR;
+
+    stream.zalloc = (alloc_func)0;
+    stream.zfree = (free_func)0;
+    stream.opaque = (voidpf)0;
+
+    if ((res = deflateInit2(&stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15 + 16, 8, Z_DEFAULT_STRATEGY)) != Z_OK)
+       return res;
+
+    res = deflate(&stream, Z_FINISH);
+
+    if (res != Z_STREAM_END) 
+    {
+        deflateEnd(&stream);
+        return res == Z_OK ? Z_BUF_ERROR : res;
+    }
+
+    *destLen = stream.total_out;
+    res = deflateEnd(&stream);
+
+    return res;
+}
+
+int gzip(MemoryStruct* data, MemoryStruct* zippedData)
+{
+   int res;
+   uLong sizeMax = compressBound(data->size) + 512;
+
+   zippedData->clear();
+   zippedData->memory = (char*)malloc(sizeMax);
+
+   if ((res = _gzip((Bytef*)zippedData->memory, &sizeMax, (Bytef*)data->memory, data->size)) != Z_OK)
+   {
+      tellZipError(res, " during compression", "");
+      return fail;
+   }
+
+   zippedData->copyAttributes(data);
+   zippedData->size = sizeMax;
+   sprintf(zippedData->contentEncoding, "gzip");
+
+   return success;
+}
+
 //*************************************************************************
 // tellZipError
 //*************************************************************************
@@ -554,11 +812,11 @@ void tellZipError(int errorCode, const char* op, const char* msg)
    {
       case Z_OK:           return;
       case Z_STREAM_END:   return;
-      case Z_MEM_ERROR:    tell(0, "Error: Not enough memory to unzip file%s!\n", op); return;
-      case Z_BUF_ERROR:    tell(0, "Error: Couldn't unzip data due to output buffer size problem%s!\n", op); return;
+      case Z_MEM_ERROR:    tell(0, "Error: Not enough memory to zip/unzip file%s!\n", op); return;
+      case Z_BUF_ERROR:    tell(0, "Error: Couldn't zip/unzip data due to output buffer size problem%s!\n", op); return;
       case Z_DATA_ERROR:   tell(0, "Error: Zipped input data corrupted%s! Details: %s\n", op, msg); return;
       case Z_STREAM_ERROR: tell(0, "Error: Invalid stream structure%s. Details: %s\n", op, msg); return;
-      default:             tell(0, "Error: Couldn't unzip data for unknown reason (%6d)%s!\n", errorCode, op); return;
+      default:             tell(0, "Error: Couldn't zip/unzip data for unknown reason (%6d)%s!\n", errorCode, op); return;
    }
 }
 
@@ -659,7 +917,7 @@ int unzip(const char* file, const char* filter, char*& buffer, int& size, char*
 
    if (r != ARCHIVE_OK)
    {
-      tell(0, "Error: Open '%s' failed - %m", file);
+      tell(0, "Error: Open '%s' failed - %s", file, strerror(errno));
       return 1;
    }
 
@@ -720,13 +978,13 @@ LogDuration::LogDuration(const char* aMessage, int aLogLevel)
 
 LogDuration::~LogDuration()
 {
-   tell(logLevel, "duration '%s' was (%dms)",
+   tell(logLevel, "duration '%s' was (%ldms)",
      message, cTimeMs::Now() - durationStart);
 }
 
 void LogDuration::show(const char* label)
 {
-   tell(logLevel, "elapsed '%s' at '%s' was (%dms)",
+   tell(logLevel, "elapsed '%s' at '%s' was (%ldms)",
         message, label, cTimeMs::Now() - durationStart);
 }
 
@@ -785,7 +1043,7 @@ int createMd5OfFile(const char* path, const char* name, md5* md5)
    
    if (!(f = fopen(file, "r")))
    {
-      tell(0, "Fatal: Can't access '%s'; %m", file);
+      tell(0, "Fatal: Can't access '%s'; %s", file, strerror(errno));
       free(file);
       return fail;
    }
@@ -810,3 +1068,137 @@ int createMd5OfFile(const char* path, const char* name, md5* md5)
 }
 
 #endif // USEMD5
+
+//***************************************************************************
+// Url Unescape
+//***************************************************************************
+/*
+ * The buffer pointed to by @dst must be at least strlen(@src) bytes.
+ * Decoding stops at the first character from @src that decodes to null.
+
+ * Path normalization will remove redundant slashes and slash+dot sequences,
+ * as well as removing path components when slash+dot+dot is found. It will
+ * keep the root slash (if one was present) and will stop normalization
+ * at the first questionmark found (so query parameters won't be normalized).
+ *
+ * @param dst       destination buffer
+ * @param src       source buffer
+ * @param normalize perform path normalization if nonzero
+ * @return          number of valid characters in @dst
+ */
+
+int urlUnescape(char* dst, const char* src, int normalize)
+{
+//    CURL* curl;
+//    int resultSize;
+
+//    if (curl_global_init(CURL_GLOBAL_ALL) != 0)
+//    {
+//       tell(0, "Error, something went wrong with curl_global_init()");
+      
+//       return fail;
+//    }
+   
+//    curl = curl_easy_init();
+   
+//    if (!curl)
+//    {
+//       tell(0, "Error, unable to get handle from curl_easy_init()");
+      
+//       return fail;
+//    }
+
+//    dst = curl_easy_unescape(curl, src, strlen(src), &resultSize);
+
+//    tell(0, " [%.40s]", src);
+
+//    tell(0, "res size %d [%.40s]", resultSize, dst);
+//    return resultSize;
+
+   char* org_dst = dst;
+   int slash_dot_dot = 0;
+   char ch, a, b;
+
+   a = 0;
+
+   do {
+      ch = *src++;
+
+      if (ch == '%' && isxdigit(a = src[0]) && isxdigit(b = src[1])) 
+      {
+         if (a < 'A') 
+            a -= '0';
+         else if 
+            (a < 'a') a -= 'A' - 10;
+         else 
+            a -= 'a' - 10;
+         
+         if (b < 'A') 
+            b -= '0';
+         else if (b < 'a') 
+            b -= 'A' - 10;
+         else 
+            b -= 'a' - 10;
+         
+         ch = 16 * a + b;
+         src += 2;
+      }
+      
+      if (normalize) 
+      {
+         switch (ch) 
+         {
+            case '/':                         // compress consecutive slashes and remove slash-dot
+               if (slash_dot_dot < 3) 
+               {
+                  
+                  dst -= slash_dot_dot;
+                  slash_dot_dot = 1;
+                  break;
+               }
+               // fall-through
+
+            case '?':                         // at start of query, stop normalizing
+               if (ch == '?')
+                  normalize = 0;
+
+               // fall-through
+
+            case '\0':                        // remove trailing slash-dot-(dot)
+               if (slash_dot_dot > 1) 
+               {
+                  dst -= slash_dot_dot;
+
+                  // remove parent directory if it was two dots
+
+                  if (slash_dot_dot == 3)
+                     while (dst > org_dst && *--dst != '/')
+                        ; //  empty body
+                  slash_dot_dot = (ch == '/') ? 1 : 0;
+
+                  // keep the root slash if any
+
+                  if (!slash_dot_dot && dst == org_dst && *dst == '/')
+                     ++dst;
+
+               }
+               break;
+
+            case '.':
+               if (slash_dot_dot == 1 || slash_dot_dot == 2) 
+               {
+                  ++slash_dot_dot;
+                  break;
+               }
+               // fall-through
+
+            default:
+               slash_dot_dot = 0;
+         }
+      }
+
+      *dst++ = ch;
+   } while(ch);
+   
+   return (dst - org_dst) - 1;
+}
diff --git a/lib/common.h b/lib/common.h
index 614dfe6..1829471 100644
--- a/lib/common.h
+++ b/lib/common.h
@@ -10,6 +10,7 @@
 
 #include <stdint.h>      // uint_64_t
 #include <stdlib.h>
+#include <string.h>
 #include <string>
 
 #include <openssl/md5.h> // MD5_*
@@ -25,7 +26,7 @@
 #endif
 
 //***************************************************************************
-// 
+// Misc
 //***************************************************************************
 
 #ifndef VDR_PLUGIN
@@ -56,49 +57,106 @@ enum Misc
    tmeSecondsPerDay = 24 * tmeSecondsPerHour
 };
 
+enum Case
+{
+   cUpper,
+   cLower
+};
+
+const char* toCase(Case cs, char* str);
+
 //***************************************************************************
 // Tell
 //***************************************************************************
 
-void tell(int eloquence, const char* format, ...);
+void __attribute__ ((format(printf, 2, 3))) tell(int eloquence, const char* format, ...);
 
 //***************************************************************************
-// MemoryStruct for curl callbacks
+// 
 //***************************************************************************
 
-struct MemoryStruct
-{
-   MemoryStruct()   { memory = 0; clear(); }
-   ~MemoryStruct()  { clear(); }
-
-   // data
+char* srealloc(void* ptr, size_t size);
 
-   char* memory;
-   size_t size;
-
-   // tag attribute
-
-   char tag[100];           // the tag to be compared 
-   char name[100];          // content name (filename)
-   int headerOnly;
+//***************************************************************************
+// MemoryStruct
+//***************************************************************************
 
-   int isEmpty() { return memory == 0; }
+struct MemoryStruct
+{
+   public:
 
-   void clear() 
-   {
-      free(memory);
-      memory = 0;
-      size = 0;
-      *tag = 0;
-      *name = 0;
-      headerOnly = no;
-   }
+      MemoryStruct()   { expireAt = time(0); memory = 0; clear(); }
+      MemoryStruct(const MemoryStruct* o)
+      {
+         size = o->size;
+         memory = (char*)malloc(size);
+         memcpy(memory, o->memory, size);
+
+         copyAttributes(o);
+      }
+      
+      void copyAttributes(const MemoryStruct* o)
+      {
+         strcpy(tag, o->tag);
+         strcpy(name, o->name);
+         strcpy(contentType, o->contentType);
+         strcpy(contentEncoding, o->contentEncoding);
+         strcpy(mimeType, o->mimeType);
+         headerOnly = o->headerOnly;
+         modTime = o->modTime;
+         expireAt = o->expireAt;
+      }
+      
+      ~MemoryStruct()  { clear(); }
+      
+      int append(const char* buf, int len)
+      {
+         memory = srealloc(memory, size+len);
+         memcpy(memory+size, buf, len);
+         size += len;
+
+         return success;
+      }
+
+      // data
+      
+      char* memory;
+      size_t size;
+      
+      // tag attribute
+      
+      char tag[100];              // the tag to be compared 
+      char name[100];             // content name (filename)
+      char contentType[100];      // e.g. text/html
+      char mimeType[100];         // 
+      char contentEncoding[100];  // 
+      int headerOnly;
+      time_t modTime;
+      time_t expireAt;
+      
+      int isEmpty() { return memory == 0; }
+      
+      void clear() 
+      {
+         free(memory);
+         memory = 0;
+         size = 0;
+         *tag = 0;
+         *name = 0;
+         *contentType = 0;
+         *contentEncoding = 0;
+         *mimeType = 0;
+         modTime = time(0);
+         // !!!! expireAt = time(0);
+         headerOnly = no;
+      }
 };
 
 //***************************************************************************
 // Tools
 //***************************************************************************
 
+double usNow();
 unsigned int getHostId();
 const char* getHostName();
 const char* getFirstIp();
@@ -120,14 +178,23 @@ std::string num2Str(int num);
 std::string l2pTime(time_t t);
 std::string ms2Dur(uint64_t t);
 const char* c2s(char c, char* buf);
+int urlUnescape(char* dst, const char* src, int normalize = yes);
+
+int storeToFile(const char* filename, const char* data, int size);
+int loadFromFile(const char* infile, MemoryStruct* data);
 
 int fileExists(const char* path);
 int fileSize(const char* path);
+time_t fileModTime(const char* path);
 int createLink(const char* link, const char* dest, int force);
 int isLink(const char* path);
+const char*  suffixOf(const char* path);
 int isEmpty(const char* str);
+const char* notNull(const char* str);
+int isZero(const char* str);
 int removeFile(const char* filename);
 int chkDir(const char* path);
+int downloadFile(const char* url, int& size, MemoryStruct* data, int timeout = 30, const char* userAgent = "libcurl-agent/1.0");
 
 #ifdef USELIBXML
   xsltStylesheetPtr loadXSLT(const char* name, const char* path, int utf8);
@@ -145,6 +212,7 @@ int chkDir(const char* path);
 //***************************************************************************
 
 int gunzip(MemoryStruct* zippedData, MemoryStruct* unzippedData);
+int gzip(MemoryStruct* data, MemoryStruct* zippedData);
 void tellZipError(int errorCode, const char* op, const char* msg);
 
 #ifdef USELIBARCHIVE
diff --git a/lib/config.c b/lib/config.c
index 9ca25f1..d53a26a 100644
--- a/lib/config.c
+++ b/lib/config.c
@@ -44,10 +44,13 @@ cEPG2VDRConfig::cEPG2VDRConfig(void)
    scheduleBoot = no;
 #else
    sstrcpy(cachePath, "/var/cache/epgd", sizeof(cachePath));
+   sstrcpy(httpPath, "/var/epgd/www", sizeof(httpPath));
    sstrcpy(pluginPath, PLGDIR, sizeof(pluginPath));
    sstrcpy(epgView, "eventsview.sql", sizeof(epgView));
+   sstrcpy(theTvDBView, "thetvdbview.sql", sizeof(epgView));
    updateThreshold = 200;
    maintanance = no;
+   httpPort = 9999;
 #endif
 
    sstrcpy(dbHost, "localhost", sizeof(dbHost));
@@ -60,4 +63,7 @@ cEPG2VDRConfig::cEPG2VDRConfig(void)
    loglevel = 1;
 
    uuid[0] = 0;
+
+   scrapEpg = yes;
+   scrapRecordings = yes;
 } 
diff --git a/lib/config.h b/lib/config.h
index e0c729c..026d9f7 100644
--- a/lib/config.h
+++ b/lib/config.h
@@ -47,10 +47,13 @@ struct cEPG2VDRConfig
       int scheduleBoot;
 #else
       char cachePath[256+TB];
+      char httpPath[256+TB];
       char pluginPath[256+TB];
       char epgView[100+TB];
+      char theTvDBView[100+TB];
       int updateThreshold;
       int maintanance;
+      int httpPort;
 #endif
 
       char dbHost[100+TB];
@@ -66,6 +69,10 @@ struct cEPG2VDRConfig
       int mainmenuFullupdate;
       int masterMode;
       char uuid[sizeUuid+TB];
+
+      int scrapEpg;
+      int scrapRecordings;
+
 };
 
 extern cEPG2VDRConfig EPG2VDRConfig;
diff --git a/lib/db.c b/lib/db.c
index 97214dc..b5e2911 100644
--- a/lib/db.c
+++ b/lib/db.c
@@ -7,7 +7,7 @@
 
 #include <stdio.h>
 
-#include <mysql/errmsg.h>
+#include <errmsg.h>
 
 #include "db.h"
 
@@ -17,6 +17,8 @@
 // DB Statement
 //***************************************************************************
 
+int cDbStatement::explain = no;
+
 cDbStatement::cDbStatement(cDbTable* aTable)
 {
    table = aTable;
@@ -30,6 +32,14 @@ cDbStatement::cDbStatement(cDbTable* aTable)
    affected = 0;
    metaResult = 0;
    bindPrefix = 0;
+   firstExec = yes;
+
+   callsPeriod = 0;
+   callsTotal = 0;
+   duration = 0;
+
+   if (connection)
+      connection->statements.append(this);
 }
 
 cDbStatement::cDbStatement(cDbConnection* aConnection, const char* stmt)
@@ -45,6 +55,22 @@ cDbStatement::cDbStatement(cDbConnection* aConnection, const char* stmt)
    affected = 0;
    metaResult = 0;
    bindPrefix = 0;
+   firstExec = yes;
+
+   callsPeriod = 0;
+   callsTotal = 0;
+   duration = 0;
+
+   if (connection)
+      connection->statements.append(this);
+}
+
+cDbStatement::~cDbStatement()  
+{ 
+   if (connection)
+      connection->statements.remove(this);
+
+   clear();
 }
 
 //***************************************************************************
@@ -61,11 +87,43 @@ int cDbStatement::execute(int noResult)
    if (!stmt)
       return connection->errorSql(connection, "execute(missing statement)");
 
+//    if (explain && firstExec)
+//    {
+//       firstExec = no;
+
+//       if (strstr(stmtTxt.c_str(), "select "))
+//       {
+//          MYSQL_RES* result;
+//          MYSQL_ROW row;
+//          string q = "explain " + stmtTxt;
+
+//          if (connection->query(q.c_str()) != success)
+//             connection->errorSql(connection, "explain ", 0);
+//          else if ((result = mysql_store_result(connection->getMySql())))
+//          {
+//             while ((row = mysql_fetch_row(result)))
+//             {
+//                tell(0, "EXPLAIN: %s) %s %s %s %s %s %s %s %s %s", 
+//                     row[0], row[1], row[2], row[3],
+//                     row[4], row[5], row[6], row[7], row[8], row[9]);
+//             }
+            
+//             mysql_free_result(result);
+//          }
+//       }
+//    }
+
    // tell(0, "execute %d [%s]", stmt, stmtTxt.c_str());
 
+   long start = usNow();
+
    if (mysql_stmt_execute(stmt))
       return connection->errorSql(connection, "execute(stmt_execute)", stmt, stmtTxt.c_str());
 
+   duration += usNow() - start;
+   callsPeriod++;
+   callsTotal++;
+
    // out binding - if needed
 
    if (outCount && !noResult)
@@ -270,7 +328,7 @@ int cDbStatement::appendBinding(cDbValue* value, BindType bt)
    if (!bindings)
       *bindings = (MYSQL_BIND*)malloc(count * sizeof(MYSQL_BIND));
    else
-      *bindings = (MYSQL_BIND*)realloc(*bindings, count * sizeof(MYSQL_BIND));
+      *bindings = (MYSQL_BIND*)srealloc(*bindings, count * sizeof(MYSQL_BIND));
 
    newBinding = &((*bindings)[count-1]);
 
@@ -361,6 +419,22 @@ int cDbStatement::prepare()
 }
 
 //***************************************************************************
+// Show Statistic
+//***************************************************************************
+
+void cDbStatement::showStat()
+{
+   if (callsPeriod)
+   {
+      tell(0, "calls %4ld in %6.2fms; total %4ld [%s]", 
+           callsPeriod, duration/1000, callsTotal, stmtTxt.c_str());
+
+      callsPeriod = 0;
+      duration = 0;
+   }
+}
+
+//***************************************************************************
 // cDbService
 //***************************************************************************
 
@@ -382,6 +456,50 @@ const char* cDbService::toString(FieldFormat t)
    return formats[t];
 }
 
+const char* cDbService::dictFormats[] =
+{
+   "int",
+   "uint",
+   "ascii",
+   "text",
+   "mlob",
+   "float",
+   "datetime",
+
+   0
+};
+
+cDbService::FieldFormat cDbService::toDictFormat(const char* format)
+{
+   for (int i = 0; i < ffCount; i++)
+      if (strcasecmp(dictFormats[i], format) == 0)
+         return (FieldFormat)i;
+
+   return ffUnknown;
+}
+
+const char* cDbService::types[] =
+{
+   "data",
+   "primary",
+   "meta",
+   "calc",
+   "autoinc",
+
+   0
+};
+
+cDbService::FieldType cDbService::toType(const char* type)
+{
+   // #TODO  !!!
+
+   for (int i = 0; i < 3; i++)
+      if (strcasecmp(types[i], type) == 0)
+         return (FieldType)i;
+
+   return ftUnknown;
+}
+
 //***************************************************************************
 // Class cDbTable
 //***************************************************************************
diff --git a/lib/db.h b/lib/db.h
index 8a365fd..34a2a62 100644
--- a/lib/db.h
+++ b/lib/db.h
@@ -13,11 +13,11 @@
 #include <stdio.h>
 #include <stdarg.h>
 #include <errno.h>
+#include <mysql/mysql.h>
 
+#include <list>
 #include <sstream>
 
-#include <mysql/mysql.h>
-
 #include "common.h"
 
 class cDbTable;
@@ -40,6 +40,8 @@ class cDbService
 
       enum FieldFormat
       {
+         ffUnknown = na,
+
          ffInt,
          ffUInt,
          ffAscii,      // -> VARCHAR
@@ -52,6 +54,8 @@ class cDbService
 
       enum FieldType
       {
+         ftUnknown    = na,
+
          ftData       = 1,
          ftPrimary    = 2,
          ftMeta       = 4,
@@ -90,7 +94,12 @@ class cDbService
       };
 
       static const char* toString(FieldFormat t);
+      static FieldFormat toDictFormat(const char* format);
       static const char* formats[];
+      static const char* dictFormats[];
+
+      static FieldType toType(const char* type);      
+      static const char* types[];
 };
 
 typedef cDbService cDBS;
@@ -374,8 +383,7 @@ class cDbStatement : public cDbService
 
       cDbStatement(cDbTable* aTable);
       cDbStatement(cDbConnection* aConnection, const char* stmt = "");
-
-      virtual ~cDbStatement()  { clear(); }
+      virtual ~cDbStatement();
       
       int execute(int noResult = no);
       int find();
@@ -404,6 +412,12 @@ class cDbStatement : public cDbService
       int getResultCount();
       const char* asText() { return stmtTxt.c_str(); }
 
+      void showStat();
+
+      // data
+
+      static int explain;          // debug explain
+
    private:
 
       void clear();
@@ -415,11 +429,49 @@ class cDbStatement : public cDbService
       cDbConnection* connection;
       cDbTable* table;
       int inCount;
-      MYSQL_BIND* inBind;   // to db
+      MYSQL_BIND* inBind;         // to db
       int outCount;
-      MYSQL_BIND* outBind;  // from db (result)
+      MYSQL_BIND* outBind;        // from db (result)
       MYSQL_RES* metaResult;
       const char* bindPrefix;
+      int firstExec;               // debug explain
+
+      unsigned long callsPeriod;
+      unsigned long callsTotal;
+      double duration;
+};
+
+//***************************************************************************
+// cDbStatements
+//***************************************************************************
+
+class cDbStatements
+{
+   public:
+
+      cDbStatements()  { statisticPeriod = time(0); }
+      ~cDbStatements() {};
+      
+      void append(cDbStatement* s)  { statements.push_back(s); }
+      void remove(cDbStatement* s)  { statements.remove(s); }
+
+      void showStat()
+      {
+         tell(0, "Statement statistic of last %ld seconds:", statisticPeriod);
+
+         for (std::list<cDbStatement*>::iterator it = statements.begin() ; it != statements.end(); ++it)
+         {
+            if (*it)
+               (*it)->showStat();
+         }
+
+         statisticPeriod = time(0);
+      }
+
+   private:
+
+      time_t statisticPeriod;
+      std::list<cDbStatement*> statements;
 };
 
 //***************************************************************************
@@ -628,7 +680,7 @@ class cDbConnection
          {
             nread += res;
             size += 1000;
-            buffer = (char*)realloc(buffer, size+1);
+            buffer = srealloc(buffer, size+1);
          }
          
          fclose(f);
@@ -704,6 +756,8 @@ class cDbConnection
 
       int errorSql(cDbConnection* mysql, const char* prefix, MYSQL_STMT* stmt = 0, const char* stmtTxt = 0);
 
+      void showStat()   { statements.showStat(); }
+
       static int init()
       {
          if (mysql_library_init(0, 0, 0)) 
@@ -729,6 +783,8 @@ class cDbConnection
 
       MYSQL* mysql;
 
+      cDbStatements statements;         // all statements of this connection
+
    private:
 
       int initialized;
diff --git a/lib/tabledef.c b/lib/tabledef.c
index 0e388f1..d9826f5 100644
--- a/lib/tabledef.c
+++ b/lib/tabledef.c
@@ -19,8 +19,8 @@ const char* cEpgdState::states[] =
 
    "busy (events)",
    "busy (match)",
-   "busy (images)",
    "busy (scraping)",
+   "busy (images)",
 
    0
 };
diff --git a/lib/tabledef.h b/lib/tabledef.h
index 614e311..9936f4e 100644
--- a/lib/tabledef.h
+++ b/lib/tabledef.h
@@ -26,13 +26,16 @@ class cEpgdState
          esStandby,
          esStopped,
 
+         // handler pause on this states!
+
          esBusy,
          esBusyEvents = esBusy,
          esBusyMatch,
+         esBusyScraping,
 
-         esBusyImages,
+         // handler don't pause on this states!
 
-         esBusyScraping,
+         esBusyImages,
 
          esCount
       };
@@ -78,6 +81,7 @@ class cUpdateState
 
       static const char* getDeletable()      { return "'A','L','P','R','I'"; }
       static const char* getNeeded()         { return "'A','L','P','C','D','R'"; }
+      static const char* getVisible()        { return "'A','L','P'"; }
 
       // checks fpr c++ code