support UTF-8

git-svn-id: https://vdr-muggle.svn.sourceforge.net/svnroot/vdr-muggle/trunk/muggle-plugin@1015 e10066b5-e1e2-0310-b819-94efdf66514b

8 files changed, 96 insertions, 4 deletions

diff --git a/README.i18n b/README.i18n
new file mode 100644
index 0000000..0c85ccb
--- /dev/null
+++ b/README.i18n
@@ -0,0 +1,41 @@
+UTF-8
+=====
+
+Muggle can handle the data using either the local 8bit character set
+UTF-8. It does this depending of the value of the environment variable
+LANG. If it ends in ".UTF-8", muggle goes into UTF-8 mode.
+
+
+File names
+==========
+
+In UTF-8 mode, muggle expects all file names to be encoded
+as UTF-8. You can easily convert all file names in /mnt/music fromk
+iso-8859-1 to UTF-8:
+
+convmv -f iso-8859-1 -t utf-8 -r /mnt/music
+
+
+Tags from mp3, flac etc.
+========================
+
+We are using the TagLib library which handles this transparently, we only
+need to tell it to return UTF-8. So if the data you import is not correct,
+either the tags in the audio file are wrong or TagLib has an error. See
+http://ktown.kde.org/~wheeler/taglib.html
+
+Please be aware that many linux programs for id3 tags do not yet handle
+UTF-8 correctly like mp3info 0.8.4 or id3v2 0.1.1: They both return
+the artist name as ISO8859-1 even if the locale is UTF-8. TagLib seems
+to handle this correctly.
+
+
+The data base
+=============
+
+We do not plan to provide on-the-fly-conversions while storing or retrieving
+data. The data will be stored in the same encoding as needed by muggle.
+This means the database should be deleted and rebuilt. The option mugglei -c
+now actually does what it should: it also deletes and recreates existing
+data bases.
+
diff --git a/README.mysql b/README.mysql
index 80df8e0..d28ff1a 100644
--- a/README.mysql
+++ b/README.mysql
@@ -85,4 +85,11 @@ Start mugglei without arguments to see a list and explanation of
 all available options. They can vary slightly depending on the
 chosen data base software.
 
+\section prereq PREREQUISITES
+
+If you get error messages containing 'Illegal mix of collations', your
+character encoding is set to UTF-8 (LANG environment variable) but
+some music file name is encoded differently. Please convert your file
+names to UTF8. For details see README.i18n.
+
 */
diff --git a/README.postgresql b/README.postgresql
index a3ea368..08f95b0 100644
--- a/README.postgresql
+++ b/README.postgresql
@@ -29,7 +29,11 @@ Please execute these steps before starting muggle the first time:
 3. createuser XXX
 4. createdb GiantDisc
 
-where XXX is the user name vdr is running under, normally vdr
+where XXX is the user name vdr is running under, normally vdr.
+You may want to define the database encoding as UTF8, see 
+http://www.postgresql.org/docs/8.3/interactive/multibyte.html.
+This is not strictly necessary - postgresql will otherwise 
+convert data between server and client automatically.
 
 Everything else will be done by muggle.
 
diff --git a/mg_db.c b/mg_db.c
index 6a53454..f511d9e 100644
--- a/mg_db.c
+++ b/mg_db.c
@@ -100,7 +100,7 @@ mgSQLString::mgSQLString(string s)
 
 mgSQLString::mgSQLString(TagLib::String s)
 {
-	Init(s.toCString());
+	Init(s.toCString(the_setup.utf8));
 }
 
 const char* 
@@ -1179,10 +1179,10 @@ mgDb::DefineGenre(const string genre)
 mgSQLString 
 mgDb::getGenre1(TagLib::FileRef& f)
 {
-	string genre1 = f.tag()->genre().toCString();
+	string genre1 = f.tag()->genre().toCString(the_setup.utf8);
 	if (genre1.empty())
 	{
-		genre1 = m_TCON.toCString();
+		genre1 = m_TCON.toCString(the_setup.utf8);
 		const char *tcon=genre1.c_str();
 		char *rparen=strchr(tcon,')');
 		if (tcon[0]=='(' && rparen)
diff --git a/mg_db_gd_mysql.c b/mg_db_gd_mysql.c
index 5c4dc46..78bee52 100644
--- a/mg_db_gd_mysql.c
+++ b/mg_db_gd_mysql.c
@@ -429,10 +429,24 @@ mgDbGd::Create()
   mgQuery q(m_db,buffer);
   if (!q.ErrorMessage().empty())
 	return false;
+
+  if (the_setup.utf8)
+  {
+        const char *cmd;
+	if (the_setup.utf8) 
+		cmd="SET NAMES utf8";
+	else
+		cmd="SET NAMES latin1";
+  	mgQuery q0(m_db,cmd);
+  	if (!q0.ErrorMessage().empty())
+		return false;
+  }
+
   sprintf(buffer,"CREATE DATABASE %s",the_setup.DbName);
   mgQuery q1(m_db,buffer);
   if (!q1.ErrorMessage().empty())
 	return false;
+
   if (!UsingEmbeddedMySQL())
 	sprintf(buffer,"grant all privileges on %s.* to vdr@localhost",
 			the_setup.DbName);
diff --git a/mg_db_gd_pg.c b/mg_db_gd_pg.c
index 05953d3..f41339b 100644
--- a/mg_db_gd_pg.c
+++ b/mg_db_gd_pg.c
@@ -254,6 +254,19 @@ mgDbGd::Commit()
 bool
 mgDbGd::Create()
 {
+
+  if (the_setup.utf8)
+  {
+        const char *cmd;
+	if (the_setup.utf8) 
+		cmd="SET NAMES UTF8";
+	else
+		cmd="SET NAMES LATIN1";
+  	mgQuery q0(m_db,cmd);
+  	if (!q0.ErrorMessage().empty())
+		return false;
+  }
+
   // create database and tables
   int len = sizeof( db_cmds ) / sizeof( char* );
   for( int i=0; i < len; i ++ )
diff --git a/mg_setup.c b/mg_setup.c
index 5b8d4d7..c71a3ca 100644
--- a/mg_setup.c
+++ b/mg_setup.c
@@ -47,6 +47,18 @@ mgSetup::mgSetup ()
     asprintf(&DbDatadir,"%s/.muggle",getenv("HOME"));
     ToplevelDir = strdup(MUSICDIR "/");
     CreateMode = false;
+    utf8 = false;
+    const char *lang = getenv("LANG");
+    if (lang)
+    {
+        const char *dot = strchr(lang, '.');
+        if (dot)
+		utf8 = strcmp(dot+1,"UTF-8")==0;
+    }
+utf8=false;
+    if (utf8)
+	mgWarning("muggle running in UTF-8 mode");
+
     DeleteStaleReferences = false;
 
     // stuff related to cover image display
diff --git a/mg_setup.h b/mg_setup.h
index 6366f5e..1053128 100644
--- a/mg_setup.h
+++ b/mg_setup.h
@@ -62,6 +62,7 @@ class mgSetup
 	bool CreateMode;
 	bool IsMugglei() const;
 	void SetMugglei();
+	bool utf8;
     private:
 	bool m_mugglei;