Ticket #9836: external-subs-utf.diff

mythtv/libs/libmythtv/textsubtitleparser.cpp

@@ -17 +17 @@
 #include <QTextCodec>
 
 // MythTV
+#include "mythlogging.h"
 #include "mythcorecontext.h"
 #include "ringbuffer.h"
 #include "textsubtitleparser.h"
@@ -118 +119 @@
 
 bool TextSubtitleParser::LoadSubtitles(QString fileName, TextSubtitles &target)
 {
+    // Open text file
+    QScopedPointer<RingBuffer> rb(RingBuffer::Create(fileName, 0, false));
+
+    if (!rb.data())
+        return false;
+
+    // ensure we are the start of the file
+    if (rb.data()->Seek(0, SEEK_SET) == -1)
+    {
+        LOG(VB_GENERAL, LOG_ERR, "Failed to seek to beginning of sub file");
+        return false;
+    }
+
+    // read a test chunk
+    QScopedPointer<char> readbuf(new char[2048]);
+    int testcount = rb.data()->Read(readbuf.data(), 2048);
+    if (testcount < 1)
+    {
+        LOG(VB_GENERAL, LOG_ERR, "Failed to read from sub file.");
+        return false;
+    }
+
+    // try and determine the text codec
+    QByteArray test(readbuf.data(), testcount);
+    QTextCodec *textCodec = QTextCodec::codecForUtfText(test, NULL);
+    if (!textCodec)
+    {
+        LOG(VB_GENERAL, LOG_WARNING, "Failed to autodetect a UTF encoding.");
+        QString codec = gCoreContext->GetSetting("SubtitleCodec", "");
+        if (!codec.isEmpty())
+            textCodec = QTextCodec::codecForName(codec.toLatin1());
+        if (!textCodec)
+            textCodec = QTextCodec::codecForName("utf-8");
+        if (!textCodec)
+        {
+            LOG(VB_GENERAL, LOG_ERR,
+                QString("Failed to find codec for subtitle file '%1'")
+                .arg(fileName));
+            return false;
+        }
+    }
+
+    LOG(VB_GENERAL, LOG_INFO, QString("Opened subtitle file '%1' with codec '%2'")
+        .arg(fileName).arg(textCodec->name().constData()));
+
+    // load the entire subtitle file, converting to unicode as we go
+    QScopedPointer<QTextDecoder> dec(textCodec->makeDecoder());
+    QString data = QString();
+    rb.data()->Seek(0, SEEK_SET);
+    int expecting = rb.data()->GetRealFileSize();
+    int size = 0;
+    while (1)
+    {
+        int res = rb.data()->Read(readbuf.data(), 2048);
+        if (res < 0)
+        {
+            LOG(VB_GENERAL, LOG_ERR, "Failed to read from subtitle file.");
+            return false;
+        }
+
+        size += res;
+        if (res == 0 && (size >= expecting))
+            break;
+        else
+            data += dec->toUnicode((const char*)readbuf.data(), res);
+    }
+
+    if (data.isEmpty())
+    {
+        LOG(VB_GENERAL, LOG_WARNING,
+            QString("Data loaded from subtitle file '%1' is empty.")
+            .arg(fileName));
+        return false;
+    }
+
+    LOG(VB_GENERAL, LOG_INFO,
+        QString("Loaded %1 bytes (expected %2) from file '%3'")
+        .arg(size).arg(expecting).arg(fileName));
+
+    // convert back to utf-8 for parsing
+    QByteArray ba = data.toUtf8();
     demux_sputext_t sub_data;
-    sub_data.rbuffer = RingBuffer::Create(fileName, 0, false);
-
-    if (!sub_data.rbuffer)
-        return false;
+    sub_data.buffer = (char*)ba.data();
+    sub_data.buffersize = data.size();
 
     subtitle_t *loaded_subs = sub_read_file(&sub_data);
     if (!loaded_subs)
     {
-        delete sub_data.rbuffer;
+        LOG(VB_GENERAL, LOG_ERR, QString("Failed to read subtitles from '%1'")
+            .arg(fileName));
         return false;
     }
 
+    LOG(VB_GENERAL, LOG_INFO, QString("Found %1 subtitles in file '%2'")
+        .arg(sub_data.num).arg(fileName));
     target.SetFrameBasedTiming(!sub_data.uses_time);
 
-    QTextCodec *textCodec = NULL;
-    QString codec = gCoreContext->GetSetting("SubtitleCodec", "");
-    if (!codec.isEmpty())
-        textCodec = QTextCodec::codecForName(codec.toLatin1());
-    if (!textCodec)
+    // convert the subtitles to our own format, free the original structures
+    // and convert back to unicode
         textCodec = QTextCodec::codecForName("utf-8");
-    if (!textCodec)
-    {
-        delete sub_data.rbuffer;
-        return false;
-    }
+    if (textCodec)
+        dec.reset(textCodec->makeDecoder());
 
-    QTextDecoder *dec = textCodec->makeDecoder();
-
-    // convert the subtitles to our own format and free the original structures
     for (int sub_i = 0; sub_i < sub_data.num; ++sub_i)
     {
         const subtitle_t *sub = &loaded_subs[sub_i];
@@ -162 +236 @@
         for (int line = 0; line < sub->lines; ++line)
         {
             const char *subLine = sub->text[line];
-            QString str = dec->toUnicode(subLine, strlen(subLine));
+            QString str;
+            if (textCodec)
+                str = dec->toUnicode(subLine, strlen(subLine));
+            else
+                str = QString(subLine);
             newsub.textLines.push_back(str);
-
             free(sub->text[line]);
         }
         target.AddSubtitle(newsub);
     }
 
-    delete dec;
-    // textCodec object is managed by Qt, do not delete...
-
     free(loaded_subs);
-    delete sub_data.rbuffer;
-
     return true;
 }

mythtv/libs/libmythtv/xine_demux_sputext.cpp

@@ -42 +42 @@
 #include <fcntl.h>
 #include <ctype.h>
 #include "xine_demux_sputext.h"
+#include "mythlogging.h"
 
 #define LOG_MODULE "demux_sputext"
 #define LOG_VERBOSE
@@ -95 +96 @@
  * Reimplementation of fgets() using the input->read() method.
  */
 static char *read_line_from_input(demux_sputext_t *demuxstr, char *line, off_t len) {
-  off_t nread = 0;
   char *s;
+  char *readpos;
   int linelen;
 
-  // Since our RemoteFile code sleeps 200ms whenever we get back less data
-  // than requested, but this code just keeps trying to read until it gets
-  // an error back, we check for empty reads so that we can stop reading
-  // when there is no more data to read
-  if (demuxstr->emptyReads == 0 && (len - demuxstr->buflen) > 512) {
-    nread = demuxstr->rbuffer->Read(
-        &demuxstr->buf[demuxstr->buflen], len - demuxstr->buflen);
-    if (nread < 0) {
-      printf("read failed.\n");
+  if (demuxstr->bufferposition >= demuxstr->buffersize)
       return NULL;
-    }
-  }
 
-  if (!nread)
-    demuxstr->emptyReads++;
+  readpos = demuxstr->buffer + demuxstr->bufferposition;
+  s = strchr(readpos, '\n');
 
-  demuxstr->buflen += nread;
-  demuxstr->buf[demuxstr->buflen] = '\0';
-
-  s = strchr(demuxstr->buf, '\n');
-
-  if (line && (s || demuxstr->buflen)) {
-
-    linelen = s ? (s - demuxstr->buf) + 1 : demuxstr->buflen;
-
-    memcpy(line, demuxstr->buf, linelen);
+  if (line && s) {
+    linelen = s - readpos + 1;
+    memcpy(line, readpos, linelen);
     line[linelen] = '\0';
-
-    memmove(demuxstr->buf, &demuxstr->buf[linelen], SUB_BUFSIZE - linelen);
-    demuxstr->buflen -= linelen;
-
+    demuxstr->bufferposition += linelen;
     return line;
   }
 
@@ -1107 +1088 @@
     sub_read_line_mpl2,
   };
 
-  /* Rewind (sub_autodetect() needs to read input from the beginning) */
-  if(demuxstr->rbuffer->Seek(0, SEEK_SET) == -1) {
-    printf("seek failed.\n");
-    return NULL;
-  }
-  demuxstr->buflen = 0;
-  demuxstr->emptyReads = 0;
-
+  demuxstr->bufferposition = 0;
   demuxstr->format=sub_autodetect (demuxstr);
   if (demuxstr->format==FORMAT_UNKNOWN) {
     return NULL;
@@ -1122 +1096 @@
 
   /*printf("Detected subtitle file format: %d\n", demuxstr->format);*/
 
-  /* Rewind */
-  if(demuxstr->rbuffer->Seek(0, SEEK_SET) == -1) {
-    printf("seek failed.\n");
-    return NULL;
-  }
-  demuxstr->buflen = 0;
-  demuxstr->emptyReads = 0;
+  demuxstr->bufferposition = 0;
 
   demuxstr->num=0;n_max=32;
   first = (subtitle_t *) malloc(n_max*sizeof(subtitle_t));
@@ -1148 +1116 @@
 
     sub = func[demuxstr->format] (demuxstr, &first[demuxstr->num]);
 
-    if (!sub) {
+    if (!sub)
       break;   /* EOF */
-    } else {
-      demuxstr->emptyReads = 0;
-    }
 
     if (sub==ERR)
       ++demuxstr->errs;

mythtv/libs/libmythtv/xine_demux_sputext.h

@@ -21 +21 @@
 
 typedef struct {
 
-  RingBuffer*        rbuffer;
-
-  int                status;
-
-  char               buf[SUB_BUFSIZE];
-  off_t              buflen;
-  off_t              emptyReads;
+  char*              buffer;
+  int                buffersize;
+  int                bufferposition;
 
   float              mpsub_position;