Ticket #11195: 11195-v1.patch

mythtv/libs/libmythtv/datadirect.cpp

@@ -25 +25 @@
 #include "exitcodes.h"
 #include "mythdownloadmanager.h"
 #include "mythtvexp.h"
+#include "mythtimer.h"
 
 #define LOC QString("DataDirect: ")
 
@@ -976 +977 @@
     auth->setPassword(GetPassword());
 }
 
-bool DataDirectProcessor::DDPost(QString    ddurl,        QString   &inputFile,
-                                 QDateTime  pstartDate,   QDateTime  pendDate,
-                                 QString   &err_txt)
+QByteArray DataDirectProcessor::DDPost(
+    const QString &url,
+    const QDateTime &startDate, const QDateTime &endDate,
+    QString &err_txt)
 {
-    if (!inputFile.isEmpty() && QFile(inputFile).exists())
-    {
-        return true;
-    }
+    QString startdatestr = startDate.toString(Qt::ISODate);
+    if (startdatestr.right(1) != "Z")
+        startdatestr += "Z"; // Z is only appended in some versions of Qt
+
+    QString enddatestr = endDate.toString(Qt::ISODate);
+    if (enddatestr.right(1) != "Z")
+        enddatestr += "Z";
 
-    QString startdatestr = pstartDate.toString(Qt::ISODate) + "Z";
-    QString enddatestr = pendDate.toString(Qt::ISODate) + "Z";
     QByteArray postdata;
     postdata  = "<?xml version='1.0' encoding='utf-8'?>\n";
     postdata += "<SOAP-ENV:Envelope\n";
@@ -1006 +1009 @@
     postdata += "</SOAP-ENV:Body>\n";
     postdata += "</SOAP-ENV:Envelope>\n";
 
-    if (inputFile.isEmpty()) {
-        inputFile = QString("/tmp/mythtv_ddp_data");
-    }
-
     QHash<QByteArray, QByteArray> headers;
     headers.insert("Accept-Encoding", "gzip");
     headers.insert("Content-Type", "application/soap+xml; charset=utf-8");
 
-    LOG(VB_GENERAL, LOG_INFO, "Downloading DataDirect feed");
+    LOG(VB_GENERAL, LOG_INFO,
+        "Downloading DataDirect feed, this can take several minutes");
 
     MythDownloadManager *manager = GetMythDownloadManager();
 
-    if (!manager->postAuth(ddurl, &postdata, &::authenticationCallback, this,
-                           &headers))
+    MythTimer downloadTimer;
+    downloadTimer.start();
+
+    if (!manager->postAuth(
+            url, &postdata, &::authenticationCallback, this, &headers))
     {
-        err_txt = QString("Download error");
-        return false;
+        err_txt = "Download error";
+        return QByteArray();
     }
 
-    LOG(VB_GENERAL, LOG_INFO, QString("Downloaded %1 bytes")
-        .arg(postdata.size()));
-
-    LOG(VB_GENERAL, LOG_INFO, "Uncompressing DataDirect feed");
+    float downloadTime = downloadTimer.elapsed() * 0.001;
 
     QByteArray uncompressed = gUncompress(postdata);
 
-    LOG(VB_GENERAL, LOG_INFO, QString("Uncompressed to %1 bytes")
-        .arg(uncompressed.size()));
-
-    if (uncompressed.size() == 0)
-        uncompressed = postdata;
-
-    QFile file(inputFile);
-    file.open(QIODevice::WriteOnly);
-    file.write(uncompressed);
-    file.close();
+    LOG(VB_GENERAL, LOG_INFO,
+        QString("Downloaded %1 megabytes in %2 seconds, "
+                "decompressed to %3 megabytes")
+        .arg(postdata.size()/(1024.0*1024.0)).arg(downloadTime)
+        .arg(uncompressed.size()/(1024.0*1024.0)));
 
     if (uncompressed.size() == 0)
     {
-        err_txt = QString("Error uncompressing data");
-        return false;
+        err_txt = "Error uncompressing data";
+        return QByteArray();
     }
 
-    return true;
+    return uncompressed;
 }
 
 bool DataDirectProcessor::GrabNextSuggestedTime(void)
@@ -1148 +1143 @@
     return nextSuggestedTime.isValid();
 }
 
-bool DataDirectProcessor::GrabData(const QDateTime &pstartDate,
-                                   const QDateTime &pendDate)
+bool DataDirectProcessor::GrabData(
+    const QDateTime &startDate, const QDateTime &endDate)
 {
-    QString msg = (pstartDate.addSecs(1) == pendDate) ? "channel" : "listing";
-    LOG(VB_GENERAL, LOG_INFO, LOC + "Grabbing " + msg + " data");
+    LOG(VB_GENERAL, LOG_INFO, LOC +
+        ((startDate.addSecs(1) == endDate) ?
+         "Grabbing channel data" : "Grabbing listing data"));
 
-    QString err = "";
     QString ddurl = m_providers[m_listingsProvider].webServiceURL;
     QString inputfile = m_inputFilename;
-    QString cache_dd_data = QString::null;
 
     if (m_cacheData)
     {
         QByteArray userid = GetUserID().toAscii();
-        cache_dd_data = m_tmpDir +
+        QString cache_dd_data = m_tmpDir +
             QString("/mythtv_dd_cache_%1_%2_UTC_%3_to_%4")
             .arg(GetListingsProvider())
             .arg(userid.constData())
-            .arg(MythDate::toString(pstartDate, MythDate::kFilename))
-            .arg(MythDate::toString(pendDate, MythDate::kFilename));
+            .arg(MythDate::toString(startDate, MythDate::kFilename))
+            .arg(MythDate::toString(endDate, MythDate::kFilename));
 
         if (QFile(cache_dd_data).exists() && m_inputFilename.isEmpty())
         {
-            LOG(VB_GENERAL, LOG_INFO, LOC + "Using DD cache");
+            LOG(VB_GENERAL, LOG_INFO, LOC + "Using cached DataDirect data");
         }
 
         if (m_inputFilename.isEmpty())
             inputfile = cache_dd_data;
     }
 
-    if (!DDPost(ddurl, inputfile, pstartDate, pendDate, err))
+    QByteArray data;
+    if (!inputfile.isEmpty())
     {
-        LOG(VB_GENERAL, LOG_ERR, LOC + QString("Failed to get data: %1")
-                .arg(err));
-        return false;
+        QFile file(inputfile);
+        if (file.open(QIODevice::ReadOnly))
+            data = file.readAll();
     }
 
-    QFile file(inputfile);
-    file.open(QIODevice::ReadOnly);
-    QByteArray data = file.readAll();
-    file.close();
+    if (data.isEmpty())
+    {
+        QString err;
+
+        data = DDPost(ddurl, startDate, endDate, err);
+
+        if (!err.isEmpty())
+        {
+            LOG(VB_GENERAL, LOG_ERR, LOC +
+                QString("Failed to get data: %1").arg(err));
+            return false;
+        }
+        else if (!inputfile.isEmpty())
+        {
+            QFile file(inputfile);
+            if (file.open(QIODevice::WriteOnly))
+                file.write(data);
+        }
+    }
 
     if (data.isEmpty())
     {
@@ -1196 +1206 @@
         return false;
     }
 
-    bool ok = true;
+    QXmlInputSource xmlsource;
+    xmlsource.setData(data);
 
-    DDStructureParser ddhandler(*this);
-    QXmlInputSource  xmlsource;
     QXmlSimpleReader xmlsimplereader;
-
-    xmlsource.setData(data);
+    DDStructureParser ddhandler(*this);
     xmlsimplereader.setContentHandler(&ddhandler);
+
+    LOG(VB_GENERAL, LOG_INFO, LOC +
+        "Parsing DataDirect XML, this can take a number of minutes");
+
     if (!xmlsimplereader.parse(xmlsource))
     {
         LOG(VB_GENERAL, LOG_ERR, LOC +
-            "DataDirect XML failed to properly parse, downloaded listings "
-            "were probably corrupt.");
-        ok = false;
+            "Failed to properly parse DataDirect XML");
+        return false;
+    }
+    else
+    {
+        LOG(VB_GENERAL, LOG_INFO, LOC + "Parsing DataDirect XML complete");
+        return true;
     }
-
-    return ok;
 }
 
 bool DataDirectProcessor::GrabLineupsOnly(void)

mythtv/libs/libmythtv/datadirect.h

@@ -414 +414 @@
     static bool Post(QString url, const PostList &list, QString documentFile,
                      QString inCookieFile, QString outCookieFile);
 
-    bool DDPost(QString    url,          QString   &inputFilename,
-                       QDateTime  pstartDate,   QDateTime  pendDate,
-                       QString   &err_txt);
+    QByteArray DDPost(
+        const QString &url,
+        const QDateTime &startDate, const QDateTime &endDate,
+        QString &err_txt);
 
 
   private: