2 #include <QCoreApplication>
8 #include <QNetworkCookie>
9 #include <QAuthenticator>
10 #include <QTextStream>
11 #include <QNetworkProxy>
12 #include <QMutexLocker>
26 #include "mythversion.h"
34 #define LOC QString("DownloadManager: ")
49 qRegisterMetaType<QNetworkReply::NetworkError>(
"QNetworkReply::NetworkError");
61 QMutexLocker lock(&
m_lock);
67 QMutexLocker lock(&
m_lock);
91 const QHash<QByteArray, QByteArray> *
m_headers {
nullptr};
93 QNetworkReply::NetworkError
m_errorCode {QNetworkReply::NoError};
159 while (!tmpDLM->getQueueThread())
162 tmpDLM->moveToThread(tmpDLM->getQueueThread());
163 tmpDLM->setRunThread();
165 while (!tmpDLM->isRunning())
195 bool downloading =
false;
196 bool itemsInQueue =
false;
197 bool itemsInCancellationQueue =
false;
198 bool waitAnyway =
false;
205 m_manager =
new QNetworkAccessManager(
this);
209 QCoreApplication::applicationName() +
"-" +
219 m_manager->cookieJar()->setParent(
nullptr);
221 QObject::connect(
m_manager, &QNetworkAccessManager::finished,
229 LOG(VB_GENERAL, LOG_DEBUG,
"Updating DLManager's Cookie Jar");
239 if (itemsInCancellationQueue)
244 QCoreApplication::processEvents();
250 if (!itemsInQueue || waitAnyway)
257 LOG(VB_FILE, LOG_DEBUG,
LOC + QString(
"waiting 200ms"));
262 LOG(VB_FILE, LOG_DEBUG,
LOC + QString(
"waiting for more items to download"));
294 if (dlInfo->
m_url.startsWith(
"myth://"))
322 const QString &
dest, QByteArray *data,
329 dlInfo->m_request = req;
330 dlInfo->m_outFile =
dest;
331 dlInfo->m_data = data;
332 dlInfo->m_caller = caller;
333 dlInfo->m_requestType = reqType;
334 dlInfo->m_reload = reload;
354 const QString &
dest, QByteArray *data,
358 const QHash<QByteArray, QByteArray> *headers,
364 dlInfo->m_request = req;
365 dlInfo->m_outFile =
dest;
366 dlInfo->m_data = data;
367 dlInfo->m_requestType = reqType;
368 dlInfo->m_reload = reload;
369 dlInfo->m_syncMode =
true;
370 dlInfo->m_authCallback = authCallbackFn;
371 dlInfo->m_authArg = authArg;
372 dlInfo->m_headers = headers;
373 dlInfo->m_finalUrl = finalUrl;
383 LOG(VB_FILE, LOG_DEBUG,
LOC + QString(
"preCache('%1')").arg(url));
384 queueItem(url,
nullptr, QString(),
nullptr,
nullptr);
398 LOG(VB_FILE, LOG_DEBUG,
LOC + QString(
"queueDownload('%1', '%2', %3)")
399 .arg(url,
dest, QString::number((
long long)caller)));
413 LOG(VB_FILE, LOG_DEBUG,
LOC + QString(
"queueDownload('%1', '%2', %3)")
414 .arg(req->url().toString()).arg((
long long)data)
415 .arg((
long long)caller));
417 queueItem(req->url().toString(), req, QString(), data, caller,
419 (QNetworkRequest::AlwaysNetwork == req->attribute(
420 QNetworkRequest::CacheLoadControlAttribute,
421 QNetworkRequest::PreferNetwork).toInt()));
443 const bool reload, QString *finalUrl)
447 nullptr,
nullptr,
nullptr, &redirected))
449 if (!redirected.isEmpty() && finalUrl !=
nullptr)
450 *finalUrl = redirected;
464 QNetworkReply *reply =
nullptr;
467 dlInfo->m_reload = reload;
468 dlInfo->m_syncMode =
true;
469 dlInfo->m_processReply =
false;
475 reply = dlInfo->m_reply;
477 dlInfo->m_reply =
nullptr;
497 LOG(VB_FILE, LOG_DEBUG,
LOC + QString(
"download('%1', '%2')")
498 .arg(req->url().toString()).arg((
long long)data));
499 return processItem(req->url().toString(), req, QString(), data,
501 (QNetworkRequest::AlwaysNetwork == req->attribute(
502 QNetworkRequest::CacheLoadControlAttribute,
503 QNetworkRequest::PreferNetwork).toInt()));
516 const bool reload,
AuthCallback authCallbackFn,
void *authArg,
517 const QHash<QByteArray, QByteArray> *headers)
533 LOG(VB_FILE, LOG_DEBUG,
LOC + QString(
"queuePost('%1', '%2')")
534 .arg(url).arg((
long long)data));
538 LOG(VB_GENERAL, LOG_ERR,
LOC +
"queuePost(), data is NULL!");
554 LOG(VB_FILE, LOG_DEBUG,
LOC + QString(
"queuePost('%1', '%2')")
555 .arg(req->url().toString()).arg((
long long)data));
559 LOG(VB_GENERAL, LOG_ERR,
LOC +
"queuePost(), data is NULL!");
563 queueItem(req->url().toString(), req, QString(), data, caller,
565 (QNetworkRequest::AlwaysNetwork == req->attribute(
566 QNetworkRequest::CacheLoadControlAttribute,
567 QNetworkRequest::PreferNetwork).toInt()));
578 LOG(VB_FILE, LOG_DEBUG,
LOC + QString(
"post('%1', '%2')")
579 .arg(url).arg((
long long)data));
583 LOG(VB_GENERAL, LOG_ERR,
LOC +
"post(), data is NULL!");
597 LOG(VB_FILE, LOG_DEBUG,
LOC + QString(
"post('%1', '%2')")
598 .arg(req->url().toString()).arg((
long long)data));
602 LOG(VB_GENERAL, LOG_ERR,
LOC +
"post(), data is NULL!");
606 return processItem(req->url().toString(), req, QString(), data,
608 (QNetworkRequest::AlwaysNetwork == req->attribute(
609 QNetworkRequest::CacheLoadControlAttribute,
610 QNetworkRequest::PreferNetwork).toInt()));
624 const QHash<QByteArray, QByteArray> *headers)
626 LOG(VB_FILE, LOG_DEBUG,
LOC + QString(
"postAuth('%1', '%2')")
627 .arg(url).arg((
long long)data));
631 LOG(VB_GENERAL, LOG_ERR,
LOC +
"postAuth(), data is NULL!");
656 static const QString kDateFormat =
"ddd, dd MMM yyyy hh:mm:ss 'GMT'";
657 QUrl qurl(dlInfo->
m_url);
658 QNetworkRequest request;
667 request.setUrl(qurl);
671 request.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
672 QNetworkRequest::AlwaysNetwork);
683 while (!(redirectLoc =
getHeader(qurl,
"Location")).isNull())
687 LOG(VB_GENERAL, LOG_WARNING, QString(
"Cache Redirection limit "
689 .arg(qurl.toString()));
692 qurl.setUrl(redirectLoc);
696 LOG(VB_NETWORK, LOG_DEBUG, QString(
"Checking cache for %1")
697 .arg(qurl.toString()));
700 QNetworkCacheMetaData urlData =
m_manager->cache()->metaData(qurl);
702 if ((urlData.isValid()) &&
703 ((!urlData.expirationDate().isValid()) ||
704 (urlData.expirationDate().toUTC().secsTo(now) < 10)))
706 QString dateString =
getHeader(urlData,
"Date");
708 if (!dateString.isNull())
712 loadDate.setTimeSpec(Qt::UTC);
713 if (loadDate.secsTo(now) <= 720)
716 LOG(VB_NETWORK, LOG_DEBUG, QString(
"Preferring cache for %1")
717 .arg(qurl.toString()));
724 request.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
725 QNetworkRequest::PreferCache);
727 if (!request.hasRawHeader(
"User-Agent"))
729 request.setRawHeader(
"User-Agent",
730 QByteArray(
"MythTV v") + MYTH_BINARY_VERSION +
731 " MythDownloadManager");
736 QHash<QByteArray, QByteArray>::const_iterator it =
738 for ( ; it != dlInfo->
m_headers->constEnd(); ++it )
740 if (!it.key().isEmpty() && !it.value().isEmpty())
742 request.setRawHeader(it.key(), it.value());
765 connect(
m_manager, &QNetworkAccessManager::authenticationRequired,
769 #if QT_VERSION < QT_VERSION_CHECK(5,15,0)
773 connect(dlInfo->
m_reply, &QNetworkReply::errorOccurred,
776 connect(dlInfo->
m_reply, &QNetworkReply::downloadProgress,
785 QAuthenticator *authenticator)
797 LOG(VB_FILE, LOG_DEBUG,
"Calling auth callback");
819 if (dlInfo->
m_url.startsWith(
"http://[fe80::",Qt::CaseInsensitive))
820 return downloadNowLinkLocal(dlInfo, deleteInfo);
832 while ((!dlInfo->
IsDone()) &&
834 (((!dlInfo->
m_url.startsWith(
"myth://")) &&
836 ((dlInfo->
m_url.startsWith(
"myth://")) &&
845 bool done = dlInfo->
IsDone();
847 done && (dlInfo->
m_errorCode == QNetworkReply::NoError);
856 LOG(VB_FILE, LOG_DEBUG,
857 LOC + QString(
"Aborting download - lack of data transfer"));
890 bool MythDownloadManager::downloadNowLinkLocal(
MythDownloadInfo *dlInfo,
bool deleteInfo)
897 LOG(VB_GENERAL, LOG_ERR,
LOC + QString(
"No data buffer provided for %1").arg(dlInfo->
m_url));
904 LOG(VB_GENERAL, LOG_ERR,
LOC + QString(
"Unsupported authentication for %1").arg(dlInfo->
m_url));
910 LOG(VB_GENERAL, LOG_ERR,
LOC + QString(
"Unsupported File output %1 for %2")
919 LOG(VB_GENERAL, LOG_ERR,
LOC + QString(
"Unsupported link-local operation %1")
920 .arg(dlInfo->
m_url));
924 QUrl url(dlInfo->
m_url);
925 QString host(url.host());
926 int port(url.port(80));
940 QByteArray* buffer = dlInfo->
m_data;
941 QHash<QByteArray, QByteArray> headers;
944 if (!headers.contains(
"User-Agent"))
945 headers.insert(
"User-Agent", QByteArray(
"MythDownloadManager v") +
946 MYTH_BINARY_VERSION);
947 headers.insert(
"Connection",
"close");
948 headers.insert(
"Accept-Encoding",
"identity");
949 if (!buffer->isEmpty())
950 headers.insert(
"Content-Length", QString::number(buffer->size()).toUtf8());
951 headers.insert(
"Host", (url.host() +
":" + QString::number(port)).toUtf8());
953 QByteArray requestMessage;
954 QString path (url.path());
955 requestMessage.append(
"POST ");
956 requestMessage.append(path.toLatin1());
957 requestMessage.append(
" HTTP/1.1\r\n");
958 QHashIterator<QByteArray, QByteArray> it(headers);
962 requestMessage.append(it.key());
963 requestMessage.append(
": ");
964 requestMessage.append(it.value());
965 requestMessage.append(
"\r\n");
967 requestMessage.append(
"\r\n");
968 if (!buffer->isEmpty())
969 requestMessage.append(*buffer);
972 socket.connectToHost(host,
static_cast<uint16_t>(port));
974 if (!socket.waitForConnected(5000))
977 ok = socket.write(requestMessage) > 0;
980 ok = socket.waitForDisconnected(5000);
983 *buffer = socket.readAll();
985 QByteArray delim(
"\r\n\r\n");
986 int delimLoc = buffer->indexOf(delim);
988 *buffer = buffer->right(buffer->size() - delimLoc - 4);
1005 LOG(VB_GENERAL, LOG_ERR,
LOC + QString(
"Link Local request failed: %1").arg(url.toString()));
1026 for (
const auto& url : qAsConst(urls))
1029 while (lit.hasNext())
1033 if (dlInfo->
m_url == url)
1056 if (QThread::currentThread() == this->thread())
1079 while (lit.hasNext())
1087 LOG(VB_FILE, LOG_DEBUG,
1088 LOC + QString(
"Aborting download - user request"));
1097 dlInfo->
m_errorCode = QNetworkReply::OperationCanceledError;
1119 dlInfo->
m_data =
nullptr;
1123 QMap <QString, MythDownloadInfo*>::iterator mit =
m_downloadInfos.begin();
1131 dlInfo->
m_data =
nullptr;
1141 auto *reply = qobject_cast<QNetworkReply *>(sender());
1142 if (reply ==
nullptr)
1145 LOG(VB_FILE, LOG_DEBUG,
LOC + QString(
"downloadError %1 ")
1146 .arg(errorCode) + reply->errorString() );
1151 reply->deleteLater();
1169 const QUrl& oldRedirectUrl)
1171 LOG(VB_FILE, LOG_DEBUG,
LOC + QString(
"redirectUrl()"));
1174 if(!possibleRedirectUrl.isEmpty() && possibleRedirectUrl != oldRedirectUrl)
1185 LOG(VB_FILE, LOG_DEBUG,
LOC + QString(
"downloadFinished(%1)")
1186 .arg((
long long)reply));
1191 reply->deleteLater();
1197 if (!dlInfo || !dlInfo->
m_reply)
1211 int statusCode = -1;
1212 static const QString kDateFormat =
"ddd, dd MMM yyyy hh:mm:ss 'GMT'";
1213 QNetworkReply *reply = dlInfo->
m_reply;
1217 QUrl possibleRedirectUrl =
1218 reply->attribute(QNetworkRequest::RedirectionTargetAttribute).toUrl();
1220 if (!possibleRedirectUrl.isEmpty() &&
1221 possibleRedirectUrl.isValid() &&
1222 possibleRedirectUrl.isRelative())
1223 possibleRedirectUrl = reply->url().resolved(possibleRedirectUrl);
1225 if (!possibleRedirectUrl.isEmpty() && dlInfo->
m_finalUrl !=
nullptr)
1226 *dlInfo->
m_finalUrl = QString(possibleRedirectUrl.toString());
1232 reply->attribute(QNetworkRequest::HttpStatusCodeAttribute);
1233 if (status.isValid())
1234 statusCode = status.toInt();
1239 (statusCode == 301 || statusCode == 302 ||
1240 statusCode == 303)))
1242 LOG(VB_FILE, LOG_DEBUG,
LOC +
1243 QString(
"downloadFinished(%1): Redirect: %2 -> %3")
1244 .arg(QString::number((
long long)dlInfo),
1245 reply->url().toString(),
1246 dlInfo->m_redirectedTo.toString()));
1249 dlInfo->m_data->clear();
1251 dlInfo->m_bytesReceived = 0;
1252 dlInfo->m_bytesTotal = 0;
1254 QNetworkRequest request(dlInfo->m_redirectedTo);
1256 if (dlInfo->m_reload)
1258 request.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
1259 QNetworkRequest::AlwaysNetwork);
1261 else if (dlInfo->m_preferCache)
1263 request.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
1264 QNetworkRequest::PreferCache);
1267 request.setRawHeader(
"User-Agent",
1268 "MythDownloadManager v" +
1269 QByteArray(MYTH_BINARY_VERSION));
1271 switch (dlInfo->m_requestType)
1274 dlInfo->m_reply =
m_manager->head(request);
1278 dlInfo->m_reply =
m_manager->get(request);
1284 #if QT_VERSION < QT_VERSION_CHECK(5,15,0)
1288 connect(dlInfo->m_reply, &QNetworkReply::errorOccurred,
1291 connect(dlInfo->m_reply, &QNetworkReply::downloadProgress,
1295 reply->deleteLater();
1299 LOG(VB_FILE, LOG_DEBUG, QString(
"downloadFinished(%1): COMPLETE: %2")
1300 .arg((
long long)dlInfo).arg(dlInfo->m_url));
1304 QUrl fileUrl = dlInfo->m_url;
1305 QString redirectLoc;
1307 while (!(redirectLoc =
getHeader(fileUrl,
"Location")).isNull())
1311 LOG(VB_GENERAL, LOG_WARNING, QString(
"Cache Redirection limit "
1313 .arg(fileUrl.toString()));
1316 fileUrl.setUrl(redirectLoc);
1321 QNetworkCacheMetaData urlData =
m_manager->cache()->metaData(fileUrl);
1323 if (
getHeader(urlData,
"Date").isNull())
1325 QNetworkCacheMetaData::RawHeaderList headers = urlData.rawHeaders();
1326 QNetworkCacheMetaData::RawHeader newheader;
1328 newheader = QNetworkCacheMetaData::RawHeader(
"Date",
1329 now.toString(kDateFormat).toLatin1());
1330 headers.append(newheader);
1331 urlData.setRawHeaders(headers);
1333 m_manager->cache()->updateMetaData(urlData);
1338 dlInfo->m_redirectedTo.clear();
1344 if (reply && dlInfo->m_processReply)
1346 bool append = (!dlInfo->m_syncMode && dlInfo->m_caller);
1347 QByteArray data = reply->readAll();
1348 dataSize = data.size();
1351 dlInfo->m_bytesReceived += dataSize;
1353 dlInfo->m_bytesReceived = dataSize;
1355 dlInfo->m_bytesTotal = dlInfo->m_bytesReceived;
1360 dlInfo->m_data->append(data);
1362 *dlInfo->m_data = data;
1364 else if (!dlInfo->m_outFile.isEmpty())
1366 saveFile(dlInfo->m_outFile, data, append);
1373 (*dlInfo->m_data) = dlInfo->m_privData;
1375 else if (!dlInfo->m_outFile.isEmpty())
1377 saveFile(dlInfo->m_outFile, dlInfo->m_privData);
1379 dlInfo->m_bytesReceived += dataSize;
1380 dlInfo->m_bytesTotal = dlInfo->m_bytesReceived;
1388 LOG(VB_GENERAL, LOG_ERR,
LOC +
1389 QString(
"ERROR download finished but failed to remove url: %1")
1390 .arg(dlInfo->m_url));
1397 dlInfo->SetDone(
true);
1399 if (!dlInfo->m_syncMode)
1401 if (dlInfo->m_caller)
1403 LOG(VB_FILE, LOG_DEBUG, QString(
"downloadFinished(%1): "
1404 "COMPLETE: %2, sending event to caller")
1405 .arg((
long long)dlInfo).arg(dlInfo->m_url));
1408 args << dlInfo->m_url;
1409 args << dlInfo->m_outFile;
1410 args << QString::number(dlInfo->m_bytesTotal);
1412 args << (reply ? reply->errorString() : QString());
1413 args << QString::number((
int)(reply ? reply->error() :
1414 dlInfo->m_errorCode));
1416 QCoreApplication::postEvent(dlInfo->m_caller,
1435 auto *reply = qobject_cast<QNetworkReply *>(sender());
1436 if (reply ==
nullptr)
1439 LOG(VB_FILE, LOG_DEBUG,
LOC +
1440 QString(
"downloadProgress(%1, %2) (for reply %3)")
1441 .arg(bytesReceived).arg(bytesTotal).arg((
long long)reply));
1454 LOG(VB_FILE, LOG_DEBUG,
LOC +
1455 QString(
"downloadProgress: %1 to %2 is at %3 of %4 bytes downloaded")
1457 .arg(bytesReceived).arg(bytesTotal));
1461 LOG(VB_FILE, LOG_DEBUG, QString(
"downloadProgress(%1): "
1462 "sending event to caller")
1463 .arg(reply->url().toString()));
1466 QByteArray data = reply->readAll();
1471 dlInfo->
m_data->append(data);
1479 args << QString::number(bytesReceived);
1480 args << QString::number(bytesTotal);
1482 QCoreApplication::postEvent(dlInfo->
m_caller,
1495 const QByteArray &data,
1498 if (outFile.isEmpty() || data.isEmpty())
1501 QFile
file(outFile);
1502 QFileInfo fileInfo(outFile);
1503 QDir qdir(fileInfo.absolutePath());
1505 if (!qdir.exists() && !qdir.mkpath(fileInfo.absolutePath()))
1507 LOG(VB_GENERAL, LOG_ERR, QString(
"Failed to create: '%1'")
1508 .arg(fileInfo.absolutePath()));
1512 QIODevice::OpenMode mode = QIODevice::Unbuffered|QIODevice::WriteOnly;
1514 mode |= QIODevice::Append;
1516 if (!
file.open(mode))
1518 LOG(VB_GENERAL, LOG_ERR, QString(
"Failed to open: '%1'") .arg(outFile));
1523 size_t remaining = data.size();
1524 uint failure_cnt = 0;
1525 while ((remaining > 0) && (failure_cnt < 5))
1527 ssize_t written =
file.write(data.data() + offset, remaining);
1537 remaining -= written;
1540 return remaining <= 0;
1554 static const QString kDateFormat =
"ddd, dd MMM yyyy hh:mm:ss 'GMT'";
1555 LOG(VB_FILE, LOG_DEBUG,
LOC + QString(
"GetLastModified('%1')").arg(url));
1560 QUrl cacheUrl = QUrl(url);
1563 QString redirectLoc;
1565 while (!(redirectLoc =
getHeader(cacheUrl,
"Location")).isNull())
1569 LOG(VB_GENERAL, LOG_WARNING, QString(
"Cache Redirection limit "
1571 .arg(cacheUrl.toString()));
1574 cacheUrl.setUrl(redirectLoc);
1579 QNetworkCacheMetaData urlData =
m_manager->cache()->metaData(cacheUrl);
1582 if (urlData.isValid() &&
1583 ((!urlData.expirationDate().isValid()) ||
1584 (urlData.expirationDate().secsTo(now) < 0)))
1586 if (urlData.lastModified().toUTC().secsTo(now) <= 3600)
1588 result = urlData.lastModified().toUTC();
1592 QString date =
getHeader(urlData,
"Date");
1595 QDateTime loadDate =
1597 loadDate.setTimeSpec(Qt::UTC);
1598 if (loadDate.secsTo(now) <= 1200)
1600 result = urlData.lastModified().toUTC();
1606 if (!result.isValid())
1609 dlInfo->
m_url = url;
1620 QNetworkRequest::LastModifiedHeader);
1621 if (lastMod.isValid())
1622 result = lastMod.toDateTime().toUTC();
1631 LOG(VB_FILE, LOG_DEBUG,
LOC + QString(
"GetLastModified('%1'): Result %2")
1632 .arg(url, result.toString()));
1660 auto *jar = qobject_cast<MythCookieJar *>(
m_manager->cookieJar());
1682 auto *inJar = qobject_cast<MythCookieJar *>(
m_manager->cookieJar());
1683 if (inJar ==
nullptr)
1699 auto *inJar = qobject_cast<MythCookieJar *>(jar);
1700 if (inJar ==
nullptr)
1718 if (inJar !=
nullptr)
1735 QNetworkCacheMetaData metadata =
m_manager->cache()->metaData(url);
1747 const QString& header)
1749 auto headers = cacheData.rawHeaders();
1750 for (
const auto& rh : qAsConst(headers))
1751 if (QString(rh.first) == header)
1762 const QList<QNetworkCookie> cookieList = old.allCookies();
1763 setAllCookies(cookieList);
1771 LOG(VB_GENERAL, LOG_DEBUG, QString(
"MythCookieJar: loading cookies from: %1").arg(
filename));
1774 if (!f.open(QIODevice::ReadOnly))
1776 LOG(VB_GENERAL, LOG_WARNING, QString(
"MythCookieJar::load() failed to open file for reading: %1").arg(
filename));
1780 QList<QNetworkCookie> cookieList;
1781 QTextStream stream(&f);
1782 while (!stream.atEnd())
1784 QString cookie = stream.readLine();
1785 cookieList << QNetworkCookie::parseCookies(cookie.toLocal8Bit());
1788 setAllCookies(cookieList);
1796 LOG(VB_GENERAL, LOG_DEBUG, QString(
"MythCookieJar: saving cookies to: %1").arg(
filename));
1799 if (!f.open(QIODevice::WriteOnly))
1801 LOG(VB_GENERAL, LOG_ERR, QString(
"MythCookieJar::save() failed to open file for writing: %1").arg(
filename));
1805 QList<QNetworkCookie> cookieList = allCookies();
1806 QTextStream stream(&f);
1808 #if QT_VERSION < QT_VERSION_CHECK(5,14,0)
1809 for (
const auto& cookie : qAsConst(cookieList))
1810 stream << cookie.toRawForm() << endl;
1812 for (
const auto& cookie : qAsConst(cookieList))
1813 stream << cookie.toRawForm() << Qt::endl;