MythTV master
mythdownloadmanager.cpp
Go to the documentation of this file.
1// qt
2#include <QCoreApplication>
3#include <QRunnable>
4#include <QString>
5#include <QByteArray>
6#include <QFile>
7#include <QDir>
8#include <QNetworkCookie>
9#include <QAuthenticator>
10#include <QTextStream>
11#include <QTimeZone>
12#include <QNetworkProxy>
13#include <QMutexLocker>
14#include <QUrl>
15#include <QTcpSocket>
16
17#include <cstdlib>
18#include <unistd.h> // for usleep()
19
20// libmythbase
21#include "compat.h"
22#include "mythcorecontext.h"
23#include "mthreadpool.h"
24#include "mythdirs.h"
25#include "mythevent.h"
26#include "mythversion.h"
27#include "remotefile.h"
28#include "mythdate.h"
29
30#include "mythdownloadmanager.h"
31#include "mythlogging.h"
32#include "portchecker.h"
33
34#define LOC QString("DownloadManager: ")
35static constexpr int CACHE_REDIRECTION_LIMIT { 10 };
36
39
44{
45 public:
48 {
49 qRegisterMetaType<QNetworkReply::NetworkError>("QNetworkReply::NetworkError");
50 }
51
53 {
54 delete m_request;
56 m_reply->deleteLater();
57 }
58
59 bool IsDone(void)
60 {
61 QMutexLocker lock(&m_lock);
62 return m_done;
63 }
64
65 void SetDone(bool done)
66 {
67 QMutexLocker lock(&m_lock);
68 m_done = done;
69 }
70
71 QString m_url;
73 QString *m_finalUrl {nullptr};
74 QNetworkRequest *m_request {nullptr};
75 QNetworkReply *m_reply {nullptr};
76 QString m_outFile;
77 QByteArray *m_data {nullptr};
78 QByteArray m_privData;
79 QObject *m_caller {nullptr};
81 bool m_reload {false};
82 bool m_preferCache {false};
83 bool m_syncMode {false};
84 bool m_processReply {true};
85 bool m_done {false};
86 qint64 m_bytesReceived {0};
87 qint64 m_bytesTotal {0};
88 QDateTime m_lastStat;
90 void *m_authArg {nullptr};
91 const QHash<QByteArray, QByteArray> *m_headers {nullptr};
92
93 QNetworkReply::NetworkError m_errorCode {QNetworkReply::NoError};
94 QMutex m_lock;
95};
96
97
101class RemoteFileDownloadThread : public QRunnable
102{
103 public:
105 MythDownloadInfo *dlInfo) :
106 m_parent(parent),
107 m_dlInfo(dlInfo) {}
108
109 void run() override // QRunnable
110 {
111 bool ok = false;
112
113 auto *rf = new RemoteFile(m_dlInfo->m_url, false, false, 0ms);
114 ok = rf->SaveAs(m_dlInfo->m_privData);
115 delete rf;
116
117 if (!ok)
118 m_dlInfo->m_errorCode = QNetworkReply::UnknownNetworkError;
119
122
124 }
125
126 private:
129};
130
134{
135 if (downloadManager)
136 {
137 delete downloadManager;
138 downloadManager = nullptr;
139 }
140}
141
146{
147 if (downloadManager)
148 return downloadManager;
149
150 QMutexLocker locker(&dmCreateLock);
151
152 // Check once more in case the download manager was created
153 // while we were securing the lock.
154 if (downloadManager)
155 return downloadManager;
156
157 auto *tmpDLM = new MythDownloadManager();
158 tmpDLM->start();
159 while (!tmpDLM->getQueueThread())
160 usleep(10000);
161
162 tmpDLM->moveToThread(tmpDLM->getQueueThread());
163 tmpDLM->setRunThread();
164
165 while (!tmpDLM->isRunning())
166 usleep(10000);
167
168 downloadManager = tmpDLM;
169
171
172 return downloadManager;
173}
174
178{
179 m_runThread = false;
180 m_queueWaitCond.wakeAll();
181
182 wait();
183
184 delete m_infoLock;
185 delete m_inCookieJar;
186}
187
192{
193 RunProlog();
194
195 bool downloading = false;
196 bool itemsInQueue = false;
197 bool itemsInCancellationQueue = false;
198 bool waitAnyway = false;
199
200 m_queueThread = QThread::currentThread();
201
202 while (!m_runThread)
203 usleep(50ms);
204
205 m_manager = new QNetworkAccessManager(this);
206 m_diskCache = new QNetworkDiskCache(this);
207 m_proxy = new QNetworkProxy();
208 m_diskCache->setCacheDirectory(GetConfDir() + "/cache/" +
209 QCoreApplication::applicationName() + "-" +
211 m_manager->setCache(m_diskCache);
212
213 // Set the proxy for the manager to be the application default proxy,
214 // which has already been setup
215 m_manager->setProxy(*m_proxy);
216
217 // make sure the cookieJar is created in the same thread as the manager
218 // and set its parent to nullptr so it can be shared between managers
219 m_manager->cookieJar()->setParent(nullptr);
220
221 QObject::connect(m_manager, &QNetworkAccessManager::finished,
222 this, qOverload<QNetworkReply*>(&MythDownloadManager::downloadFinished));
223
224 m_isRunning = true;
225 while (m_runThread)
226 {
227 if (m_inCookieJar)
228 {
229 LOG(VB_GENERAL, LOG_DEBUG, "Updating DLManager's Cookie Jar");
231 }
232 m_infoLock->lock();
233 LOG(VB_FILE, LOG_DEBUG, LOC + QString("items downloading %1").arg(m_downloadInfos.count()));
234 LOG(VB_FILE, LOG_DEBUG, LOC + QString("items queued %1").arg(m_downloadQueue.count()));
235 downloading = !m_downloadInfos.isEmpty();
236 itemsInCancellationQueue = !m_cancellationQueue.isEmpty();
237 m_infoLock->unlock();
238
239 if (itemsInCancellationQueue)
240 {
242 }
243 if (downloading)
244 QCoreApplication::processEvents();
245
246 m_infoLock->lock();
247 itemsInQueue = !m_downloadQueue.isEmpty();
248 m_infoLock->unlock();
249
250 if (!itemsInQueue || waitAnyway)
251 {
252 waitAnyway = false;
253 m_queueWaitLock.lock();
254
255 if (downloading)
256 {
257 LOG(VB_FILE, LOG_DEBUG, LOC + QString("waiting 200ms"));
259 }
260 else
261 {
262 LOG(VB_FILE, LOG_DEBUG, LOC + QString("waiting for more items to download"));
264 }
265
266 m_queueWaitLock.unlock();
267 }
268
269 m_infoLock->lock();
270 if (!m_downloadQueue.isEmpty())
271 {
272 MythDownloadInfo *dlInfo = m_downloadQueue.front();
273
274 m_downloadQueue.pop_front();
275
276 if (!dlInfo)
277 {
278 m_infoLock->unlock();
279 continue;
280 }
281
282 if (m_downloadInfos.contains(dlInfo->m_url))
283 {
284 // Push request to the end of the queue to let others process.
285 // If this is the only item in the queue, force the loop to
286 // wait a little.
287 if (m_downloadQueue.isEmpty())
288 waitAnyway = true;
289 m_downloadQueue.push_back(dlInfo);
290 m_infoLock->unlock();
291 continue;
292 }
293
294 if (dlInfo->m_url.startsWith("myth://"))
295 downloadRemoteFile(dlInfo);
296 else
297 {
298 QMutexLocker cLock(&m_cookieLock);
300 }
301
302 m_downloadInfos[dlInfo->m_url] = dlInfo;
303 }
304 m_infoLock->unlock();
305 }
306 m_isRunning = false;
307
308 RunEpilog();
309}
310
321void MythDownloadManager::queueItem(const QString &url, QNetworkRequest *req,
322 const QString &dest, QByteArray *data,
323 QObject *caller, const MRequestType reqType,
324 const bool reload)
325{
326 auto *dlInfo = new MythDownloadInfo;
327
328 dlInfo->m_url = url;
329 dlInfo->m_request = req;
330 dlInfo->m_outFile = dest;
331 dlInfo->m_data = data;
332 dlInfo->m_caller = caller;
333 dlInfo->m_requestType = reqType;
334 dlInfo->m_reload = reload;
335
336 QMutexLocker locker(m_infoLock);
337 m_downloadQueue.push_back(dlInfo);
338 m_queueWaitCond.wakeAll();
339}
340
353bool MythDownloadManager::processItem(const QString &url, QNetworkRequest *req,
354 const QString &dest, QByteArray *data,
355 const MRequestType reqType,
356 const bool reload,
357 AuthCallback authCallbackFn, void *authArg,
358 const QHash<QByteArray, QByteArray> *headers,
359 QString *finalUrl)
360{
361 auto *dlInfo = new MythDownloadInfo;
362
363 dlInfo->m_url = url;
364 dlInfo->m_request = req;
365 dlInfo->m_outFile = dest;
366 dlInfo->m_data = data;
367 dlInfo->m_requestType = reqType;
368 dlInfo->m_reload = reload;
369 dlInfo->m_syncMode = true;
370 dlInfo->m_authCallback = authCallbackFn;
371 dlInfo->m_authArg = authArg;
372 dlInfo->m_headers = headers;
373 dlInfo->m_finalUrl = finalUrl;
374
375 return downloadNow(dlInfo, true);
376}
377
381void MythDownloadManager::preCache(const QString &url)
382{
383 LOG(VB_FILE, LOG_DEBUG, LOC + QString("preCache('%1')").arg(url));
384 queueItem(url, nullptr, QString(), nullptr, nullptr);
385}
386
394 const QString &dest,
395 QObject *caller,
396 const bool reload)
397{
398 LOG(VB_FILE, LOG_DEBUG, LOC + QString("queueDownload('%1', '%2', %3)")
399 .arg(url, dest, QString::number((long long)caller)));
400
401 queueItem(url, nullptr, dest, nullptr, caller, kRequestGet, reload);
402}
403
409void MythDownloadManager::queueDownload(QNetworkRequest *req,
410 QByteArray *data,
411 QObject *caller)
412{
413 LOG(VB_FILE, LOG_DEBUG, LOC + QString("queueDownload('%1', '%2', %3)")
414 .arg(req->url().toString()).arg((long long)data)
415 .arg((long long)caller));
416
417 queueItem(req->url().toString(), req, QString(), data, caller,
419 (QNetworkRequest::AlwaysNetwork == req->attribute(
420 QNetworkRequest::CacheLoadControlAttribute,
421 QNetworkRequest::PreferNetwork).toInt()));
422}
423
430bool MythDownloadManager::download(const QString &url, const QString &dest,
431 const bool reload)
432{
433 return processItem(url, nullptr, dest, nullptr, kRequestGet, reload);
434}
435
442bool MythDownloadManager::download(const QString &url, QByteArray *data,
443 const bool reload, QString *finalUrl)
444{
445 QString redirected;
446 if (!processItem(url, nullptr, QString(), data, kRequestGet, reload,
447 nullptr, nullptr, nullptr, &redirected))
448 return false;
449 if (!redirected.isEmpty() && finalUrl != nullptr)
450 *finalUrl = redirected;
451 return true;
452}
453
460QNetworkReply *MythDownloadManager::download(const QString &url,
461 const bool reload)
462{
463 auto *dlInfo = new MythDownloadInfo;
464 QNetworkReply *reply = nullptr;
465
466 dlInfo->m_url = url;
467 dlInfo->m_reload = reload;
468 dlInfo->m_syncMode = true;
469 dlInfo->m_processReply = false;
470
471 if (downloadNow(dlInfo, false))
472 {
473 if (dlInfo->m_reply)
474 {
475 reply = dlInfo->m_reply;
476 // prevent dlInfo dtor from deleting the reply
477 dlInfo->m_reply = nullptr;
478
479 delete dlInfo;
480
481 return reply;
482 }
483
484 delete dlInfo;
485 }
486
487 return nullptr;
488}
489
495bool MythDownloadManager::download(QNetworkRequest *req, QByteArray *data)
496{
497 LOG(VB_FILE, LOG_DEBUG, LOC + QString("download('%1', '%2')")
498 .arg(req->url().toString()).arg((long long)data));
499 return processItem(req->url().toString(), req, QString(), data,
501 (QNetworkRequest::AlwaysNetwork == req->attribute(
502 QNetworkRequest::CacheLoadControlAttribute,
503 QNetworkRequest::PreferNetwork).toInt()));
504}
505
515bool MythDownloadManager::downloadAuth(const QString &url, const QString &dest,
516 const bool reload, AuthCallback authCallbackFn, void *authArg,
517 const QHash<QByteArray, QByteArray> *headers)
518{
519 return processItem(url, nullptr, dest, nullptr, kRequestGet, reload, authCallbackFn,
520 authArg, headers);
521}
522
523
529void MythDownloadManager::queuePost(const QString &url,
530 QByteArray *data,
531 QObject *caller)
532{
533 LOG(VB_FILE, LOG_DEBUG, LOC + QString("queuePost('%1', '%2')")
534 .arg(url).arg((long long)data));
535
536 if (!data)
537 {
538 LOG(VB_GENERAL, LOG_ERR, LOC + "queuePost(), data is NULL!");
539 return;
540 }
541
542 queueItem(url, nullptr, QString(), data, caller, kRequestPost);
543}
544
550void MythDownloadManager::queuePost(QNetworkRequest *req,
551 QByteArray *data,
552 QObject *caller)
553{
554 LOG(VB_FILE, LOG_DEBUG, LOC + QString("queuePost('%1', '%2')")
555 .arg(req->url().toString()).arg((long long)data));
556
557 if (!data)
558 {
559 LOG(VB_GENERAL, LOG_ERR, LOC + "queuePost(), data is NULL!");
560 return;
561 }
562
563 queueItem(req->url().toString(), req, QString(), data, caller,
565 (QNetworkRequest::AlwaysNetwork == req->attribute(
566 QNetworkRequest::CacheLoadControlAttribute,
567 QNetworkRequest::PreferNetwork).toInt()));
568
569}
570
576bool MythDownloadManager::post(const QString &url, QByteArray *data)
577{
578 LOG(VB_FILE, LOG_DEBUG, LOC + QString("post('%1', '%2')")
579 .arg(url).arg((long long)data));
580
581 if (!data)
582 {
583 LOG(VB_GENERAL, LOG_ERR, LOC + "post(), data is NULL!");
584 return false;
585 }
586
587 return processItem(url, nullptr, QString(), data, kRequestPost);
588}
589
595bool MythDownloadManager::post(QNetworkRequest *req, QByteArray *data)
596{
597 LOG(VB_FILE, LOG_DEBUG, LOC + QString("post('%1', '%2')")
598 .arg(req->url().toString()).arg((long long)data));
599
600 if (!data)
601 {
602 LOG(VB_GENERAL, LOG_ERR, LOC + "post(), data is NULL!");
603 return false;
604 }
605
606 return processItem(req->url().toString(), req, QString(), data,
608 (QNetworkRequest::AlwaysNetwork == req->attribute(
609 QNetworkRequest::CacheLoadControlAttribute,
610 QNetworkRequest::PreferNetwork).toInt()));
611
612}
613
622bool MythDownloadManager::postAuth(const QString &url, QByteArray *data,
623 AuthCallback authCallbackFn, void *authArg,
624 const QHash<QByteArray, QByteArray> *headers)
625{
626 LOG(VB_FILE, LOG_DEBUG, LOC + QString("postAuth('%1', '%2')")
627 .arg(url).arg((long long)data));
628
629 if (!data)
630 {
631 LOG(VB_GENERAL, LOG_ERR, LOC + "postAuth(), data is NULL!");
632 return false;
633 }
634
635 return processItem(url, nullptr, nullptr, data, kRequestPost, false, authCallbackFn,
636 authArg, headers);
637}
638
643{
644 auto *dlThread = new RemoteFileDownloadThread(this, dlInfo);
645 MThreadPool::globalInstance()->start(dlThread, "RemoteFileDownload");
646}
647
652{
653 if (!dlInfo)
654 return;
655
656 static const QString kDateFormat = "ddd, dd MMM yyyy hh:mm:ss 'GMT'";
657 QUrl qurl(dlInfo->m_url);
658 QNetworkRequest request;
659
660 if (dlInfo->m_request)
661 {
662 request = *dlInfo->m_request;
663 delete dlInfo->m_request;
664 dlInfo->m_request = nullptr;
665 }
666 else
667 {
668 request.setUrl(qurl);
669 }
670
671 if (dlInfo->m_reload)
672 {
673 request.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
674 QNetworkRequest::AlwaysNetwork);
675 }
676 else
677 {
678 // Prefer the in-cache item if one exists and it is less than 5 minutes
679 // old and it will not expire in the next 10 seconds
680 QDateTime now = MythDate::current();
681
682 // Handle redirects, we want the metadata of the file headers
683 QString redirectLoc;
684 int limit = 0;
685 while (!(redirectLoc = getHeader(qurl, "Location")).isNull())
686 {
687 if (limit == CACHE_REDIRECTION_LIMIT)
688 {
689 LOG(VB_GENERAL, LOG_WARNING, QString("Cache Redirection limit "
690 "reached for %1")
691 .arg(qurl.toString()));
692 return;
693 }
694 qurl.setUrl(redirectLoc);
695 limit++;
696 }
697
698 LOG(VB_NETWORK, LOG_DEBUG, QString("Checking cache for %1")
699 .arg(qurl.toString()));
700
701 m_infoLock->lock();
702 QNetworkCacheMetaData urlData = m_manager->cache()->metaData(qurl);
703 m_infoLock->unlock();
704 if ((urlData.isValid()) &&
705 ((!urlData.expirationDate().isValid()) ||
706 (urlData.expirationDate().toUTC().secsTo(now) < 10)))
707 {
708 QString dateString = getHeader(urlData, "Date");
709
710 if (!dateString.isNull())
711 {
712 QDateTime loadDate =
713 MythDate::fromString(dateString, kDateFormat);
714#if QT_VERSION < QT_VERSION_CHECK(6,5,0)
715 loadDate.setTimeSpec(Qt::UTC);
716#else
717 loadDate.setTimeZone(QTimeZone(QTimeZone::UTC));
718#endif
719 if (loadDate.secsTo(now) <= 720)
720 {
721 dlInfo->m_preferCache = true;
722 LOG(VB_NETWORK, LOG_DEBUG, QString("Preferring cache for %1")
723 .arg(qurl.toString()));
724 }
725 }
726 }
727 }
728
729 if (dlInfo->m_preferCache)
730 request.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
731 QNetworkRequest::PreferCache);
732
733 if (!request.hasRawHeader("User-Agent"))
734 {
735 request.setRawHeader("User-Agent",
736 QByteArray("MythTV v") + MYTH_BINARY_VERSION +
737 " MythDownloadManager");
738 }
739
740 if (dlInfo->m_headers)
741 {
742 QHash<QByteArray, QByteArray>::const_iterator it =
743 dlInfo->m_headers->constBegin();
744 for ( ; it != dlInfo->m_headers->constEnd(); ++it )
745 {
746 if (!it.key().isEmpty() && !it.value().isEmpty())
747 {
748 request.setRawHeader(it.key(), it.value());
749 }
750 }
751 }
752
753 switch (dlInfo->m_requestType)
754 {
755 case kRequestPost :
756 dlInfo->m_reply = m_manager->post(request, *dlInfo->m_data);
757 break;
758 case kRequestHead :
759 dlInfo->m_reply = m_manager->head(request);
760 break;
761 case kRequestGet :
762 default:
763 dlInfo->m_reply = m_manager->get(request);
764 break;
765 }
766
767 m_downloadReplies[dlInfo->m_reply] = dlInfo;
768
769 if (dlInfo->m_authCallback)
770 {
771 connect(m_manager, &QNetworkAccessManager::authenticationRequired,
773 }
774
775 connect(dlInfo->m_reply, &QNetworkReply::errorOccurred,
777 connect(dlInfo->m_reply, &QNetworkReply::downloadProgress,
779}
780
785void MythDownloadManager::authCallback(QNetworkReply *reply,
786 QAuthenticator *authenticator)
787{
788 if (!reply)
789 return;
790
791 MythDownloadInfo *dlInfo = m_downloadReplies[reply];
792
793 if (!dlInfo)
794 return;
795
796 if (dlInfo->m_authCallback)
797 {
798 LOG(VB_FILE, LOG_DEBUG, "Calling auth callback");
799 dlInfo->m_authCallback(reply, authenticator, dlInfo->m_authArg);
800 }
801}
802
810{
811 if (!dlInfo)
812 return false;
813
814 dlInfo->m_syncMode = true;
815
816 // Special handling for link-local
817 // Not needed for Windows because windows does not need
818 // the scope id.
819#ifndef _WIN32
820 if (dlInfo->m_url.startsWith("http://[fe80::",Qt::CaseInsensitive))
821 return downloadNowLinkLocal(dlInfo, deleteInfo);
822#endif
823 m_infoLock->lock();
824 m_downloadQueue.push_back(dlInfo);
825 m_infoLock->unlock();
826 m_queueWaitCond.wakeAll();
827
828 // timeout myth:// RemoteFile transfers 20 seconds from now
829 // timeout non-myth:// QNetworkAccessManager transfers 60 seconds after
830 // their last progress update
831 QDateTime startedAt = MythDate::current();
832 m_infoLock->lock();
833 while ((!dlInfo->IsDone()) &&
834 (dlInfo->m_errorCode == QNetworkReply::NoError) &&
835 (((!dlInfo->m_url.startsWith("myth://")) &&
836 (MythDate::secsInPast(dlInfo->m_lastStat) < 60s)) ||
837 ((dlInfo->m_url.startsWith("myth://")) &&
838 (MythDate::secsInPast(startedAt) < 20s))))
839 {
840 m_infoLock->unlock();
841 m_queueWaitLock.lock();
843 m_queueWaitLock.unlock();
844 m_infoLock->lock();
845 }
846 bool done = dlInfo->IsDone();
847 bool success =
848 done && (dlInfo->m_errorCode == QNetworkReply::NoError);
849
850 if (!done)
851 {
852 dlInfo->m_data = nullptr; // Prevent downloadFinished() from updating
853 dlInfo->m_syncMode = false; // Let downloadFinished() cleanup for us
854 if ((dlInfo->m_reply) &&
855 (dlInfo->m_errorCode == QNetworkReply::NoError))
856 {
857 LOG(VB_FILE, LOG_DEBUG,
858 LOC + QString("Aborting download - lack of data transfer"));
859 dlInfo->m_reply->abort();
860 }
861 }
862 else if (deleteInfo)
863 {
864 delete dlInfo;
865 }
866
867 m_infoLock->unlock();
868
869 return success;
870}
871
872#ifndef _WIN32
893bool MythDownloadManager::downloadNowLinkLocal(MythDownloadInfo *dlInfo, bool deleteInfo)
894{
895 bool ok = true;
896
897 // No buffer - no reply...
898 if (!dlInfo->m_data)
899 {
900 LOG(VB_GENERAL, LOG_ERR, LOC + QString("No data buffer provided for %1").arg(dlInfo->m_url));
901 ok = false;
902 }
903
904 // Only certain features are supported here
905 if (dlInfo->m_authCallback || dlInfo->m_authArg)
906 {
907 LOG(VB_GENERAL, LOG_ERR, LOC + QString("Unsupported authentication for %1").arg(dlInfo->m_url));
908 ok = false;
909 }
910
911 if (ok && !dlInfo->m_outFile.isEmpty())
912 {
913 LOG(VB_GENERAL, LOG_ERR, LOC + QString("Unsupported File output %1 for %2")
914 .arg(dlInfo->m_outFile, dlInfo->m_url));
915 ok = false;
916 }
917
918 if (ok && (!deleteInfo || dlInfo->m_requestType == kRequestHead))
919 {
920 // We do not have the ability to return a network reply in dlInfo
921 // so if we are asked to do that, return an error.
922 LOG(VB_GENERAL, LOG_ERR, LOC + QString("Unsupported link-local operation %1")
923 .arg(dlInfo->m_url));
924 ok = false;
925 }
926
927 QUrl url(dlInfo->m_url);
928 QString host(url.host());
929 int port(url.port(80));
930 if (ok && PortChecker::resolveLinkLocal(host, port))
931 {
932 QString reqType;
933 switch (dlInfo->m_requestType)
934 {
935 case kRequestPost :
936 reqType = "POST";
937 break;
938 case kRequestGet :
939 default:
940 reqType = "GET";
941 break;
942 }
943 QByteArray* buffer = dlInfo->m_data;
944 QHash<QByteArray, QByteArray> headers;
945 if (dlInfo->m_headers)
946 headers = *dlInfo->m_headers;
947 if (!headers.contains("User-Agent"))
948 headers.insert("User-Agent", QByteArray("MythDownloadManager v") +
949 MYTH_BINARY_VERSION);
950 headers.insert("Connection", "close");
951 headers.insert("Accept-Encoding", "identity");
952 if (!buffer->isEmpty())
953 headers.insert("Content-Length", QString::number(buffer->size()).toUtf8());
954 headers.insert("Host", (url.host() + ":" + QString::number(port)).toUtf8());
955
956 QByteArray requestMessage;
957 QString path (url.path());
958 requestMessage.append("POST ");
959 requestMessage.append(path.toLatin1());
960 requestMessage.append(" HTTP/1.1\r\n");
961 QHashIterator<QByteArray, QByteArray> it(headers);
962 while (it.hasNext())
963 {
964 it.next();
965 requestMessage.append(it.key());
966 requestMessage.append(": ");
967 requestMessage.append(it.value());
968 requestMessage.append("\r\n");
969 }
970 requestMessage.append("\r\n");
971 if (!buffer->isEmpty())
972 requestMessage.append(*buffer);
973
974 QTcpSocket socket;
975 socket.connectToHost(host, static_cast<uint16_t>(port));
976 // QT Warning - this may not work on Windows
977 if (!socket.waitForConnected(5000))
978 ok = false;
979 if (ok)
980 ok = socket.write(requestMessage) > 0;
981 if (ok)
982 // QT Warning - this may not work on Windows
983 ok = socket.waitForDisconnected(5000);
984 if (ok)
985 {
986 *buffer = socket.readAll();
987 // Find the start of the content
988 QByteArray delim("\r\n\r\n");
989 int delimLoc = buffer->indexOf(delim);
990 if (delimLoc > -1)
991 *buffer = buffer->right(buffer->size() - delimLoc - 4);
992 else
993 ok=false;
994 }
995 socket.close();
996 }
997 else
998 {
999 ok = false;
1000 }
1001
1002 if (deleteInfo)
1003 delete dlInfo;
1004
1005 if (ok)
1006 return true;
1007
1008 LOG(VB_GENERAL, LOG_ERR, LOC + QString("Link Local request failed: %1").arg(url.toString()));
1009 return false;
1010}
1011#endif
1012
1017void MythDownloadManager::cancelDownload(const QString &url, bool block)
1018{
1019 cancelDownload(QStringList(url), block);
1020}
1021
1026void MythDownloadManager::cancelDownload(const QStringList &urls, bool block)
1027{
1028 m_infoLock->lock();
1029 for (const auto& url : std::as_const(urls))
1030 {
1031 QMutableListIterator<MythDownloadInfo*> lit(m_downloadQueue);
1032 while (lit.hasNext())
1033 {
1034 lit.next();
1035 MythDownloadInfo *dlInfo = lit.value();
1036 if (dlInfo->m_url == url)
1037 {
1038 if (!m_cancellationQueue.contains(dlInfo))
1039 m_cancellationQueue.append(dlInfo);
1040 lit.remove();
1041 }
1042 }
1043
1044 if (m_downloadInfos.contains(url))
1045 {
1046 MythDownloadInfo *dlInfo = m_downloadInfos[url];
1047
1048 if (!m_cancellationQueue.contains(dlInfo))
1049 m_cancellationQueue.append(dlInfo);
1050
1051 if (dlInfo->m_reply)
1052 m_downloadReplies.remove(dlInfo->m_reply);
1053
1054 m_downloadInfos.remove(url);
1055 }
1056 }
1057 m_infoLock->unlock();
1058
1059 if (QThread::currentThread() == this->thread())
1060 {
1062 return;
1063 }
1064
1065 // wake-up running thread
1066 m_queueWaitCond.wakeAll();
1067
1068 if (!block)
1069 return;
1070
1071 while (!m_cancellationQueue.isEmpty())
1072 {
1073 usleep(50ms); // re-test in another 50ms
1074 }
1075}
1076
1078{
1079 QMutexLocker locker(m_infoLock);
1080
1081 QMutableListIterator<MythDownloadInfo*> lit(m_cancellationQueue);
1082 while (lit.hasNext())
1083 {
1084 lit.next();
1085 MythDownloadInfo *dlInfo = lit.value();
1086 dlInfo->m_lock.lock();
1087
1088 if (dlInfo->m_reply)
1089 {
1090 LOG(VB_FILE, LOG_DEBUG,
1091 LOC + QString("Aborting download - user request"));
1092 dlInfo->m_reply->abort();
1093 }
1094 lit.remove();
1095 if (dlInfo->m_done)
1096 {
1097 dlInfo->m_lock.unlock();
1098 continue;
1099 }
1100 dlInfo->m_errorCode = QNetworkReply::OperationCanceledError;
1101 dlInfo->m_done = true;
1102 dlInfo->m_lock.unlock();
1103 }
1104}
1105
1111{
1112 QMutexLocker locker(m_infoLock);
1113
1114 QList <MythDownloadInfo*>::iterator lit = m_downloadQueue.begin();
1115 for (; lit != m_downloadQueue.end(); ++lit)
1116 {
1117 MythDownloadInfo *dlInfo = *lit;
1118 if (dlInfo->m_caller == caller)
1119 {
1120 dlInfo->m_caller = nullptr;
1121 dlInfo->m_outFile = QString();
1122 dlInfo->m_data = nullptr;
1123 }
1124 }
1125
1126 QMap <QString, MythDownloadInfo*>::iterator mit = m_downloadInfos.begin();
1127 for (; mit != m_downloadInfos.end(); ++mit)
1128 {
1129 MythDownloadInfo *dlInfo = mit.value();
1130 if (dlInfo->m_caller == caller)
1131 {
1132 dlInfo->m_caller = nullptr;
1133 dlInfo->m_outFile = QString();
1134 dlInfo->m_data = nullptr;
1135 }
1136 }
1137}
1138
1142void MythDownloadManager::downloadError(QNetworkReply::NetworkError errorCode)
1143{
1144 auto *reply = qobject_cast<QNetworkReply *>(sender());
1145 if (reply == nullptr)
1146 return;
1147
1148 LOG(VB_FILE, LOG_DEBUG, LOC + QString("downloadError %1 ")
1149 .arg(errorCode) + reply->errorString() );
1150
1151 QMutexLocker locker(m_infoLock);
1152 if (!m_downloadReplies.contains(reply))
1153 {
1154 reply->deleteLater();
1155 return;
1156 }
1157
1158 MythDownloadInfo *dlInfo = m_downloadReplies[reply];
1159
1160 if (!dlInfo)
1161 return;
1162
1163 dlInfo->m_errorCode = errorCode;
1164}
1165
1171QUrl MythDownloadManager::redirectUrl(const QUrl& possibleRedirectUrl,
1172 const QUrl& oldRedirectUrl)
1173{
1174 LOG(VB_FILE, LOG_DEBUG, LOC + QString("redirectUrl()"));
1175 QUrl redirectUrl;
1176
1177 if(!possibleRedirectUrl.isEmpty() && possibleRedirectUrl != oldRedirectUrl)
1178 redirectUrl = possibleRedirectUrl;
1179
1180 return redirectUrl;
1181}
1182
1187{
1188 LOG(VB_FILE, LOG_DEBUG, LOC + QString("downloadFinished(%1)")
1189 .arg((long long)reply));
1190
1191 QMutexLocker locker(m_infoLock);
1192 if (!m_downloadReplies.contains(reply))
1193 {
1194 reply->deleteLater();
1195 return;
1196 }
1197
1198 MythDownloadInfo *dlInfo = m_downloadReplies[reply];
1199
1200 if (!dlInfo || !dlInfo->m_reply)
1201 return;
1202
1203 downloadFinished(dlInfo);
1204}
1205
1210{
1211 if (!dlInfo)
1212 return;
1213
1214 int statusCode = -1;
1215 static const QString kDateFormat = "ddd, dd MMM yyyy hh:mm:ss 'GMT'";
1216 QNetworkReply *reply = dlInfo->m_reply;
1217
1218 if (reply)
1219 {
1220 QUrl possibleRedirectUrl =
1221 reply->attribute(QNetworkRequest::RedirectionTargetAttribute).toUrl();
1222
1223 if (!possibleRedirectUrl.isEmpty() &&
1224 possibleRedirectUrl.isValid() &&
1225 possibleRedirectUrl.isRelative()) // Turn relative Url to absolute
1226 possibleRedirectUrl = reply->url().resolved(possibleRedirectUrl);
1227
1228 if (!possibleRedirectUrl.isEmpty() && dlInfo->m_finalUrl != nullptr)
1229 *dlInfo->m_finalUrl = QString(possibleRedirectUrl.toString());
1230
1231 dlInfo->m_redirectedTo =
1232 redirectUrl(possibleRedirectUrl, dlInfo->m_redirectedTo);
1233
1234 QVariant status =
1235 reply->attribute(QNetworkRequest::HttpStatusCodeAttribute);
1236 if (status.isValid())
1237 statusCode = status.toInt();
1238 }
1239
1240 if(reply && !dlInfo->m_redirectedTo.isEmpty() &&
1241 ((dlInfo->m_requestType != kRequestPost) ||
1242 (statusCode == 301 || statusCode == 302 ||
1243 statusCode == 303)))
1244 {
1245 LOG(VB_FILE, LOG_DEBUG, LOC +
1246 QString("downloadFinished(%1): Redirect: %2 -> %3")
1247 .arg(QString::number((long long)dlInfo),
1248 reply->url().toString(),
1249 dlInfo->m_redirectedTo.toString()));
1250
1251 if (dlInfo->m_data)
1252 dlInfo->m_data->clear();
1253
1254 dlInfo->m_bytesReceived = 0;
1255 dlInfo->m_bytesTotal = 0;
1256
1257 QNetworkRequest request(dlInfo->m_redirectedTo);
1258
1259 if (dlInfo->m_reload)
1260 {
1261 request.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
1262 QNetworkRequest::AlwaysNetwork);
1263 }
1264 else if (dlInfo->m_preferCache)
1265 {
1266 request.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
1267 QNetworkRequest::PreferCache);
1268 }
1269
1270 request.setRawHeader("User-Agent",
1271 "MythDownloadManager v" +
1272 QByteArray(MYTH_BINARY_VERSION));
1273
1274 switch (dlInfo->m_requestType)
1275 {
1276 case kRequestHead :
1277 dlInfo->m_reply = m_manager->head(request);
1278 break;
1279 case kRequestGet :
1280 default:
1281 dlInfo->m_reply = m_manager->get(request);
1282 break;
1283 }
1284
1285 m_downloadReplies[dlInfo->m_reply] = dlInfo;
1286
1287 connect(dlInfo->m_reply, &QNetworkReply::errorOccurred,
1289 connect(dlInfo->m_reply, &QNetworkReply::downloadProgress,
1291
1292 m_downloadReplies.remove(reply);
1293 reply->deleteLater();
1294 }
1295 else
1296 {
1297 LOG(VB_FILE, LOG_DEBUG, QString("downloadFinished(%1): COMPLETE: %2")
1298 .arg((long long)dlInfo).arg(dlInfo->m_url));
1299
1300 // HACK Insert a Date header into the cached metadata if one doesn't
1301 // already exist
1302 QUrl fileUrl = dlInfo->m_url;
1303 QString redirectLoc;
1304 int limit = 0;
1305 while (!(redirectLoc = getHeader(fileUrl, "Location")).isNull())
1306 {
1307 if (limit == CACHE_REDIRECTION_LIMIT)
1308 {
1309 LOG(VB_GENERAL, LOG_WARNING, QString("Cache Redirection limit "
1310 "reached for %1")
1311 .arg(fileUrl.toString()));
1312 return;
1313 }
1314 fileUrl.setUrl(redirectLoc);
1315 limit++;
1316 }
1317
1318 m_infoLock->lock();
1319 QNetworkCacheMetaData urlData = m_manager->cache()->metaData(fileUrl);
1320 m_infoLock->unlock();
1321 if (getHeader(urlData, "Date").isNull())
1322 {
1323 QNetworkCacheMetaData::RawHeaderList headers = urlData.rawHeaders();
1324 QNetworkCacheMetaData::RawHeader newheader;
1325 QDateTime now = MythDate::current();
1326 newheader = QNetworkCacheMetaData::RawHeader("Date",
1327 now.toString(kDateFormat).toLatin1());
1328 headers.append(newheader);
1329 urlData.setRawHeaders(headers);
1330 m_infoLock->lock();
1331 m_manager->cache()->updateMetaData(urlData);
1332 m_infoLock->unlock();
1333 }
1334 // End HACK
1335
1336 dlInfo->m_redirectedTo.clear();
1337
1338 int dataSize = -1;
1339
1340 // If we downloaded via the QNetworkAccessManager
1341 // AND the caller isn't handling the reply directly
1342 if (reply && dlInfo->m_processReply)
1343 {
1344 bool append = (!dlInfo->m_syncMode && dlInfo->m_caller);
1345 QByteArray data = reply->readAll();
1346 dataSize = data.size();
1347
1348 if (append)
1349 dlInfo->m_bytesReceived += dataSize;
1350 else
1351 dlInfo->m_bytesReceived = dataSize;
1352
1353 dlInfo->m_bytesTotal = dlInfo->m_bytesReceived;
1354
1355 if (dlInfo->m_data)
1356 {
1357 if (append)
1358 dlInfo->m_data->append(data);
1359 else
1360 *dlInfo->m_data = data;
1361 }
1362 else if (!dlInfo->m_outFile.isEmpty())
1363 {
1364 saveFile(dlInfo->m_outFile, data, append);
1365 }
1366 }
1367 else if (!reply) // If we downloaded via RemoteFile
1368 {
1369 if (dlInfo->m_data)
1370 {
1371 (*dlInfo->m_data) = dlInfo->m_privData;
1372 }
1373 else if (!dlInfo->m_outFile.isEmpty())
1374 {
1375 saveFile(dlInfo->m_outFile, dlInfo->m_privData);
1376 }
1377 dlInfo->m_bytesReceived += dataSize;
1378 dlInfo->m_bytesTotal = dlInfo->m_bytesReceived;
1379 }
1380 // else we downloaded via QNetworkAccessManager
1381 // AND the caller is handling the reply
1382
1383 m_infoLock->lock();
1384 if (!m_downloadInfos.remove(dlInfo->m_url))
1385 {
1386 LOG(VB_GENERAL, LOG_ERR, LOC +
1387 QString("ERROR download finished but failed to remove url: %1")
1388 .arg(dlInfo->m_url));
1389 }
1390
1391 if (reply)
1392 m_downloadReplies.remove(reply);
1393 m_infoLock->unlock();
1394
1395 dlInfo->SetDone(true);
1396
1397 if (!dlInfo->m_syncMode)
1398 {
1399 if (dlInfo->m_caller)
1400 {
1401 LOG(VB_FILE, LOG_DEBUG, QString("downloadFinished(%1): "
1402 "COMPLETE: %2, sending event to caller")
1403 .arg((long long)dlInfo).arg(dlInfo->m_url));
1404
1405 QStringList args;
1406 args << dlInfo->m_url;
1407 args << dlInfo->m_outFile;
1408 args << QString::number(dlInfo->m_bytesTotal);
1409 // placeholder for error string
1410 args << (reply ? reply->errorString() : QString());
1411 args << QString::number((int)(reply ? reply->error() :
1412 dlInfo->m_errorCode));
1413
1414 QCoreApplication::postEvent(dlInfo->m_caller,
1415 new MythEvent("DOWNLOAD_FILE FINISHED", args));
1416 }
1417
1418 delete dlInfo;
1419 }
1420
1421 m_queueWaitCond.wakeAll();
1422 }
1423}
1424
1431 qint64 bytesTotal)
1432{
1433 auto *reply = qobject_cast<QNetworkReply *>(sender());
1434 if (reply == nullptr)
1435 return;
1436
1437 LOG(VB_FILE, LOG_DEBUG, LOC +
1438 QString("downloadProgress(%1, %2) (for reply %3)")
1439 .arg(bytesReceived).arg(bytesTotal).arg((long long)reply));
1440
1441 QMutexLocker locker(m_infoLock);
1442 if (!m_downloadReplies.contains(reply))
1443 return;
1444
1445 MythDownloadInfo *dlInfo = m_downloadReplies[reply];
1446
1447 if (!dlInfo)
1448 return;
1449
1450 dlInfo->m_lastStat = MythDate::current();
1451
1452 LOG(VB_FILE, LOG_DEBUG, LOC +
1453 QString("downloadProgress: %1 to %2 is at %3 of %4 bytes downloaded")
1454 .arg(dlInfo->m_url, dlInfo->m_outFile)
1455 .arg(bytesReceived).arg(bytesTotal));
1456
1457 if (!dlInfo->m_syncMode && dlInfo->m_caller)
1458 {
1459 LOG(VB_FILE, LOG_DEBUG, QString("downloadProgress(%1): "
1460 "sending event to caller")
1461 .arg(reply->url().toString()));
1462
1463 bool appendToFile = (dlInfo->m_bytesReceived != 0);
1464 QByteArray data = reply->readAll();
1465 if (!dlInfo->m_outFile.isEmpty())
1466 saveFile(dlInfo->m_outFile, data, appendToFile);
1467
1468 if (dlInfo->m_data)
1469 dlInfo->m_data->append(data);
1470
1471 dlInfo->m_bytesReceived = bytesReceived;
1472 dlInfo->m_bytesTotal = bytesTotal;
1473
1474 QStringList args;
1475 args << dlInfo->m_url;
1476 args << dlInfo->m_outFile;
1477 args << QString::number(bytesReceived);
1478 args << QString::number(bytesTotal);
1479
1480 QCoreApplication::postEvent(dlInfo->m_caller,
1481 new MythEvent("DOWNLOAD_FILE UPDATE", args));
1482 }
1483}
1484
1492bool MythDownloadManager::saveFile(const QString &outFile,
1493 const QByteArray &data,
1494 const bool append)
1495{
1496 if (outFile.isEmpty() || data.isEmpty())
1497 return false;
1498
1499 QFile file(outFile);
1500 QFileInfo fileInfo(outFile);
1501 QDir qdir(fileInfo.absolutePath());
1502
1503 if (!qdir.exists() && !qdir.mkpath(fileInfo.absolutePath()))
1504 {
1505 LOG(VB_GENERAL, LOG_ERR, QString("Failed to create: '%1'")
1506 .arg(fileInfo.absolutePath()));
1507 return false;
1508 }
1509
1510 QIODevice::OpenMode mode = QIODevice::Unbuffered|QIODevice::WriteOnly;
1511 if (append)
1512 mode |= QIODevice::Append;
1513
1514 if (!file.open(mode))
1515 {
1516 LOG(VB_GENERAL, LOG_ERR, QString("Failed to open: '%1'") .arg(outFile));
1517 return false;
1518 }
1519
1520 off_t offset = 0;
1521 size_t remaining = data.size();
1522 uint failure_cnt = 0;
1523 while ((remaining > 0) && (failure_cnt < 5))
1524 {
1525 ssize_t written = file.write(data.data() + offset, remaining);
1526 if (written < 0)
1527 {
1528 failure_cnt++;
1529 usleep(50ms);
1530 continue;
1531 }
1532
1533 failure_cnt = 0;
1534 offset += written;
1535 remaining -= written;
1536 }
1537
1538 return remaining <= 0;
1539}
1540
1545QDateTime MythDownloadManager::GetLastModified(const QString &url)
1546{
1547 // If the header has not expired and
1548 // the last modification date is less than 1 hours old or if
1549 // the cache object is less than 20 minutes old,
1550 // then use the cached header otherwise redownload the header
1551
1552 static const QString kDateFormat = "ddd, dd MMM yyyy hh:mm:ss 'GMT'";
1553 LOG(VB_FILE, LOG_DEBUG, LOC + QString("GetLastModified('%1')").arg(url));
1554 QDateTime result;
1555
1556 QDateTime now = MythDate::current();
1557
1558 QUrl cacheUrl = QUrl(url);
1559
1560 // Deal with redirects, we want the cached data for the final url
1561 QString redirectLoc;
1562 int limit = 0;
1563 while (!(redirectLoc = getHeader(cacheUrl, "Location")).isNull())
1564 {
1565 if (limit == CACHE_REDIRECTION_LIMIT)
1566 {
1567 LOG(VB_GENERAL, LOG_WARNING, QString("Cache Redirection limit "
1568 "reached for %1")
1569 .arg(cacheUrl.toString()));
1570 return result;
1571 }
1572 cacheUrl.setUrl(redirectLoc);
1573 limit++;
1574 }
1575
1576 m_infoLock->lock();
1577 QNetworkCacheMetaData urlData = m_manager->cache()->metaData(cacheUrl);
1578 m_infoLock->unlock();
1579
1580 if (urlData.isValid() &&
1581 ((!urlData.expirationDate().isValid()) ||
1582 (urlData.expirationDate().secsTo(now) < 0)))
1583 {
1584 if (urlData.lastModified().toUTC().secsTo(now) <= 3600) // 1 Hour
1585 {
1586 result = urlData.lastModified().toUTC();
1587 }
1588 else
1589 {
1590 QString date = getHeader(urlData, "Date");
1591 if (!date.isNull())
1592 {
1593 QDateTime loadDate =
1594 MythDate::fromString(date, kDateFormat);
1595#if QT_VERSION < QT_VERSION_CHECK(6,5,0)
1596 loadDate.setTimeSpec(Qt::UTC);
1597#else
1598 loadDate.setTimeZone(QTimeZone(QTimeZone::UTC));
1599#endif
1600 if (loadDate.secsTo(now) <= 1200) // 20 Minutes
1601 {
1602 result = urlData.lastModified().toUTC();
1603 }
1604 }
1605 }
1606 }
1607
1608 if (!result.isValid())
1609 {
1610 auto *dlInfo = new MythDownloadInfo;
1611 dlInfo->m_url = url;
1612 dlInfo->m_syncMode = true;
1613 // Head request, we only want to inspect the headers
1614 dlInfo->m_requestType = kRequestHead;
1615
1616 if (downloadNow(dlInfo, false))
1617 {
1618 if (dlInfo->m_reply)
1619 {
1620 QVariant lastMod =
1621 dlInfo->m_reply->header(
1622 QNetworkRequest::LastModifiedHeader);
1623 if (lastMod.isValid())
1624 result = lastMod.toDateTime().toUTC();
1625 }
1626
1627 // downloadNow() will set a flag to trigger downloadFinished()
1628 // to delete the dlInfo if the download times out
1629 delete dlInfo;
1630 }
1631 }
1632
1633 LOG(VB_FILE, LOG_DEBUG, LOC + QString("GetLastModified('%1'): Result %2")
1634 .arg(url, result.toString()));
1635
1636 return result;
1637}
1638
1639
1644{
1645 QMutexLocker locker(&m_cookieLock);
1646
1647 auto *jar = new MythCookieJar;
1648 jar->load(filename);
1649 m_manager->setCookieJar(jar);
1650}
1651
1656{
1657 QMutexLocker locker(&m_cookieLock);
1658
1659 if (!m_manager->cookieJar())
1660 return;
1661
1662 auto *jar = qobject_cast<MythCookieJar *>(m_manager->cookieJar());
1663 if (jar == nullptr)
1664 return;
1665 jar->save(filename);
1666}
1667
1668void MythDownloadManager::setCookieJar(QNetworkCookieJar *cookieJar)
1669{
1670 QMutexLocker locker(&m_cookieLock);
1671 m_manager->setCookieJar(cookieJar);
1672}
1673
1678{
1679 QMutexLocker locker(&m_cookieLock);
1680
1681 if (!m_manager->cookieJar())
1682 return nullptr;
1683
1684 auto *inJar = qobject_cast<MythCookieJar *>(m_manager->cookieJar());
1685 if (inJar == nullptr)
1686 return nullptr;
1687 auto *outJar = new MythCookieJar;
1688 outJar->copyAllCookies(*inJar);
1689
1690 return outJar;
1691}
1692
1696void MythDownloadManager::refreshCookieJar(QNetworkCookieJar *jar)
1697{
1698 QMutexLocker locker(&m_cookieLock);
1699 delete m_inCookieJar;
1700
1701 auto *inJar = qobject_cast<MythCookieJar *>(jar);
1702 if (inJar == nullptr)
1703 return;
1704
1705 auto *outJar = new MythCookieJar;
1706 outJar->copyAllCookies(*inJar);
1707 m_inCookieJar = outJar;
1708
1709 QMutexLocker locker2(&m_queueWaitLock);
1710 m_queueWaitCond.wakeAll();
1711}
1712
1716{
1717 QMutexLocker locker(&m_cookieLock);
1718
1719 auto *inJar = qobject_cast<MythCookieJar *>(m_inCookieJar);
1720 if (inJar != nullptr)
1721 {
1722 auto *outJar = new MythCookieJar;
1723 outJar->copyAllCookies(*inJar);
1724 m_manager->setCookieJar(outJar);
1725 }
1726
1727 delete m_inCookieJar;
1728 m_inCookieJar = nullptr;
1729}
1730
1731QString MythDownloadManager::getHeader(const QUrl& url, const QString& header)
1732{
1733 if (!m_manager || !m_manager->cache())
1734 return {};
1735
1736 m_infoLock->lock();
1737 QNetworkCacheMetaData metadata = m_manager->cache()->metaData(url);
1738 m_infoLock->unlock();
1739
1740 return getHeader(metadata, header);
1741}
1742
1748QString MythDownloadManager::getHeader(const QNetworkCacheMetaData &cacheData,
1749 const QString& header)
1750{
1751 auto headers = cacheData.rawHeaders();
1752 for (const auto& rh : std::as_const(headers))
1753 if (QString(rh.first) == header)
1754 return {rh.second};
1755 return {};
1756}
1757
1758
1763{
1764 const QList<QNetworkCookie> cookieList = old.allCookies();
1765 setAllCookies(cookieList);
1766}
1767
1771void MythCookieJar::load(const QString &filename)
1772{
1773 LOG(VB_GENERAL, LOG_DEBUG, QString("MythCookieJar: loading cookies from: %1").arg(filename));
1774
1775 QFile f(filename);
1776 if (!f.open(QIODevice::ReadOnly))
1777 {
1778 LOG(VB_GENERAL, LOG_WARNING, QString("MythCookieJar::load() failed to open file for reading: %1").arg(filename));
1779 return;
1780 }
1781
1782 QList<QNetworkCookie> cookieList;
1783 QTextStream stream(&f);
1784 while (!stream.atEnd())
1785 {
1786 QString cookie = stream.readLine();
1787 cookieList << QNetworkCookie::parseCookies(cookie.toLocal8Bit());
1788 }
1789
1790 setAllCookies(cookieList);
1791}
1792
1796void MythCookieJar::save(const QString &filename)
1797{
1798 LOG(VB_GENERAL, LOG_DEBUG, QString("MythCookieJar: saving cookies to: %1").arg(filename));
1799
1800 QFile f(filename);
1801 if (!f.open(QIODevice::WriteOnly))
1802 {
1803 LOG(VB_GENERAL, LOG_ERR, QString("MythCookieJar::save() failed to open file for writing: %1").arg(filename));
1804 return;
1805 }
1806
1807 QList<QNetworkCookie> cookieList = allCookies();
1808 QTextStream stream(&f);
1809
1810 for (const auto& cookie : std::as_const(cookieList))
1811 stream << cookie.toRawForm() << Qt::endl;
1812}
1813
1814
1815/* vim: set expandtab tabstop=4 shiftwidth=4: */
static MThreadPool * globalInstance(void)
void start(QRunnable *runnable, const QString &debugName, int priority=0)
void RunProlog(void)
Sets up a thread, call this if you reimplement run().
Definition: mthread.cpp:196
static void usleep(std::chrono::microseconds time)
Definition: mthread.cpp:335
void RunEpilog(void)
Cleans up a thread's resources, call this if you reimplement run().
Definition: mthread.cpp:209
bool wait(std::chrono::milliseconds time=std::chrono::milliseconds::max())
Wait for the MThread to exit, with a maximum timeout.
Definition: mthread.cpp:300
A subclassed QNetworkCookieJar that allows for reading and writing cookie files that contain raw form...
void load(const QString &filename)
Loads the cookie jar from a cookie file.
void save(const QString &filename)
Saves the cookie jar to a cookie file.
void copyAllCookies(MythCookieJar &old)
Copies all cookies from one MythCookieJar to another.
QString GetHostName(void)
const QHash< QByteArray, QByteArray > * m_headers
QNetworkReply::NetworkError m_errorCode
AuthCallback m_authCallback
MRequestType m_requestType
void SetDone(bool done)
QNetworkRequest * m_request
QNetworkReply * m_reply
QNetworkCookieJar * copyCookieJar(void)
Copy from one cookie jar to another.
void preCache(const QString &url)
Downloads a URL but doesn't store the resulting data anywhere.
QMap< QString, MythDownloadInfo * > m_downloadInfos
void downloadRemoteFile(MythDownloadInfo *dlInfo)
Triggers a myth:// URI download in the background via RemoteFile.
void queueItem(const QString &url, QNetworkRequest *req, const QString &dest, QByteArray *data, QObject *caller, MRequestType reqType=kRequestGet, bool reload=false)
Adds a request to the download queue.
~MythDownloadManager() override
Destructor for MythDownloadManager.
bool postAuth(const QString &url, QByteArray *data, AuthCallback authCallback, void *authArg, const QHash< QByteArray, QByteArray > *headers=nullptr)
Posts data to a url via the QNetworkAccessManager.
QNetworkCookieJar * m_inCookieJar
QNetworkDiskCache * m_diskCache
static bool saveFile(const QString &outFile, const QByteArray &data, bool append=false)
Saves a QByteArray of data to a given filename.
QWaitCondition m_queueWaitCond
bool downloadNow(MythDownloadInfo *dlInfo, bool deleteInfo=true)
Download helper for download() blocking methods.
QString getHeader(const QUrl &url, const QString &header)
void downloadQNetworkRequest(MythDownloadInfo *dlInfo)
Downloads a QNetworkRequest via the QNetworkAccessManager.
void queuePost(const QString &url, QByteArray *data, QObject *caller)
Queues a post to a URL via the QNetworkAccessManager.
void loadCookieJar(const QString &filename)
Loads the cookie jar from a cookie file.
void downloadProgress(qint64 bytesReceived, qint64 bytesTotal)
Slot to process download update events.
void updateCookieJar(void)
Update the cookie jar from the temporary cookie jar.
bool post(const QString &url, QByteArray *data)
Posts data to a url via the QNetworkAccessManager.
bool processItem(const QString &url, QNetworkRequest *req, const QString &dest, QByteArray *data, MRequestType reqType=kRequestGet, bool reload=false, AuthCallback authCallback=nullptr, void *authArg=nullptr, const QHash< QByteArray, QByteArray > *headers=nullptr, QString *finalUrl=nullptr)
Processes a network request immediately and waits for a response.
QList< MythDownloadInfo * > m_cancellationQueue
QRecursiveMutex * m_infoLock
static QUrl redirectUrl(const QUrl &possibleRedirectUrl, const QUrl &oldRedirectUrl)
Checks whether we were redirected to the given URL.
friend class RemoteFileDownloadThread
QMap< QNetworkReply *, MythDownloadInfo * > m_downloadReplies
void authCallback(QNetworkReply *reply, QAuthenticator *authenticator)
Signal handler for authentication requests.
QList< MythDownloadInfo * > m_downloadQueue
void saveCookieJar(const QString &filename)
Saves the cookie jar to a cookie file.
bool downloadAuth(const QString &url, const QString &dest, bool reload=false, AuthCallback authCallback=nullptr, void *authArg=nullptr, const QHash< QByteArray, QByteArray > *headers=nullptr)
Downloads a URL to a file in blocking mode.
QNetworkAccessManager * m_manager
void setCookieJar(QNetworkCookieJar *cookieJar)
void run(void) override
Runs a loop to process incoming download requests and triggers download events to be processed.
void cancelDownload(const QString &url, bool block=true)
Cancel a queued or current download.
void removeListener(QObject *caller)
Disconnects the specified caller from any existing MythDownloadInfo instances.
void refreshCookieJar(QNetworkCookieJar *jar)
Refresh the temporary cookie jar from another cookie jar.
void queueDownload(const QString &url, const QString &dest, QObject *caller, bool reload=false)
Adds a url to the download queue.
QDateTime GetLastModified(const QString &url)
Gets the Last Modified timestamp for a URI.
void downloadError(QNetworkReply::NetworkError errorCode)
Slot to process download error events.
void downloadFinished(QNetworkReply *reply)
Slot to process download finished events.
bool download(const QString &url, const QString &dest, bool reload=false)
Downloads a URL to a file in blocking mode.
This class is used as a container for messages.
Definition: mythevent.h:17
static bool resolveLinkLocal(QString &host, int port, std::chrono::milliseconds timeLimit=30s)
Convenience method to resolve link-local address.
MythDownloadManager * m_parent
RemoteFileDownloadThread(MythDownloadManager *parent, MythDownloadInfo *dlInfo)
unsigned int uint
Definition: freesurround.h:24
unsigned short uint16_t
Definition: iso6937tables.h:3
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
QString GetConfDir(void)
Definition: mythdirs.cpp:263
#define LOC
QMutex dmCreateLock
static constexpr int CACHE_REDIRECTION_LIMIT
void ShutdownMythDownloadManager(void)
Deletes the running MythDownloadManager at program exit.
MythDownloadManager * GetMythDownloadManager(void)
Gets the pointer to the MythDownloadManager singleton.
MythDownloadManager * downloadManager
MRequestType
@ kRequestPost
@ kRequestHead
@ kRequestGet
void(*)(QNetworkReply *, QAuthenticator *, void *) AuthCallback
#define off_t
#define LOG(_MASK_, _LEVEL_, _QSTRING_)
Definition: mythlogging.h:39
std::chrono::seconds secsInPast(const QDateTime &past)
Definition: mythdate.cpp:212
QDateTime fromString(const QString &dtstr)
Converts kFilename && kISODate formats to QDateTime.
Definition: mythdate.cpp:39
QDateTime current(bool stripped)
Returns current Date and Time in UTC.
Definition: mythdate.cpp:15