MythTV  master
mythdownloadmanager.cpp
Go to the documentation of this file.
1 // qt
2 #include <QCoreApplication>
3 #include <QRunnable>
4 #include <QString>
5 #include <QByteArray>
6 #include <QFile>
7 #include <QDir>
8 #include <QNetworkCookie>
9 #include <QAuthenticator>
10 #include <QTextStream>
11 #include <QNetworkProxy>
12 #include <QMutexLocker>
13 #include <QUrl>
14 #include <QTcpSocket>
15 
16 #include <cstdlib>
17 #include <unistd.h> // for usleep()
18 
19 // libmythbase
20 #include "compat.h"
21 #include "mythcorecontext.h"
22 #include "mythcoreutil.h"
23 #include "mthreadpool.h"
24 #include "mythdirs.h"
25 #include "mythevent.h"
26 #include "mythversion.h"
27 #include "remotefile.h"
28 #include "mythdate.h"
29 
30 #include "mythdownloadmanager.h"
31 #include "mythlogging.h"
32 #include "portchecker.h"
33 
34 #define LOC QString("DownloadManager: ")
35 #define CACHE_REDIRECTION_LIMIT 10
36 
38 QMutex dmCreateLock;
39 
44 {
45  public:
48  {
49  qRegisterMetaType<QNetworkReply::NetworkError>("QNetworkReply::NetworkError");
50  }
51 
53  {
54  delete m_request;
55  if (m_reply && m_processReply)
56  m_reply->deleteLater();
57  }
58 
59  bool IsDone(void)
60  {
61  QMutexLocker lock(&m_lock);
62  return m_done;
63  }
64 
65  void SetDone(bool done)
66  {
67  QMutexLocker lock(&m_lock);
68  m_done = done;
69  }
70 
71  QString m_url;
73  QString *m_finalUrl {nullptr};
74  QNetworkRequest *m_request {nullptr};
75  QNetworkReply *m_reply {nullptr};
76  QString m_outFile;
77  QByteArray *m_data {nullptr};
78  QByteArray m_privData;
79  QObject *m_caller {nullptr};
81  bool m_reload {false};
82  bool m_preferCache {false};
83  bool m_syncMode {false};
84  bool m_processReply {true};
85  bool m_done {false};
86  qint64 m_bytesReceived {0};
87  qint64 m_bytesTotal {0};
88  QDateTime m_lastStat;
90  void *m_authArg {nullptr};
91  const QHash<QByteArray, QByteArray> *m_headers {nullptr};
92 
93  QNetworkReply::NetworkError m_errorCode {QNetworkReply::NoError};
94  QMutex m_lock;
95 };
96 
97 
101 class RemoteFileDownloadThread : public QRunnable
102 {
103  public:
105  MythDownloadInfo *dlInfo) :
106  m_parent(parent),
107  m_dlInfo(dlInfo) {}
108 
109  void run() override // QRunnable
110  {
111  bool ok = false;
112 
113  auto *rf = new RemoteFile(m_dlInfo->m_url, false, false, 0);
114  ok = rf->SaveAs(m_dlInfo->m_privData);
115  delete rf;
116 
117  if (!ok)
118  m_dlInfo->m_errorCode = QNetworkReply::UnknownNetworkError;
119 
122 
124  }
125 
126  private:
129 };
130 
134 {
135  if (downloadManager)
136  {
137  delete downloadManager;
138  downloadManager = nullptr;
139  }
140 }
141 
146 {
147  if (downloadManager)
148  return downloadManager;
149 
150  QMutexLocker locker(&dmCreateLock);
151 
152  // Check once more in case the download manager was created
153  // while we were securing the lock.
154  if (downloadManager)
155  return downloadManager;
156 
157  auto *tmpDLM = new MythDownloadManager();
158  tmpDLM->start();
159  while (!tmpDLM->getQueueThread())
160  usleep(10000);
161 
162  tmpDLM->moveToThread(tmpDLM->getQueueThread());
163  tmpDLM->setRunThread();
164 
165  while (!tmpDLM->isRunning())
166  usleep(10000);
167 
168  downloadManager = tmpDLM;
169 
171 
172  return downloadManager;
173 }
174 
178 {
179  m_runThread = false;
180  m_queueWaitCond.wakeAll();
181 
182  wait();
183 
184  delete m_infoLock;
185  delete m_inCookieJar;
186 }
187 
192 {
193  RunProlog();
194 
195  bool downloading = false;
196  bool itemsInQueue = false;
197  bool itemsInCancellationQueue = false;
198  bool waitAnyway = false;
199 
200  m_queueThread = QThread::currentThread();
201 
202  while (!m_runThread)
203  usleep(50000);
204 
205  m_manager = new QNetworkAccessManager(this);
206  m_diskCache = new QNetworkDiskCache(this);
207  m_proxy = new QNetworkProxy();
208  m_diskCache->setCacheDirectory(GetConfDir() + "/cache/" +
209  QCoreApplication::applicationName() + "-" +
211  m_manager->setCache(m_diskCache);
212 
213  // Set the proxy for the manager to be the application default proxy,
214  // which has already been setup
215  m_manager->setProxy(*m_proxy);
216 
217  // make sure the cookieJar is created in the same thread as the manager
218  // and set its parent to nullptr so it can be shared between managers
219  m_manager->cookieJar()->setParent(nullptr);
220 
221  QObject::connect(m_manager, &QNetworkAccessManager::finished,
222  this, qOverload<QNetworkReply*>(&MythDownloadManager::downloadFinished));
223 
224  m_isRunning = true;
225  while (m_runThread)
226  {
227  if (m_inCookieJar)
228  {
229  LOG(VB_GENERAL, LOG_DEBUG, "Updating DLManager's Cookie Jar");
230  updateCookieJar();
231  }
232  m_infoLock->lock();
233  LOG(VB_FILE, LOG_DEBUG, LOC + QString("items downloading %1").arg(m_downloadInfos.count()));
234  LOG(VB_FILE, LOG_DEBUG, LOC + QString("items queued %1").arg(m_downloadQueue.count()));
235  downloading = !m_downloadInfos.isEmpty();
236  itemsInCancellationQueue = !m_cancellationQueue.isEmpty();
237  m_infoLock->unlock();
238 
239  if (itemsInCancellationQueue)
240  {
242  }
243  if (downloading)
244  QCoreApplication::processEvents();
245 
246  m_infoLock->lock();
247  itemsInQueue = !m_downloadQueue.isEmpty();
248  m_infoLock->unlock();
249 
250  if (!itemsInQueue || waitAnyway)
251  {
252  waitAnyway = false;
253  m_queueWaitLock.lock();
254 
255  if (downloading)
256  {
257  LOG(VB_FILE, LOG_DEBUG, LOC + QString("waiting 200ms"));
258  m_queueWaitCond.wait(&m_queueWaitLock, 200);
259  }
260  else
261  {
262  LOG(VB_FILE, LOG_DEBUG, LOC + QString("waiting for more items to download"));
264  }
265 
266  m_queueWaitLock.unlock();
267  }
268 
269  m_infoLock->lock();
270  if (!m_downloadQueue.isEmpty())
271  {
272  MythDownloadInfo *dlInfo = m_downloadQueue.front();
273 
274  m_downloadQueue.pop_front();
275 
276  if (!dlInfo)
277  {
278  m_infoLock->unlock();
279  continue;
280  }
281 
282  if (m_downloadInfos.contains(dlInfo->m_url))
283  {
284  // Push request to the end of the queue to let others process.
285  // If this is the only item in the queue, force the loop to
286  // wait a little.
287  if (m_downloadQueue.isEmpty())
288  waitAnyway = true;
289  m_downloadQueue.push_back(dlInfo);
290  m_infoLock->unlock();
291  continue;
292  }
293 
294  if (dlInfo->m_url.startsWith("myth://"))
295  downloadRemoteFile(dlInfo);
296  else
297  {
298  QMutexLocker cLock(&m_cookieLock);
299  downloadQNetworkRequest(dlInfo);
300  }
301 
302  m_downloadInfos[dlInfo->m_url] = dlInfo;
303  }
304  m_infoLock->unlock();
305  }
306  m_isRunning = false;
307 
308  RunEpilog();
309 }
310 
321 void MythDownloadManager::queueItem(const QString &url, QNetworkRequest *req,
322  const QString &dest, QByteArray *data,
323  QObject *caller, const MRequestType reqType,
324  const bool reload)
325 {
326  auto *dlInfo = new MythDownloadInfo;
327 
328  dlInfo->m_url = url;
329  dlInfo->m_request = req;
330  dlInfo->m_outFile = dest;
331  dlInfo->m_data = data;
332  dlInfo->m_caller = caller;
333  dlInfo->m_requestType = reqType;
334  dlInfo->m_reload = reload;
335 
336  QMutexLocker locker(m_infoLock);
337  m_downloadQueue.push_back(dlInfo);
338  m_queueWaitCond.wakeAll();
339 }
340 
353 bool MythDownloadManager::processItem(const QString &url, QNetworkRequest *req,
354  const QString &dest, QByteArray *data,
355  const MRequestType reqType,
356  const bool reload,
357  AuthCallback authCallbackFn, void *authArg,
358  const QHash<QByteArray, QByteArray> *headers,
359  QString *finalUrl)
360 {
361  auto *dlInfo = new MythDownloadInfo;
362 
363  dlInfo->m_url = url;
364  dlInfo->m_request = req;
365  dlInfo->m_outFile = dest;
366  dlInfo->m_data = data;
367  dlInfo->m_requestType = reqType;
368  dlInfo->m_reload = reload;
369  dlInfo->m_syncMode = true;
370  dlInfo->m_authCallback = authCallbackFn;
371  dlInfo->m_authArg = authArg;
372  dlInfo->m_headers = headers;
373  dlInfo->m_finalUrl = finalUrl;
374 
375  return downloadNow(dlInfo, true);
376 }
377 
381 void MythDownloadManager::preCache(const QString &url)
382 {
383  LOG(VB_FILE, LOG_DEBUG, LOC + QString("preCache('%1')").arg(url));
384  queueItem(url, nullptr, QString(), nullptr, nullptr);
385 }
386 
393 void MythDownloadManager::queueDownload(const QString &url,
394  const QString &dest,
395  QObject *caller,
396  const bool reload)
397 {
398  LOG(VB_FILE, LOG_DEBUG, LOC + QString("queueDownload('%1', '%2', %3)")
399  .arg(url).arg(dest).arg((long long)caller));
400 
401  queueItem(url, nullptr, dest, nullptr, caller, kRequestGet, reload);
402 }
403 
409 void MythDownloadManager::queueDownload(QNetworkRequest *req,
410  QByteArray *data,
411  QObject *caller)
412 {
413  LOG(VB_FILE, LOG_DEBUG, LOC + QString("queueDownload('%1', '%2', %3)")
414  .arg(req->url().toString()).arg((long long)data)
415  .arg((long long)caller));
416 
417  queueItem(req->url().toString(), req, QString(), data, caller,
418  kRequestGet,
419  (QNetworkRequest::AlwaysNetwork == req->attribute(
420  QNetworkRequest::CacheLoadControlAttribute,
421  QNetworkRequest::PreferNetwork).toInt()));
422 }
423 
430 bool MythDownloadManager::download(const QString &url, const QString &dest,
431  const bool reload)
432 {
433  return processItem(url, nullptr, dest, nullptr, kRequestGet, reload);
434 }
435 
442 bool MythDownloadManager::download(const QString &url, QByteArray *data,
443  const bool reload, QString *finalUrl)
444 {
445  QString redirected;
446  if (!processItem(url, nullptr, QString(), data, kRequestGet, reload,
447  nullptr, nullptr, nullptr, &redirected))
448  return false;
449  if (!redirected.isEmpty() && finalUrl != nullptr)
450  *finalUrl = redirected;
451  return true;
452 }
453 
460 QNetworkReply *MythDownloadManager::download(const QString &url,
461  const bool reload)
462 {
463  auto *dlInfo = new MythDownloadInfo;
464  QNetworkReply *reply = nullptr;
465 
466  dlInfo->m_url = url;
467  dlInfo->m_reload = reload;
468  dlInfo->m_syncMode = true;
469  dlInfo->m_processReply = false;
470 
471  if (downloadNow(dlInfo, false))
472  {
473  if (dlInfo->m_reply)
474  {
475  reply = dlInfo->m_reply;
476  // prevent dlInfo dtor from deleting the reply
477  dlInfo->m_reply = nullptr;
478 
479  delete dlInfo;
480 
481  return reply;
482  }
483 
484  delete dlInfo;
485  }
486 
487  return nullptr;
488 }
489 
495 bool MythDownloadManager::download(QNetworkRequest *req, QByteArray *data)
496 {
497  LOG(VB_FILE, LOG_DEBUG, LOC + QString("download('%1', '%2')")
498  .arg(req->url().toString()).arg((long long)data));
499  return processItem(req->url().toString(), req, QString(), data,
500  kRequestGet,
501  (QNetworkRequest::AlwaysNetwork == req->attribute(
502  QNetworkRequest::CacheLoadControlAttribute,
503  QNetworkRequest::PreferNetwork).toInt()));
504 }
505 
515 bool MythDownloadManager::downloadAuth(const QString &url, const QString &dest,
516  const bool reload, AuthCallback authCallbackFn, void *authArg,
517  const QHash<QByteArray, QByteArray> *headers)
518 {
519  return processItem(url, nullptr, dest, nullptr, kRequestGet, reload, authCallbackFn,
520  authArg, headers);
521 }
522 
523 
529 void MythDownloadManager::queuePost(const QString &url,
530  QByteArray *data,
531  QObject *caller)
532 {
533  LOG(VB_FILE, LOG_DEBUG, LOC + QString("queuePost('%1', '%2')")
534  .arg(url).arg((long long)data));
535 
536  if (!data)
537  {
538  LOG(VB_GENERAL, LOG_ERR, LOC + "queuePost(), data is NULL!");
539  return;
540  }
541 
542  queueItem(url, nullptr, QString(), data, caller, kRequestPost);
543 }
544 
550 void MythDownloadManager::queuePost(QNetworkRequest *req,
551  QByteArray *data,
552  QObject *caller)
553 {
554  LOG(VB_FILE, LOG_DEBUG, LOC + QString("queuePost('%1', '%2')")
555  .arg(req->url().toString()).arg((long long)data));
556 
557  if (!data)
558  {
559  LOG(VB_GENERAL, LOG_ERR, LOC + "queuePost(), data is NULL!");
560  return;
561  }
562 
563  queueItem(req->url().toString(), req, QString(), data, caller,
564  kRequestPost,
565  (QNetworkRequest::AlwaysNetwork == req->attribute(
566  QNetworkRequest::CacheLoadControlAttribute,
567  QNetworkRequest::PreferNetwork).toInt()));
568 
569 }
570 
576 bool MythDownloadManager::post(const QString &url, QByteArray *data)
577 {
578  LOG(VB_FILE, LOG_DEBUG, LOC + QString("post('%1', '%2')")
579  .arg(url).arg((long long)data));
580 
581  if (!data)
582  {
583  LOG(VB_GENERAL, LOG_ERR, LOC + "post(), data is NULL!");
584  return false;
585  }
586 
587  return processItem(url, nullptr, QString(), data, kRequestPost);
588 }
589 
595 bool MythDownloadManager::post(QNetworkRequest *req, QByteArray *data)
596 {
597  LOG(VB_FILE, LOG_DEBUG, LOC + QString("post('%1', '%2')")
598  .arg(req->url().toString()).arg((long long)data));
599 
600  if (!data)
601  {
602  LOG(VB_GENERAL, LOG_ERR, LOC + "post(), data is NULL!");
603  return false;
604  }
605 
606  return processItem(req->url().toString(), req, QString(), data,
607  kRequestPost,
608  (QNetworkRequest::AlwaysNetwork == req->attribute(
609  QNetworkRequest::CacheLoadControlAttribute,
610  QNetworkRequest::PreferNetwork).toInt()));
611 
612 }
613 
622 bool MythDownloadManager::postAuth(const QString &url, QByteArray *data,
623  AuthCallback authCallbackFn, void *authArg,
624  const QHash<QByteArray, QByteArray> *headers)
625 {
626  LOG(VB_FILE, LOG_DEBUG, LOC + QString("postAuth('%1', '%2')")
627  .arg(url).arg((long long)data));
628 
629  if (!data)
630  {
631  LOG(VB_GENERAL, LOG_ERR, LOC + "postAuth(), data is NULL!");
632  return false;
633  }
634 
635  return processItem(url, nullptr, nullptr, data, kRequestPost, false, authCallbackFn,
636  authArg, headers);
637 }
638 
643 {
644  auto *dlThread = new RemoteFileDownloadThread(this, dlInfo);
645  MThreadPool::globalInstance()->start(dlThread, "RemoteFileDownload");
646 }
647 
652 {
653  if (!dlInfo)
654  return;
655 
656  static const QString kDateFormat = "ddd, dd MMM yyyy hh:mm:ss 'GMT'";
657  QUrl qurl(dlInfo->m_url);
658  QNetworkRequest request;
659 
660  if (dlInfo->m_request)
661  {
662  request = *dlInfo->m_request;
663  delete dlInfo->m_request;
664  dlInfo->m_request = nullptr;
665  }
666  else
667  request.setUrl(qurl);
668 
669  if (dlInfo->m_reload)
670  {
671  request.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
672  QNetworkRequest::AlwaysNetwork);
673  }
674  else
675  {
676  // Prefer the in-cache item if one exists and it is less than 5 minutes
677  // old and it will not expire in the next 10 seconds
678  QDateTime now = MythDate::current();
679 
680  // Handle redirects, we want the metadata of the file headers
681  QString redirectLoc;
682  int limit = 0;
683  while (!(redirectLoc = getHeader(qurl, "Location")).isNull())
684  {
685  if (limit == CACHE_REDIRECTION_LIMIT)
686  {
687  LOG(VB_GENERAL, LOG_WARNING, QString("Cache Redirection limit "
688  "reached for %1")
689  .arg(qurl.toString()));
690  return;
691  }
692  qurl.setUrl(redirectLoc);
693  limit++;
694  }
695 
696  LOG(VB_NETWORK, LOG_DEBUG, QString("Checking cache for %1")
697  .arg(qurl.toString()));
698 
699  m_infoLock->lock();
700  QNetworkCacheMetaData urlData = m_manager->cache()->metaData(qurl);
701  m_infoLock->unlock();
702  if ((urlData.isValid()) &&
703  ((!urlData.expirationDate().isValid()) ||
704  (urlData.expirationDate().toUTC().secsTo(now) < 10)))
705  {
706  QString dateString = getHeader(urlData, "Date");
707 
708  if (!dateString.isNull())
709  {
710  QDateTime loadDate =
711  MythDate::fromString(dateString, kDateFormat);
712  loadDate.setTimeSpec(Qt::UTC);
713  if (loadDate.secsTo(now) <= 720)
714  {
715  dlInfo->m_preferCache = true;
716  LOG(VB_NETWORK, LOG_DEBUG, QString("Preferring cache for %1")
717  .arg(qurl.toString()));
718  }
719  }
720  }
721  }
722 
723  if (dlInfo->m_preferCache)
724  request.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
725  QNetworkRequest::PreferCache);
726 
727  if (!request.hasRawHeader("User-Agent"))
728  {
729  request.setRawHeader("User-Agent",
730  "MythTV v" MYTH_BINARY_VERSION
731  " MythDownloadManager");
732  }
733 
734  if (dlInfo->m_headers)
735  {
736  QHash<QByteArray, QByteArray>::const_iterator it =
737  dlInfo->m_headers->constBegin();
738  for ( ; it != dlInfo->m_headers->constEnd(); ++it )
739  {
740  if (!it.key().isEmpty() && !it.value().isEmpty())
741  {
742  request.setRawHeader(it.key(), it.value());
743  }
744  }
745  }
746 
747  switch (dlInfo->m_requestType)
748  {
749  case kRequestPost :
750  dlInfo->m_reply = m_manager->post(request, *dlInfo->m_data);
751  break;
752  case kRequestHead :
753  dlInfo->m_reply = m_manager->head(request);
754  break;
755  case kRequestGet :
756  default:
757  dlInfo->m_reply = m_manager->get(request);
758  break;
759  }
760 
761  m_downloadReplies[dlInfo->m_reply] = dlInfo;
762 
763  if (dlInfo->m_authCallback)
764  {
765  connect(m_manager, &QNetworkAccessManager::authenticationRequired,
767  }
768 
769 #if QT_VERSION < QT_VERSION_CHECK(5,15,0)
770  connect(dlInfo->m_reply, qOverload<QNetworkReply::NetworkError>(&QNetworkReply::error),
772 #else
773  connect(dlInfo->m_reply, &QNetworkReply::errorOccurred,
775 #endif
776  connect(dlInfo->m_reply, &QNetworkReply::downloadProgress,
778 }
779 
784 void MythDownloadManager::authCallback(QNetworkReply *reply,
785  QAuthenticator *authenticator)
786 {
787  if (!reply)
788  return;
789 
790  MythDownloadInfo *dlInfo = m_downloadReplies[reply];
791 
792  if (!dlInfo)
793  return;
794 
795  if (dlInfo->m_authCallback)
796  {
797  LOG(VB_FILE, LOG_DEBUG, "Calling auth callback");
798  dlInfo->m_authCallback(reply, authenticator, dlInfo->m_authArg);
799  }
800 }
801 
809 {
810  if (!dlInfo)
811  return false;
812 
813  dlInfo->m_syncMode = true;
814 
815  // Special handling for link-local
816  // Not needed for Windows because windows does not need
817  // the scope id.
818 #ifndef _WIN32
819  if (dlInfo->m_url.startsWith("http://[fe80::",Qt::CaseInsensitive))
820  return downloadNowLinkLocal(dlInfo, deleteInfo);
821 #endif
822  m_infoLock->lock();
823  m_downloadQueue.push_back(dlInfo);
824  m_infoLock->unlock();
825  m_queueWaitCond.wakeAll();
826 
827  // timeout myth:// RemoteFile transfers 20 seconds from now
828  // timeout non-myth:// QNetworkAccessManager transfers 60 seconds after
829  // their last progress update
830  QDateTime startedAt = MythDate::current();
831  m_infoLock->lock();
832  while ((!dlInfo->IsDone()) &&
833  (dlInfo->m_errorCode == QNetworkReply::NoError) &&
834  (((!dlInfo->m_url.startsWith("myth://")) &&
835  (dlInfo->m_lastStat.secsTo(MythDate::current()) < 60)) ||
836  ((dlInfo->m_url.startsWith("myth://")) &&
837  (startedAt.secsTo(MythDate::current()) < 20))))
838  {
839  m_infoLock->unlock();
840  m_queueWaitLock.lock();
841  m_queueWaitCond.wait(&m_queueWaitLock, 200);
842  m_queueWaitLock.unlock();
843  m_infoLock->lock();
844  }
845  bool done = dlInfo->IsDone();
846  bool success =
847  done && (dlInfo->m_errorCode == QNetworkReply::NoError);
848 
849  if (!done)
850  {
851  dlInfo->m_data = nullptr; // Prevent downloadFinished() from updating
852  dlInfo->m_syncMode = false; // Let downloadFinished() cleanup for us
853  if ((dlInfo->m_reply) &&
854  (dlInfo->m_errorCode == QNetworkReply::NoError))
855  {
856  LOG(VB_FILE, LOG_DEBUG,
857  LOC + QString("Aborting download - lack of data transfer"));
858  dlInfo->m_reply->abort();
859  }
860  }
861  else if (deleteInfo)
862  delete dlInfo;
863 
864  m_infoLock->unlock();
865 
866  return success;
867 }
868 
869 #ifndef _WIN32
870 
890 bool MythDownloadManager::downloadNowLinkLocal(MythDownloadInfo *dlInfo, bool deleteInfo)
891 {
892  bool ok = true;
893 
894  // No buffer - no reply...
895  if (!dlInfo->m_data)
896  {
897  LOG(VB_GENERAL, LOG_ERR, LOC + QString("No data buffer provided for %1").arg(dlInfo->m_url));
898  ok = false;
899  }
900 
901  // Only certain features are supported here
902  if (dlInfo->m_authCallback || dlInfo->m_authArg)
903  {
904  LOG(VB_GENERAL, LOG_ERR, LOC + QString("Unsupported authentication for %1").arg(dlInfo->m_url));
905  ok = false;
906  }
907 
908  if (ok && !dlInfo->m_outFile.isEmpty())
909  {
910  LOG(VB_GENERAL, LOG_ERR, LOC + QString("Unsupported File output %1 for %2")
911  .arg(dlInfo->m_outFile).arg(dlInfo->m_url));
912  ok = false;
913  }
914 
915  if (ok && (!deleteInfo || dlInfo->m_requestType == kRequestHead))
916  {
917  // We do not have the ability to return a network reply in dlInfo
918  // so if we are asked to do that, return an error.
919  LOG(VB_GENERAL, LOG_ERR, LOC + QString("Unsupported link-local operation %1")
920  .arg(dlInfo->m_url));
921  ok = false;
922  }
923 
924  QUrl url(dlInfo->m_url);
925  QString host(url.host());
926  int port(url.port(80));
927  if (ok && PortChecker::resolveLinkLocal(host, port))
928  {
929  QString reqType;
930  switch (dlInfo->m_requestType)
931  {
932  case kRequestPost :
933  reqType = "POST";
934  break;
935  case kRequestGet :
936  default:
937  reqType = "GET";
938  break;
939  }
940  QByteArray* buffer = dlInfo->m_data;
941  QHash<QByteArray, QByteArray> headers;
942  if (dlInfo->m_headers)
943  headers = *dlInfo->m_headers;
944  if (!headers.contains("User-Agent"))
945  headers.insert("User-Agent", "MythDownloadManager v" MYTH_BINARY_VERSION);
946  headers.insert("Connection", "close");
947  headers.insert("Accept-Encoding", "identity");
948  if (!buffer->isEmpty())
949  headers.insert("Content-Length", QString::number(buffer->size()).toUtf8());
950  headers.insert("Host", (url.host() + ":" + QString::number(port)).toUtf8());
951 
952  QByteArray requestMessage;
953  QString path (url.path());
954  requestMessage.append("POST ");
955  requestMessage.append(path);
956  requestMessage.append(" HTTP/1.1\r\n");
957  QHashIterator<QByteArray, QByteArray> it(headers);
958  while (it.hasNext())
959  {
960  it.next();
961  requestMessage.append(it.key());
962  requestMessage.append(": ");
963  requestMessage.append(it.value());
964  requestMessage.append("\r\n");
965  }
966  requestMessage.append("\r\n");
967  if (!buffer->isEmpty())
968  requestMessage.append(*buffer);
969 
970  QTcpSocket socket;
971  socket.connectToHost(host, static_cast<uint16_t>(port));
972  // QT Warning - this may not work on Windows
973  if (!socket.waitForConnected(5000))
974  ok = false;
975  if (ok)
976  ok = socket.write(requestMessage) > 0;
977  if (ok)
978  // QT Warning - this may not work on Windows
979  ok = socket.waitForDisconnected(5000);
980  if (ok)
981  {
982  *buffer = socket.readAll();
983  // Find the start of the content
984  QByteArray delim("\r\n\r\n");
985  int delimLoc = buffer->indexOf(delim);
986  if (delimLoc > -1)
987  *buffer = buffer->right(buffer->size() - delimLoc - 4);
988  else
989  ok=false;
990  }
991  socket.close();
992  }
993  else
994  {
995  ok = false;
996  }
997 
998  if (deleteInfo)
999  delete dlInfo;
1000 
1001  if (ok)
1002  return true;
1003 
1004  LOG(VB_GENERAL, LOG_ERR, LOC + QString("Link Local request failed: %1").arg(url.toString()));
1005  return false;
1006 }
1007 #endif
1008 
1013 void MythDownloadManager::cancelDownload(const QString &url, bool block)
1014 {
1015  cancelDownload(QStringList(url), block);
1016 }
1017 
1022 void MythDownloadManager::cancelDownload(const QStringList &urls, bool block)
1023 {
1024  m_infoLock->lock();
1025  for (const auto& url : qAsConst(urls))
1026  {
1027  QMutableListIterator<MythDownloadInfo*> lit(m_downloadQueue);
1028  while (lit.hasNext())
1029  {
1030  lit.next();
1031  MythDownloadInfo *dlInfo = lit.value();
1032  if (dlInfo->m_url == url)
1033  {
1034  if (!m_cancellationQueue.contains(dlInfo))
1035  m_cancellationQueue.append(dlInfo);
1036  lit.remove();
1037  }
1038  }
1039 
1040  if (m_downloadInfos.contains(url))
1041  {
1042  MythDownloadInfo *dlInfo = m_downloadInfos[url];
1043 
1044  if (!m_cancellationQueue.contains(dlInfo))
1045  m_cancellationQueue.append(dlInfo);
1046 
1047  if (dlInfo->m_reply)
1048  m_downloadReplies.remove(dlInfo->m_reply);
1049 
1050  m_downloadInfos.remove(url);
1051  }
1052  }
1053  m_infoLock->unlock();
1054 
1055  if (QThread::currentThread() == this->thread())
1056  {
1057  downloadCanceled();
1058  return;
1059  }
1060 
1061  // wake-up running thread
1062  m_queueWaitCond.wakeAll();
1063 
1064  if (!block)
1065  return;
1066 
1067  while (!m_cancellationQueue.isEmpty())
1068  {
1069  usleep(50000); // re-test in another 50ms
1070  }
1071 }
1072 
1074 {
1075  QMutexLocker locker(m_infoLock);
1076 
1077  QMutableListIterator<MythDownloadInfo*> lit(m_cancellationQueue);
1078  while (lit.hasNext())
1079  {
1080  lit.next();
1081  MythDownloadInfo *dlInfo = lit.value();
1082  dlInfo->m_lock.lock();
1083 
1084  if (dlInfo->m_reply)
1085  {
1086  LOG(VB_FILE, LOG_DEBUG,
1087  LOC + QString("Aborting download - user request"));
1088  dlInfo->m_reply->abort();
1089  }
1090  lit.remove();
1091  if (dlInfo->m_done)
1092  {
1093  dlInfo->m_lock.unlock();
1094  continue;
1095  }
1096  dlInfo->m_errorCode = QNetworkReply::OperationCanceledError;
1097  dlInfo->m_done = true;
1098  dlInfo->m_lock.unlock();
1099  }
1100 }
1101 
1107 {
1108  QMutexLocker locker(m_infoLock);
1109 
1110  QList <MythDownloadInfo*>::iterator lit = m_downloadQueue.begin();
1111  for (; lit != m_downloadQueue.end(); ++lit)
1112  {
1113  MythDownloadInfo *dlInfo = *lit;
1114  if (dlInfo->m_caller == caller)
1115  {
1116  dlInfo->m_caller = nullptr;
1117  dlInfo->m_outFile = QString();
1118  dlInfo->m_data = nullptr;
1119  }
1120  }
1121 
1122  QMap <QString, MythDownloadInfo*>::iterator mit = m_downloadInfos.begin();
1123  for (; mit != m_downloadInfos.end(); ++mit)
1124  {
1125  MythDownloadInfo *dlInfo = mit.value();
1126  if (dlInfo->m_caller == caller)
1127  {
1128  dlInfo->m_caller = nullptr;
1129  dlInfo->m_outFile = QString();
1130  dlInfo->m_data = nullptr;
1131  }
1132  }
1133 }
1134 
1138 void MythDownloadManager::downloadError(QNetworkReply::NetworkError errorCode)
1139 {
1140  auto *reply = qobject_cast<QNetworkReply *>(sender());
1141  if (reply == nullptr)
1142  return;
1143 
1144  LOG(VB_FILE, LOG_DEBUG, LOC + QString("downloadError %1 ")
1145  .arg(errorCode) + reply->errorString() );
1146 
1147  QMutexLocker locker(m_infoLock);
1148  if (!m_downloadReplies.contains(reply))
1149  {
1150  reply->deleteLater();
1151  return;
1152  }
1153 
1154  MythDownloadInfo *dlInfo = m_downloadReplies[reply];
1155 
1156  if (!dlInfo)
1157  return;
1158 
1159  dlInfo->m_errorCode = errorCode;
1160 }
1161 
1167 QUrl MythDownloadManager::redirectUrl(const QUrl& possibleRedirectUrl,
1168  const QUrl& oldRedirectUrl)
1169 {
1170  LOG(VB_FILE, LOG_DEBUG, LOC + QString("redirectUrl()"));
1171  QUrl redirectUrl;
1172 
1173  if(!possibleRedirectUrl.isEmpty() && possibleRedirectUrl != oldRedirectUrl)
1174  redirectUrl = possibleRedirectUrl;
1175 
1176  return redirectUrl;
1177 }
1178 
1182 void MythDownloadManager::downloadFinished(QNetworkReply* reply)
1183 {
1184  LOG(VB_FILE, LOG_DEBUG, LOC + QString("downloadFinished(%1)")
1185  .arg((long long)reply));
1186 
1187  QMutexLocker locker(m_infoLock);
1188  if (!m_downloadReplies.contains(reply))
1189  {
1190  reply->deleteLater();
1191  return;
1192  }
1193 
1194  MythDownloadInfo *dlInfo = m_downloadReplies[reply];
1195 
1196  if (!dlInfo || !dlInfo->m_reply)
1197  return;
1198 
1199  downloadFinished(dlInfo);
1200 }
1201 
1206 {
1207  if (!dlInfo)
1208  return;
1209 
1210  int statusCode = -1;
1211  static const QString kDateFormat = "ddd, dd MMM yyyy hh:mm:ss 'GMT'";
1212  QNetworkReply *reply = dlInfo->m_reply;
1213 
1214  if (reply)
1215  {
1216  QUrl possibleRedirectUrl =
1217  reply->attribute(QNetworkRequest::RedirectionTargetAttribute).toUrl();
1218 
1219  if (!possibleRedirectUrl.isEmpty() &&
1220  possibleRedirectUrl.isValid() &&
1221  possibleRedirectUrl.isRelative()) // Turn relative Url to absolute
1222  possibleRedirectUrl = reply->url().resolved(possibleRedirectUrl);
1223 
1224  if (!possibleRedirectUrl.isEmpty() && dlInfo->m_finalUrl != nullptr)
1225  *dlInfo->m_finalUrl = QString(possibleRedirectUrl.toString());
1226 
1227  dlInfo->m_redirectedTo =
1228  redirectUrl(possibleRedirectUrl, dlInfo->m_redirectedTo);
1229 
1230  QVariant status =
1231  reply->attribute(QNetworkRequest::HttpStatusCodeAttribute);
1232  if (status.isValid())
1233  statusCode = status.toInt();
1234  }
1235 
1236  if(reply && !dlInfo->m_redirectedTo.isEmpty() &&
1237  ((dlInfo->m_requestType != kRequestPost) ||
1238  (statusCode == 301 || statusCode == 302 ||
1239  statusCode == 303)))
1240  {
1241  LOG(VB_FILE, LOG_DEBUG, LOC +
1242  QString("downloadFinished(%1): Redirect: %2 -> %3")
1243  .arg((long long)dlInfo)
1244  .arg(reply->url().toString())
1245  .arg(dlInfo->m_redirectedTo.toString()));
1246 
1247  if (dlInfo->m_data)
1248  dlInfo->m_data->clear();
1249 
1250  dlInfo->m_bytesReceived = 0;
1251  dlInfo->m_bytesTotal = 0;
1252 
1253  QNetworkRequest request(dlInfo->m_redirectedTo);
1254 
1255  if (dlInfo->m_reload)
1256  {
1257  request.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
1258  QNetworkRequest::AlwaysNetwork);
1259  }
1260  else if (dlInfo->m_preferCache)
1261  {
1262  request.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
1263  QNetworkRequest::PreferCache);
1264  }
1265 
1266  request.setRawHeader("User-Agent",
1267  "MythDownloadManager v" MYTH_BINARY_VERSION);
1268 
1269  switch (dlInfo->m_requestType)
1270  {
1271  case kRequestHead :
1272  dlInfo->m_reply = m_manager->head(request);
1273  break;
1274  case kRequestGet :
1275  default:
1276  dlInfo->m_reply = m_manager->get(request);
1277  break;
1278  }
1279 
1280  m_downloadReplies[dlInfo->m_reply] = dlInfo;
1281 
1282 #if QT_VERSION < QT_VERSION_CHECK(5,15,0)
1283  connect(dlInfo->m_reply, qOverload<QNetworkReply::NetworkError>(&QNetworkReply::error),
1285 #else
1286  connect(dlInfo->m_reply, &QNetworkReply::errorOccurred,
1288 #endif
1289  connect(dlInfo->m_reply, &QNetworkReply::downloadProgress,
1291 
1292  m_downloadReplies.remove(reply);
1293  reply->deleteLater();
1294  }
1295  else
1296  {
1297  LOG(VB_FILE, LOG_DEBUG, QString("downloadFinished(%1): COMPLETE: %2")
1298  .arg((long long)dlInfo).arg(dlInfo->m_url));
1299 
1300  // HACK Insert a Date header into the cached metadata if one doesn't
1301  // already exist
1302  QUrl fileUrl = dlInfo->m_url;
1303  QString redirectLoc;
1304  int limit = 0;
1305  while (!(redirectLoc = getHeader(fileUrl, "Location")).isNull())
1306  {
1307  if (limit == CACHE_REDIRECTION_LIMIT)
1308  {
1309  LOG(VB_GENERAL, LOG_WARNING, QString("Cache Redirection limit "
1310  "reached for %1")
1311  .arg(fileUrl.toString()));
1312  return;
1313  }
1314  fileUrl.setUrl(redirectLoc);
1315  limit++;
1316  }
1317 
1318  m_infoLock->lock();
1319  QNetworkCacheMetaData urlData = m_manager->cache()->metaData(fileUrl);
1320  m_infoLock->unlock();
1321  if (getHeader(urlData, "Date").isNull())
1322  {
1323  QNetworkCacheMetaData::RawHeaderList headers = urlData.rawHeaders();
1324  QNetworkCacheMetaData::RawHeader newheader;
1325  QDateTime now = MythDate::current();
1326  newheader = QNetworkCacheMetaData::RawHeader("Date",
1327  now.toString(kDateFormat).toLatin1());
1328  headers.append(newheader);
1329  urlData.setRawHeaders(headers);
1330  m_infoLock->lock();
1331  m_manager->cache()->updateMetaData(urlData);
1332  m_infoLock->unlock();
1333  }
1334  // End HACK
1335 
1336  dlInfo->m_redirectedTo.clear();
1337 
1338  int dataSize = -1;
1339 
1340  // If we downloaded via the QNetworkAccessManager
1341  // AND the caller isn't handling the reply directly
1342  if (reply && dlInfo->m_processReply)
1343  {
1344  bool append = (!dlInfo->m_syncMode && dlInfo->m_caller);
1345  QByteArray data = reply->readAll();
1346  dataSize = data.size();
1347 
1348  if (append)
1349  dlInfo->m_bytesReceived += dataSize;
1350  else
1351  dlInfo->m_bytesReceived = dataSize;
1352 
1353  dlInfo->m_bytesTotal = dlInfo->m_bytesReceived;
1354 
1355  if (dlInfo->m_data)
1356  {
1357  if (append)
1358  dlInfo->m_data->append(data);
1359  else
1360  *dlInfo->m_data = data;
1361  }
1362  else if (!dlInfo->m_outFile.isEmpty())
1363  {
1364  saveFile(dlInfo->m_outFile, data, append);
1365  }
1366  }
1367  else if (!reply) // If we downloaded via RemoteFile
1368  {
1369  if (dlInfo->m_data)
1370  {
1371  (*dlInfo->m_data) = dlInfo->m_privData;
1372  }
1373  else if (!dlInfo->m_outFile.isEmpty())
1374  {
1375  saveFile(dlInfo->m_outFile, dlInfo->m_privData);
1376  }
1377  dlInfo->m_bytesReceived += dataSize;
1378  dlInfo->m_bytesTotal = dlInfo->m_bytesReceived;
1379  }
1380  // else we downloaded via QNetworkAccessManager
1381  // AND the caller is handling the reply
1382 
1383  m_infoLock->lock();
1384  if (!m_downloadInfos.remove(dlInfo->m_url))
1385  {
1386  LOG(VB_GENERAL, LOG_ERR, LOC +
1387  QString("ERROR download finished but failed to remove url: %1")
1388  .arg(dlInfo->m_url));
1389  }
1390 
1391  if (reply)
1392  m_downloadReplies.remove(reply);
1393  m_infoLock->unlock();
1394 
1395  dlInfo->SetDone(true);
1396 
1397  if (!dlInfo->m_syncMode)
1398  {
1399  if (dlInfo->m_caller)
1400  {
1401  LOG(VB_FILE, LOG_DEBUG, QString("downloadFinished(%1): "
1402  "COMPLETE: %2, sending event to caller")
1403  .arg((long long)dlInfo).arg(dlInfo->m_url));
1404 
1405  QStringList args;
1406  args << dlInfo->m_url;
1407  args << dlInfo->m_outFile;
1408  args << QString::number(dlInfo->m_bytesTotal);
1409  // placeholder for error string
1410  args << (reply ? reply->errorString() : QString());
1411  args << QString::number((int)(reply ? reply->error() :
1412  dlInfo->m_errorCode));
1413 
1414  QCoreApplication::postEvent(dlInfo->m_caller,
1415  new MythEvent("DOWNLOAD_FILE FINISHED", args));
1416  }
1417 
1418  delete dlInfo;
1419  }
1420 
1421  m_queueWaitCond.wakeAll();
1422  }
1423 }
1424 
1430 void MythDownloadManager::downloadProgress(qint64 bytesReceived,
1431  qint64 bytesTotal)
1432 {
1433  auto *reply = qobject_cast<QNetworkReply *>(sender());
1434  if (reply == nullptr)
1435  return;
1436 
1437  LOG(VB_FILE, LOG_DEBUG, LOC +
1438  QString("downloadProgress(%1, %2) (for reply %3)")
1439  .arg(bytesReceived).arg(bytesTotal).arg((long long)reply));
1440 
1441  QMutexLocker locker(m_infoLock);
1442  if (!m_downloadReplies.contains(reply))
1443  return;
1444 
1445  MythDownloadInfo *dlInfo = m_downloadReplies[reply];
1446 
1447  if (!dlInfo)
1448  return;
1449 
1450  dlInfo->m_lastStat = MythDate::current();
1451 
1452  LOG(VB_FILE, LOG_DEBUG, LOC +
1453  QString("downloadProgress: %1 to %2 is at %3 of %4 bytes downloaded")
1454  .arg(dlInfo->m_url).arg(dlInfo->m_outFile)
1455  .arg(bytesReceived).arg(bytesTotal));
1456 
1457  if (!dlInfo->m_syncMode && dlInfo->m_caller)
1458  {
1459  LOG(VB_FILE, LOG_DEBUG, QString("downloadProgress(%1): "
1460  "sending event to caller")
1461  .arg(reply->url().toString()));
1462 
1463  bool appendToFile = (dlInfo->m_bytesReceived != 0);
1464  QByteArray data = reply->readAll();
1465  if (!dlInfo->m_outFile.isEmpty())
1466  saveFile(dlInfo->m_outFile, data, appendToFile);
1467 
1468  if (dlInfo->m_data)
1469  dlInfo->m_data->append(data);
1470 
1471  dlInfo->m_bytesReceived = bytesReceived;
1472  dlInfo->m_bytesTotal = bytesTotal;
1473 
1474  QStringList args;
1475  args << dlInfo->m_url;
1476  args << dlInfo->m_outFile;
1477  args << QString::number(bytesReceived);
1478  args << QString::number(bytesTotal);
1479 
1480  QCoreApplication::postEvent(dlInfo->m_caller,
1481  new MythEvent("DOWNLOAD_FILE UPDATE", args));
1482  }
1483 }
1484 
1492 bool MythDownloadManager::saveFile(const QString &outFile,
1493  const QByteArray &data,
1494  const bool append)
1495 {
1496  if (outFile.isEmpty() || data.isEmpty())
1497  return false;
1498 
1499  QFile file(outFile);
1500  QFileInfo fileInfo(outFile);
1501  QDir qdir(fileInfo.absolutePath());
1502 
1503  if (!qdir.exists() && !qdir.mkpath(fileInfo.absolutePath()))
1504  {
1505  LOG(VB_GENERAL, LOG_ERR, QString("Failed to create: '%1'")
1506  .arg(fileInfo.absolutePath()));
1507  return false;
1508  }
1509 
1510  QIODevice::OpenMode mode = QIODevice::Unbuffered|QIODevice::WriteOnly;
1511  if (append)
1512  mode |= QIODevice::Append;
1513 
1514  if (!file.open(mode))
1515  {
1516  LOG(VB_GENERAL, LOG_ERR, QString("Failed to open: '%1'") .arg(outFile));
1517  return false;
1518  }
1519 
1520  off_t offset = 0;
1521  size_t remaining = data.size();
1522  uint failure_cnt = 0;
1523  while ((remaining > 0) && (failure_cnt < 5))
1524  {
1525  ssize_t written = file.write(data.data() + offset, remaining);
1526  if (written < 0)
1527  {
1528  failure_cnt++;
1529  usleep(50000);
1530  continue;
1531  }
1532 
1533  failure_cnt = 0;
1534  offset += written;
1535  remaining -= written;
1536  }
1537 
1538  return remaining <= 0;
1539 }
1540 
1545 QDateTime MythDownloadManager::GetLastModified(const QString &url)
1546 {
1547  // If the header has not expired and
1548  // the last modification date is less than 1 hours old or if
1549  // the cache object is less than 20 minutes old,
1550  // then use the cached header otherwise redownload the header
1551 
1552  static const QString kDateFormat = "ddd, dd MMM yyyy hh:mm:ss 'GMT'";
1553  LOG(VB_FILE, LOG_DEBUG, LOC + QString("GetLastModified('%1')").arg(url));
1554  QDateTime result;
1555 
1556  QDateTime now = MythDate::current();
1557 
1558  QUrl cacheUrl = QUrl(url);
1559 
1560  // Deal with redirects, we want the cached data for the final url
1561  QString redirectLoc;
1562  int limit = 0;
1563  while (!(redirectLoc = getHeader(cacheUrl, "Location")).isNull())
1564  {
1565  if (limit == CACHE_REDIRECTION_LIMIT)
1566  {
1567  LOG(VB_GENERAL, LOG_WARNING, QString("Cache Redirection limit "
1568  "reached for %1")
1569  .arg(cacheUrl.toString()));
1570  return result;
1571  }
1572  cacheUrl.setUrl(redirectLoc);
1573  limit++;
1574  }
1575 
1576  m_infoLock->lock();
1577  QNetworkCacheMetaData urlData = m_manager->cache()->metaData(cacheUrl);
1578  m_infoLock->unlock();
1579 
1580  if (urlData.isValid() &&
1581  ((!urlData.expirationDate().isValid()) ||
1582  (urlData.expirationDate().secsTo(now) < 0)))
1583  {
1584  if (urlData.lastModified().toUTC().secsTo(now) <= 3600) // 1 Hour
1585  {
1586  result = urlData.lastModified().toUTC();
1587  }
1588  else
1589  {
1590  QString date = getHeader(urlData, "Date");
1591  if (!date.isNull())
1592  {
1593  QDateTime loadDate =
1594  MythDate::fromString(date, kDateFormat);
1595  loadDate.setTimeSpec(Qt::UTC);
1596  if (loadDate.secsTo(now) <= 1200) // 20 Minutes
1597  {
1598  result = urlData.lastModified().toUTC();
1599  }
1600  }
1601  }
1602  }
1603 
1604  if (!result.isValid())
1605  {
1606  auto *dlInfo = new MythDownloadInfo;
1607  dlInfo->m_url = url;
1608  dlInfo->m_syncMode = true;
1609  // Head request, we only want to inspect the headers
1610  dlInfo->m_requestType = kRequestHead;
1611 
1612  if (downloadNow(dlInfo, false))
1613  {
1614  if (dlInfo->m_reply)
1615  {
1616  QVariant lastMod =
1617  dlInfo->m_reply->header(
1618  QNetworkRequest::LastModifiedHeader);
1619  if (lastMod.isValid())
1620  result = lastMod.toDateTime().toUTC();
1621  }
1622 
1623  // downloadNow() will set a flag to trigger downloadFinished()
1624  // to delete the dlInfo if the download times out
1625  delete dlInfo;
1626  }
1627  }
1628 
1629  LOG(VB_FILE, LOG_DEBUG, LOC + QString("GetLastModified('%1'): Result %2")
1630  .arg(url).arg(result.toString()));
1631 
1632  return result;
1633 }
1634 
1635 
1640 {
1641  QMutexLocker locker(&m_cookieLock);
1642 
1643  auto *jar = new MythCookieJar;
1644  jar->load(filename);
1645  m_manager->setCookieJar(jar);
1646 }
1647 
1652 {
1653  QMutexLocker locker(&m_cookieLock);
1654 
1655  if (!m_manager->cookieJar())
1656  return;
1657 
1658  auto *jar = qobject_cast<MythCookieJar *>(m_manager->cookieJar());
1659  if (jar == nullptr)
1660  return;
1661  jar->save(filename);
1662 }
1663 
1664 void MythDownloadManager::setCookieJar(QNetworkCookieJar *cookieJar)
1665 {
1666  QMutexLocker locker(&m_cookieLock);
1667  m_manager->setCookieJar(cookieJar);
1668 }
1669 
1673 QNetworkCookieJar *MythDownloadManager::copyCookieJar(void)
1674 {
1675  QMutexLocker locker(&m_cookieLock);
1676 
1677  if (!m_manager->cookieJar())
1678  return nullptr;
1679 
1680  auto *inJar = qobject_cast<MythCookieJar *>(m_manager->cookieJar());
1681  if (inJar == nullptr)
1682  return nullptr;
1683  auto *outJar = new MythCookieJar;
1684  outJar->copyAllCookies(*inJar);
1685 
1686  return outJar;
1687 }
1688 
1692 void MythDownloadManager::refreshCookieJar(QNetworkCookieJar *jar)
1693 {
1694  QMutexLocker locker(&m_cookieLock);
1695  delete m_inCookieJar;
1696 
1697  auto *inJar = qobject_cast<MythCookieJar *>(jar);
1698  if (inJar == nullptr)
1699  return;
1700 
1701  auto *outJar = new MythCookieJar;
1702  outJar->copyAllCookies(*inJar);
1703  m_inCookieJar = outJar;
1704 
1705  QMutexLocker locker2(&m_queueWaitLock);
1706  m_queueWaitCond.wakeAll();
1707 }
1708 
1712 {
1713  QMutexLocker locker(&m_cookieLock);
1714 
1715  auto *inJar = qobject_cast<MythCookieJar *>(m_inCookieJar);
1716  if (inJar != nullptr)
1717  {
1718  auto *outJar = new MythCookieJar;
1719  outJar->copyAllCookies(*inJar);
1720  m_manager->setCookieJar(outJar);
1721  }
1722 
1723  delete m_inCookieJar;
1724  m_inCookieJar = nullptr;
1725 }
1726 
1727 QString MythDownloadManager::getHeader(const QUrl& url, const QString& header)
1728 {
1729  if (!m_manager || !m_manager->cache())
1730  return QString();
1731 
1732  m_infoLock->lock();
1733  QNetworkCacheMetaData metadata = m_manager->cache()->metaData(url);
1734  m_infoLock->unlock();
1735 
1736  return getHeader(metadata, header);
1737 }
1738 
1744 QString MythDownloadManager::getHeader(const QNetworkCacheMetaData &cacheData,
1745  const QString& header)
1746 {
1747  auto headers = cacheData.rawHeaders();
1748  for (const auto& rh : qAsConst(headers))
1749  if (QString(rh.first) == header)
1750  return QString(rh.second);
1751  return QString();
1752 }
1753 
1754 
1759 {
1760  const QList<QNetworkCookie> cookieList = old.allCookies();
1761  setAllCookies(cookieList);
1762 }
1763 
1767 void MythCookieJar::load(const QString &filename)
1768 {
1769  LOG(VB_GENERAL, LOG_DEBUG, QString("MythCookieJar: loading cookies from: %1").arg(filename));
1770 
1771  QFile f(filename);
1772  if (!f.open(QIODevice::ReadOnly))
1773  {
1774  LOG(VB_GENERAL, LOG_WARNING, QString("MythCookieJar::load() failed to open file for reading: %1").arg(filename));
1775  return;
1776  }
1777 
1778  QList<QNetworkCookie> cookieList;
1779  QTextStream stream(&f);
1780  while (!stream.atEnd())
1781  {
1782  QString cookie = stream.readLine();
1783  cookieList << QNetworkCookie::parseCookies(cookie.toLocal8Bit());
1784  }
1785 
1786  setAllCookies(cookieList);
1787 }
1788 
1792 void MythCookieJar::save(const QString &filename)
1793 {
1794  LOG(VB_GENERAL, LOG_DEBUG, QString("MythCookieJar: saving cookies to: %1").arg(filename));
1795 
1796  QFile f(filename);
1797  if (!f.open(QIODevice::WriteOnly))
1798  {
1799  LOG(VB_GENERAL, LOG_ERR, QString("MythCookieJar::save() failed to open file for writing: %1").arg(filename));
1800  return;
1801  }
1802 
1803  QList<QNetworkCookie> cookieList = allCookies();
1804  QTextStream stream(&f);
1805 
1806 #if QT_VERSION < QT_VERSION_CHECK(5,14,0)
1807  for (const auto& cookie : qAsConst(cookieList))
1808  stream << cookie.toRawForm() << endl;
1809 #else
1810  for (const auto& cookie : qAsConst(cookieList))
1811  stream << cookie.toRawForm() << Qt::endl;
1812 #endif
1813 }
1814 
1815 
1816 /* vim: set expandtab tabstop=4 shiftwidth=4: */
MythDownloadInfo::m_reload
bool m_reload
Definition: mythdownloadmanager.cpp:81
MythDownloadManager::m_cookieLock
QMutex m_cookieLock
Definition: mythdownloadmanager.h:176
build_compdb.args
args
Definition: build_compdb.py:11
mythevent.h
build_compdb.dest
dest
Definition: build_compdb.py:9
fileInfo
QFileInfo fileInfo(filename)
MythDownloadManager::processItem
bool processItem(const QString &url, QNetworkRequest *req, const QString &dest, QByteArray *data, MRequestType reqType=kRequestGet, bool reload=false, AuthCallback authCallback=nullptr, void *authArg=nullptr, const QHash< QByteArray, QByteArray > *headers=nullptr, QString *finalUrl=nullptr)
Processes a network request immediately and waits for a response.
Definition: mythdownloadmanager.cpp:353
error
static void error(const char *str,...)
Definition: vbi.cpp:42
downloadManager
MythDownloadManager * downloadManager
Definition: mythdownloadmanager.cpp:37
MythDownloadInfo::m_lastStat
QDateTime m_lastStat
Definition: mythdownloadmanager.cpp:88
MythDownloadInfo::m_data
QByteArray * m_data
Definition: mythdownloadmanager.cpp:77
MythCookieJar::save
void save(const QString &filename)
Saves the cookie jar to a cookie file.
Definition: mythdownloadmanager.cpp:1792
MythDownloadInfo::m_privData
QByteArray m_privData
Definition: mythdownloadmanager.cpp:78
MythDownloadManager::m_downloadInfos
QMap< QString, MythDownloadInfo * > m_downloadInfos
Definition: mythdownloadmanager.h:165
dmCreateLock
QMutex dmCreateLock
Definition: mythdownloadmanager.cpp:38
MythDownloadManager::removeListener
void removeListener(QObject *caller)
Disconnects the specified caller from any existing MythDownloadInfo instances.
Definition: mythdownloadmanager.cpp:1106
ShutdownMythDownloadManager
void ShutdownMythDownloadManager(void)
Deletes the running MythDownloadManager at program exit.
Definition: mythdownloadmanager.cpp:133
RemoteFileDownloadThread
Definition: mythdownloadmanager.cpp:102
RemoteFileDownloadThread::run
void run() override
Definition: mythdownloadmanager.cpp:109
MythEvent
This class is used as a container for messages.
Definition: mythevent.h:17
mythcoreutil.h
MythDownloadInfo::m_lock
QMutex m_lock
Definition: mythdownloadmanager.cpp:94
MythDownloadManager::queueDownload
void queueDownload(const QString &url, const QString &dest, QObject *caller, bool reload=false)
Adds a url to the download queue.
Definition: mythdownloadmanager.cpp:393
arg
arg(title).arg(filename).arg(doDelete))
MythDownloadManager::m_diskCache
QNetworkDiskCache * m_diskCache
Definition: mythdownloadmanager.h:158
MythDownloadManager::m_proxy
QNetworkProxy * m_proxy
Definition: mythdownloadmanager.h:159
RemoteFile
Definition: remotefile.h:18
MythDownloadManager::postAuth
bool postAuth(const QString &url, QByteArray *data, AuthCallback authCallback, void *authArg, const QHash< QByteArray, QByteArray > *headers=nullptr)
Posts data to a url via the QNetworkAccessManager.
Definition: mythdownloadmanager.cpp:622
kRequestPost
@ kRequestPost
Definition: mythdownloadmanager.h:42
LOG
#define LOG(_MASK_, _LEVEL_, _QSTRING_)
Definition: mythlogging.h:23
MThread::RunProlog
void RunProlog(void)
Sets up a thread, call this if you reimplement run().
Definition: mthread.cpp:198
MythDownloadManager::downloadNow
bool downloadNow(MythDownloadInfo *dlInfo, bool deleteInfo=true)
Download helper for download() blocking methods.
Definition: mythdownloadmanager.cpp:808
MythDownloadManager
Definition: mythdownloadmanager.h:48
MythDownloadInfo::SetDone
void SetDone(bool done)
Definition: mythdownloadmanager.cpp:65
MythDownloadManager::saveFile
static bool saveFile(const QString &outFile, const QByteArray &data, bool append=false)
Saves a QByteArray of data to a given filename.
Definition: mythdownloadmanager.cpp:1492
MythDownloadInfo::IsDone
bool IsDone(void)
Definition: mythdownloadmanager.cpp:59
build_compdb.file
file
Definition: build_compdb.py:55
MythDownloadInfo::m_request
QNetworkRequest * m_request
Definition: mythdownloadmanager.cpp:74
mythdirs.h
RemoteFileDownloadThread::RemoteFileDownloadThread
RemoteFileDownloadThread(MythDownloadManager *parent, MythDownloadInfo *dlInfo)
Definition: mythdownloadmanager.cpp:104
MythDownloadInfo::m_bytesTotal
qint64 m_bytesTotal
Definition: mythdownloadmanager.cpp:87
MythDownloadInfo::m_processReply
bool m_processReply
Definition: mythdownloadmanager.cpp:84
MythDownloadManager::authCallback
void authCallback(QNetworkReply *reply, QAuthenticator *authenticator)
Signal handler for authentication requests.
Definition: mythdownloadmanager.cpp:784
MythDate::current
QDateTime current(bool stripped)
Returns current Date and Time in UTC.
Definition: mythdate.cpp:10
MythDownloadInfo
Definition: mythdownloadmanager.cpp:44
MythDownloadInfo::m_bytesReceived
qint64 m_bytesReceived
Definition: mythdownloadmanager.cpp:86
mythversion.h
mythdate.h
mythlogging.h
kRequestHead
@ kRequestHead
Definition: mythdownloadmanager.h:41
GetConfDir
QString GetConfDir(void)
Definition: mythdirs.cpp:224
PortChecker::resolveLinkLocal
static bool resolveLinkLocal(QString &host, int port, int timeLimit=30000)
Convenience method to resolve link-local address.
Definition: portchecker.cpp:228
MythDownloadInfo::m_authArg
void * m_authArg
Definition: mythdownloadmanager.cpp:90
MythDownloadManager::queuePost
void queuePost(const QString &url, QByteArray *data, QObject *caller)
Queues a post to a URL via the QNetworkAccessManager.
Definition: mythdownloadmanager.cpp:529
remotefile.h
MythDownloadManager::run
void run(void) override
Runs a loop to process incoming download requests and triggers download events to be processed.
Definition: mythdownloadmanager.cpp:191
LOC
#define LOC
Definition: mythdownloadmanager.cpp:34
MythDownloadManager::m_isRunning
bool m_isRunning
Definition: mythdownloadmanager.h:173
compat.h
MythDownloadInfo::m_finalUrl
QString * m_finalUrl
Definition: mythdownloadmanager.cpp:73
MythDownloadManager::downloadError
void downloadError(QNetworkReply::NetworkError errorCode)
Slot to process download error events.
Definition: mythdownloadmanager.cpp:1138
MythDownloadInfo::m_errorCode
QNetworkReply::NetworkError m_errorCode
Definition: mythdownloadmanager.cpp:93
MythDownloadManager::m_runThread
bool m_runThread
Definition: mythdownloadmanager.h:172
filename
QString filename
Definition: mythplugins/mytharchive/mytharchivehelper/main.cpp:637
MThread::RunEpilog
void RunEpilog(void)
Cleans up a thread's resources, call this if you reimplement run().
Definition: mthread.cpp:211
MythDownloadManager::download
bool download(const QString &url, const QString &dest, bool reload=false)
Downloads a URL to a file in blocking mode.
Definition: mythdownloadmanager.cpp:430
MythDownloadManager::getHeader
QString getHeader(const QUrl &url, const QString &header)
Definition: mythdownloadmanager.cpp:1727
MythCookieJar::copyAllCookies
void copyAllCookies(MythCookieJar &old)
Copies all cookies from one MythCookieJar to another.
Definition: mythdownloadmanager.cpp:1758
MythDownloadManager::m_infoLock
QMutex * m_infoLock
Definition: mythdownloadmanager.h:164
kRequestGet
@ kRequestGet
Definition: mythdownloadmanager.h:40
MythDownloadManager::setCookieJar
void setCookieJar(QNetworkCookieJar *cookieJar)
Definition: mythdownloadmanager.cpp:1664
MythDownloadManager::downloadProgress
void downloadProgress(qint64 bytesReceived, qint64 bytesTotal)
Slot to process download update events.
Definition: mythdownloadmanager.cpp:1430
f
QTextStream t & f
Definition: mythplugins/mytharchive/mytharchivehelper/main.cpp:603
portchecker.h
MythDownloadManager::cancelDownload
void cancelDownload(const QString &url, bool block=true)
Cancel a queued or current download.
Definition: mythdownloadmanager.cpp:1013
MythDownloadInfo::m_syncMode
bool m_syncMode
Definition: mythdownloadmanager.cpp:83
uint
unsigned int uint
Definition: compat.h:140
MythDownloadManager::downloadRemoteFile
void downloadRemoteFile(MythDownloadInfo *dlInfo)
Triggers a myth:// URI download in the background via RemoteFile.
Definition: mythdownloadmanager.cpp:642
gCoreContext
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
Definition: mythcorecontext.cpp:56
MythDownloadManager::redirectUrl
static QUrl redirectUrl(const QUrl &possibleRedirectUrl, const QUrl &oldRedirectUrl)
Checks whether we were redirected to the given URL.
Definition: mythdownloadmanager.cpp:1167
MythDownloadManager::downloadFinished
void downloadFinished(QNetworkReply *reply)
Slot to process download finished events.
Definition: mythdownloadmanager.cpp:1182
MythDownloadInfo::~MythDownloadInfo
~MythDownloadInfo()
Definition: mythdownloadmanager.cpp:52
MythDate::fromString
QDateTime fromString(const QString &dtstr)
Converts kFilename && kISODate formats to QDateTime.
Definition: mythdate.cpp:30
MythDownloadManager::~MythDownloadManager
~MythDownloadManager() override
Destructor for MythDownloadManager.
Definition: mythdownloadmanager.cpp:177
off_t
#define off_t
Definition: mythiowrapper.cpp:238
mthreadpool.h
MYTH_BINARY_VERSION
#define MYTH_BINARY_VERSION
Update this whenever the plug-in ABI changes.
Definition: mythversion.h:15
MythDownloadInfo::m_reply
QNetworkReply * m_reply
Definition: mythdownloadmanager.cpp:75
MythDownloadInfo::m_preferCache
bool m_preferCache
Definition: mythdownloadmanager.cpp:82
MythDownloadManager::preCache
void preCache(const QString &url)
Downloads a URL but doesn't store the resulting data anywhere.
Definition: mythdownloadmanager.cpp:381
MythDownloadManager::m_manager
QNetworkAccessManager * m_manager
Definition: mythdownloadmanager.h:157
MythDownloadManager::downloadAuth
bool downloadAuth(const QString &url, const QString &dest, bool reload=false, AuthCallback authCallback=nullptr, void *authArg=nullptr, const QHash< QByteArray, QByteArray > *headers=nullptr)
Downloads a URL to a file in blocking mode.
Definition: mythdownloadmanager.cpp:515
MythDownloadManager::loadCookieJar
void loadCookieJar(const QString &filename)
Loads the cookie jar from a cookie file.
Definition: mythdownloadmanager.cpp:1639
MythDownloadManager::m_inCookieJar
QNetworkCookieJar * m_inCookieJar
Definition: mythdownloadmanager.h:175
mythcorecontext.h
MythDownloadManager::copyCookieJar
QNetworkCookieJar * copyCookieJar(void)
Copy from one cookie jar to another.
Definition: mythdownloadmanager.cpp:1673
MythDownloadInfo::m_headers
const QHash< QByteArray, QByteArray > * m_headers
Definition: mythdownloadmanager.cpp:91
MythDownloadInfo::m_authCallback
AuthCallback m_authCallback
Definition: mythdownloadmanager.cpp:89
MythDate
Definition: mythdate.cpp:8
MythDownloadManager::m_queueThread
QThread * m_queueThread
Definition: mythdownloadmanager.h:170
MythDownloadManager::m_queueWaitCond
QWaitCondition m_queueWaitCond
Definition: mythdownloadmanager.h:161
MythDownloadInfo::m_done
bool m_done
Definition: mythdownloadmanager.cpp:85
MythDownloadManager::RemoteFileDownloadThread
friend class RemoteFileDownloadThread
Definition: mythdownloadmanager.h:178
MythDownloadManager::refreshCookieJar
void refreshCookieJar(QNetworkCookieJar *jar)
Refresh the temporary cookie jar from another cookie jar.
Definition: mythdownloadmanager.cpp:1692
AuthCallback
void(*)(QNetworkReply *, QAuthenticator *, void *) AuthCallback
Definition: mythdownloadmanager.h:45
MythCookieJar
A subclassed QNetworkCookieJar that allows for reading and writing cookie files that contain raw form...
Definition: mythdownloadmanager.h:29
MythDownloadManager::downloadCanceled
void downloadCanceled(void)
Definition: mythdownloadmanager.cpp:1073
MythDownloadInfo::m_requestType
MRequestType m_requestType
Definition: mythdownloadmanager.cpp:80
MythCoreContext::GetHostName
QString GetHostName(void)
Definition: mythcorecontext.cpp:855
uint16_t
unsigned short uint16_t
Definition: iso6937tables.h:3
MythDownloadManager::saveCookieJar
void saveCookieJar(const QString &filename)
Saves the cookie jar to a cookie file.
Definition: mythdownloadmanager.cpp:1651
MythDownloadManager::GetLastModified
QDateTime GetLastModified(const QString &url)
Gets the Last Modified timestamp for a URI.
Definition: mythdownloadmanager.cpp:1545
MythDownloadInfo::MythDownloadInfo
MythDownloadInfo()
Definition: mythdownloadmanager.cpp:46
MythDownloadManager::updateCookieJar
void updateCookieJar(void)
Update the cookie jar from the temporary cookie jar.
Definition: mythdownloadmanager.cpp:1711
MythDownloadManager::downloadQNetworkRequest
void downloadQNetworkRequest(MythDownloadInfo *dlInfo)
Downloads a QNetworkRequest via the QNetworkAccessManager.
Definition: mythdownloadmanager.cpp:651
MythCookieJar::load
void load(const QString &filename)
Loads the cookie jar from a cookie file.
Definition: mythdownloadmanager.cpp:1767
MThread::wait
bool wait(unsigned long time=ULONG_MAX)
Wait for the MThread to exit, with a maximum timeout.
Definition: mthread.cpp:305
mythdownloadmanager.h
MythDownloadInfo::m_redirectedTo
QUrl m_redirectedTo
Definition: mythdownloadmanager.cpp:72
MythDownloadManager::post
bool post(const QString &url, QByteArray *data)
Posts data to a url via the QNetworkAccessManager.
Definition: mythdownloadmanager.cpp:576
MThreadPool::globalInstance
static MThreadPool * globalInstance(void)
Definition: mthreadpool.cpp:306
MythDownloadInfo::m_outFile
QString m_outFile
Definition: mythdownloadmanager.cpp:76
MythDownloadInfo::m_caller
QObject * m_caller
Definition: mythdownloadmanager.cpp:79
MythDownloadManager::queueItem
void queueItem(const QString &url, QNetworkRequest *req, const QString &dest, QByteArray *data, QObject *caller, MRequestType reqType=kRequestGet, bool reload=false)
Adds a request to the download queue.
Definition: mythdownloadmanager.cpp:321
RemoteFileDownloadThread::m_dlInfo
MythDownloadInfo * m_dlInfo
Definition: mythdownloadmanager.cpp:128
MythDownloadManager::m_cancellationQueue
QList< MythDownloadInfo * > m_cancellationQueue
Definition: mythdownloadmanager.h:168
CACHE_REDIRECTION_LIMIT
#define CACHE_REDIRECTION_LIMIT
Definition: mythdownloadmanager.cpp:35
MythDownloadManager::m_queueWaitLock
QMutex m_queueWaitLock
Definition: mythdownloadmanager.h:162
RemoteFileDownloadThread::m_parent
MythDownloadManager * m_parent
Definition: mythdownloadmanager.cpp:127
MThread::usleep
static void usleep(unsigned long time)
Definition: mthread.cpp:342
MThreadPool::start
void start(QRunnable *runnable, const QString &debugName, int priority=0)
Definition: mthreadpool.cpp:341
MythDownloadInfo::m_url
QString m_url
Definition: mythdownloadmanager.cpp:71
MythDownloadManager::m_downloadQueue
QList< MythDownloadInfo * > m_downloadQueue
Definition: mythdownloadmanager.h:167
MythDownloadManager::m_downloadReplies
QMap< QNetworkReply *, MythDownloadInfo * > m_downloadReplies
Definition: mythdownloadmanager.h:166
GetMythDownloadManager
MythDownloadManager * GetMythDownloadManager(void)
Gets the pointer to the MythDownloadManager singleton.
Definition: mythdownloadmanager.cpp:145
MRequestType
MRequestType
Definition: mythdownloadmanager.h:39