MythTV  master
mythdownloadmanager.cpp
Go to the documentation of this file.
1 // qt
2 #include <QCoreApplication>
3 #include <QRunnable>
4 #include <QString>
5 #include <QByteArray>
6 #include <QFile>
7 #include <QDir>
8 #include <QNetworkCookie>
9 #include <QAuthenticator>
10 #include <QTextStream>
11 #include <QNetworkProxy>
12 #include <QMutexLocker>
13 #include <QUrl>
14 #include <QTcpSocket>
15 
16 #include <cstdlib>
17 #include <unistd.h> // for usleep()
18 
19 // libmythbase
20 #include "compat.h"
21 #include "mythcorecontext.h"
22 #include "mythcoreutil.h"
23 #include "mthreadpool.h"
24 #include "mythdirs.h"
25 #include "mythevent.h"
26 #include "mythversion.h"
27 #include "remotefile.h"
28 #include "mythdate.h"
29 
30 #include "mythdownloadmanager.h"
31 #include "mythlogging.h"
32 #include "portchecker.h"
33 
34 #define LOC QString("DownloadManager: ")
35 static constexpr int CACHE_REDIRECTION_LIMIT { 10 };
36 
38 QMutex dmCreateLock;
39 
44 {
45  public:
48  {
49  qRegisterMetaType<QNetworkReply::NetworkError>("QNetworkReply::NetworkError");
50  }
51 
53  {
54  delete m_request;
55  if (m_reply && m_processReply)
56  m_reply->deleteLater();
57  }
58 
59  bool IsDone(void)
60  {
61  QMutexLocker lock(&m_lock);
62  return m_done;
63  }
64 
65  void SetDone(bool done)
66  {
67  QMutexLocker lock(&m_lock);
68  m_done = done;
69  }
70 
71  QString m_url;
73  QString *m_finalUrl {nullptr};
74  QNetworkRequest *m_request {nullptr};
75  QNetworkReply *m_reply {nullptr};
76  QString m_outFile;
77  QByteArray *m_data {nullptr};
78  QByteArray m_privData;
79  QObject *m_caller {nullptr};
81  bool m_reload {false};
82  bool m_preferCache {false};
83  bool m_syncMode {false};
84  bool m_processReply {true};
85  bool m_done {false};
86  qint64 m_bytesReceived {0};
87  qint64 m_bytesTotal {0};
88  QDateTime m_lastStat;
90  void *m_authArg {nullptr};
91  const QHash<QByteArray, QByteArray> *m_headers {nullptr};
92 
93  QNetworkReply::NetworkError m_errorCode {QNetworkReply::NoError};
94  QMutex m_lock;
95 };
96 
97 
101 class RemoteFileDownloadThread : public QRunnable
102 {
103  public:
105  MythDownloadInfo *dlInfo) :
106  m_parent(parent),
107  m_dlInfo(dlInfo) {}
108 
109  void run() override // QRunnable
110  {
111  bool ok = false;
112 
113  auto *rf = new RemoteFile(m_dlInfo->m_url, false, false, 0ms);
114  ok = rf->SaveAs(m_dlInfo->m_privData);
115  delete rf;
116 
117  if (!ok)
118  m_dlInfo->m_errorCode = QNetworkReply::UnknownNetworkError;
119 
122 
124  }
125 
126  private:
129 };
130 
134 {
135  if (downloadManager)
136  {
137  delete downloadManager;
138  downloadManager = nullptr;
139  }
140 }
141 
146 {
147  if (downloadManager)
148  return downloadManager;
149 
150  QMutexLocker locker(&dmCreateLock);
151 
152  // Check once more in case the download manager was created
153  // while we were securing the lock.
154  if (downloadManager)
155  return downloadManager;
156 
157  auto *tmpDLM = new MythDownloadManager();
158  tmpDLM->start();
159  while (!tmpDLM->getQueueThread())
160  usleep(10000);
161 
162  tmpDLM->moveToThread(tmpDLM->getQueueThread());
163  tmpDLM->setRunThread();
164 
165  while (!tmpDLM->isRunning())
166  usleep(10000);
167 
168  downloadManager = tmpDLM;
169 
171 
172  return downloadManager;
173 }
174 
178 {
179  m_runThread = false;
180  m_queueWaitCond.wakeAll();
181 
182  wait();
183 
184  delete m_infoLock;
185  delete m_inCookieJar;
186 }
187 
192 {
193  RunProlog();
194 
195  bool downloading = false;
196  bool itemsInQueue = false;
197  bool itemsInCancellationQueue = false;
198  bool waitAnyway = false;
199 
200  m_queueThread = QThread::currentThread();
201 
202  while (!m_runThread)
203  usleep(50ms);
204 
205  m_manager = new QNetworkAccessManager(this);
206  m_diskCache = new QNetworkDiskCache(this);
207  m_proxy = new QNetworkProxy();
208  m_diskCache->setCacheDirectory(GetConfDir() + "/cache/" +
209  QCoreApplication::applicationName() + "-" +
211  m_manager->setCache(m_diskCache);
212 
213  // Set the proxy for the manager to be the application default proxy,
214  // which has already been setup
215  m_manager->setProxy(*m_proxy);
216 
217  // make sure the cookieJar is created in the same thread as the manager
218  // and set its parent to nullptr so it can be shared between managers
219  m_manager->cookieJar()->setParent(nullptr);
220 
221  QObject::connect(m_manager, &QNetworkAccessManager::finished,
222  this, qOverload<QNetworkReply*>(&MythDownloadManager::downloadFinished));
223 
224  m_isRunning = true;
225  while (m_runThread)
226  {
227  if (m_inCookieJar)
228  {
229  LOG(VB_GENERAL, LOG_DEBUG, "Updating DLManager's Cookie Jar");
230  updateCookieJar();
231  }
232  m_infoLock->lock();
233  LOG(VB_FILE, LOG_DEBUG, LOC + QString("items downloading %1").arg(m_downloadInfos.count()));
234  LOG(VB_FILE, LOG_DEBUG, LOC + QString("items queued %1").arg(m_downloadQueue.count()));
235  downloading = !m_downloadInfos.isEmpty();
236  itemsInCancellationQueue = !m_cancellationQueue.isEmpty();
237  m_infoLock->unlock();
238 
239  if (itemsInCancellationQueue)
240  {
242  }
243  if (downloading)
244  QCoreApplication::processEvents();
245 
246  m_infoLock->lock();
247  itemsInQueue = !m_downloadQueue.isEmpty();
248  m_infoLock->unlock();
249 
250  if (!itemsInQueue || waitAnyway)
251  {
252  waitAnyway = false;
253  m_queueWaitLock.lock();
254 
255  if (downloading)
256  {
257  LOG(VB_FILE, LOG_DEBUG, LOC + QString("waiting 200ms"));
258  m_queueWaitCond.wait(&m_queueWaitLock, 200);
259  }
260  else
261  {
262  LOG(VB_FILE, LOG_DEBUG, LOC + QString("waiting for more items to download"));
264  }
265 
266  m_queueWaitLock.unlock();
267  }
268 
269  m_infoLock->lock();
270  if (!m_downloadQueue.isEmpty())
271  {
272  MythDownloadInfo *dlInfo = m_downloadQueue.front();
273 
274  m_downloadQueue.pop_front();
275 
276  if (!dlInfo)
277  {
278  m_infoLock->unlock();
279  continue;
280  }
281 
282  if (m_downloadInfos.contains(dlInfo->m_url))
283  {
284  // Push request to the end of the queue to let others process.
285  // If this is the only item in the queue, force the loop to
286  // wait a little.
287  if (m_downloadQueue.isEmpty())
288  waitAnyway = true;
289  m_downloadQueue.push_back(dlInfo);
290  m_infoLock->unlock();
291  continue;
292  }
293 
294  if (dlInfo->m_url.startsWith("myth://"))
295  downloadRemoteFile(dlInfo);
296  else
297  {
298  QMutexLocker cLock(&m_cookieLock);
299  downloadQNetworkRequest(dlInfo);
300  }
301 
302  m_downloadInfos[dlInfo->m_url] = dlInfo;
303  }
304  m_infoLock->unlock();
305  }
306  m_isRunning = false;
307 
308  RunEpilog();
309 }
310 
321 void MythDownloadManager::queueItem(const QString &url, QNetworkRequest *req,
322  const QString &dest, QByteArray *data,
323  QObject *caller, const MRequestType reqType,
324  const bool reload)
325 {
326  auto *dlInfo = new MythDownloadInfo;
327 
328  dlInfo->m_url = url;
329  dlInfo->m_request = req;
330  dlInfo->m_outFile = dest;
331  dlInfo->m_data = data;
332  dlInfo->m_caller = caller;
333  dlInfo->m_requestType = reqType;
334  dlInfo->m_reload = reload;
335 
336  QMutexLocker locker(m_infoLock);
337  m_downloadQueue.push_back(dlInfo);
338  m_queueWaitCond.wakeAll();
339 }
340 
353 bool MythDownloadManager::processItem(const QString &url, QNetworkRequest *req,
354  const QString &dest, QByteArray *data,
355  const MRequestType reqType,
356  const bool reload,
357  AuthCallback authCallbackFn, void *authArg,
358  const QHash<QByteArray, QByteArray> *headers,
359  QString *finalUrl)
360 {
361  auto *dlInfo = new MythDownloadInfo;
362 
363  dlInfo->m_url = url;
364  dlInfo->m_request = req;
365  dlInfo->m_outFile = dest;
366  dlInfo->m_data = data;
367  dlInfo->m_requestType = reqType;
368  dlInfo->m_reload = reload;
369  dlInfo->m_syncMode = true;
370  dlInfo->m_authCallback = authCallbackFn;
371  dlInfo->m_authArg = authArg;
372  dlInfo->m_headers = headers;
373  dlInfo->m_finalUrl = finalUrl;
374 
375  return downloadNow(dlInfo, true);
376 }
377 
381 void MythDownloadManager::preCache(const QString &url)
382 {
383  LOG(VB_FILE, LOG_DEBUG, LOC + QString("preCache('%1')").arg(url));
384  queueItem(url, nullptr, QString(), nullptr, nullptr);
385 }
386 
393 void MythDownloadManager::queueDownload(const QString &url,
394  const QString &dest,
395  QObject *caller,
396  const bool reload)
397 {
398  LOG(VB_FILE, LOG_DEBUG, LOC + QString("queueDownload('%1', '%2', %3)")
399  .arg(url, dest, QString::number((long long)caller)));
400 
401  queueItem(url, nullptr, dest, nullptr, caller, kRequestGet, reload);
402 }
403 
409 void MythDownloadManager::queueDownload(QNetworkRequest *req,
410  QByteArray *data,
411  QObject *caller)
412 {
413  LOG(VB_FILE, LOG_DEBUG, LOC + QString("queueDownload('%1', '%2', %3)")
414  .arg(req->url().toString()).arg((long long)data)
415  .arg((long long)caller));
416 
417  queueItem(req->url().toString(), req, QString(), data, caller,
418  kRequestGet,
419  (QNetworkRequest::AlwaysNetwork == req->attribute(
420  QNetworkRequest::CacheLoadControlAttribute,
421  QNetworkRequest::PreferNetwork).toInt()));
422 }
423 
430 bool MythDownloadManager::download(const QString &url, const QString &dest,
431  const bool reload)
432 {
433  return processItem(url, nullptr, dest, nullptr, kRequestGet, reload);
434 }
435 
442 bool MythDownloadManager::download(const QString &url, QByteArray *data,
443  const bool reload, QString *finalUrl)
444 {
445  QString redirected;
446  if (!processItem(url, nullptr, QString(), data, kRequestGet, reload,
447  nullptr, nullptr, nullptr, &redirected))
448  return false;
449  if (!redirected.isEmpty() && finalUrl != nullptr)
450  *finalUrl = redirected;
451  return true;
452 }
453 
460 QNetworkReply *MythDownloadManager::download(const QString &url,
461  const bool reload)
462 {
463  auto *dlInfo = new MythDownloadInfo;
464  QNetworkReply *reply = nullptr;
465 
466  dlInfo->m_url = url;
467  dlInfo->m_reload = reload;
468  dlInfo->m_syncMode = true;
469  dlInfo->m_processReply = false;
470 
471  if (downloadNow(dlInfo, false))
472  {
473  if (dlInfo->m_reply)
474  {
475  reply = dlInfo->m_reply;
476  // prevent dlInfo dtor from deleting the reply
477  dlInfo->m_reply = nullptr;
478 
479  delete dlInfo;
480 
481  return reply;
482  }
483 
484  delete dlInfo;
485  }
486 
487  return nullptr;
488 }
489 
495 bool MythDownloadManager::download(QNetworkRequest *req, QByteArray *data)
496 {
497  LOG(VB_FILE, LOG_DEBUG, LOC + QString("download('%1', '%2')")
498  .arg(req->url().toString()).arg((long long)data));
499  return processItem(req->url().toString(), req, QString(), data,
500  kRequestGet,
501  (QNetworkRequest::AlwaysNetwork == req->attribute(
502  QNetworkRequest::CacheLoadControlAttribute,
503  QNetworkRequest::PreferNetwork).toInt()));
504 }
505 
515 bool MythDownloadManager::downloadAuth(const QString &url, const QString &dest,
516  const bool reload, AuthCallback authCallbackFn, void *authArg,
517  const QHash<QByteArray, QByteArray> *headers)
518 {
519  return processItem(url, nullptr, dest, nullptr, kRequestGet, reload, authCallbackFn,
520  authArg, headers);
521 }
522 
523 
529 void MythDownloadManager::queuePost(const QString &url,
530  QByteArray *data,
531  QObject *caller)
532 {
533  LOG(VB_FILE, LOG_DEBUG, LOC + QString("queuePost('%1', '%2')")
534  .arg(url).arg((long long)data));
535 
536  if (!data)
537  {
538  LOG(VB_GENERAL, LOG_ERR, LOC + "queuePost(), data is NULL!");
539  return;
540  }
541 
542  queueItem(url, nullptr, QString(), data, caller, kRequestPost);
543 }
544 
550 void MythDownloadManager::queuePost(QNetworkRequest *req,
551  QByteArray *data,
552  QObject *caller)
553 {
554  LOG(VB_FILE, LOG_DEBUG, LOC + QString("queuePost('%1', '%2')")
555  .arg(req->url().toString()).arg((long long)data));
556 
557  if (!data)
558  {
559  LOG(VB_GENERAL, LOG_ERR, LOC + "queuePost(), data is NULL!");
560  return;
561  }
562 
563  queueItem(req->url().toString(), req, QString(), data, caller,
564  kRequestPost,
565  (QNetworkRequest::AlwaysNetwork == req->attribute(
566  QNetworkRequest::CacheLoadControlAttribute,
567  QNetworkRequest::PreferNetwork).toInt()));
568 
569 }
570 
576 bool MythDownloadManager::post(const QString &url, QByteArray *data)
577 {
578  LOG(VB_FILE, LOG_DEBUG, LOC + QString("post('%1', '%2')")
579  .arg(url).arg((long long)data));
580 
581  if (!data)
582  {
583  LOG(VB_GENERAL, LOG_ERR, LOC + "post(), data is NULL!");
584  return false;
585  }
586 
587  return processItem(url, nullptr, QString(), data, kRequestPost);
588 }
589 
595 bool MythDownloadManager::post(QNetworkRequest *req, QByteArray *data)
596 {
597  LOG(VB_FILE, LOG_DEBUG, LOC + QString("post('%1', '%2')")
598  .arg(req->url().toString()).arg((long long)data));
599 
600  if (!data)
601  {
602  LOG(VB_GENERAL, LOG_ERR, LOC + "post(), data is NULL!");
603  return false;
604  }
605 
606  return processItem(req->url().toString(), req, QString(), data,
607  kRequestPost,
608  (QNetworkRequest::AlwaysNetwork == req->attribute(
609  QNetworkRequest::CacheLoadControlAttribute,
610  QNetworkRequest::PreferNetwork).toInt()));
611 
612 }
613 
622 bool MythDownloadManager::postAuth(const QString &url, QByteArray *data,
623  AuthCallback authCallbackFn, void *authArg,
624  const QHash<QByteArray, QByteArray> *headers)
625 {
626  LOG(VB_FILE, LOG_DEBUG, LOC + QString("postAuth('%1', '%2')")
627  .arg(url).arg((long long)data));
628 
629  if (!data)
630  {
631  LOG(VB_GENERAL, LOG_ERR, LOC + "postAuth(), data is NULL!");
632  return false;
633  }
634 
635  return processItem(url, nullptr, nullptr, data, kRequestPost, false, authCallbackFn,
636  authArg, headers);
637 }
638 
643 {
644  auto *dlThread = new RemoteFileDownloadThread(this, dlInfo);
645  MThreadPool::globalInstance()->start(dlThread, "RemoteFileDownload");
646 }
647 
652 {
653  if (!dlInfo)
654  return;
655 
656  static const QString kDateFormat = "ddd, dd MMM yyyy hh:mm:ss 'GMT'";
657  QUrl qurl(dlInfo->m_url);
658  QNetworkRequest request;
659 
660  if (dlInfo->m_request)
661  {
662  request = *dlInfo->m_request;
663  delete dlInfo->m_request;
664  dlInfo->m_request = nullptr;
665  }
666  else
667  request.setUrl(qurl);
668 
669  if (dlInfo->m_reload)
670  {
671  request.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
672  QNetworkRequest::AlwaysNetwork);
673  }
674  else
675  {
676  // Prefer the in-cache item if one exists and it is less than 5 minutes
677  // old and it will not expire in the next 10 seconds
678  QDateTime now = MythDate::current();
679 
680  // Handle redirects, we want the metadata of the file headers
681  QString redirectLoc;
682  int limit = 0;
683  while (!(redirectLoc = getHeader(qurl, "Location")).isNull())
684  {
685  if (limit == CACHE_REDIRECTION_LIMIT)
686  {
687  LOG(VB_GENERAL, LOG_WARNING, QString("Cache Redirection limit "
688  "reached for %1")
689  .arg(qurl.toString()));
690  return;
691  }
692  qurl.setUrl(redirectLoc);
693  limit++;
694  }
695 
696  LOG(VB_NETWORK, LOG_DEBUG, QString("Checking cache for %1")
697  .arg(qurl.toString()));
698 
699  m_infoLock->lock();
700  QNetworkCacheMetaData urlData = m_manager->cache()->metaData(qurl);
701  m_infoLock->unlock();
702  if ((urlData.isValid()) &&
703  ((!urlData.expirationDate().isValid()) ||
704  (urlData.expirationDate().toUTC().secsTo(now) < 10)))
705  {
706  QString dateString = getHeader(urlData, "Date");
707 
708  if (!dateString.isNull())
709  {
710  QDateTime loadDate =
711  MythDate::fromString(dateString, kDateFormat);
712  loadDate.setTimeSpec(Qt::UTC);
713  if (loadDate.secsTo(now) <= 720)
714  {
715  dlInfo->m_preferCache = true;
716  LOG(VB_NETWORK, LOG_DEBUG, QString("Preferring cache for %1")
717  .arg(qurl.toString()));
718  }
719  }
720  }
721  }
722 
723  if (dlInfo->m_preferCache)
724  request.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
725  QNetworkRequest::PreferCache);
726 
727  if (!request.hasRawHeader("User-Agent"))
728  {
729  request.setRawHeader("User-Agent",
730  QByteArray("MythTV v") + MYTH_BINARY_VERSION +
731  " MythDownloadManager");
732  }
733 
734  if (dlInfo->m_headers)
735  {
736  QHash<QByteArray, QByteArray>::const_iterator it =
737  dlInfo->m_headers->constBegin();
738  for ( ; it != dlInfo->m_headers->constEnd(); ++it )
739  {
740  if (!it.key().isEmpty() && !it.value().isEmpty())
741  {
742  request.setRawHeader(it.key(), it.value());
743  }
744  }
745  }
746 
747  switch (dlInfo->m_requestType)
748  {
749  case kRequestPost :
750  dlInfo->m_reply = m_manager->post(request, *dlInfo->m_data);
751  break;
752  case kRequestHead :
753  dlInfo->m_reply = m_manager->head(request);
754  break;
755  case kRequestGet :
756  default:
757  dlInfo->m_reply = m_manager->get(request);
758  break;
759  }
760 
761  m_downloadReplies[dlInfo->m_reply] = dlInfo;
762 
763  if (dlInfo->m_authCallback)
764  {
765  connect(m_manager, &QNetworkAccessManager::authenticationRequired,
767  }
768 
769 #if QT_VERSION < QT_VERSION_CHECK(5,15,0)
770  connect(dlInfo->m_reply, qOverload<QNetworkReply::NetworkError>(&QNetworkReply::error),
772 #else
773  connect(dlInfo->m_reply, &QNetworkReply::errorOccurred,
775 #endif
776  connect(dlInfo->m_reply, &QNetworkReply::downloadProgress,
778 }
779 
784 void MythDownloadManager::authCallback(QNetworkReply *reply,
785  QAuthenticator *authenticator)
786 {
787  if (!reply)
788  return;
789 
790  MythDownloadInfo *dlInfo = m_downloadReplies[reply];
791 
792  if (!dlInfo)
793  return;
794 
795  if (dlInfo->m_authCallback)
796  {
797  LOG(VB_FILE, LOG_DEBUG, "Calling auth callback");
798  dlInfo->m_authCallback(reply, authenticator, dlInfo->m_authArg);
799  }
800 }
801 
809 {
810  if (!dlInfo)
811  return false;
812 
813  dlInfo->m_syncMode = true;
814 
815  // Special handling for link-local
816  // Not needed for Windows because windows does not need
817  // the scope id.
818 #ifndef _WIN32
819  if (dlInfo->m_url.startsWith("http://[fe80::",Qt::CaseInsensitive))
820  return downloadNowLinkLocal(dlInfo, deleteInfo);
821 #endif
822  m_infoLock->lock();
823  m_downloadQueue.push_back(dlInfo);
824  m_infoLock->unlock();
825  m_queueWaitCond.wakeAll();
826 
827  // timeout myth:// RemoteFile transfers 20 seconds from now
828  // timeout non-myth:// QNetworkAccessManager transfers 60 seconds after
829  // their last progress update
830  QDateTime startedAt = MythDate::current();
831  m_infoLock->lock();
832  while ((!dlInfo->IsDone()) &&
833  (dlInfo->m_errorCode == QNetworkReply::NoError) &&
834  (((!dlInfo->m_url.startsWith("myth://")) &&
835  (MythDate::secsInPast(dlInfo->m_lastStat) < 60s)) ||
836  ((dlInfo->m_url.startsWith("myth://")) &&
837  (MythDate::secsInPast(startedAt) < 20s))))
838  {
839  m_infoLock->unlock();
840  m_queueWaitLock.lock();
841  m_queueWaitCond.wait(&m_queueWaitLock, 200);
842  m_queueWaitLock.unlock();
843  m_infoLock->lock();
844  }
845  bool done = dlInfo->IsDone();
846  bool success =
847  done && (dlInfo->m_errorCode == QNetworkReply::NoError);
848 
849  if (!done)
850  {
851  dlInfo->m_data = nullptr; // Prevent downloadFinished() from updating
852  dlInfo->m_syncMode = false; // Let downloadFinished() cleanup for us
853  if ((dlInfo->m_reply) &&
854  (dlInfo->m_errorCode == QNetworkReply::NoError))
855  {
856  LOG(VB_FILE, LOG_DEBUG,
857  LOC + QString("Aborting download - lack of data transfer"));
858  dlInfo->m_reply->abort();
859  }
860  }
861  else if (deleteInfo)
862  delete dlInfo;
863 
864  m_infoLock->unlock();
865 
866  return success;
867 }
868 
869 #ifndef _WIN32
870 
890 bool MythDownloadManager::downloadNowLinkLocal(MythDownloadInfo *dlInfo, bool deleteInfo)
891 {
892  bool ok = true;
893 
894  // No buffer - no reply...
895  if (!dlInfo->m_data)
896  {
897  LOG(VB_GENERAL, LOG_ERR, LOC + QString("No data buffer provided for %1").arg(dlInfo->m_url));
898  ok = false;
899  }
900 
901  // Only certain features are supported here
902  if (dlInfo->m_authCallback || dlInfo->m_authArg)
903  {
904  LOG(VB_GENERAL, LOG_ERR, LOC + QString("Unsupported authentication for %1").arg(dlInfo->m_url));
905  ok = false;
906  }
907 
908  if (ok && !dlInfo->m_outFile.isEmpty())
909  {
910  LOG(VB_GENERAL, LOG_ERR, LOC + QString("Unsupported File output %1 for %2")
911  .arg(dlInfo->m_outFile, dlInfo->m_url));
912  ok = false;
913  }
914 
915  if (ok && (!deleteInfo || dlInfo->m_requestType == kRequestHead))
916  {
917  // We do not have the ability to return a network reply in dlInfo
918  // so if we are asked to do that, return an error.
919  LOG(VB_GENERAL, LOG_ERR, LOC + QString("Unsupported link-local operation %1")
920  .arg(dlInfo->m_url));
921  ok = false;
922  }
923 
924  QUrl url(dlInfo->m_url);
925  QString host(url.host());
926  int port(url.port(80));
927  if (ok && PortChecker::resolveLinkLocal(host, port))
928  {
929  QString reqType;
930  switch (dlInfo->m_requestType)
931  {
932  case kRequestPost :
933  reqType = "POST";
934  break;
935  case kRequestGet :
936  default:
937  reqType = "GET";
938  break;
939  }
940  QByteArray* buffer = dlInfo->m_data;
941  QHash<QByteArray, QByteArray> headers;
942  if (dlInfo->m_headers)
943  headers = *dlInfo->m_headers;
944  if (!headers.contains("User-Agent"))
945  headers.insert("User-Agent", QByteArray("MythDownloadManager v") +
946  MYTH_BINARY_VERSION);
947  headers.insert("Connection", "close");
948  headers.insert("Accept-Encoding", "identity");
949  if (!buffer->isEmpty())
950  headers.insert("Content-Length", QString::number(buffer->size()).toUtf8());
951  headers.insert("Host", (url.host() + ":" + QString::number(port)).toUtf8());
952 
953  QByteArray requestMessage;
954  QString path (url.path());
955  requestMessage.append("POST ");
956  requestMessage.append(path.toLatin1());
957  requestMessage.append(" HTTP/1.1\r\n");
958  QHashIterator<QByteArray, QByteArray> it(headers);
959  while (it.hasNext())
960  {
961  it.next();
962  requestMessage.append(it.key());
963  requestMessage.append(": ");
964  requestMessage.append(it.value());
965  requestMessage.append("\r\n");
966  }
967  requestMessage.append("\r\n");
968  if (!buffer->isEmpty())
969  requestMessage.append(*buffer);
970 
971  QTcpSocket socket;
972  socket.connectToHost(host, static_cast<uint16_t>(port));
973  // QT Warning - this may not work on Windows
974  if (!socket.waitForConnected(5000))
975  ok = false;
976  if (ok)
977  ok = socket.write(requestMessage) > 0;
978  if (ok)
979  // QT Warning - this may not work on Windows
980  ok = socket.waitForDisconnected(5000);
981  if (ok)
982  {
983  *buffer = socket.readAll();
984  // Find the start of the content
985  QByteArray delim("\r\n\r\n");
986  int delimLoc = buffer->indexOf(delim);
987  if (delimLoc > -1)
988  *buffer = buffer->right(buffer->size() - delimLoc - 4);
989  else
990  ok=false;
991  }
992  socket.close();
993  }
994  else
995  {
996  ok = false;
997  }
998 
999  if (deleteInfo)
1000  delete dlInfo;
1001 
1002  if (ok)
1003  return true;
1004 
1005  LOG(VB_GENERAL, LOG_ERR, LOC + QString("Link Local request failed: %1").arg(url.toString()));
1006  return false;
1007 }
1008 #endif
1009 
1014 void MythDownloadManager::cancelDownload(const QString &url, bool block)
1015 {
1016  cancelDownload(QStringList(url), block);
1017 }
1018 
1023 void MythDownloadManager::cancelDownload(const QStringList &urls, bool block)
1024 {
1025  m_infoLock->lock();
1026  for (const auto& url : qAsConst(urls))
1027  {
1028  QMutableListIterator<MythDownloadInfo*> lit(m_downloadQueue);
1029  while (lit.hasNext())
1030  {
1031  lit.next();
1032  MythDownloadInfo *dlInfo = lit.value();
1033  if (dlInfo->m_url == url)
1034  {
1035  if (!m_cancellationQueue.contains(dlInfo))
1036  m_cancellationQueue.append(dlInfo);
1037  lit.remove();
1038  }
1039  }
1040 
1041  if (m_downloadInfos.contains(url))
1042  {
1043  MythDownloadInfo *dlInfo = m_downloadInfos[url];
1044 
1045  if (!m_cancellationQueue.contains(dlInfo))
1046  m_cancellationQueue.append(dlInfo);
1047 
1048  if (dlInfo->m_reply)
1049  m_downloadReplies.remove(dlInfo->m_reply);
1050 
1051  m_downloadInfos.remove(url);
1052  }
1053  }
1054  m_infoLock->unlock();
1055 
1056  if (QThread::currentThread() == this->thread())
1057  {
1058  downloadCanceled();
1059  return;
1060  }
1061 
1062  // wake-up running thread
1063  m_queueWaitCond.wakeAll();
1064 
1065  if (!block)
1066  return;
1067 
1068  while (!m_cancellationQueue.isEmpty())
1069  {
1070  usleep(50ms); // re-test in another 50ms
1071  }
1072 }
1073 
1075 {
1076  QMutexLocker locker(m_infoLock);
1077 
1078  QMutableListIterator<MythDownloadInfo*> lit(m_cancellationQueue);
1079  while (lit.hasNext())
1080  {
1081  lit.next();
1082  MythDownloadInfo *dlInfo = lit.value();
1083  dlInfo->m_lock.lock();
1084 
1085  if (dlInfo->m_reply)
1086  {
1087  LOG(VB_FILE, LOG_DEBUG,
1088  LOC + QString("Aborting download - user request"));
1089  dlInfo->m_reply->abort();
1090  }
1091  lit.remove();
1092  if (dlInfo->m_done)
1093  {
1094  dlInfo->m_lock.unlock();
1095  continue;
1096  }
1097  dlInfo->m_errorCode = QNetworkReply::OperationCanceledError;
1098  dlInfo->m_done = true;
1099  dlInfo->m_lock.unlock();
1100  }
1101 }
1102 
1108 {
1109  QMutexLocker locker(m_infoLock);
1110 
1111  QList <MythDownloadInfo*>::iterator lit = m_downloadQueue.begin();
1112  for (; lit != m_downloadQueue.end(); ++lit)
1113  {
1114  MythDownloadInfo *dlInfo = *lit;
1115  if (dlInfo->m_caller == caller)
1116  {
1117  dlInfo->m_caller = nullptr;
1118  dlInfo->m_outFile = QString();
1119  dlInfo->m_data = nullptr;
1120  }
1121  }
1122 
1123  QMap <QString, MythDownloadInfo*>::iterator mit = m_downloadInfos.begin();
1124  for (; mit != m_downloadInfos.end(); ++mit)
1125  {
1126  MythDownloadInfo *dlInfo = mit.value();
1127  if (dlInfo->m_caller == caller)
1128  {
1129  dlInfo->m_caller = nullptr;
1130  dlInfo->m_outFile = QString();
1131  dlInfo->m_data = nullptr;
1132  }
1133  }
1134 }
1135 
1139 void MythDownloadManager::downloadError(QNetworkReply::NetworkError errorCode)
1140 {
1141  auto *reply = qobject_cast<QNetworkReply *>(sender());
1142  if (reply == nullptr)
1143  return;
1144 
1145  LOG(VB_FILE, LOG_DEBUG, LOC + QString("downloadError %1 ")
1146  .arg(errorCode) + reply->errorString() );
1147 
1148  QMutexLocker locker(m_infoLock);
1149  if (!m_downloadReplies.contains(reply))
1150  {
1151  reply->deleteLater();
1152  return;
1153  }
1154 
1155  MythDownloadInfo *dlInfo = m_downloadReplies[reply];
1156 
1157  if (!dlInfo)
1158  return;
1159 
1160  dlInfo->m_errorCode = errorCode;
1161 }
1162 
1168 QUrl MythDownloadManager::redirectUrl(const QUrl& possibleRedirectUrl,
1169  const QUrl& oldRedirectUrl)
1170 {
1171  LOG(VB_FILE, LOG_DEBUG, LOC + QString("redirectUrl()"));
1172  QUrl redirectUrl;
1173 
1174  if(!possibleRedirectUrl.isEmpty() && possibleRedirectUrl != oldRedirectUrl)
1175  redirectUrl = possibleRedirectUrl;
1176 
1177  return redirectUrl;
1178 }
1179 
1183 void MythDownloadManager::downloadFinished(QNetworkReply* reply)
1184 {
1185  LOG(VB_FILE, LOG_DEBUG, LOC + QString("downloadFinished(%1)")
1186  .arg((long long)reply));
1187 
1188  QMutexLocker locker(m_infoLock);
1189  if (!m_downloadReplies.contains(reply))
1190  {
1191  reply->deleteLater();
1192  return;
1193  }
1194 
1195  MythDownloadInfo *dlInfo = m_downloadReplies[reply];
1196 
1197  if (!dlInfo || !dlInfo->m_reply)
1198  return;
1199 
1200  downloadFinished(dlInfo);
1201 }
1202 
1207 {
1208  if (!dlInfo)
1209  return;
1210 
1211  int statusCode = -1;
1212  static const QString kDateFormat = "ddd, dd MMM yyyy hh:mm:ss 'GMT'";
1213  QNetworkReply *reply = dlInfo->m_reply;
1214 
1215  if (reply)
1216  {
1217  QUrl possibleRedirectUrl =
1218  reply->attribute(QNetworkRequest::RedirectionTargetAttribute).toUrl();
1219 
1220  if (!possibleRedirectUrl.isEmpty() &&
1221  possibleRedirectUrl.isValid() &&
1222  possibleRedirectUrl.isRelative()) // Turn relative Url to absolute
1223  possibleRedirectUrl = reply->url().resolved(possibleRedirectUrl);
1224 
1225  if (!possibleRedirectUrl.isEmpty() && dlInfo->m_finalUrl != nullptr)
1226  *dlInfo->m_finalUrl = QString(possibleRedirectUrl.toString());
1227 
1228  dlInfo->m_redirectedTo =
1229  redirectUrl(possibleRedirectUrl, dlInfo->m_redirectedTo);
1230 
1231  QVariant status =
1232  reply->attribute(QNetworkRequest::HttpStatusCodeAttribute);
1233  if (status.isValid())
1234  statusCode = status.toInt();
1235  }
1236 
1237  if(reply && !dlInfo->m_redirectedTo.isEmpty() &&
1238  ((dlInfo->m_requestType != kRequestPost) ||
1239  (statusCode == 301 || statusCode == 302 ||
1240  statusCode == 303)))
1241  {
1242  LOG(VB_FILE, LOG_DEBUG, LOC +
1243  QString("downloadFinished(%1): Redirect: %2 -> %3")
1244  .arg(QString::number((long long)dlInfo),
1245  reply->url().toString(),
1246  dlInfo->m_redirectedTo.toString()));
1247 
1248  if (dlInfo->m_data)
1249  dlInfo->m_data->clear();
1250 
1251  dlInfo->m_bytesReceived = 0;
1252  dlInfo->m_bytesTotal = 0;
1253 
1254  QNetworkRequest request(dlInfo->m_redirectedTo);
1255 
1256  if (dlInfo->m_reload)
1257  {
1258  request.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
1259  QNetworkRequest::AlwaysNetwork);
1260  }
1261  else if (dlInfo->m_preferCache)
1262  {
1263  request.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
1264  QNetworkRequest::PreferCache);
1265  }
1266 
1267  request.setRawHeader("User-Agent",
1268  "MythDownloadManager v" +
1269  QByteArray(MYTH_BINARY_VERSION));
1270 
1271  switch (dlInfo->m_requestType)
1272  {
1273  case kRequestHead :
1274  dlInfo->m_reply = m_manager->head(request);
1275  break;
1276  case kRequestGet :
1277  default:
1278  dlInfo->m_reply = m_manager->get(request);
1279  break;
1280  }
1281 
1282  m_downloadReplies[dlInfo->m_reply] = dlInfo;
1283 
1284 #if QT_VERSION < QT_VERSION_CHECK(5,15,0)
1285  connect(dlInfo->m_reply, qOverload<QNetworkReply::NetworkError>(&QNetworkReply::error),
1287 #else
1288  connect(dlInfo->m_reply, &QNetworkReply::errorOccurred,
1290 #endif
1291  connect(dlInfo->m_reply, &QNetworkReply::downloadProgress,
1293 
1294  m_downloadReplies.remove(reply);
1295  reply->deleteLater();
1296  }
1297  else
1298  {
1299  LOG(VB_FILE, LOG_DEBUG, QString("downloadFinished(%1): COMPLETE: %2")
1300  .arg((long long)dlInfo).arg(dlInfo->m_url));
1301 
1302  // HACK Insert a Date header into the cached metadata if one doesn't
1303  // already exist
1304  QUrl fileUrl = dlInfo->m_url;
1305  QString redirectLoc;
1306  int limit = 0;
1307  while (!(redirectLoc = getHeader(fileUrl, "Location")).isNull())
1308  {
1309  if (limit == CACHE_REDIRECTION_LIMIT)
1310  {
1311  LOG(VB_GENERAL, LOG_WARNING, QString("Cache Redirection limit "
1312  "reached for %1")
1313  .arg(fileUrl.toString()));
1314  return;
1315  }
1316  fileUrl.setUrl(redirectLoc);
1317  limit++;
1318  }
1319 
1320  m_infoLock->lock();
1321  QNetworkCacheMetaData urlData = m_manager->cache()->metaData(fileUrl);
1322  m_infoLock->unlock();
1323  if (getHeader(urlData, "Date").isNull())
1324  {
1325  QNetworkCacheMetaData::RawHeaderList headers = urlData.rawHeaders();
1326  QNetworkCacheMetaData::RawHeader newheader;
1327  QDateTime now = MythDate::current();
1328  newheader = QNetworkCacheMetaData::RawHeader("Date",
1329  now.toString(kDateFormat).toLatin1());
1330  headers.append(newheader);
1331  urlData.setRawHeaders(headers);
1332  m_infoLock->lock();
1333  m_manager->cache()->updateMetaData(urlData);
1334  m_infoLock->unlock();
1335  }
1336  // End HACK
1337 
1338  dlInfo->m_redirectedTo.clear();
1339 
1340  int dataSize = -1;
1341 
1342  // If we downloaded via the QNetworkAccessManager
1343  // AND the caller isn't handling the reply directly
1344  if (reply && dlInfo->m_processReply)
1345  {
1346  bool append = (!dlInfo->m_syncMode && dlInfo->m_caller);
1347  QByteArray data = reply->readAll();
1348  dataSize = data.size();
1349 
1350  if (append)
1351  dlInfo->m_bytesReceived += dataSize;
1352  else
1353  dlInfo->m_bytesReceived = dataSize;
1354 
1355  dlInfo->m_bytesTotal = dlInfo->m_bytesReceived;
1356 
1357  if (dlInfo->m_data)
1358  {
1359  if (append)
1360  dlInfo->m_data->append(data);
1361  else
1362  *dlInfo->m_data = data;
1363  }
1364  else if (!dlInfo->m_outFile.isEmpty())
1365  {
1366  saveFile(dlInfo->m_outFile, data, append);
1367  }
1368  }
1369  else if (!reply) // If we downloaded via RemoteFile
1370  {
1371  if (dlInfo->m_data)
1372  {
1373  (*dlInfo->m_data) = dlInfo->m_privData;
1374  }
1375  else if (!dlInfo->m_outFile.isEmpty())
1376  {
1377  saveFile(dlInfo->m_outFile, dlInfo->m_privData);
1378  }
1379  dlInfo->m_bytesReceived += dataSize;
1380  dlInfo->m_bytesTotal = dlInfo->m_bytesReceived;
1381  }
1382  // else we downloaded via QNetworkAccessManager
1383  // AND the caller is handling the reply
1384 
1385  m_infoLock->lock();
1386  if (!m_downloadInfos.remove(dlInfo->m_url))
1387  {
1388  LOG(VB_GENERAL, LOG_ERR, LOC +
1389  QString("ERROR download finished but failed to remove url: %1")
1390  .arg(dlInfo->m_url));
1391  }
1392 
1393  if (reply)
1394  m_downloadReplies.remove(reply);
1395  m_infoLock->unlock();
1396 
1397  dlInfo->SetDone(true);
1398 
1399  if (!dlInfo->m_syncMode)
1400  {
1401  if (dlInfo->m_caller)
1402  {
1403  LOG(VB_FILE, LOG_DEBUG, QString("downloadFinished(%1): "
1404  "COMPLETE: %2, sending event to caller")
1405  .arg((long long)dlInfo).arg(dlInfo->m_url));
1406 
1407  QStringList args;
1408  args << dlInfo->m_url;
1409  args << dlInfo->m_outFile;
1410  args << QString::number(dlInfo->m_bytesTotal);
1411  // placeholder for error string
1412  args << (reply ? reply->errorString() : QString());
1413  args << QString::number((int)(reply ? reply->error() :
1414  dlInfo->m_errorCode));
1415 
1416  QCoreApplication::postEvent(dlInfo->m_caller,
1417  new MythEvent("DOWNLOAD_FILE FINISHED", args));
1418  }
1419 
1420  delete dlInfo;
1421  }
1422 
1423  m_queueWaitCond.wakeAll();
1424  }
1425 }
1426 
1432 void MythDownloadManager::downloadProgress(qint64 bytesReceived,
1433  qint64 bytesTotal)
1434 {
1435  auto *reply = qobject_cast<QNetworkReply *>(sender());
1436  if (reply == nullptr)
1437  return;
1438 
1439  LOG(VB_FILE, LOG_DEBUG, LOC +
1440  QString("downloadProgress(%1, %2) (for reply %3)")
1441  .arg(bytesReceived).arg(bytesTotal).arg((long long)reply));
1442 
1443  QMutexLocker locker(m_infoLock);
1444  if (!m_downloadReplies.contains(reply))
1445  return;
1446 
1447  MythDownloadInfo *dlInfo = m_downloadReplies[reply];
1448 
1449  if (!dlInfo)
1450  return;
1451 
1452  dlInfo->m_lastStat = MythDate::current();
1453 
1454  LOG(VB_FILE, LOG_DEBUG, LOC +
1455  QString("downloadProgress: %1 to %2 is at %3 of %4 bytes downloaded")
1456  .arg(dlInfo->m_url, dlInfo->m_outFile)
1457  .arg(bytesReceived).arg(bytesTotal));
1458 
1459  if (!dlInfo->m_syncMode && dlInfo->m_caller)
1460  {
1461  LOG(VB_FILE, LOG_DEBUG, QString("downloadProgress(%1): "
1462  "sending event to caller")
1463  .arg(reply->url().toString()));
1464 
1465  bool appendToFile = (dlInfo->m_bytesReceived != 0);
1466  QByteArray data = reply->readAll();
1467  if (!dlInfo->m_outFile.isEmpty())
1468  saveFile(dlInfo->m_outFile, data, appendToFile);
1469 
1470  if (dlInfo->m_data)
1471  dlInfo->m_data->append(data);
1472 
1473  dlInfo->m_bytesReceived = bytesReceived;
1474  dlInfo->m_bytesTotal = bytesTotal;
1475 
1476  QStringList args;
1477  args << dlInfo->m_url;
1478  args << dlInfo->m_outFile;
1479  args << QString::number(bytesReceived);
1480  args << QString::number(bytesTotal);
1481 
1482  QCoreApplication::postEvent(dlInfo->m_caller,
1483  new MythEvent("DOWNLOAD_FILE UPDATE", args));
1484  }
1485 }
1486 
1494 bool MythDownloadManager::saveFile(const QString &outFile,
1495  const QByteArray &data,
1496  const bool append)
1497 {
1498  if (outFile.isEmpty() || data.isEmpty())
1499  return false;
1500 
1501  QFile file(outFile);
1502  QFileInfo fileInfo(outFile);
1503  QDir qdir(fileInfo.absolutePath());
1504 
1505  if (!qdir.exists() && !qdir.mkpath(fileInfo.absolutePath()))
1506  {
1507  LOG(VB_GENERAL, LOG_ERR, QString("Failed to create: '%1'")
1508  .arg(fileInfo.absolutePath()));
1509  return false;
1510  }
1511 
1512  QIODevice::OpenMode mode = QIODevice::Unbuffered|QIODevice::WriteOnly;
1513  if (append)
1514  mode |= QIODevice::Append;
1515 
1516  if (!file.open(mode))
1517  {
1518  LOG(VB_GENERAL, LOG_ERR, QString("Failed to open: '%1'") .arg(outFile));
1519  return false;
1520  }
1521 
1522  off_t offset = 0;
1523  size_t remaining = data.size();
1524  uint failure_cnt = 0;
1525  while ((remaining > 0) && (failure_cnt < 5))
1526  {
1527  ssize_t written = file.write(data.data() + offset, remaining);
1528  if (written < 0)
1529  {
1530  failure_cnt++;
1531  usleep(50ms);
1532  continue;
1533  }
1534 
1535  failure_cnt = 0;
1536  offset += written;
1537  remaining -= written;
1538  }
1539 
1540  return remaining <= 0;
1541 }
1542 
1547 QDateTime MythDownloadManager::GetLastModified(const QString &url)
1548 {
1549  // If the header has not expired and
1550  // the last modification date is less than 1 hours old or if
1551  // the cache object is less than 20 minutes old,
1552  // then use the cached header otherwise redownload the header
1553 
1554  static const QString kDateFormat = "ddd, dd MMM yyyy hh:mm:ss 'GMT'";
1555  LOG(VB_FILE, LOG_DEBUG, LOC + QString("GetLastModified('%1')").arg(url));
1556  QDateTime result;
1557 
1558  QDateTime now = MythDate::current();
1559 
1560  QUrl cacheUrl = QUrl(url);
1561 
1562  // Deal with redirects, we want the cached data for the final url
1563  QString redirectLoc;
1564  int limit = 0;
1565  while (!(redirectLoc = getHeader(cacheUrl, "Location")).isNull())
1566  {
1567  if (limit == CACHE_REDIRECTION_LIMIT)
1568  {
1569  LOG(VB_GENERAL, LOG_WARNING, QString("Cache Redirection limit "
1570  "reached for %1")
1571  .arg(cacheUrl.toString()));
1572  return result;
1573  }
1574  cacheUrl.setUrl(redirectLoc);
1575  limit++;
1576  }
1577 
1578  m_infoLock->lock();
1579  QNetworkCacheMetaData urlData = m_manager->cache()->metaData(cacheUrl);
1580  m_infoLock->unlock();
1581 
1582  if (urlData.isValid() &&
1583  ((!urlData.expirationDate().isValid()) ||
1584  (urlData.expirationDate().secsTo(now) < 0)))
1585  {
1586  if (urlData.lastModified().toUTC().secsTo(now) <= 3600) // 1 Hour
1587  {
1588  result = urlData.lastModified().toUTC();
1589  }
1590  else
1591  {
1592  QString date = getHeader(urlData, "Date");
1593  if (!date.isNull())
1594  {
1595  QDateTime loadDate =
1596  MythDate::fromString(date, kDateFormat);
1597  loadDate.setTimeSpec(Qt::UTC);
1598  if (loadDate.secsTo(now) <= 1200) // 20 Minutes
1599  {
1600  result = urlData.lastModified().toUTC();
1601  }
1602  }
1603  }
1604  }
1605 
1606  if (!result.isValid())
1607  {
1608  auto *dlInfo = new MythDownloadInfo;
1609  dlInfo->m_url = url;
1610  dlInfo->m_syncMode = true;
1611  // Head request, we only want to inspect the headers
1612  dlInfo->m_requestType = kRequestHead;
1613 
1614  if (downloadNow(dlInfo, false))
1615  {
1616  if (dlInfo->m_reply)
1617  {
1618  QVariant lastMod =
1619  dlInfo->m_reply->header(
1620  QNetworkRequest::LastModifiedHeader);
1621  if (lastMod.isValid())
1622  result = lastMod.toDateTime().toUTC();
1623  }
1624 
1625  // downloadNow() will set a flag to trigger downloadFinished()
1626  // to delete the dlInfo if the download times out
1627  delete dlInfo;
1628  }
1629  }
1630 
1631  LOG(VB_FILE, LOG_DEBUG, LOC + QString("GetLastModified('%1'): Result %2")
1632  .arg(url, result.toString()));
1633 
1634  return result;
1635 }
1636 
1637 
1642 {
1643  QMutexLocker locker(&m_cookieLock);
1644 
1645  auto *jar = new MythCookieJar;
1646  jar->load(filename);
1647  m_manager->setCookieJar(jar);
1648 }
1649 
1654 {
1655  QMutexLocker locker(&m_cookieLock);
1656 
1657  if (!m_manager->cookieJar())
1658  return;
1659 
1660  auto *jar = qobject_cast<MythCookieJar *>(m_manager->cookieJar());
1661  if (jar == nullptr)
1662  return;
1663  jar->save(filename);
1664 }
1665 
1666 void MythDownloadManager::setCookieJar(QNetworkCookieJar *cookieJar)
1667 {
1668  QMutexLocker locker(&m_cookieLock);
1669  m_manager->setCookieJar(cookieJar);
1670 }
1671 
1675 QNetworkCookieJar *MythDownloadManager::copyCookieJar(void)
1676 {
1677  QMutexLocker locker(&m_cookieLock);
1678 
1679  if (!m_manager->cookieJar())
1680  return nullptr;
1681 
1682  auto *inJar = qobject_cast<MythCookieJar *>(m_manager->cookieJar());
1683  if (inJar == nullptr)
1684  return nullptr;
1685  auto *outJar = new MythCookieJar;
1686  outJar->copyAllCookies(*inJar);
1687 
1688  return outJar;
1689 }
1690 
1694 void MythDownloadManager::refreshCookieJar(QNetworkCookieJar *jar)
1695 {
1696  QMutexLocker locker(&m_cookieLock);
1697  delete m_inCookieJar;
1698 
1699  auto *inJar = qobject_cast<MythCookieJar *>(jar);
1700  if (inJar == nullptr)
1701  return;
1702 
1703  auto *outJar = new MythCookieJar;
1704  outJar->copyAllCookies(*inJar);
1705  m_inCookieJar = outJar;
1706 
1707  QMutexLocker locker2(&m_queueWaitLock);
1708  m_queueWaitCond.wakeAll();
1709 }
1710 
1714 {
1715  QMutexLocker locker(&m_cookieLock);
1716 
1717  auto *inJar = qobject_cast<MythCookieJar *>(m_inCookieJar);
1718  if (inJar != nullptr)
1719  {
1720  auto *outJar = new MythCookieJar;
1721  outJar->copyAllCookies(*inJar);
1722  m_manager->setCookieJar(outJar);
1723  }
1724 
1725  delete m_inCookieJar;
1726  m_inCookieJar = nullptr;
1727 }
1728 
1729 QString MythDownloadManager::getHeader(const QUrl& url, const QString& header)
1730 {
1731  if (!m_manager || !m_manager->cache())
1732  return {};
1733 
1734  m_infoLock->lock();
1735  QNetworkCacheMetaData metadata = m_manager->cache()->metaData(url);
1736  m_infoLock->unlock();
1737 
1738  return getHeader(metadata, header);
1739 }
1740 
1746 QString MythDownloadManager::getHeader(const QNetworkCacheMetaData &cacheData,
1747  const QString& header)
1748 {
1749  auto headers = cacheData.rawHeaders();
1750  for (const auto& rh : qAsConst(headers))
1751  if (QString(rh.first) == header)
1752  return {rh.second};
1753  return {};
1754 }
1755 
1756 
1761 {
1762  const QList<QNetworkCookie> cookieList = old.allCookies();
1763  setAllCookies(cookieList);
1764 }
1765 
1769 void MythCookieJar::load(const QString &filename)
1770 {
1771  LOG(VB_GENERAL, LOG_DEBUG, QString("MythCookieJar: loading cookies from: %1").arg(filename));
1772 
1773  QFile f(filename);
1774  if (!f.open(QIODevice::ReadOnly))
1775  {
1776  LOG(VB_GENERAL, LOG_WARNING, QString("MythCookieJar::load() failed to open file for reading: %1").arg(filename));
1777  return;
1778  }
1779 
1780  QList<QNetworkCookie> cookieList;
1781  QTextStream stream(&f);
1782  while (!stream.atEnd())
1783  {
1784  QString cookie = stream.readLine();
1785  cookieList << QNetworkCookie::parseCookies(cookie.toLocal8Bit());
1786  }
1787 
1788  setAllCookies(cookieList);
1789 }
1790 
1794 void MythCookieJar::save(const QString &filename)
1795 {
1796  LOG(VB_GENERAL, LOG_DEBUG, QString("MythCookieJar: saving cookies to: %1").arg(filename));
1797 
1798  QFile f(filename);
1799  if (!f.open(QIODevice::WriteOnly))
1800  {
1801  LOG(VB_GENERAL, LOG_ERR, QString("MythCookieJar::save() failed to open file for writing: %1").arg(filename));
1802  return;
1803  }
1804 
1805  QList<QNetworkCookie> cookieList = allCookies();
1806  QTextStream stream(&f);
1807 
1808 #if QT_VERSION < QT_VERSION_CHECK(5,14,0)
1809  for (const auto& cookie : qAsConst(cookieList))
1810  stream << cookie.toRawForm() << endl;
1811 #else
1812  for (const auto& cookie : qAsConst(cookieList))
1813  stream << cookie.toRawForm() << Qt::endl;
1814 #endif
1815 }
1816 
1817 
1818 /* vim: set expandtab tabstop=4 shiftwidth=4: */
MythDownloadInfo::m_reload
bool m_reload
Definition: mythdownloadmanager.cpp:81
MythDownloadManager::m_cookieLock
QMutex m_cookieLock
Definition: mythdownloadmanager.h:188
build_compdb.args
args
Definition: build_compdb.py:11
mythevent.h
build_compdb.dest
dest
Definition: build_compdb.py:9
MythDownloadManager::processItem
bool processItem(const QString &url, QNetworkRequest *req, const QString &dest, QByteArray *data, MRequestType reqType=kRequestGet, bool reload=false, AuthCallback authCallback=nullptr, void *authArg=nullptr, const QHash< QByteArray, QByteArray > *headers=nullptr, QString *finalUrl=nullptr)
Processes a network request immediately and waits for a response.
Definition: mythdownloadmanager.cpp:353
error
static void error(const char *str,...)
Definition: vbi.cpp:36
downloadManager
MythDownloadManager * downloadManager
Definition: mythdownloadmanager.cpp:37
MythDownloadInfo::m_lastStat
QDateTime m_lastStat
Definition: mythdownloadmanager.cpp:88
MythDownloadInfo::m_data
QByteArray * m_data
Definition: mythdownloadmanager.cpp:77
MythCookieJar::save
void save(const QString &filename)
Saves the cookie jar to a cookie file.
Definition: mythdownloadmanager.cpp:1794
MythDownloadInfo::m_privData
QByteArray m_privData
Definition: mythdownloadmanager.cpp:78
MThread::wait
bool wait(std::chrono::milliseconds time=std::chrono::milliseconds::max())
Wait for the MThread to exit, with a maximum timeout.
Definition: mthread.cpp:300
MythDownloadManager::m_downloadInfos
QMap< QString, MythDownloadInfo * > m_downloadInfos
Definition: mythdownloadmanager.h:177
dmCreateLock
QMutex dmCreateLock
Definition: mythdownloadmanager.cpp:38
MythDownloadManager::removeListener
void removeListener(QObject *caller)
Disconnects the specified caller from any existing MythDownloadInfo instances.
Definition: mythdownloadmanager.cpp:1107
ShutdownMythDownloadManager
void ShutdownMythDownloadManager(void)
Deletes the running MythDownloadManager at program exit.
Definition: mythdownloadmanager.cpp:133
RemoteFileDownloadThread
Definition: mythdownloadmanager.cpp:101
RemoteFileDownloadThread::run
void run() override
Definition: mythdownloadmanager.cpp:109
MythEvent
This class is used as a container for messages.
Definition: mythevent.h:16
mythcoreutil.h
MThread::usleep
static void usleep(std::chrono::microseconds time)
Definition: mthread.cpp:335
MythDownloadInfo::m_lock
QMutex m_lock
Definition: mythdownloadmanager.cpp:94
MythDownloadManager::queueDownload
void queueDownload(const QString &url, const QString &dest, QObject *caller, bool reload=false)
Adds a url to the download queue.
Definition: mythdownloadmanager.cpp:393
MythDownloadManager::m_diskCache
QNetworkDiskCache * m_diskCache
Definition: mythdownloadmanager.h:166
MythDownloadManager::m_proxy
QNetworkProxy * m_proxy
Definition: mythdownloadmanager.h:167
RemoteFile
Definition: remotefile.h:17
MythDownloadManager::postAuth
bool postAuth(const QString &url, QByteArray *data, AuthCallback authCallback, void *authArg, const QHash< QByteArray, QByteArray > *headers=nullptr)
Posts data to a url via the QNetworkAccessManager.
Definition: mythdownloadmanager.cpp:622
kRequestPost
@ kRequestPost
Definition: mythdownloadmanager.h:45
LOG
#define LOG(_MASK_, _LEVEL_, _QSTRING_)
Definition: mythlogging.h:39
MThread::RunProlog
void RunProlog(void)
Sets up a thread, call this if you reimplement run().
Definition: mthread.cpp:196
MythDownloadManager::downloadNow
bool downloadNow(MythDownloadInfo *dlInfo, bool deleteInfo=true)
Download helper for download() blocking methods.
Definition: mythdownloadmanager.cpp:808
MythDownloadManager
Definition: mythdownloadmanager.h:50
MythDownloadInfo::SetDone
void SetDone(bool done)
Definition: mythdownloadmanager.cpp:65
MythDownloadManager::saveFile
static bool saveFile(const QString &outFile, const QByteArray &data, bool append=false)
Saves a QByteArray of data to a given filename.
Definition: mythdownloadmanager.cpp:1494
MythDownloadInfo::IsDone
bool IsDone(void)
Definition: mythdownloadmanager.cpp:59
build_compdb.file
file
Definition: build_compdb.py:55
MythDownloadInfo::m_request
QNetworkRequest * m_request
Definition: mythdownloadmanager.cpp:74
mythdirs.h
RemoteFileDownloadThread::RemoteFileDownloadThread
RemoteFileDownloadThread(MythDownloadManager *parent, MythDownloadInfo *dlInfo)
Definition: mythdownloadmanager.cpp:104
MythDownloadInfo::m_bytesTotal
qint64 m_bytesTotal
Definition: mythdownloadmanager.cpp:87
MythDownloadInfo::m_processReply
bool m_processReply
Definition: mythdownloadmanager.cpp:84
MythDownloadManager::authCallback
void authCallback(QNetworkReply *reply, QAuthenticator *authenticator)
Signal handler for authentication requests.
Definition: mythdownloadmanager.cpp:784
MythDate::current
QDateTime current(bool stripped)
Returns current Date and Time in UTC.
Definition: mythdate.cpp:14
MythDownloadInfo
Definition: mythdownloadmanager.cpp:43
MythDownloadInfo::m_bytesReceived
qint64 m_bytesReceived
Definition: mythdownloadmanager.cpp:86
mythdate.h
mythlogging.h
kRequestHead
@ kRequestHead
Definition: mythdownloadmanager.h:44
GetConfDir
QString GetConfDir(void)
Definition: mythdirs.cpp:256
MythDownloadInfo::m_authArg
void * m_authArg
Definition: mythdownloadmanager.cpp:90
MythDownloadManager::queuePost
void queuePost(const QString &url, QByteArray *data, QObject *caller)
Queues a post to a URL via the QNetworkAccessManager.
Definition: mythdownloadmanager.cpp:529
remotefile.h
MythDownloadManager::run
void run(void) override
Runs a loop to process incoming download requests and triggers download events to be processed.
Definition: mythdownloadmanager.cpp:191
LOC
#define LOC
Definition: mythdownloadmanager.cpp:34
MythDownloadManager::m_isRunning
bool m_isRunning
Definition: mythdownloadmanager.h:185
compat.h
MythDownloadInfo::m_finalUrl
QString * m_finalUrl
Definition: mythdownloadmanager.cpp:73
MythDownloadManager::downloadError
void downloadError(QNetworkReply::NetworkError errorCode)
Slot to process download error events.
Definition: mythdownloadmanager.cpp:1139
MythDownloadInfo::m_errorCode
QNetworkReply::NetworkError m_errorCode
Definition: mythdownloadmanager.cpp:93
MythDownloadManager::m_runThread
bool m_runThread
Definition: mythdownloadmanager.h:184
MThread::RunEpilog
void RunEpilog(void)
Cleans up a thread's resources, call this if you reimplement run().
Definition: mthread.cpp:209
MythDownloadManager::download
bool download(const QString &url, const QString &dest, bool reload=false)
Downloads a URL to a file in blocking mode.
Definition: mythdownloadmanager.cpp:430
MythDownloadManager::getHeader
QString getHeader(const QUrl &url, const QString &header)
Definition: mythdownloadmanager.cpp:1729
MythCookieJar::copyAllCookies
void copyAllCookies(MythCookieJar &old)
Copies all cookies from one MythCookieJar to another.
Definition: mythdownloadmanager.cpp:1760
kRequestGet
@ kRequestGet
Definition: mythdownloadmanager.h:43
MythDownloadManager::setCookieJar
void setCookieJar(QNetworkCookieJar *cookieJar)
Definition: mythdownloadmanager.cpp:1666
MythDownloadManager::downloadProgress
void downloadProgress(qint64 bytesReceived, qint64 bytesTotal)
Slot to process download update events.
Definition: mythdownloadmanager.cpp:1432
portchecker.h
MythDownloadManager::cancelDownload
void cancelDownload(const QString &url, bool block=true)
Cancel a queued or current download.
Definition: mythdownloadmanager.cpp:1014
PortChecker::resolveLinkLocal
static bool resolveLinkLocal(QString &host, int port, std::chrono::milliseconds timeLimit=30s)
Convenience method to resolve link-local address.
Definition: portchecker.cpp:226
MythDownloadInfo::m_syncMode
bool m_syncMode
Definition: mythdownloadmanager.cpp:83
uint
unsigned int uint
Definition: compat.h:81
MythDownloadManager::downloadRemoteFile
void downloadRemoteFile(MythDownloadInfo *dlInfo)
Triggers a myth:// URI download in the background via RemoteFile.
Definition: mythdownloadmanager.cpp:642
gCoreContext
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
Definition: mythcorecontext.cpp:55
MythDownloadManager::redirectUrl
static QUrl redirectUrl(const QUrl &possibleRedirectUrl, const QUrl &oldRedirectUrl)
Checks whether we were redirected to the given URL.
Definition: mythdownloadmanager.cpp:1168
MythDownloadManager::downloadFinished
void downloadFinished(QNetworkReply *reply)
Slot to process download finished events.
Definition: mythdownloadmanager.cpp:1183
MythDownloadInfo::~MythDownloadInfo
~MythDownloadInfo()
Definition: mythdownloadmanager.cpp:52
MythDate::fromString
QDateTime fromString(const QString &dtstr)
Converts kFilename && kISODate formats to QDateTime.
Definition: mythdate.cpp:34
MythDownloadManager::~MythDownloadManager
~MythDownloadManager() override
Destructor for MythDownloadManager.
Definition: mythdownloadmanager.cpp:177
off_t
#define off_t
Definition: mythiowrapper.cpp:241
mthreadpool.h
MythDownloadInfo::m_reply
QNetworkReply * m_reply
Definition: mythdownloadmanager.cpp:75
MythDownloadInfo::m_preferCache
bool m_preferCache
Definition: mythdownloadmanager.cpp:82
MythDownloadManager::preCache
void preCache(const QString &url)
Downloads a URL but doesn't store the resulting data anywhere.
Definition: mythdownloadmanager.cpp:381
MythDownloadManager::m_manager
QNetworkAccessManager * m_manager
Definition: mythdownloadmanager.h:165
MythDownloadManager::downloadAuth
bool downloadAuth(const QString &url, const QString &dest, bool reload=false, AuthCallback authCallback=nullptr, void *authArg=nullptr, const QHash< QByteArray, QByteArray > *headers=nullptr)
Downloads a URL to a file in blocking mode.
Definition: mythdownloadmanager.cpp:515
MythDownloadManager::loadCookieJar
void loadCookieJar(const QString &filename)
Loads the cookie jar from a cookie file.
Definition: mythdownloadmanager.cpp:1641
MythDownloadManager::m_inCookieJar
QNetworkCookieJar * m_inCookieJar
Definition: mythdownloadmanager.h:187
MythDate::secsInPast
std::chrono::seconds secsInPast(const QDateTime &past)
Definition: mythdate.cpp:203
mythcorecontext.h
MythDownloadManager::m_infoLock
QRecursiveMutex * m_infoLock
Definition: mythdownloadmanager.h:175
MythDownloadManager::copyCookieJar
QNetworkCookieJar * copyCookieJar(void)
Copy from one cookie jar to another.
Definition: mythdownloadmanager.cpp:1675
MythDownloadInfo::m_headers
const QHash< QByteArray, QByteArray > * m_headers
Definition: mythdownloadmanager.cpp:91
MythDownloadInfo::m_authCallback
AuthCallback m_authCallback
Definition: mythdownloadmanager.cpp:89
MythDate
Definition: mythdate.cpp:11
MythDownloadManager::m_queueThread
QThread * m_queueThread
Definition: mythdownloadmanager.h:182
MythDownloadManager::m_queueWaitCond
QWaitCondition m_queueWaitCond
Definition: mythdownloadmanager.h:169
MythDownloadInfo::m_done
bool m_done
Definition: mythdownloadmanager.cpp:85
MythDownloadManager::RemoteFileDownloadThread
friend class RemoteFileDownloadThread
Definition: mythdownloadmanager.h:190
MythDownloadManager::refreshCookieJar
void refreshCookieJar(QNetworkCookieJar *jar)
Refresh the temporary cookie jar from another cookie jar.
Definition: mythdownloadmanager.cpp:1694
AuthCallback
void(*)(QNetworkReply *, QAuthenticator *, void *) AuthCallback
Definition: mythdownloadmanager.h:48
MythCookieJar
A subclassed QNetworkCookieJar that allows for reading and writing cookie files that contain raw form...
Definition: mythdownloadmanager.h:31
MythDownloadManager::downloadCanceled
void downloadCanceled(void)
Definition: mythdownloadmanager.cpp:1074
MythDownloadInfo::m_requestType
MRequestType m_requestType
Definition: mythdownloadmanager.cpp:80
MythCoreContext::GetHostName
QString GetHostName(void)
Definition: mythcorecontext.cpp:837
uint16_t
unsigned short uint16_t
Definition: iso6937tables.h:3
MythDownloadManager::saveCookieJar
void saveCookieJar(const QString &filename)
Saves the cookie jar to a cookie file.
Definition: mythdownloadmanager.cpp:1653
MythDownloadManager::GetLastModified
QDateTime GetLastModified(const QString &url)
Gets the Last Modified timestamp for a URI.
Definition: mythdownloadmanager.cpp:1547
MythDownloadInfo::MythDownloadInfo
MythDownloadInfo()
Definition: mythdownloadmanager.cpp:46
MythDownloadManager::updateCookieJar
void updateCookieJar(void)
Update the cookie jar from the temporary cookie jar.
Definition: mythdownloadmanager.cpp:1713
MythDownloadManager::downloadQNetworkRequest
void downloadQNetworkRequest(MythDownloadInfo *dlInfo)
Downloads a QNetworkRequest via the QNetworkAccessManager.
Definition: mythdownloadmanager.cpp:651
MythCookieJar::load
void load(const QString &filename)
Loads the cookie jar from a cookie file.
Definition: mythdownloadmanager.cpp:1769
mythdownloadmanager.h
MythDownloadInfo::m_redirectedTo
QUrl m_redirectedTo
Definition: mythdownloadmanager.cpp:72
MythDownloadManager::post
bool post(const QString &url, QByteArray *data)
Posts data to a url via the QNetworkAccessManager.
Definition: mythdownloadmanager.cpp:576
build_compdb.filename
filename
Definition: build_compdb.py:21
MThreadPool::globalInstance
static MThreadPool * globalInstance(void)
Definition: mthreadpool.cpp:317
MythDownloadInfo::m_outFile
QString m_outFile
Definition: mythdownloadmanager.cpp:76
MythDownloadInfo::m_caller
QObject * m_caller
Definition: mythdownloadmanager.cpp:79
MythDownloadManager::queueItem
void queueItem(const QString &url, QNetworkRequest *req, const QString &dest, QByteArray *data, QObject *caller, MRequestType reqType=kRequestGet, bool reload=false)
Adds a request to the download queue.
Definition: mythdownloadmanager.cpp:321
CACHE_REDIRECTION_LIMIT
static constexpr int CACHE_REDIRECTION_LIMIT
Definition: mythdownloadmanager.cpp:35
RemoteFileDownloadThread::m_dlInfo
MythDownloadInfo * m_dlInfo
Definition: mythdownloadmanager.cpp:128
MythDownloadManager::m_cancellationQueue
QList< MythDownloadInfo * > m_cancellationQueue
Definition: mythdownloadmanager.h:180
MythDownloadManager::m_queueWaitLock
QMutex m_queueWaitLock
Definition: mythdownloadmanager.h:170
RemoteFileDownloadThread::m_parent
MythDownloadManager * m_parent
Definition: mythdownloadmanager.cpp:127
MThreadPool::start
void start(QRunnable *runnable, const QString &debugName, int priority=0)
Definition: mthreadpool.cpp:352
MythDownloadInfo::m_url
QString m_url
Definition: mythdownloadmanager.cpp:71
MythDownloadManager::m_downloadQueue
QList< MythDownloadInfo * > m_downloadQueue
Definition: mythdownloadmanager.h:179
MythDownloadManager::m_downloadReplies
QMap< QNetworkReply *, MythDownloadInfo * > m_downloadReplies
Definition: mythdownloadmanager.h:178
GetMythDownloadManager
MythDownloadManager * GetMythDownloadManager(void)
Gets the pointer to the MythDownloadManager singleton.
Definition: mythdownloadmanager.cpp:145
MRequestType
MRequestType
Definition: mythdownloadmanager.h:42