MythTV  master
mythdownloadmanager.cpp
Go to the documentation of this file.
1 // qt
2 #include <QCoreApplication>
3 #include <QRunnable>
4 #include <QString>
5 #include <QByteArray>
6 #include <QFile>
7 #include <QDir>
8 #include <QNetworkCookieJar>
9 #include <QNetworkCookie>
10 #include <QAuthenticator>
11 #include <QTextStream>
12 #include <QNetworkProxy>
13 #include <QMutexLocker>
14 #include <QUrl>
15 #include <QTcpSocket>
16 
17 #include <cstdlib>
18 #include <unistd.h> // for usleep()
19 
20 // libmythbase
21 #include "compat.h"
22 #include "mythcorecontext.h"
23 #include "mythcoreutil.h"
24 #include "mthreadpool.h"
25 #include "mythdirs.h"
26 #include "mythevent.h"
27 #include "mythversion.h"
28 #include "remotefile.h"
29 #include "mythdate.h"
30 
31 #include "mythdownloadmanager.h"
32 #include "mythlogging.h"
33 #include "portchecker.h"
34 
35 using namespace std;
36 
37 #define LOC QString("DownloadManager: ")
38 #define CACHE_REDIRECTION_LIMIT 10
39 
41 QMutex dmCreateLock;
42 
47 {
48  public:
50  m_lastStat(MythDate::current())
51  {
52  qRegisterMetaType<QNetworkReply::NetworkError>("QNetworkReply::NetworkError");
53  }
54 
56  {
57  delete m_request;
58  if (m_reply && m_processReply)
59  m_reply->deleteLater();
60  }
61 
62  bool IsDone(void)
63  {
64  QMutexLocker lock(&m_lock);
65  return m_done;
66  }
67 
68  void SetDone(bool done)
69  {
70  QMutexLocker lock(&m_lock);
71  m_done = done;
72  }
73 
74  QString m_url;
76  QString *m_finalUrl {nullptr};
77  QNetworkRequest *m_request {nullptr};
78  QNetworkReply *m_reply {nullptr};
79  QString m_outFile;
80  QByteArray *m_data {nullptr};
81  QByteArray m_privData;
82  QObject *m_caller {nullptr};
83  MRequestType m_requestType {kRequestGet};
84  bool m_reload {false};
85  bool m_preferCache {false};
86  bool m_syncMode {false};
87  bool m_processReply {true};
88  bool m_done {false};
89  qint64 m_bytesReceived {0};
90  qint64 m_bytesTotal {0};
91  QDateTime m_lastStat;
92  AuthCallback m_authCallback {nullptr};
93  void *m_authArg {nullptr};
94  const QHash<QByteArray, QByteArray> *m_headers {nullptr};
95 
96  QNetworkReply::NetworkError m_errorCode {QNetworkReply::NoError};
97  QMutex m_lock;
98 };
99 
100 
105 class MythCookieJar : public QNetworkCookieJar
106 {
107  public:
108  MythCookieJar() = default;
109  void copyAllCookies(MythCookieJar &old);
110  void load(const QString &filename);
111  void save(const QString &filename);
112 };
113 
117 class RemoteFileDownloadThread : public QRunnable
118 {
119  public:
121  MythDownloadInfo *dlInfo) :
122  m_parent(parent),
123  m_dlInfo(dlInfo) {}
124 
125  void run() override // QRunnable
126  {
127  bool ok = false;
128 
129  auto *rf = new RemoteFile(m_dlInfo->m_url, false, false, 0);
130  ok = rf->SaveAs(m_dlInfo->m_privData);
131  delete rf;
132 
133  if (!ok)
134  m_dlInfo->m_errorCode = QNetworkReply::UnknownNetworkError;
135 
136  m_dlInfo->m_bytesReceived = m_dlInfo->m_privData.size();
137  m_dlInfo->m_bytesTotal = m_dlInfo->m_bytesReceived;
138 
139  m_parent->downloadFinished(m_dlInfo);
140  }
141 
142  private:
143  MythDownloadManager *m_parent {nullptr};
144  MythDownloadInfo *m_dlInfo {nullptr};
145 };
146 
150 {
151  if (downloadManager)
152  {
153  delete downloadManager;
154  downloadManager = nullptr;
155  }
156 }
157 
162 {
163  if (downloadManager)
164  return downloadManager;
165 
166  QMutexLocker locker(&dmCreateLock);
167 
168  // Check once more in case the download manager was created
169  // while we were securing the lock.
170  if (downloadManager)
171  return downloadManager;
172 
173  auto *tmpDLM = new MythDownloadManager();
174  tmpDLM->start();
175  while (!tmpDLM->getQueueThread())
176  usleep(10000);
177 
178  tmpDLM->moveToThread(tmpDLM->getQueueThread());
179  tmpDLM->setRunThread();
180 
181  while (!tmpDLM->isRunning())
182  usleep(10000);
183 
184  downloadManager = tmpDLM;
185 
187 
188  return downloadManager;
189 }
190 
194 {
195  m_runThread = false;
196  m_queueWaitCond.wakeAll();
197 
198  wait();
199 
200  delete m_infoLock;
201  delete m_inCookieJar;
202 }
203 
208 {
209  RunProlog();
210 
211  bool downloading = false;
212  bool itemsInQueue = false;
213  bool itemsInCancellationQueue = false;
214  bool waitAnyway = false;
215 
216  m_queueThread = QThread::currentThread();
217 
218  while (!m_runThread)
219  usleep(50000);
220 
221  m_manager = new QNetworkAccessManager(this);
222  m_diskCache = new QNetworkDiskCache(this);
223  m_proxy = new QNetworkProxy();
224  m_diskCache->setCacheDirectory(GetConfDir() + "/cache/" +
225  QCoreApplication::applicationName() + "-" +
227  m_manager->setCache(m_diskCache);
228 
229  // Set the proxy for the manager to be the application default proxy,
230  // which has already been setup
231  m_manager->setProxy(*m_proxy);
232 
233  // make sure the cookieJar is created in the same thread as the manager
234  // and set its parent to nullptr so it can be shared between managers
235  m_manager->cookieJar()->setParent(nullptr);
236 
237  QObject::connect(m_manager, SIGNAL(finished(QNetworkReply*)), this,
238  SLOT(downloadFinished(QNetworkReply*)));
239 
240  m_isRunning = true;
241  while (m_runThread)
242  {
243  if (m_inCookieJar)
244  {
245  LOG(VB_GENERAL, LOG_DEBUG, "Updating DLManager's Cookie Jar");
246  updateCookieJar();
247  }
248  m_infoLock->lock();
249  LOG(VB_FILE, LOG_DEBUG, LOC + QString("items downloading %1").arg(m_downloadInfos.count()));
250  LOG(VB_FILE, LOG_DEBUG, LOC + QString("items queued %1").arg(m_downloadQueue.count()));
251  downloading = !m_downloadInfos.isEmpty();
252  itemsInCancellationQueue = !m_cancellationQueue.isEmpty();
253  m_infoLock->unlock();
254 
255  if (itemsInCancellationQueue)
256  {
257  downloadCanceled();
258  }
259  if (downloading)
260  QCoreApplication::processEvents();
261 
262  m_infoLock->lock();
263  itemsInQueue = !m_downloadQueue.isEmpty();
264  m_infoLock->unlock();
265 
266  if (!itemsInQueue || waitAnyway)
267  {
268  waitAnyway = false;
269  m_queueWaitLock.lock();
270 
271  if (downloading)
272  {
273  LOG(VB_FILE, LOG_DEBUG, LOC + QString("waiting 200ms"));
274  m_queueWaitCond.wait(&m_queueWaitLock, 200);
275  }
276  else
277  {
278  LOG(VB_FILE, LOG_DEBUG, LOC + QString("waiting for more items to download"));
279  m_queueWaitCond.wait(&m_queueWaitLock);
280  }
281 
282  m_queueWaitLock.unlock();
283  }
284 
285  m_infoLock->lock();
286  if (!m_downloadQueue.isEmpty())
287  {
288  MythDownloadInfo *dlInfo = m_downloadQueue.front();
289 
290  m_downloadQueue.pop_front();
291 
292  if (!dlInfo)
293  {
294  m_infoLock->unlock();
295  continue;
296  }
297 
298  if (m_downloadInfos.contains(dlInfo->m_url))
299  {
300  // Push request to the end of the queue to let others process.
301  // If this is the only item in the queue, force the loop to
302  // wait a little.
303  if (m_downloadQueue.isEmpty())
304  waitAnyway = true;
305  m_downloadQueue.push_back(dlInfo);
306  m_infoLock->unlock();
307  continue;
308  }
309 
310  if (dlInfo->m_url.startsWith("myth://"))
311  downloadRemoteFile(dlInfo);
312  else
313  {
314  QMutexLocker cLock(&m_cookieLock);
315  downloadQNetworkRequest(dlInfo);
316  }
317 
318  m_downloadInfos[dlInfo->m_url] = dlInfo;
319  }
320  m_infoLock->unlock();
321  }
322  m_isRunning = false;
323 
324  RunEpilog();
325 }
326 
337 void MythDownloadManager::queueItem(const QString &url, QNetworkRequest *req,
338  const QString &dest, QByteArray *data,
339  QObject *caller, const MRequestType reqType,
340  const bool reload)
341 {
342  auto *dlInfo = new MythDownloadInfo;
343 
344  dlInfo->m_url = url;
345  dlInfo->m_request = req;
346  dlInfo->m_outFile = dest;
347  dlInfo->m_data = data;
348  dlInfo->m_caller = caller;
349  dlInfo->m_requestType = reqType;
350  dlInfo->m_reload = reload;
351 
352  QMutexLocker locker(m_infoLock);
353  m_downloadQueue.push_back(dlInfo);
354  m_queueWaitCond.wakeAll();
355 }
356 
369 bool MythDownloadManager::processItem(const QString &url, QNetworkRequest *req,
370  const QString &dest, QByteArray *data,
371  const MRequestType reqType,
372  const bool reload,
373  AuthCallback authCallbackFn, void *authArg,
374  const QHash<QByteArray, QByteArray> *headers,
375  QString *finalUrl)
376 {
377  auto *dlInfo = new MythDownloadInfo;
378 
379  dlInfo->m_url = url;
380  dlInfo->m_request = req;
381  dlInfo->m_outFile = dest;
382  dlInfo->m_data = data;
383  dlInfo->m_requestType = reqType;
384  dlInfo->m_reload = reload;
385  dlInfo->m_syncMode = true;
386  dlInfo->m_authCallback = authCallbackFn;
387  dlInfo->m_authArg = authArg;
388  dlInfo->m_headers = headers;
389  dlInfo->m_finalUrl = finalUrl;
390 
391  return downloadNow(dlInfo, true);
392 }
393 
397 void MythDownloadManager::preCache(const QString &url)
398 {
399  LOG(VB_FILE, LOG_DEBUG, LOC + QString("preCache('%1')").arg(url));
400  queueItem(url, nullptr, QString(), nullptr, nullptr);
401 }
402 
409 void MythDownloadManager::queueDownload(const QString &url,
410  const QString &dest,
411  QObject *caller,
412  const bool reload)
413 {
414  LOG(VB_FILE, LOG_DEBUG, LOC + QString("queueDownload('%1', '%2', %3)")
415  .arg(url).arg(dest).arg((long long)caller));
416 
417  queueItem(url, nullptr, dest, nullptr, caller, kRequestGet, reload);
418 }
419 
425 void MythDownloadManager::queueDownload(QNetworkRequest *req,
426  QByteArray *data,
427  QObject *caller)
428 {
429  LOG(VB_FILE, LOG_DEBUG, LOC + QString("queueDownload('%1', '%2', %3)")
430  .arg(req->url().toString()).arg((long long)data)
431  .arg((long long)caller));
432 
433  queueItem(req->url().toString(), req, QString(), data, caller,
434  kRequestGet,
435  (QNetworkRequest::AlwaysNetwork == req->attribute(
436  QNetworkRequest::CacheLoadControlAttribute,
437  QNetworkRequest::PreferNetwork).toInt()));
438 }
439 
446 bool MythDownloadManager::download(const QString &url, const QString &dest,
447  const bool reload)
448 {
449  return processItem(url, nullptr, dest, nullptr, kRequestGet, reload);
450 }
451 
458 bool MythDownloadManager::download(const QString &url, QByteArray *data,
459  const bool reload, QString *finalUrl)
460 {
461  QString redirected;
462  if (!processItem(url, nullptr, QString(), data, kRequestGet, reload,
463  nullptr, nullptr, nullptr, &redirected))
464  return false;
465  if (!redirected.isEmpty() && finalUrl != nullptr)
466  *finalUrl = redirected;
467  return true;
468 }
469 
476 QNetworkReply *MythDownloadManager::download(const QString &url,
477  const bool reload)
478 {
479  auto *dlInfo = new MythDownloadInfo;
480  QNetworkReply *reply = nullptr;
481 
482  dlInfo->m_url = url;
483  dlInfo->m_reload = reload;
484  dlInfo->m_syncMode = true;
485  dlInfo->m_processReply = false;
486 
487  if (downloadNow(dlInfo, false))
488  {
489  if (dlInfo->m_reply)
490  {
491  reply = dlInfo->m_reply;
492  // prevent dlInfo dtor from deleting the reply
493  dlInfo->m_reply = nullptr;
494 
495  delete dlInfo;
496 
497  return reply;
498  }
499 
500  delete dlInfo;
501  }
502 
503  return nullptr;
504 }
505 
511 bool MythDownloadManager::download(QNetworkRequest *req, QByteArray *data)
512 {
513  LOG(VB_FILE, LOG_DEBUG, LOC + QString("download('%1', '%2')")
514  .arg(req->url().toString()).arg((long long)data));
515  return processItem(req->url().toString(), req, QString(), data,
516  kRequestGet,
517  (QNetworkRequest::AlwaysNetwork == req->attribute(
518  QNetworkRequest::CacheLoadControlAttribute,
519  QNetworkRequest::PreferNetwork).toInt()));
520 }
521 
531 bool MythDownloadManager::downloadAuth(const QString &url, const QString &dest,
532  const bool reload, AuthCallback authCallbackFn, void *authArg,
533  const QHash<QByteArray, QByteArray> *headers)
534 {
535  return processItem(url, nullptr, dest, nullptr, kRequestGet, reload, authCallbackFn,
536  authArg, headers);
537 }
538 
539 
545 void MythDownloadManager::queuePost(const QString &url,
546  QByteArray *data,
547  QObject *caller)
548 {
549  LOG(VB_FILE, LOG_DEBUG, LOC + QString("queuePost('%1', '%2')")
550  .arg(url).arg((long long)data));
551 
552  if (!data)
553  {
554  LOG(VB_GENERAL, LOG_ERR, LOC + "queuePost(), data is NULL!");
555  return;
556  }
557 
558  queueItem(url, nullptr, QString(), data, caller, kRequestPost);
559 }
560 
566 void MythDownloadManager::queuePost(QNetworkRequest *req,
567  QByteArray *data,
568  QObject *caller)
569 {
570  LOG(VB_FILE, LOG_DEBUG, LOC + QString("queuePost('%1', '%2')")
571  .arg(req->url().toString()).arg((long long)data));
572 
573  if (!data)
574  {
575  LOG(VB_GENERAL, LOG_ERR, LOC + "queuePost(), data is NULL!");
576  return;
577  }
578 
579  queueItem(req->url().toString(), req, QString(), data, caller,
580  kRequestPost,
581  (QNetworkRequest::AlwaysNetwork == req->attribute(
582  QNetworkRequest::CacheLoadControlAttribute,
583  QNetworkRequest::PreferNetwork).toInt()));
584 
585 }
586 
592 bool MythDownloadManager::post(const QString &url, QByteArray *data)
593 {
594  LOG(VB_FILE, LOG_DEBUG, LOC + QString("post('%1', '%2')")
595  .arg(url).arg((long long)data));
596 
597  if (!data)
598  {
599  LOG(VB_GENERAL, LOG_ERR, LOC + "post(), data is NULL!");
600  return false;
601  }
602 
603  return processItem(url, nullptr, QString(), data, kRequestPost);
604 }
605 
611 bool MythDownloadManager::post(QNetworkRequest *req, QByteArray *data)
612 {
613  LOG(VB_FILE, LOG_DEBUG, LOC + QString("post('%1', '%2')")
614  .arg(req->url().toString()).arg((long long)data));
615 
616  if (!data)
617  {
618  LOG(VB_GENERAL, LOG_ERR, LOC + "post(), data is NULL!");
619  return false;
620  }
621 
622  return processItem(req->url().toString(), req, QString(), data,
623  kRequestPost,
624  (QNetworkRequest::AlwaysNetwork == req->attribute(
625  QNetworkRequest::CacheLoadControlAttribute,
626  QNetworkRequest::PreferNetwork).toInt()));
627 
628 }
629 
638 bool MythDownloadManager::postAuth(const QString &url, QByteArray *data,
639  AuthCallback authCallbackFn, void *authArg,
640  const QHash<QByteArray, QByteArray> *headers)
641 {
642  LOG(VB_FILE, LOG_DEBUG, LOC + QString("postAuth('%1', '%2')")
643  .arg(url).arg((long long)data));
644 
645  if (!data)
646  {
647  LOG(VB_GENERAL, LOG_ERR, LOC + "postAuth(), data is NULL!");
648  return false;
649  }
650 
651  return processItem(url, nullptr, nullptr, data, kRequestPost, false, authCallbackFn,
652  authArg, headers);
653 }
654 
659 {
660  auto *dlThread = new RemoteFileDownloadThread(this, dlInfo);
661  MThreadPool::globalInstance()->start(dlThread, "RemoteFileDownload");
662 }
663 
668 {
669  if (!dlInfo)
670  return;
671 
672  static const QString kDateFormat = "ddd, dd MMM yyyy hh:mm:ss 'GMT'";
673  QUrl qurl(dlInfo->m_url);
674  QNetworkRequest request;
675 
676  if (dlInfo->m_request)
677  {
678  request = *dlInfo->m_request;
679  delete dlInfo->m_request;
680  dlInfo->m_request = nullptr;
681  }
682  else
683  request.setUrl(qurl);
684 
685  if (dlInfo->m_reload)
686  {
687  request.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
688  QNetworkRequest::AlwaysNetwork);
689  }
690  else
691  {
692  // Prefer the in-cache item if one exists and it is less than 5 minutes
693  // old and it will not expire in the next 10 seconds
694  QDateTime now = MythDate::current();
695 
696  // Handle redirects, we want the metadata of the file headers
697  QString redirectLoc;
698  int limit = 0;
699  while (!(redirectLoc = getHeader(qurl, "Location")).isNull())
700  {
701  if (limit == CACHE_REDIRECTION_LIMIT)
702  {
703  LOG(VB_GENERAL, LOG_WARNING, QString("Cache Redirection limit "
704  "reached for %1")
705  .arg(qurl.toString()));
706  return;
707  }
708  qurl.setUrl(redirectLoc);
709  limit++;
710  }
711 
712  LOG(VB_NETWORK, LOG_DEBUG, QString("Checking cache for %1")
713  .arg(qurl.toString()));
714 
715  m_infoLock->lock();
716  QNetworkCacheMetaData urlData = m_manager->cache()->metaData(qurl);
717  m_infoLock->unlock();
718  if ((urlData.isValid()) &&
719  ((!urlData.expirationDate().isValid()) ||
720  (urlData.expirationDate().toUTC().secsTo(now) < 10)))
721  {
722  QString dateString = getHeader(urlData, "Date");
723 
724  if (!dateString.isNull())
725  {
726  QDateTime loadDate =
727  MythDate::fromString(dateString, kDateFormat);
728  loadDate.setTimeSpec(Qt::UTC);
729  if (loadDate.secsTo(now) <= 720)
730  {
731  dlInfo->m_preferCache = true;
732  LOG(VB_NETWORK, LOG_DEBUG, QString("Preferring cache for %1")
733  .arg(qurl.toString()));
734  }
735  }
736  }
737  }
738 
739  if (dlInfo->m_preferCache)
740  request.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
741  QNetworkRequest::PreferCache);
742 
743  if (!request.hasRawHeader("User-Agent"))
744  {
745  request.setRawHeader("User-Agent",
746  "MythTV v" MYTH_BINARY_VERSION
747  " MythDownloadManager");
748  }
749 
750  if (dlInfo->m_headers)
751  {
752  QHash<QByteArray, QByteArray>::const_iterator it =
753  dlInfo->m_headers->constBegin();
754  for ( ; it != dlInfo->m_headers->constEnd(); ++it )
755  {
756  if (!it.key().isEmpty() && !it.value().isEmpty())
757  {
758  request.setRawHeader(it.key(), it.value());
759  }
760  }
761  }
762 
763  switch (dlInfo->m_requestType)
764  {
765  case kRequestPost :
766  dlInfo->m_reply = m_manager->post(request, *dlInfo->m_data);
767  break;
768  case kRequestHead :
769  dlInfo->m_reply = m_manager->head(request);
770  break;
771  case kRequestGet :
772  default:
773  dlInfo->m_reply = m_manager->get(request);
774  break;
775  }
776 
777  m_downloadReplies[dlInfo->m_reply] = dlInfo;
778 
779  if (dlInfo->m_authCallback)
780  {
781  connect(m_manager, SIGNAL(authenticationRequired(QNetworkReply *,
782  QAuthenticator *)),
783  this, SLOT(authCallback(QNetworkReply *, QAuthenticator *)));
784  }
785 
786  connect(dlInfo->m_reply, SIGNAL(error(QNetworkReply::NetworkError)), this,
787  SLOT(downloadError(QNetworkReply::NetworkError)));
788  connect(dlInfo->m_reply, SIGNAL(downloadProgress(qint64, qint64)),
789  this, SLOT(downloadProgress(qint64, qint64)));
790 }
791 
796 void MythDownloadManager::authCallback(QNetworkReply *reply,
797  QAuthenticator *authenticator)
798 {
799  if (!reply)
800  return;
801 
802  MythDownloadInfo *dlInfo = m_downloadReplies[reply];
803 
804  if (!dlInfo)
805  return;
806 
807  if (dlInfo->m_authCallback)
808  {
809  LOG(VB_FILE, LOG_DEBUG, "Calling auth callback");
810  dlInfo->m_authCallback(reply, authenticator, dlInfo->m_authArg);
811  }
812 }
813 
821 {
822  if (!dlInfo)
823  return false;
824 
825  dlInfo->m_syncMode = true;
826 
827  // Special handling for link-local
828  // Not needed for Windows because windows does not need
829  // the scope id.
830 #ifndef _WIN32
831  if (dlInfo->m_url.startsWith("http://[fe80::",Qt::CaseInsensitive))
832  return downloadNowLinkLocal(dlInfo,deleteInfo);
833 #endif
834  m_infoLock->lock();
835  m_downloadQueue.push_back(dlInfo);
836  m_infoLock->unlock();
837  m_queueWaitCond.wakeAll();
838 
839  // timeout myth:// RemoteFile transfers 20 seconds from now
840  // timeout non-myth:// QNetworkAccessManager transfers 60 seconds after
841  // their last progress update
842  QDateTime startedAt = MythDate::current();
843  m_infoLock->lock();
844  while ((!dlInfo->IsDone()) &&
845  (dlInfo->m_errorCode == QNetworkReply::NoError) &&
846  (((!dlInfo->m_url.startsWith("myth://")) &&
847  (dlInfo->m_lastStat.secsTo(MythDate::current()) < 60)) ||
848  ((dlInfo->m_url.startsWith("myth://")) &&
849  (startedAt.secsTo(MythDate::current()) < 20))))
850  {
851  m_infoLock->unlock();
852  m_queueWaitLock.lock();
853  m_queueWaitCond.wait(&m_queueWaitLock, 200);
854  m_queueWaitLock.unlock();
855  m_infoLock->lock();
856  }
857  bool done = dlInfo->IsDone();
858  bool success =
859  done && (dlInfo->m_errorCode == QNetworkReply::NoError);
860 
861  if (!done)
862  {
863  dlInfo->m_data = nullptr; // Prevent downloadFinished() from updating
864  dlInfo->m_syncMode = false; // Let downloadFinished() cleanup for us
865  if ((dlInfo->m_reply) &&
866  (dlInfo->m_errorCode == QNetworkReply::NoError))
867  {
868  LOG(VB_FILE, LOG_DEBUG,
869  LOC + QString("Aborting download - lack of data transfer"));
870  dlInfo->m_reply->abort();
871  }
872  }
873  else if (deleteInfo)
874  delete dlInfo;
875 
876  m_infoLock->unlock();
877 
878  return success;
879 }
880 
881 #ifndef _WIN32
882 
902 bool MythDownloadManager::downloadNowLinkLocal(MythDownloadInfo *dlInfo, bool deleteInfo)
903 {
904  bool isOK = true;
905 
906  // Only certain features are supported here
907  if (dlInfo->m_authCallback || dlInfo->m_authArg)
908  {
909  LOG(VB_GENERAL, LOG_ERR, LOC +
910  QString("Unsupported authentication for %1").arg(dlInfo->m_url));
911  isOK = false;
912  }
913  if (!dlInfo->m_outFile.isEmpty())
914  {
915  LOG(VB_GENERAL, LOG_ERR, LOC +
916  QString("Unsupported File output %1 for %2")
917  .arg(dlInfo->m_outFile).arg(dlInfo->m_url));
918  isOK = false;
919  }
920 
921  if (!deleteInfo || dlInfo->m_requestType == kRequestHead)
922  {
923  // We do not have the ability to return a network reply in dlInfo
924  // so if we are asked to do that, return an error.
925  LOG(VB_GENERAL, LOG_ERR, LOC +
926  QString("Unsupported link-local operation %1")
927  .arg(dlInfo->m_url));
928  isOK = false;
929  }
930 
931  QUrl url(dlInfo->m_url);
932  QString host(url.host());
933  int port(url.port(80));
934  if (isOK && PortChecker::resolveLinkLocal(host, port))
935  {
936  QString reqType;
937  switch (dlInfo->m_requestType)
938  {
939  case kRequestPost :
940  reqType = "POST";
941  break;
942  case kRequestGet :
943  default:
944  reqType = "GET";
945  break;
946  }
947  QByteArray *aBuffer = dlInfo->m_data;
948  QHash<QByteArray, QByteArray> headers;
949  if (dlInfo->m_headers)
950  headers = *dlInfo->m_headers;
951  if (!headers.contains("User-Agent"))
952  headers.insert("User-Agent",
953  "MythDownloadManager v" MYTH_BINARY_VERSION);
954  headers.insert("Connection", "close");
955  headers.insert("Accept-Encoding", "identity");
956  if (aBuffer && !aBuffer->isEmpty())
957  headers.insert("Content-Length",
958  (QString::number(aBuffer->size())).toUtf8());
959  headers.insert("Host",
960  (url.host()+":"+QString::number(port)).toUtf8());
961 
962  QByteArray requestMessage;
963  QString path (url.path());
964  requestMessage.append("POST ");
965  requestMessage.append(path);
966  requestMessage.append(" HTTP/1.1\r\n");
967  QHashIterator<QByteArray, QByteArray> it(headers);
968  while (it.hasNext())
969  {
970  it.next();
971  requestMessage.append(it.key());
972  requestMessage.append(": ");
973  requestMessage.append(it.value());
974  requestMessage.append("\r\n");
975  }
976  requestMessage.append("\r\n");
977  if (aBuffer && !aBuffer->isEmpty())
978  {
979  requestMessage.append(*aBuffer);
980  }
981  QTcpSocket socket;
982  socket.connectToHost(host, port);
983  // QT Warning - this may not work on Windows
984  if (!socket.waitForConnected(5000))
985  isOK = false;
986  if (isOK)
987  isOK = (socket.write(requestMessage) > 0);
988  if (isOK)
989  // QT Warning - this may not work on Windows
990  isOK = socket.waitForDisconnected(5000);
991  if (isOK)
992  {
993  *aBuffer = socket.readAll();
994  // Find the start of the content
995  QByteArray delim("\r\n\r\n");
996  int delimLoc=aBuffer->indexOf(delim);
997  if (delimLoc > -1)
998  {
999  *aBuffer = aBuffer->right
1000  (aBuffer->size()-delimLoc-4);
1001  }
1002  else
1003  {
1004  isOK=false;
1005  }
1006  }
1007  socket.close();
1008  }
1009  else
1010  isOK = false;
1011 
1012  if (deleteInfo)
1013  delete dlInfo;
1014 
1015  if (isOK)
1016  return true;
1017  LOG(VB_GENERAL, LOG_ERR, LOC + QString("Link Local request failed: %1")
1018  .arg(url.toString()));
1019  return false;
1020 }
1021 #endif
1022 
1027 void MythDownloadManager::cancelDownload(const QString &url, bool block)
1028 {
1029  cancelDownload(QStringList(url), block);
1030 }
1031 
1036 void MythDownloadManager::cancelDownload(const QStringList &urls, bool block)
1037 {
1038  m_infoLock->lock();
1039  for (const auto& url : qAsConst(urls))
1040  {
1041  QMutableListIterator<MythDownloadInfo*> lit(m_downloadQueue);
1042  while (lit.hasNext())
1043  {
1044  lit.next();
1045  MythDownloadInfo *dlInfo = lit.value();
1046  if (dlInfo->m_url == url)
1047  {
1048  if (!m_cancellationQueue.contains(dlInfo))
1049  m_cancellationQueue.append(dlInfo);
1050  lit.remove();
1051  }
1052  }
1053 
1054  if (m_downloadInfos.contains(url))
1055  {
1056  MythDownloadInfo *dlInfo = m_downloadInfos[url];
1057 
1058  if (!m_cancellationQueue.contains(dlInfo))
1059  m_cancellationQueue.append(dlInfo);
1060 
1061  if (dlInfo->m_reply)
1062  m_downloadReplies.remove(dlInfo->m_reply);
1063 
1064  m_downloadInfos.remove(url);
1065  }
1066  }
1067  m_infoLock->unlock();
1068 
1069  if (QThread::currentThread() == this->thread())
1070  {
1071  downloadCanceled();
1072  return;
1073  }
1074 
1075  // wake-up running thread
1076  m_queueWaitCond.wakeAll();
1077 
1078  if (!block)
1079  return;
1080 
1081  while (!m_cancellationQueue.isEmpty())
1082  {
1083  usleep(50000); // re-test in another 50ms
1084  }
1085 }
1086 
1088 {
1089  QMutexLocker locker(m_infoLock);
1090 
1091  QMutableListIterator<MythDownloadInfo*> lit(m_cancellationQueue);
1092  while (lit.hasNext())
1093  {
1094  lit.next();
1095  MythDownloadInfo *dlInfo = lit.value();
1096  dlInfo->m_lock.lock();
1097 
1098  if (dlInfo->m_reply)
1099  {
1100  LOG(VB_FILE, LOG_DEBUG,
1101  LOC + QString("Aborting download - user request"));
1102  dlInfo->m_reply->abort();
1103  }
1104  lit.remove();
1105  if (dlInfo->m_done)
1106  {
1107  dlInfo->m_lock.unlock();
1108  continue;
1109  }
1110  dlInfo->m_errorCode = QNetworkReply::OperationCanceledError;
1111  dlInfo->m_done = true;
1112  dlInfo->m_lock.unlock();
1113  }
1114 }
1115 
1121 {
1122  QMutexLocker locker(m_infoLock);
1123 
1124  QList <MythDownloadInfo*>::iterator lit = m_downloadQueue.begin();
1125  for (; lit != m_downloadQueue.end(); ++lit)
1126  {
1127  MythDownloadInfo *dlInfo = *lit;
1128  if (dlInfo->m_caller == caller)
1129  {
1130  dlInfo->m_caller = nullptr;
1131  dlInfo->m_outFile = QString();
1132  dlInfo->m_data = nullptr;
1133  }
1134  }
1135 
1136  QMap <QString, MythDownloadInfo*>::iterator mit = m_downloadInfos.begin();
1137  for (; mit != m_downloadInfos.end(); ++mit)
1138  {
1139  MythDownloadInfo *dlInfo = mit.value();
1140  if (dlInfo->m_caller == caller)
1141  {
1142  dlInfo->m_caller = nullptr;
1143  dlInfo->m_outFile = QString();
1144  dlInfo->m_data = nullptr;
1145  }
1146  }
1147 }
1148 
1152 void MythDownloadManager::downloadError(QNetworkReply::NetworkError errorCode)
1153 {
1154  auto *reply = dynamic_cast<QNetworkReply *>(sender());
1155  if (reply == nullptr)
1156  return;
1157 
1158  LOG(VB_FILE, LOG_DEBUG, LOC + QString("downloadError %1 ")
1159  .arg(errorCode) + reply->errorString() );
1160 
1161  QMutexLocker locker(m_infoLock);
1162  if (!m_downloadReplies.contains(reply))
1163  {
1164  reply->deleteLater();
1165  return;
1166  }
1167 
1168  MythDownloadInfo *dlInfo = m_downloadReplies[reply];
1169 
1170  if (!dlInfo)
1171  return;
1172 
1173  dlInfo->m_errorCode = errorCode;
1174 }
1175 
1181 QUrl MythDownloadManager::redirectUrl(const QUrl& possibleRedirectUrl,
1182  const QUrl& oldRedirectUrl)
1183 {
1184  LOG(VB_FILE, LOG_DEBUG, LOC + QString("redirectUrl()"));
1185  QUrl redirectUrl;
1186 
1187  if(!possibleRedirectUrl.isEmpty() && possibleRedirectUrl != oldRedirectUrl)
1188  redirectUrl = possibleRedirectUrl;
1189 
1190  return redirectUrl;
1191 }
1192 
1196 void MythDownloadManager::downloadFinished(QNetworkReply* reply)
1197 {
1198  LOG(VB_FILE, LOG_DEBUG, LOC + QString("downloadFinished(%1)")
1199  .arg((long long)reply));
1200 
1201  QMutexLocker locker(m_infoLock);
1202  if (!m_downloadReplies.contains(reply))
1203  {
1204  reply->deleteLater();
1205  return;
1206  }
1207 
1208  MythDownloadInfo *dlInfo = m_downloadReplies[reply];
1209 
1210  if (!dlInfo || !dlInfo->m_reply)
1211  return;
1212 
1213  downloadFinished(dlInfo);
1214 }
1215 
1220 {
1221  if (!dlInfo)
1222  return;
1223 
1224  int statusCode = -1;
1225  static const QString kDateFormat = "ddd, dd MMM yyyy hh:mm:ss 'GMT'";
1226  QNetworkReply *reply = dlInfo->m_reply;
1227 
1228  if (reply)
1229  {
1230  QUrl possibleRedirectUrl =
1231  reply->attribute(QNetworkRequest::RedirectionTargetAttribute).toUrl();
1232 
1233  if (!possibleRedirectUrl.isEmpty() &&
1234  possibleRedirectUrl.isValid() &&
1235  possibleRedirectUrl.isRelative()) // Turn relative Url to absolute
1236  possibleRedirectUrl = reply->url().resolved(possibleRedirectUrl);
1237 
1238  if (!possibleRedirectUrl.isEmpty() && dlInfo->m_finalUrl != nullptr)
1239  *dlInfo->m_finalUrl = QString(possibleRedirectUrl.toString());
1240 
1241  dlInfo->m_redirectedTo =
1242  redirectUrl(possibleRedirectUrl, dlInfo->m_redirectedTo);
1243 
1244  QVariant status =
1245  reply->attribute(QNetworkRequest::HttpStatusCodeAttribute);
1246  if (status.isValid())
1247  statusCode = status.toInt();
1248  }
1249 
1250  if(reply && !dlInfo->m_redirectedTo.isEmpty() &&
1251  ((dlInfo->m_requestType != kRequestPost) ||
1252  (statusCode == 301 || statusCode == 302 ||
1253  statusCode == 303)))
1254  {
1255  LOG(VB_FILE, LOG_DEBUG, LOC +
1256  QString("downloadFinished(%1): Redirect: %2 -> %3")
1257  .arg((long long)dlInfo)
1258  .arg(reply->url().toString())
1259  .arg(dlInfo->m_redirectedTo.toString()));
1260 
1261  if (dlInfo->m_data)
1262  dlInfo->m_data->clear();
1263 
1264  dlInfo->m_bytesReceived = 0;
1265  dlInfo->m_bytesTotal = 0;
1266 
1267  QNetworkRequest request(dlInfo->m_redirectedTo);
1268 
1269  if (dlInfo->m_reload)
1270  {
1271  request.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
1272  QNetworkRequest::AlwaysNetwork);
1273  }
1274  else if (dlInfo->m_preferCache)
1275  {
1276  request.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
1277  QNetworkRequest::PreferCache);
1278  }
1279 
1280  request.setRawHeader("User-Agent",
1281  "MythDownloadManager v" MYTH_BINARY_VERSION);
1282 
1283  switch (dlInfo->m_requestType)
1284  {
1285  case kRequestHead :
1286  dlInfo->m_reply = m_manager->head(request);
1287  break;
1288  case kRequestGet :
1289  default:
1290  dlInfo->m_reply = m_manager->get(request);
1291  break;
1292  }
1293 
1294  m_downloadReplies[dlInfo->m_reply] = dlInfo;
1295 
1296  connect(dlInfo->m_reply, SIGNAL(error(QNetworkReply::NetworkError)),
1297  this, SLOT(downloadError(QNetworkReply::NetworkError)));
1298  connect(dlInfo->m_reply, SIGNAL(downloadProgress(qint64, qint64)),
1299  this, SLOT(downloadProgress(qint64, qint64)));
1300 
1301  m_downloadReplies.remove(reply);
1302  reply->deleteLater();
1303  }
1304  else
1305  {
1306  LOG(VB_FILE, LOG_DEBUG, QString("downloadFinished(%1): COMPLETE: %2")
1307  .arg((long long)dlInfo).arg(dlInfo->m_url));
1308 
1309  // HACK Insert a Date header into the cached metadata if one doesn't
1310  // already exist
1311  QUrl fileUrl = dlInfo->m_url;
1312  QString redirectLoc;
1313  int limit = 0;
1314  while (!(redirectLoc = getHeader(fileUrl, "Location")).isNull())
1315  {
1316  if (limit == CACHE_REDIRECTION_LIMIT)
1317  {
1318  LOG(VB_GENERAL, LOG_WARNING, QString("Cache Redirection limit "
1319  "reached for %1")
1320  .arg(fileUrl.toString()));
1321  return;
1322  }
1323  fileUrl.setUrl(redirectLoc);
1324  limit++;
1325  }
1326 
1327  m_infoLock->lock();
1328  QNetworkCacheMetaData urlData = m_manager->cache()->metaData(fileUrl);
1329  m_infoLock->unlock();
1330  if (getHeader(urlData, "Date").isNull())
1331  {
1332  QNetworkCacheMetaData::RawHeaderList headers = urlData.rawHeaders();
1333  QNetworkCacheMetaData::RawHeader newheader;
1334  QDateTime now = MythDate::current();
1335  newheader = QNetworkCacheMetaData::RawHeader("Date",
1336  now.toString(kDateFormat).toLatin1());
1337  headers.append(newheader);
1338  urlData.setRawHeaders(headers);
1339  m_infoLock->lock();
1340  m_manager->cache()->updateMetaData(urlData);
1341  m_infoLock->unlock();
1342  }
1343  // End HACK
1344 
1345  dlInfo->m_redirectedTo.clear();
1346 
1347  int dataSize = -1;
1348 
1349  // If we downloaded via the QNetworkAccessManager
1350  // AND the caller isn't handling the reply directly
1351  if (reply && dlInfo->m_processReply)
1352  {
1353  bool append = (!dlInfo->m_syncMode && dlInfo->m_caller);
1354  QByteArray data = reply->readAll();
1355  dataSize = data.size();
1356 
1357  if (append)
1358  dlInfo->m_bytesReceived += dataSize;
1359  else
1360  dlInfo->m_bytesReceived = dataSize;
1361 
1362  dlInfo->m_bytesTotal = dlInfo->m_bytesReceived;
1363 
1364  if (dlInfo->m_data)
1365  {
1366  if (append)
1367  dlInfo->m_data->append(data);
1368  else
1369  *dlInfo->m_data = data;
1370  }
1371  else if (!dlInfo->m_outFile.isEmpty())
1372  {
1373  saveFile(dlInfo->m_outFile, data, append);
1374  }
1375  }
1376  else if (!reply) // If we downloaded via RemoteFile
1377  {
1378  if (dlInfo->m_data)
1379  {
1380  (*dlInfo->m_data) = dlInfo->m_privData;
1381  }
1382  else if (!dlInfo->m_outFile.isEmpty())
1383  {
1384  saveFile(dlInfo->m_outFile, dlInfo->m_privData);
1385  }
1386  dlInfo->m_bytesReceived += dataSize;
1387  dlInfo->m_bytesTotal = dlInfo->m_bytesReceived;
1388  }
1389  // else we downloaded via QNetworkAccessManager
1390  // AND the caller is handling the reply
1391 
1392  m_infoLock->lock();
1393  if (!m_downloadInfos.remove(dlInfo->m_url))
1394  {
1395  LOG(VB_GENERAL, LOG_ERR, LOC +
1396  QString("ERROR download finished but failed to remove url: %1")
1397  .arg(dlInfo->m_url));
1398  }
1399 
1400  if (reply)
1401  m_downloadReplies.remove(reply);
1402  m_infoLock->unlock();
1403 
1404  dlInfo->SetDone(true);
1405 
1406  if (!dlInfo->m_syncMode)
1407  {
1408  if (dlInfo->m_caller)
1409  {
1410  LOG(VB_FILE, LOG_DEBUG, QString("downloadFinished(%1): "
1411  "COMPLETE: %2, sending event to caller")
1412  .arg((long long)dlInfo).arg(dlInfo->m_url));
1413 
1414  QStringList args;
1415  args << dlInfo->m_url;
1416  args << dlInfo->m_outFile;
1417  args << QString::number(dlInfo->m_bytesTotal);
1418  // placeholder for error string
1419  args << (reply ? reply->errorString() : QString());
1420  args << QString::number((int)(reply ? reply->error() :
1421  dlInfo->m_errorCode));
1422 
1423  QCoreApplication::postEvent(dlInfo->m_caller,
1424  new MythEvent("DOWNLOAD_FILE FINISHED", args));
1425  }
1426 
1427  delete dlInfo;
1428  }
1429 
1430  m_queueWaitCond.wakeAll();
1431  }
1432 }
1433 
1439 void MythDownloadManager::downloadProgress(qint64 bytesReceived,
1440  qint64 bytesTotal)
1441 {
1442  auto *reply = dynamic_cast<QNetworkReply *>(sender());
1443  if (reply == nullptr)
1444  return;
1445 
1446  LOG(VB_FILE, LOG_DEBUG, LOC +
1447  QString("downloadProgress(%1, %2) (for reply %3)")
1448  .arg(bytesReceived).arg(bytesTotal).arg((long long)reply));
1449 
1450  QMutexLocker locker(m_infoLock);
1451  if (!m_downloadReplies.contains(reply))
1452  return;
1453 
1454  MythDownloadInfo *dlInfo = m_downloadReplies[reply];
1455 
1456  if (!dlInfo)
1457  return;
1458 
1459  dlInfo->m_lastStat = MythDate::current();
1460 
1461  LOG(VB_FILE, LOG_DEBUG, LOC +
1462  QString("downloadProgress: %1 to %2 is at %3 of %4 bytes downloaded")
1463  .arg(dlInfo->m_url).arg(dlInfo->m_outFile)
1464  .arg(bytesReceived).arg(bytesTotal));
1465 
1466  if (!dlInfo->m_syncMode && dlInfo->m_caller)
1467  {
1468  LOG(VB_FILE, LOG_DEBUG, QString("downloadProgress(%1): "
1469  "sending event to caller")
1470  .arg(reply->url().toString()));
1471 
1472  bool appendToFile = (dlInfo->m_bytesReceived != 0);
1473  QByteArray data = reply->readAll();
1474  if (!dlInfo->m_outFile.isEmpty())
1475  saveFile(dlInfo->m_outFile, data, appendToFile);
1476 
1477  if (dlInfo->m_data)
1478  dlInfo->m_data->append(data);
1479 
1480  dlInfo->m_bytesReceived = bytesReceived;
1481  dlInfo->m_bytesTotal = bytesTotal;
1482 
1483  QStringList args;
1484  args << dlInfo->m_url;
1485  args << dlInfo->m_outFile;
1486  args << QString::number(bytesReceived);
1487  args << QString::number(bytesTotal);
1488 
1489  QCoreApplication::postEvent(dlInfo->m_caller,
1490  new MythEvent("DOWNLOAD_FILE UPDATE", args));
1491  }
1492 }
1493 
1501 bool MythDownloadManager::saveFile(const QString &outFile,
1502  const QByteArray &data,
1503  const bool append)
1504 {
1505  if (outFile.isEmpty() || !data.size())
1506  return false;
1507 
1508  QFile file(outFile);
1509  QFileInfo fileInfo(outFile);
1510  QDir qdir(fileInfo.absolutePath());
1511 
1512  if (!qdir.exists() && !qdir.mkpath(fileInfo.absolutePath()))
1513  {
1514  LOG(VB_GENERAL, LOG_ERR, QString("Failed to create: '%1'")
1515  .arg(fileInfo.absolutePath()));
1516  return false;
1517  }
1518 
1519  QIODevice::OpenMode mode = QIODevice::Unbuffered|QIODevice::WriteOnly;
1520  if (append)
1521  mode |= QIODevice::Append;
1522 
1523  if (!file.open(mode))
1524  {
1525  LOG(VB_GENERAL, LOG_ERR, QString("Failed to open: '%1'") .arg(outFile));
1526  return false;
1527  }
1528 
1529  off_t offset = 0;
1530  size_t remaining = data.size();
1531  uint failure_cnt = 0;
1532  while ((remaining > 0) && (failure_cnt < 5))
1533  {
1534  ssize_t written = file.write(data.data() + offset, remaining);
1535  if (written < 0)
1536  {
1537  failure_cnt++;
1538  usleep(50000);
1539  continue;
1540  }
1541 
1542  failure_cnt = 0;
1543  offset += written;
1544  remaining -= written;
1545  }
1546 
1547  return remaining <= 0;
1548 }
1549 
1554 QDateTime MythDownloadManager::GetLastModified(const QString &url)
1555 {
1556  // If the header has not expired and
1557  // the last modification date is less than 1 hours old or if
1558  // the cache object is less than 20 minutes old,
1559  // then use the cached header otherwise redownload the header
1560 
1561  static const QString kDateFormat = "ddd, dd MMM yyyy hh:mm:ss 'GMT'";
1562  LOG(VB_FILE, LOG_DEBUG, LOC + QString("GetLastModified('%1')").arg(url));
1563  QDateTime result;
1564 
1565  QDateTime now = MythDate::current();
1566 
1567  QUrl cacheUrl = QUrl(url);
1568 
1569  // Deal with redirects, we want the cached data for the final url
1570  QString redirectLoc;
1571  int limit = 0;
1572  while (!(redirectLoc = getHeader(cacheUrl, "Location")).isNull())
1573  {
1574  if (limit == CACHE_REDIRECTION_LIMIT)
1575  {
1576  LOG(VB_GENERAL, LOG_WARNING, QString("Cache Redirection limit "
1577  "reached for %1")
1578  .arg(cacheUrl.toString()));
1579  return result;
1580  }
1581  cacheUrl.setUrl(redirectLoc);
1582  limit++;
1583  }
1584 
1585  m_infoLock->lock();
1586  QNetworkCacheMetaData urlData = m_manager->cache()->metaData(cacheUrl);
1587  m_infoLock->unlock();
1588 
1589  if (urlData.isValid() &&
1590  ((!urlData.expirationDate().isValid()) ||
1591  (urlData.expirationDate().secsTo(now) < 0)))
1592  {
1593  if (urlData.lastModified().toUTC().secsTo(now) <= 3600) // 1 Hour
1594  {
1595  result = urlData.lastModified().toUTC();
1596  }
1597  else
1598  {
1599  QString date = getHeader(urlData, "Date");
1600  if (!date.isNull())
1601  {
1602  QDateTime loadDate =
1603  MythDate::fromString(date, kDateFormat);
1604  loadDate.setTimeSpec(Qt::UTC);
1605  if (loadDate.secsTo(now) <= 1200) // 20 Minutes
1606  {
1607  result = urlData.lastModified().toUTC();
1608  }
1609  }
1610  }
1611  }
1612 
1613  if (!result.isValid())
1614  {
1615  auto *dlInfo = new MythDownloadInfo;
1616  dlInfo->m_url = url;
1617  dlInfo->m_syncMode = true;
1618  // Head request, we only want to inspect the headers
1619  dlInfo->m_requestType = kRequestHead;
1620 
1621  if (downloadNow(dlInfo, false))
1622  {
1623  if (dlInfo->m_reply)
1624  {
1625  QVariant lastMod =
1626  dlInfo->m_reply->header(
1627  QNetworkRequest::LastModifiedHeader);
1628  if (lastMod.isValid())
1629  result = lastMod.toDateTime().toUTC();
1630  }
1631 
1632  // downloadNow() will set a flag to trigger downloadFinished()
1633  // to delete the dlInfo if the download times out
1634  delete dlInfo;
1635  }
1636  }
1637 
1638  LOG(VB_FILE, LOG_DEBUG, LOC + QString("GetLastModified('%1'): Result %2")
1639  .arg(url).arg(result.toString()));
1640 
1641  return result;
1642 }
1643 
1644 
1649 {
1650  QMutexLocker locker(&m_cookieLock);
1651 
1652  auto *jar = new MythCookieJar;
1653  jar->load(filename);
1654  m_manager->setCookieJar(jar);
1655 }
1656 
1661 {
1662  QMutexLocker locker(&m_cookieLock);
1663 
1664  if (!m_manager->cookieJar())
1665  return;
1666 
1667  auto *jar = dynamic_cast<MythCookieJar *>(m_manager->cookieJar());
1668  if (jar == nullptr)
1669  return;
1670  jar->save(filename);
1671 }
1672 
1673 void MythDownloadManager::setCookieJar(QNetworkCookieJar *cookieJar)
1674 {
1675  QMutexLocker locker(&m_cookieLock);
1676  m_manager->setCookieJar(cookieJar);
1677 }
1678 
1682 QNetworkCookieJar *MythDownloadManager::copyCookieJar(void)
1683 {
1684  QMutexLocker locker(&m_cookieLock);
1685 
1686  if (!m_manager->cookieJar())
1687  return nullptr;
1688 
1689  auto *inJar = dynamic_cast<MythCookieJar *>(m_manager->cookieJar());
1690  if (inJar == nullptr)
1691  return nullptr;
1692  auto *outJar = new MythCookieJar;
1693  outJar->copyAllCookies(*inJar);
1694 
1695  return static_cast<QNetworkCookieJar *>(outJar);
1696 }
1697 
1701 void MythDownloadManager::refreshCookieJar(QNetworkCookieJar *jar)
1702 {
1703  QMutexLocker locker(&m_cookieLock);
1704  delete m_inCookieJar;
1705 
1706  auto *inJar = dynamic_cast<MythCookieJar *>(jar);
1707  if (inJar == nullptr)
1708  return;
1709 
1710  auto *outJar = new MythCookieJar;
1711  outJar->copyAllCookies(*inJar);
1712  m_inCookieJar = static_cast<QNetworkCookieJar *>(outJar);
1713 
1714  QMutexLocker locker2(&m_queueWaitLock);
1715  m_queueWaitCond.wakeAll();
1716 }
1717 
1721 {
1722  QMutexLocker locker(&m_cookieLock);
1723 
1724  auto *inJar = dynamic_cast<MythCookieJar *>(m_inCookieJar);
1725  if (inJar != nullptr)
1726  {
1727  auto *outJar = new MythCookieJar;
1728  outJar->copyAllCookies(*inJar);
1729  m_manager->setCookieJar(static_cast<QNetworkCookieJar *>(outJar));
1730  }
1731 
1732  delete m_inCookieJar;
1733  m_inCookieJar = nullptr;
1734 }
1735 
1736 QString MythDownloadManager::getHeader(const QUrl& url, const QString& header)
1737 {
1738  if (!m_manager || !m_manager->cache())
1739  return QString();
1740 
1741  m_infoLock->lock();
1742  QNetworkCacheMetaData metadata = m_manager->cache()->metaData(url);
1743  m_infoLock->unlock();
1744 
1745  return getHeader(metadata, header);
1746 }
1747 
1753 QString MythDownloadManager::getHeader(const QNetworkCacheMetaData &cacheData,
1754  const QString& header)
1755 {
1756  for (const auto& rh : cacheData.rawHeaders())
1757  if (QString(rh.first) == header)
1758  return QString(rh.second);
1759  return QString();
1760 }
1761 
1762 
1767 {
1768  const QList<QNetworkCookie> cookieList = old.allCookies();
1769  setAllCookies(cookieList);
1770 }
1771 
1775 void MythCookieJar::load(const QString &filename)
1776 {
1777  LOG(VB_GENERAL, LOG_DEBUG, QString("MythCookieJar: loading cookies from: %1").arg(filename));
1778 
1779  QFile f(filename);
1780  if (!f.open(QIODevice::ReadOnly))
1781  {
1782  LOG(VB_GENERAL, LOG_WARNING, QString("MythCookieJar::load() failed to open file for reading: %1").arg(filename));
1783  return;
1784  }
1785 
1786  QList<QNetworkCookie> cookieList;
1787  QTextStream stream(&f);
1788  while (!stream.atEnd())
1789  {
1790  QString cookie = stream.readLine();
1791  cookieList << QNetworkCookie::parseCookies(cookie.toLocal8Bit());
1792  }
1793 
1794  setAllCookies(cookieList);
1795 }
1796 
1800 void MythCookieJar::save(const QString &filename)
1801 {
1802  LOG(VB_GENERAL, LOG_DEBUG, QString("MythCookieJar: saving cookies to: %1").arg(filename));
1803 
1804  QFile f(filename);
1805  if (!f.open(QIODevice::WriteOnly))
1806  {
1807  LOG(VB_GENERAL, LOG_ERR, QString("MythCookieJar::save() failed to open file for writing: %1").arg(filename));
1808  return;
1809  }
1810 
1811  QList<QNetworkCookie> cookieList = allCookies();
1812  QTextStream stream(&f);
1813 
1814  for (const auto& cookie : qAsConst(cookieList))
1815  stream << cookie.toRawForm() << endl;
1816 }
1817 
1818 
1819 /* vim: set expandtab tabstop=4 shiftwidth=4: */
MythDownloadInfo::m_reload
bool m_reload
Definition: mythdownloadmanager.cpp:84
build_compdb.args
args
Definition: build_compdb.py:11
mythevent.h
build_compdb.dest
dest
Definition: build_compdb.py:9
MythCookieJar::MythCookieJar
MythCookieJar()=default
fileInfo
QFileInfo fileInfo(filename)
MythDownloadManager::processItem
bool processItem(const QString &url, QNetworkRequest *req, const QString &dest, QByteArray *data, MRequestType reqType=kRequestGet, bool reload=false, AuthCallback authCallback=nullptr, void *authArg=nullptr, const QHash< QByteArray, QByteArray > *headers=nullptr, QString *finalUrl=nullptr)
Processes a network request immediately and waits for a response.
Definition: mythdownloadmanager.cpp:369
error
static void error(const char *str,...)
Definition: vbi.cpp:42
downloadManager
MythDownloadManager * downloadManager
Definition: mythdownloadmanager.cpp:40
MythDownloadInfo::m_lastStat
QDateTime m_lastStat
Definition: mythdownloadmanager.cpp:91
MythDownloadInfo::m_data
QByteArray * m_data
Definition: mythdownloadmanager.cpp:80
MythCookieJar::save
void save(const QString &filename)
Saves the cookie jar to a cookie file.
Definition: mythdownloadmanager.cpp:1800
downloadRemoteFile
static QString downloadRemoteFile(const QString &cmd, const QString &url, const QString &storageGroup, const QString &filename)
Definition: mythcoreutil.cpp:280
MythDownloadInfo::m_privData
QByteArray m_privData
Definition: mythdownloadmanager.cpp:81
dmCreateLock
QMutex dmCreateLock
Definition: mythdownloadmanager.cpp:41
MythDownloadManager::removeListener
void removeListener(QObject *caller)
Disconnects the specified caller from any existing MythDownloadInfo instances.
Definition: mythdownloadmanager.cpp:1120
ShutdownMythDownloadManager
void ShutdownMythDownloadManager(void)
Deletes the running MythDownloadManager at program exit.
Definition: mythdownloadmanager.cpp:149
RemoteFileDownloadThread
Definition: mythdownloadmanager.cpp:118
RemoteFileDownloadThread::run
void run() override
Definition: mythdownloadmanager.cpp:125
MythEvent
This class is used as a container for messages.
Definition: mythevent.h:17
mythcoreutil.h
MythDownloadInfo::m_lock
QMutex m_lock
Definition: mythdownloadmanager.cpp:97
MythDownloadManager::queueDownload
void queueDownload(const QString &url, const QString &dest, QObject *caller, bool reload=false)
Adds a url to the download queue.
Definition: mythdownloadmanager.cpp:409
arg
arg(title).arg(filename).arg(doDelete))
RemoteFile
Definition: remotefile.h:18
MythDownloadManager::postAuth
bool postAuth(const QString &url, QByteArray *data, AuthCallback authCallback, void *authArg, const QHash< QByteArray, QByteArray > *headers=nullptr)
Posts data to a url via the QNetworkAccessManager.
Definition: mythdownloadmanager.cpp:638
kRequestPost
@ kRequestPost
Definition: mythdownloadmanager.h:27
LOG
#define LOG(_MASK_, _LEVEL_, _QSTRING_)
Definition: mythlogging.h:23
MythDownloadManager::downloadNow
bool downloadNow(MythDownloadInfo *dlInfo, bool deleteInfo=true)
Download helper for download() blocking methods.
Definition: mythdownloadmanager.cpp:820
MythDownloadManager
Definition: mythdownloadmanager.h:33
MythDownloadInfo::SetDone
void SetDone(bool done)
Definition: mythdownloadmanager.cpp:68
MythDownloadManager::saveFile
static bool saveFile(const QString &outFile, const QByteArray &data, bool append=false)
Saves a QByteArray of data to a given filename.
Definition: mythdownloadmanager.cpp:1501
MythDownloadInfo::IsDone
bool IsDone(void)
Definition: mythdownloadmanager.cpp:62
build_compdb.file
file
Definition: build_compdb.py:55
MythDownloadInfo::m_request
QNetworkRequest * m_request
Definition: mythdownloadmanager.cpp:77
mythdirs.h
RemoteFileDownloadThread::RemoteFileDownloadThread
RemoteFileDownloadThread(MythDownloadManager *parent, MythDownloadInfo *dlInfo)
Definition: mythdownloadmanager.cpp:120
MythDownloadInfo::m_bytesTotal
qint64 m_bytesTotal
Definition: mythdownloadmanager.cpp:90
MythDownloadManager::authCallback
void authCallback(QNetworkReply *reply, QAuthenticator *authenticator)
Signal handler for authentication requests.
Definition: mythdownloadmanager.cpp:796
MythDate::current
QDateTime current(bool stripped)
Returns current Date and Time in UTC.
Definition: mythdate.cpp:10
MythDownloadInfo
Definition: mythdownloadmanager.cpp:47
MythDownloadInfo::m_bytesReceived
qint64 m_bytesReceived
Definition: mythdownloadmanager.cpp:89
mythversion.h
mythdate.h
mythlogging.h
kRequestHead
@ kRequestHead
Definition: mythdownloadmanager.h:26
GetConfDir
QString GetConfDir(void)
Definition: mythdirs.cpp:224
PortChecker::resolveLinkLocal
static bool resolveLinkLocal(QString &host, int port, int timeLimit=30000)
Convenience method to resolve link-local address.
Definition: portchecker.cpp:228
MythDownloadInfo::m_authArg
void * m_authArg
Definition: mythdownloadmanager.cpp:93
MythDownloadManager::queuePost
void queuePost(const QString &url, QByteArray *data, QObject *caller)
Queues a post to a URL via the QNetworkAccessManager.
Definition: mythdownloadmanager.cpp:545
remotefile.h
MythDownloadManager::run
void run(void) override
Runs a loop to process incoming download requests and triggers download events to be processed.
Definition: mythdownloadmanager.cpp:207
LOC
#define LOC
Definition: mythdownloadmanager.cpp:37
compat.h
MythDownloadInfo::m_finalUrl
QString * m_finalUrl
Definition: mythdownloadmanager.cpp:76
MythDownloadManager::downloadError
void downloadError(QNetworkReply::NetworkError errorCode)
Slot to process download error events.
Definition: mythdownloadmanager.cpp:1152
MythDownloadInfo::m_errorCode
QNetworkReply::NetworkError m_errorCode
Definition: mythdownloadmanager.cpp:96
filename
QString filename
Definition: mythplugins/mytharchive/mytharchivehelper/main.cpp:634
MythDownloadManager::download
bool download(const QString &url, const QString &dest, bool reload=false)
Downloads a URL to a file in blocking mode.
Definition: mythdownloadmanager.cpp:446
MythDownloadManager::getHeader
QString getHeader(const QUrl &url, const QString &header)
Definition: mythdownloadmanager.cpp:1736
MythCookieJar::copyAllCookies
void copyAllCookies(MythCookieJar &old)
Copies all cookies from one MythCookieJar to another.
Definition: mythdownloadmanager.cpp:1766
kRequestGet
@ kRequestGet
Definition: mythdownloadmanager.h:25
MythDownloadManager::setCookieJar
void setCookieJar(QNetworkCookieJar *cookieJar)
Definition: mythdownloadmanager.cpp:1673
MythDownloadManager::downloadProgress
void downloadProgress(qint64 bytesReceived, qint64 bytesTotal)
Slot to process download update events.
Definition: mythdownloadmanager.cpp:1439
f
QTextStream t & f
Definition: mythplugins/mytharchive/mytharchivehelper/main.cpp:600
portchecker.h
MythDownloadManager::cancelDownload
void cancelDownload(const QString &url, bool block=true)
Cancel a queued or current download.
Definition: mythdownloadmanager.cpp:1027
MythDownloadInfo::m_syncMode
bool m_syncMode
Definition: mythdownloadmanager.cpp:86
uint
unsigned int uint
Definition: compat.h:140
MythDownloadManager::downloadRemoteFile
void downloadRemoteFile(MythDownloadInfo *dlInfo)
Triggers a myth:// URI download in the background via RemoteFile.
Definition: mythdownloadmanager.cpp:658
gCoreContext
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
Definition: mythcorecontext.cpp:57
MythDownloadManager::redirectUrl
static QUrl redirectUrl(const QUrl &possibleRedirectUrl, const QUrl &oldRedirectUrl)
Checks whether we were redirected to the given URL.
Definition: mythdownloadmanager.cpp:1181
MythDownloadManager::downloadFinished
void downloadFinished(QNetworkReply *reply)
Slot to process download finished events.
Definition: mythdownloadmanager.cpp:1196
MythDownloadInfo::~MythDownloadInfo
~MythDownloadInfo()
Definition: mythdownloadmanager.cpp:55
MythDate::fromString
QDateTime fromString(const QString &dtstr)
Converts kFilename && kISODate formats to QDateTime.
Definition: mythdate.cpp:30
MythDownloadManager::~MythDownloadManager
~MythDownloadManager() override
Destructor for MythDownloadManager.
Definition: mythdownloadmanager.cpp:193
off_t
#define off_t
Definition: mythiowrapper.cpp:238
mthreadpool.h
MYTH_BINARY_VERSION
#define MYTH_BINARY_VERSION
Update this whenever the plug-in ABI changes.
Definition: mythversion.h:15
MythDownloadInfo::m_reply
QNetworkReply * m_reply
Definition: mythdownloadmanager.cpp:78
MythDownloadInfo::m_preferCache
bool m_preferCache
Definition: mythdownloadmanager.cpp:85
MythDownloadManager::preCache
void preCache(const QString &url)
Downloads a URL but doesn't store the resulting data anywhere.
Definition: mythdownloadmanager.cpp:397
MythDownloadManager::downloadAuth
bool downloadAuth(const QString &url, const QString &dest, bool reload=false, AuthCallback authCallback=nullptr, void *authArg=nullptr, const QHash< QByteArray, QByteArray > *headers=nullptr)
Downloads a URL to a file in blocking mode.
Definition: mythdownloadmanager.cpp:531
MythDownloadManager::loadCookieJar
void loadCookieJar(const QString &filename)
Loads the cookie jar from a cookie file.
Definition: mythdownloadmanager.cpp:1648
mythcorecontext.h
MythDownloadManager::copyCookieJar
QNetworkCookieJar * copyCookieJar(void)
Copy from one cookie jar to another.
Definition: mythdownloadmanager.cpp:1682
MythDownloadInfo::m_headers
const QHash< QByteArray, QByteArray > * m_headers
Definition: mythdownloadmanager.cpp:94
MythDownloadInfo::m_authCallback
AuthCallback m_authCallback
Definition: mythdownloadmanager.cpp:92
MythDate
Definition: mythdate.cpp:8
MythDownloadInfo::m_done
bool m_done
Definition: mythdownloadmanager.cpp:88
MythDownloadManager::refreshCookieJar
void refreshCookieJar(QNetworkCookieJar *jar)
Refresh the temporary cookie jar from another cookie jar.
Definition: mythdownloadmanager.cpp:1701
AuthCallback
void(*)(QNetworkReply *, QAuthenticator *, void *) AuthCallback
Definition: mythdownloadmanager.h:30
MythCookieJar
A subclassed QNetworkCookieJar that allows for reading and writing cookie files that contain raw form...
Definition: mythdownloadmanager.cpp:106
MythDownloadManager::downloadCanceled
void downloadCanceled(void)
Definition: mythdownloadmanager.cpp:1087
MythDownloadInfo::m_requestType
MRequestType m_requestType
Definition: mythdownloadmanager.cpp:83
MythCoreContext::GetHostName
QString GetHostName(void)
Definition: mythcorecontext.cpp:856
MythDownloadManager::saveCookieJar
void saveCookieJar(const QString &filename)
Saves the cookie jar to a cookie file.
Definition: mythdownloadmanager.cpp:1660
MythDownloadManager::GetLastModified
QDateTime GetLastModified(const QString &url)
Gets the Last Modified timestamp for a URI.
Definition: mythdownloadmanager.cpp:1554
MythDownloadInfo::MythDownloadInfo
MythDownloadInfo()
Definition: mythdownloadmanager.cpp:49
MythDownloadManager::updateCookieJar
void updateCookieJar(void)
Update the cookie jar from the temporary cookie jar.
Definition: mythdownloadmanager.cpp:1720
MythDownloadManager::downloadQNetworkRequest
void downloadQNetworkRequest(MythDownloadInfo *dlInfo)
Downloads a QNetworkRequest via the QNetworkAccessManager.
Definition: mythdownloadmanager.cpp:667
MythCookieJar::load
void load(const QString &filename)
Loads the cookie jar from a cookie file.
Definition: mythdownloadmanager.cpp:1775
mythdownloadmanager.h
MythDownloadInfo::m_redirectedTo
QUrl m_redirectedTo
Definition: mythdownloadmanager.cpp:75
MythDownloadManager::post
bool post(const QString &url, QByteArray *data)
Posts data to a url via the QNetworkAccessManager.
Definition: mythdownloadmanager.cpp:592
MThreadPool::globalInstance
static MThreadPool * globalInstance(void)
Definition: mthreadpool.cpp:307
MythDownloadInfo::m_outFile
QString m_outFile
Definition: mythdownloadmanager.cpp:79
MythDownloadInfo::m_caller
QObject * m_caller
Definition: mythdownloadmanager.cpp:82
MythDownloadManager::queueItem
void queueItem(const QString &url, QNetworkRequest *req, const QString &dest, QByteArray *data, QObject *caller, MRequestType reqType=kRequestGet, bool reload=false)
Adds a request to the download queue.
Definition: mythdownloadmanager.cpp:337
CACHE_REDIRECTION_LIMIT
#define CACHE_REDIRECTION_LIMIT
Definition: mythdownloadmanager.cpp:38
MThreadPool::start
void start(QRunnable *runnable, const QString &debugName, int priority=0)
Definition: mthreadpool.cpp:342
MythDownloadInfo::m_url
QString m_url
Definition: mythdownloadmanager.cpp:74
GetMythDownloadManager
MythDownloadManager * GetMythDownloadManager(void)
Gets the pointer to the MythDownloadManager singleton.
Definition: mythdownloadmanager.cpp:161
MRequestType
MRequestType
Definition: mythdownloadmanager.h:24