MythTV  master
mythdownloadmanager.cpp
Go to the documentation of this file.
1 // qt
2 #include <QCoreApplication>
3 #include <QRunnable>
4 #include <QString>
5 #include <QByteArray>
6 #include <QFile>
7 #include <QDir>
8 #include <QNetworkCookieJar>
9 #include <QNetworkCookie>
10 #include <QAuthenticator>
11 #include <QTextStream>
12 #include <QNetworkProxy>
13 #include <QMutexLocker>
14 #include <QUrl>
15 #include <QTcpSocket>
16 
17 #include <cstdlib>
18 #include <unistd.h> // for usleep()
19 
20 // libmythbase
21 #include "compat.h"
22 #include "mythcorecontext.h"
23 #include "mythcoreutil.h"
24 #include "mthreadpool.h"
25 #include "mythdirs.h"
26 #include "mythevent.h"
27 #include "mythversion.h"
28 #include "remotefile.h"
29 #include "mythdate.h"
30 
31 #include "mythdownloadmanager.h"
32 #include "mythlogging.h"
33 #include "portchecker.h"
34 
35 using namespace std;
36 
37 #define LOC QString("DownloadManager: ")
38 #define CACHE_REDIRECTION_LIMIT 10
39 
41 QMutex dmCreateLock;
42 
47 {
48  public:
50  m_lastStat(MythDate::current())
51  {
52  qRegisterMetaType<QNetworkReply::NetworkError>("QNetworkReply::NetworkError");
53  }
54 
56  {
57  delete m_request;
58  if (m_reply && m_processReply)
59  m_reply->deleteLater();
60  }
61 
62  bool IsDone(void)
63  {
64  QMutexLocker lock(&m_lock);
65  return m_done;
66  }
67 
68  void SetDone(bool done)
69  {
70  QMutexLocker lock(&m_lock);
71  m_done = done;
72  }
73 
74  QString m_url;
76  QString *m_finalUrl {nullptr};
77  QNetworkRequest *m_request {nullptr};
78  QNetworkReply *m_reply {nullptr};
79  QString m_outFile;
80  QByteArray *m_data {nullptr};
81  QByteArray m_privData;
82  QObject *m_caller {nullptr};
83  MRequestType m_requestType {kRequestGet};
84  bool m_reload {false};
85  bool m_preferCache {false};
86  bool m_syncMode {false};
87  bool m_processReply {true};
88  bool m_done {false};
89  qint64 m_bytesReceived {0};
90  qint64 m_bytesTotal {0};
91  QDateTime m_lastStat;
92  AuthCallback m_authCallback {nullptr};
93  void *m_authArg {nullptr};
94  const QHash<QByteArray, QByteArray> *m_headers {nullptr};
95 
96  QNetworkReply::NetworkError m_errorCode {QNetworkReply::NoError};
97  QMutex m_lock;
98 };
99 
100 
105 class MythCookieJar : public QNetworkCookieJar
106 {
107  public:
108  MythCookieJar() = default;
109  void copyAllCookies(MythCookieJar &old);
110  void load(const QString &filename);
111  void save(const QString &filename);
112 };
113 
117 class RemoteFileDownloadThread : public QRunnable
118 {
119  public:
121  MythDownloadInfo *dlInfo) :
122  m_parent(parent),
123  m_dlInfo(dlInfo) {}
124 
125  void run() override // QRunnable
126  {
127  bool ok = false;
128 
129  auto *rf = new RemoteFile(m_dlInfo->m_url, false, false, 0);
130  ok = rf->SaveAs(m_dlInfo->m_privData);
131  delete rf;
132 
133  if (!ok)
134  m_dlInfo->m_errorCode = QNetworkReply::UnknownNetworkError;
135 
136  m_dlInfo->m_bytesReceived = m_dlInfo->m_privData.size();
137  m_dlInfo->m_bytesTotal = m_dlInfo->m_bytesReceived;
138 
139  m_parent->downloadFinished(m_dlInfo);
140  }
141 
142  private:
143  MythDownloadManager *m_parent {nullptr};
144  MythDownloadInfo *m_dlInfo {nullptr};
145 };
146 
150 {
151  if (downloadManager)
152  {
153  delete downloadManager;
154  downloadManager = nullptr;
155  }
156 }
157 
162 {
163  if (downloadManager)
164  return downloadManager;
165 
166  QMutexLocker locker(&dmCreateLock);
167 
168  // Check once more in case the download manager was created
169  // while we were securing the lock.
170  if (downloadManager)
171  return downloadManager;
172 
173  auto *tmpDLM = new MythDownloadManager();
174  tmpDLM->start();
175  while (!tmpDLM->getQueueThread())
176  usleep(10000);
177 
178  tmpDLM->moveToThread(tmpDLM->getQueueThread());
179  tmpDLM->setRunThread();
180 
181  while (!tmpDLM->isRunning())
182  usleep(10000);
183 
184  downloadManager = tmpDLM;
185 
187 
188  return downloadManager;
189 }
190 
194 {
195  m_runThread = false;
196  m_queueWaitCond.wakeAll();
197 
198  wait();
199 
200  delete m_infoLock;
201  delete m_inCookieJar;
202 }
203 
208 {
209  RunProlog();
210 
211  bool downloading = false;
212  bool itemsInQueue = false;
213  bool itemsInCancellationQueue = false;
214  bool waitAnyway = false;
215 
216  m_queueThread = QThread::currentThread();
217 
218  while (!m_runThread)
219  usleep(50000);
220 
221  m_manager = new QNetworkAccessManager(this);
222  m_diskCache = new QNetworkDiskCache(this);
223  m_proxy = new QNetworkProxy();
224  m_diskCache->setCacheDirectory(GetConfDir() + "/cache/" +
225  QCoreApplication::applicationName() + "-" +
227  m_manager->setCache(m_diskCache);
228 
229  // Set the proxy for the manager to be the application default proxy,
230  // which has already been setup
231  m_manager->setProxy(*m_proxy);
232 
233  // make sure the cookieJar is created in the same thread as the manager
234  // and set its parent to nullptr so it can be shared between managers
235  m_manager->cookieJar()->setParent(nullptr);
236 
237  QObject::connect(m_manager, SIGNAL(finished(QNetworkReply*)), this,
238  SLOT(downloadFinished(QNetworkReply*)));
239 
240  m_isRunning = true;
241  while (m_runThread)
242  {
243  if (m_inCookieJar)
244  {
245  LOG(VB_GENERAL, LOG_DEBUG, "Updating DLManager's Cookie Jar");
246  updateCookieJar();
247  }
248  m_infoLock->lock();
249  LOG(VB_FILE, LOG_DEBUG, LOC + QString("items downloading %1").arg(m_downloadInfos.count()));
250  LOG(VB_FILE, LOG_DEBUG, LOC + QString("items queued %1").arg(m_downloadQueue.count()));
251  downloading = !m_downloadInfos.isEmpty();
252  itemsInCancellationQueue = !m_cancellationQueue.isEmpty();
253  m_infoLock->unlock();
254 
255  if (itemsInCancellationQueue)
256  {
257  downloadCanceled();
258  }
259  if (downloading)
260  QCoreApplication::processEvents();
261 
262  m_infoLock->lock();
263  itemsInQueue = !m_downloadQueue.isEmpty();
264  m_infoLock->unlock();
265 
266  if (!itemsInQueue || waitAnyway)
267  {
268  waitAnyway = false;
269  m_queueWaitLock.lock();
270 
271  if (downloading)
272  {
273  LOG(VB_FILE, LOG_DEBUG, LOC + QString("waiting 200ms"));
274  m_queueWaitCond.wait(&m_queueWaitLock, 200);
275  }
276  else
277  {
278  LOG(VB_FILE, LOG_DEBUG, LOC + QString("waiting for more items to download"));
279  m_queueWaitCond.wait(&m_queueWaitLock);
280  }
281 
282  m_queueWaitLock.unlock();
283  }
284 
285  m_infoLock->lock();
286  if (!m_downloadQueue.isEmpty())
287  {
288  MythDownloadInfo *dlInfo = m_downloadQueue.front();
289 
290  m_downloadQueue.pop_front();
291 
292  if (!dlInfo)
293  {
294  m_infoLock->unlock();
295  continue;
296  }
297 
298  if (m_downloadInfos.contains(dlInfo->m_url))
299  {
300  // Push request to the end of the queue to let others process.
301  // If this is the only item in the queue, force the loop to
302  // wait a little.
303  if (m_downloadQueue.isEmpty())
304  waitAnyway = true;
305  m_downloadQueue.push_back(dlInfo);
306  m_infoLock->unlock();
307  continue;
308  }
309 
310  if (dlInfo->m_url.startsWith("myth://"))
311  downloadRemoteFile(dlInfo);
312  else
313  {
314  QMutexLocker cLock(&m_cookieLock);
315  downloadQNetworkRequest(dlInfo);
316  }
317 
318  m_downloadInfos[dlInfo->m_url] = dlInfo;
319  }
320  m_infoLock->unlock();
321  }
322  m_isRunning = false;
323 
324  RunEpilog();
325 }
326 
337 void MythDownloadManager::queueItem(const QString &url, QNetworkRequest *req,
338  const QString &dest, QByteArray *data,
339  QObject *caller, const MRequestType reqType,
340  const bool reload)
341 {
342  auto *dlInfo = new MythDownloadInfo;
343 
344  dlInfo->m_url = url;
345  dlInfo->m_request = req;
346  dlInfo->m_outFile = dest;
347  dlInfo->m_data = data;
348  dlInfo->m_caller = caller;
349  dlInfo->m_requestType = reqType;
350  dlInfo->m_reload = reload;
351 
352  QMutexLocker locker(m_infoLock);
353  m_downloadQueue.push_back(dlInfo);
354  m_queueWaitCond.wakeAll();
355 }
356 
369 bool MythDownloadManager::processItem(const QString &url, QNetworkRequest *req,
370  const QString &dest, QByteArray *data,
371  const MRequestType reqType,
372  const bool reload,
373  AuthCallback authCallbackFn, void *authArg,
374  const QHash<QByteArray, QByteArray> *headers,
375  QString *finalUrl)
376 {
377  auto *dlInfo = new MythDownloadInfo;
378 
379  dlInfo->m_url = url;
380  dlInfo->m_request = req;
381  dlInfo->m_outFile = dest;
382  dlInfo->m_data = data;
383  dlInfo->m_requestType = reqType;
384  dlInfo->m_reload = reload;
385  dlInfo->m_syncMode = true;
386  dlInfo->m_authCallback = authCallbackFn;
387  dlInfo->m_authArg = authArg;
388  dlInfo->m_headers = headers;
389  dlInfo->m_finalUrl = finalUrl;
390 
391  return downloadNow(dlInfo, true);
392 }
393 
397 void MythDownloadManager::preCache(const QString &url)
398 {
399  LOG(VB_FILE, LOG_DEBUG, LOC + QString("preCache('%1')").arg(url));
400  queueItem(url, nullptr, QString(), nullptr, nullptr);
401 }
402 
409 void MythDownloadManager::queueDownload(const QString &url,
410  const QString &dest,
411  QObject *caller,
412  const bool reload)
413 {
414  LOG(VB_FILE, LOG_DEBUG, LOC + QString("queueDownload('%1', '%2', %3)")
415  .arg(url).arg(dest).arg((long long)caller));
416 
417  queueItem(url, nullptr, dest, nullptr, caller, kRequestGet, reload);
418 }
419 
425 void MythDownloadManager::queueDownload(QNetworkRequest *req,
426  QByteArray *data,
427  QObject *caller)
428 {
429  LOG(VB_FILE, LOG_DEBUG, LOC + QString("queueDownload('%1', '%2', %3)")
430  .arg(req->url().toString()).arg((long long)data)
431  .arg((long long)caller));
432 
433  queueItem(req->url().toString(), req, QString(), data, caller,
434  kRequestGet,
435  (QNetworkRequest::AlwaysNetwork == req->attribute(
436  QNetworkRequest::CacheLoadControlAttribute,
437  QNetworkRequest::PreferNetwork).toInt()));
438 }
439 
446 bool MythDownloadManager::download(const QString &url, const QString &dest,
447  const bool reload)
448 {
449  return processItem(url, nullptr, dest, nullptr, kRequestGet, reload);
450 }
451 
458 bool MythDownloadManager::download(const QString &url, QByteArray *data,
459  const bool reload, QString *finalUrl)
460 {
461  QString redirected;
462  if (!processItem(url, nullptr, QString(), data, kRequestGet, reload,
463  nullptr, nullptr, nullptr, &redirected))
464  return false;
465  if (!redirected.isEmpty() && finalUrl != nullptr)
466  *finalUrl = redirected;
467  return true;
468 }
469 
476 QNetworkReply *MythDownloadManager::download(const QString &url,
477  const bool reload)
478 {
479  auto *dlInfo = new MythDownloadInfo;
480  QNetworkReply *reply = nullptr;
481 
482  dlInfo->m_url = url;
483  dlInfo->m_reload = reload;
484  dlInfo->m_syncMode = true;
485  dlInfo->m_processReply = false;
486 
487  if (downloadNow(dlInfo, false))
488  {
489  if (dlInfo->m_reply)
490  {
491  reply = dlInfo->m_reply;
492  // prevent dlInfo dtor from deleting the reply
493  dlInfo->m_reply = nullptr;
494 
495  delete dlInfo;
496 
497  return reply;
498  }
499 
500  delete dlInfo;
501  }
502 
503  return nullptr;
504 }
505 
511 bool MythDownloadManager::download(QNetworkRequest *req, QByteArray *data)
512 {
513  LOG(VB_FILE, LOG_DEBUG, LOC + QString("download('%1', '%2')")
514  .arg(req->url().toString()).arg((long long)data));
515  return processItem(req->url().toString(), req, QString(), data,
516  kRequestGet,
517  (QNetworkRequest::AlwaysNetwork == req->attribute(
518  QNetworkRequest::CacheLoadControlAttribute,
519  QNetworkRequest::PreferNetwork).toInt()));
520 }
521 
531 bool MythDownloadManager::downloadAuth(const QString &url, const QString &dest,
532  const bool reload, AuthCallback authCallbackFn, void *authArg,
533  const QHash<QByteArray, QByteArray> *headers)
534 {
535  return processItem(url, nullptr, dest, nullptr, kRequestGet, reload, authCallbackFn,
536  authArg, headers);
537 }
538 
539 
545 void MythDownloadManager::queuePost(const QString &url,
546  QByteArray *data,
547  QObject *caller)
548 {
549  LOG(VB_FILE, LOG_DEBUG, LOC + QString("queuePost('%1', '%2')")
550  .arg(url).arg((long long)data));
551 
552  if (!data)
553  {
554  LOG(VB_GENERAL, LOG_ERR, LOC + "queuePost(), data is NULL!");
555  return;
556  }
557 
558  queueItem(url, nullptr, QString(), data, caller, kRequestPost);
559 }
560 
566 void MythDownloadManager::queuePost(QNetworkRequest *req,
567  QByteArray *data,
568  QObject *caller)
569 {
570  LOG(VB_FILE, LOG_DEBUG, LOC + QString("queuePost('%1', '%2')")
571  .arg(req->url().toString()).arg((long long)data));
572 
573  if (!data)
574  {
575  LOG(VB_GENERAL, LOG_ERR, LOC + "queuePost(), data is NULL!");
576  return;
577  }
578 
579  queueItem(req->url().toString(), req, QString(), data, caller,
580  kRequestPost,
581  (QNetworkRequest::AlwaysNetwork == req->attribute(
582  QNetworkRequest::CacheLoadControlAttribute,
583  QNetworkRequest::PreferNetwork).toInt()));
584 
585 }
586 
592 bool MythDownloadManager::post(const QString &url, QByteArray *data)
593 {
594  LOG(VB_FILE, LOG_DEBUG, LOC + QString("post('%1', '%2')")
595  .arg(url).arg((long long)data));
596 
597  if (!data)
598  {
599  LOG(VB_GENERAL, LOG_ERR, LOC + "post(), data is NULL!");
600  return false;
601  }
602 
603  return processItem(url, nullptr, QString(), data, kRequestPost);
604 }
605 
611 bool MythDownloadManager::post(QNetworkRequest *req, QByteArray *data)
612 {
613  LOG(VB_FILE, LOG_DEBUG, LOC + QString("post('%1', '%2')")
614  .arg(req->url().toString()).arg((long long)data));
615 
616  if (!data)
617  {
618  LOG(VB_GENERAL, LOG_ERR, LOC + "post(), data is NULL!");
619  return false;
620  }
621 
622  return processItem(req->url().toString(), req, QString(), data,
623  kRequestPost,
624  (QNetworkRequest::AlwaysNetwork == req->attribute(
625  QNetworkRequest::CacheLoadControlAttribute,
626  QNetworkRequest::PreferNetwork).toInt()));
627 
628 }
629 
638 bool MythDownloadManager::postAuth(const QString &url, QByteArray *data,
639  AuthCallback authCallbackFn, void *authArg,
640  const QHash<QByteArray, QByteArray> *headers)
641 {
642  LOG(VB_FILE, LOG_DEBUG, LOC + QString("postAuth('%1', '%2')")
643  .arg(url).arg((long long)data));
644 
645  if (!data)
646  {
647  LOG(VB_GENERAL, LOG_ERR, LOC + "postAuth(), data is NULL!");
648  return false;
649  }
650 
651  return processItem(url, nullptr, nullptr, data, kRequestPost, false, authCallbackFn,
652  authArg, headers);
653 }
654 
659 {
660  auto *dlThread = new RemoteFileDownloadThread(this, dlInfo);
661  MThreadPool::globalInstance()->start(dlThread, "RemoteFileDownload");
662 }
663 
668 {
669  if (!dlInfo)
670  return;
671 
672  static constexpr char kDateFormat[] = "ddd, dd MMM yyyy hh:mm:ss 'GMT'";
673  QUrl qurl(dlInfo->m_url);
674  QNetworkRequest request;
675 
676  if (dlInfo->m_request)
677  {
678  request = *dlInfo->m_request;
679  delete dlInfo->m_request;
680  dlInfo->m_request = nullptr;
681  }
682  else
683  request.setUrl(qurl);
684 
685  if (dlInfo->m_reload)
686  {
687  request.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
688  QNetworkRequest::AlwaysNetwork);
689  }
690  else
691  {
692  // Prefer the in-cache item if one exists and it is less than 5 minutes
693  // old and it will not expire in the next 10 seconds
694  QDateTime now = MythDate::current();
695 
696  // Handle redirects, we want the metadata of the file headers
697  QString redirectLoc;
698  int limit = 0;
699  while (!(redirectLoc = getHeader(qurl, "Location")).isNull())
700  {
701  if (limit == CACHE_REDIRECTION_LIMIT)
702  {
703  LOG(VB_GENERAL, LOG_WARNING, QString("Cache Redirection limit "
704  "reached for %1")
705  .arg(qurl.toString()));
706  return;
707  }
708  qurl.setUrl(redirectLoc);
709  limit++;
710  }
711 
712  LOG(VB_NETWORK, LOG_DEBUG, QString("Checking cache for %1")
713  .arg(qurl.toString()));
714 
715  m_infoLock->lock();
716  QNetworkCacheMetaData urlData = m_manager->cache()->metaData(qurl);
717  m_infoLock->unlock();
718  if ((urlData.isValid()) &&
719  ((!urlData.expirationDate().isValid()) ||
720  (urlData.expirationDate().toUTC().secsTo(now) < 10)))
721  {
722  QString dateString = getHeader(urlData, "Date");
723 
724  if (!dateString.isNull())
725  {
726  QDateTime loadDate =
727  MythDate::fromString(dateString, kDateFormat);
728  loadDate.setTimeSpec(Qt::UTC);
729  if (loadDate.secsTo(now) <= 720)
730  {
731  dlInfo->m_preferCache = true;
732  LOG(VB_NETWORK, LOG_DEBUG, QString("Preferring cache for %1")
733  .arg(qurl.toString()));
734  }
735  }
736  }
737  }
738 
739  if (dlInfo->m_preferCache)
740  request.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
741  QNetworkRequest::PreferCache);
742 
743  if (!request.hasRawHeader("User-Agent"))
744  {
745  request.setRawHeader("User-Agent",
746  "MythTV v" MYTH_BINARY_VERSION
747  " MythDownloadManager");
748  }
749 
750  if (dlInfo->m_headers)
751  {
752  QHash<QByteArray, QByteArray>::const_iterator it =
753  dlInfo->m_headers->constBegin();
754  for ( ; it != dlInfo->m_headers->constEnd(); ++it )
755  {
756  if (!it.key().isEmpty() && !it.value().isEmpty())
757  {
758  request.setRawHeader(it.key(), it.value());
759  }
760  }
761  }
762 
763  switch (dlInfo->m_requestType)
764  {
765  case kRequestPost :
766  dlInfo->m_reply = m_manager->post(request, *dlInfo->m_data);
767  break;
768  case kRequestHead :
769  dlInfo->m_reply = m_manager->head(request);
770  break;
771  case kRequestGet :
772  default:
773  dlInfo->m_reply = m_manager->get(request);
774  break;
775  }
776 
777  m_downloadReplies[dlInfo->m_reply] = dlInfo;
778 
779  if (dlInfo->m_authCallback)
780  {
781  connect(m_manager, SIGNAL(authenticationRequired(QNetworkReply *,
782  QAuthenticator *)),
783  this, SLOT(authCallback(QNetworkReply *, QAuthenticator *)));
784  }
785 
786  connect(dlInfo->m_reply, SIGNAL(error(QNetworkReply::NetworkError)), this,
787  SLOT(downloadError(QNetworkReply::NetworkError)));
788  connect(dlInfo->m_reply, SIGNAL(downloadProgress(qint64, qint64)),
789  this, SLOT(downloadProgress(qint64, qint64)));
790 }
791 
796 void MythDownloadManager::authCallback(QNetworkReply *reply,
797  QAuthenticator *authenticator)
798 {
799  if (!reply)
800  return;
801 
802  MythDownloadInfo *dlInfo = m_downloadReplies[reply];
803 
804  if (!dlInfo)
805  return;
806 
807  if (dlInfo->m_authCallback)
808  {
809  LOG(VB_FILE, LOG_DEBUG, "Calling auth callback");
810  dlInfo->m_authCallback(reply, authenticator, dlInfo->m_authArg);
811  }
812 }
813 
821 {
822  if (!dlInfo)
823  return false;
824 
825  dlInfo->m_syncMode = true;
826 
827  // Special handling for link-local
828  // Not needed for Windows because windows does not need
829  // the scope id.
830 #ifndef _WIN32
831  if (dlInfo->m_url.startsWith("http://[fe80::",Qt::CaseInsensitive))
832  return downloadNowLinkLocal(dlInfo,deleteInfo);
833 #endif
834  m_infoLock->lock();
835  m_downloadQueue.push_back(dlInfo);
836  m_infoLock->unlock();
837  m_queueWaitCond.wakeAll();
838 
839  // timeout myth:// RemoteFile transfers 20 seconds from now
840  // timeout non-myth:// QNetworkAccessManager transfers 60 seconds after
841  // their last progress update
842  QDateTime startedAt = MythDate::current();
843  m_infoLock->lock();
844  while ((!dlInfo->IsDone()) &&
845  (dlInfo->m_errorCode == QNetworkReply::NoError) &&
846  (((!dlInfo->m_url.startsWith("myth://")) &&
847  (dlInfo->m_lastStat.secsTo(MythDate::current()) < 60)) ||
848  ((dlInfo->m_url.startsWith("myth://")) &&
849  (startedAt.secsTo(MythDate::current()) < 20))))
850  {
851  m_infoLock->unlock();
852  m_queueWaitLock.lock();
853  m_queueWaitCond.wait(&m_queueWaitLock, 200);
854  m_queueWaitLock.unlock();
855  m_infoLock->lock();
856  }
857  bool done = dlInfo->IsDone();
858  bool success =
859  done && (dlInfo->m_errorCode == QNetworkReply::NoError);
860 
861  if (!done)
862  {
863  dlInfo->m_data = nullptr; // Prevent downloadFinished() from updating
864  dlInfo->m_syncMode = false; // Let downloadFinished() cleanup for us
865  if ((dlInfo->m_reply) &&
866  (dlInfo->m_errorCode == QNetworkReply::NoError))
867  {
868  LOG(VB_FILE, LOG_DEBUG,
869  LOC + QString("Aborting download - lack of data transfer"));
870  dlInfo->m_reply->abort();
871  }
872  }
873  else if (deleteInfo)
874  delete dlInfo;
875 
876  m_infoLock->unlock();
877 
878  return success;
879 }
880 
881 #ifndef _WIN32
882 
902 bool MythDownloadManager::downloadNowLinkLocal(MythDownloadInfo *dlInfo, bool deleteInfo)
903 {
904  bool isOK = true;
905 
906  // Only certain features are supported here
907  if (dlInfo->m_authCallback || dlInfo->m_authArg)
908  {
909  LOG(VB_GENERAL, LOG_ERR, LOC +
910  QString("Unsupported authentication for %1").arg(dlInfo->m_url));
911  isOK = false;
912  }
913  if (!dlInfo->m_outFile.isEmpty())
914  {
915  LOG(VB_GENERAL, LOG_ERR, LOC +
916  QString("Unsupported File output %1 for %2")
917  .arg(dlInfo->m_outFile).arg(dlInfo->m_url));
918  isOK = false;
919  }
920 
921  if (!deleteInfo || dlInfo->m_requestType == kRequestHead)
922  {
923  // We do not have the ability to return a network reply in dlInfo
924  // so if we are asked to do that, return an error.
925  LOG(VB_GENERAL, LOG_ERR, LOC +
926  QString("Unsupported link-local operation %1")
927  .arg(dlInfo->m_url));
928  isOK = false;
929  }
930 
931  QUrl url(dlInfo->m_url);
932  QString host(url.host());
933  int port(url.port(80));
934  if (isOK && PortChecker::resolveLinkLocal(host, port))
935  {
936  QString reqType;
937  switch (dlInfo->m_requestType)
938  {
939  case kRequestPost :
940  reqType = "POST";
941  break;
942  case kRequestGet :
943  default:
944  reqType = "GET";
945  break;
946  }
947  QByteArray *aBuffer = dlInfo->m_data;
948  QHash<QByteArray, QByteArray> headers;
949  if (dlInfo->m_headers)
950  headers = *dlInfo->m_headers;
951  if (!headers.contains("User-Agent"))
952  headers.insert("User-Agent",
953  "MythDownloadManager v" MYTH_BINARY_VERSION);
954  headers.insert("Connection", "close");
955  headers.insert("Accept-Encoding", "identity");
956  if (aBuffer && !aBuffer->isEmpty())
957  headers.insert("Content-Length",
958  (QString::number(aBuffer->size())).toUtf8());
959  headers.insert("Host",
960  (url.host()+":"+QString::number(port)).toUtf8());
961 
962  QByteArray requestMessage;
963  QString path (url.path());
964  requestMessage.append("POST ");
965  requestMessage.append(path);
966  requestMessage.append(" HTTP/1.1\r\n");
967  QHashIterator<QByteArray, QByteArray> it(headers);
968  while (it.hasNext())
969  {
970  it.next();
971  requestMessage.append(it.key());
972  requestMessage.append(": ");
973  requestMessage.append(it.value());
974  requestMessage.append("\r\n");
975  }
976  requestMessage.append("\r\n");
977  if (aBuffer && !aBuffer->isEmpty())
978  {
979  requestMessage.append(*aBuffer);
980  }
981  QTcpSocket socket;
982  socket.connectToHost(host, port);
983  // QT Warning - this may not work on Windows
984  if (!socket.waitForConnected(5000))
985  isOK = false;
986  if (isOK)
987  isOK = (socket.write(requestMessage) > 0);
988  if (isOK)
989  // QT Warning - this may not work on Windows
990  isOK = socket.waitForDisconnected(5000);
991  if (isOK)
992  {
993  *aBuffer = socket.readAll();
994  // Find the start of the content
995  QByteArray delim("\r\n\r\n");
996  int delimLoc=aBuffer->indexOf(delim);
997  if (delimLoc > -1)
998  *aBuffer = aBuffer->right
999  (aBuffer->size()-delimLoc-4);
1000  else
1001  isOK=false;
1002  }
1003  socket.close();
1004  }
1005  else
1006  isOK = false;
1007 
1008  if (deleteInfo)
1009  delete dlInfo;
1010 
1011  if (isOK)
1012  return true;
1013  LOG(VB_GENERAL, LOG_ERR, LOC + QString("Link Local request failed: %1")
1014  .arg(url.toString()));
1015  return false;
1016 }
1017 #endif
1018 
1023 void MythDownloadManager::cancelDownload(const QString &url, bool block)
1024 {
1025  cancelDownload(QStringList(url), block);
1026 }
1027 
1032 void MythDownloadManager::cancelDownload(const QStringList &urls, bool block)
1033 {
1034  m_infoLock->lock();
1035  foreach (QString url, urls)
1036  {
1037  QMutableListIterator<MythDownloadInfo*> lit(m_downloadQueue);
1038  while (lit.hasNext())
1039  {
1040  lit.next();
1041  MythDownloadInfo *dlInfo = lit.value();
1042  if (dlInfo->m_url == url)
1043  {
1044  if (!m_cancellationQueue.contains(dlInfo))
1045  m_cancellationQueue.append(dlInfo);
1046  lit.remove();
1047  }
1048  }
1049 
1050  if (m_downloadInfos.contains(url))
1051  {
1052  MythDownloadInfo *dlInfo = m_downloadInfos[url];
1053 
1054  if (!m_cancellationQueue.contains(dlInfo))
1055  m_cancellationQueue.append(dlInfo);
1056 
1057  if (dlInfo->m_reply)
1058  m_downloadReplies.remove(dlInfo->m_reply);
1059 
1060  m_downloadInfos.remove(url);
1061  }
1062  }
1063  m_infoLock->unlock();
1064 
1065  if (QThread::currentThread() == this->thread())
1066  {
1067  downloadCanceled();
1068  return;
1069  }
1070 
1071  // wake-up running thread
1072  m_queueWaitCond.wakeAll();
1073 
1074  if (!block)
1075  return;
1076 
1077  while (!m_cancellationQueue.isEmpty())
1078  {
1079  usleep(50000); // re-test in another 50ms
1080  }
1081 }
1082 
1084 {
1085  QMutexLocker locker(m_infoLock);
1086 
1087  QMutableListIterator<MythDownloadInfo*> lit(m_cancellationQueue);
1088  while (lit.hasNext())
1089  {
1090  lit.next();
1091  MythDownloadInfo *dlInfo = lit.value();
1092  dlInfo->m_lock.lock();
1093 
1094  if (dlInfo->m_reply)
1095  {
1096  LOG(VB_FILE, LOG_DEBUG,
1097  LOC + QString("Aborting download - user request"));
1098  dlInfo->m_reply->abort();
1099  }
1100  lit.remove();
1101  if (dlInfo->m_done)
1102  {
1103  dlInfo->m_lock.unlock();
1104  continue;
1105  }
1106  dlInfo->m_errorCode = QNetworkReply::OperationCanceledError;
1107  dlInfo->m_done = true;
1108  dlInfo->m_lock.unlock();
1109  }
1110 }
1111 
1117 {
1118  QMutexLocker locker(m_infoLock);
1119 
1120  QList <MythDownloadInfo*>::iterator lit = m_downloadQueue.begin();
1121  for (; lit != m_downloadQueue.end(); ++lit)
1122  {
1123  MythDownloadInfo *dlInfo = *lit;
1124  if (dlInfo->m_caller == caller)
1125  {
1126  dlInfo->m_caller = nullptr;
1127  dlInfo->m_outFile = QString();
1128  dlInfo->m_data = nullptr;
1129  }
1130  }
1131 
1132  QMap <QString, MythDownloadInfo*>::iterator mit = m_downloadInfos.begin();
1133  for (; mit != m_downloadInfos.end(); ++mit)
1134  {
1135  MythDownloadInfo *dlInfo = mit.value();
1136  if (dlInfo->m_caller == caller)
1137  {
1138  dlInfo->m_caller = nullptr;
1139  dlInfo->m_outFile = QString();
1140  dlInfo->m_data = nullptr;
1141  }
1142  }
1143 }
1144 
1148 void MythDownloadManager::downloadError(QNetworkReply::NetworkError errorCode)
1149 {
1150  auto reply = dynamic_cast<QNetworkReply *>(sender());
1151  if (reply == nullptr)
1152  return;
1153 
1154  LOG(VB_FILE, LOG_DEBUG, LOC + QString("downloadError %1 ")
1155  .arg(errorCode) + reply->errorString() );
1156 
1157  QMutexLocker locker(m_infoLock);
1158  if (!m_downloadReplies.contains(reply))
1159  {
1160  reply->deleteLater();
1161  return;
1162  }
1163 
1164  MythDownloadInfo *dlInfo = m_downloadReplies[reply];
1165 
1166  if (!dlInfo)
1167  return;
1168 
1169  dlInfo->m_errorCode = errorCode;
1170 }
1171 
1177 QUrl MythDownloadManager::redirectUrl(const QUrl& possibleRedirectUrl,
1178  const QUrl& oldRedirectUrl)
1179 {
1180  LOG(VB_FILE, LOG_DEBUG, LOC + QString("redirectUrl()"));
1181  QUrl redirectUrl;
1182 
1183  if(!possibleRedirectUrl.isEmpty() && possibleRedirectUrl != oldRedirectUrl)
1184  redirectUrl = possibleRedirectUrl;
1185 
1186  return redirectUrl;
1187 }
1188 
1192 void MythDownloadManager::downloadFinished(QNetworkReply* reply)
1193 {
1194  LOG(VB_FILE, LOG_DEBUG, LOC + QString("downloadFinished(%1)")
1195  .arg((long long)reply));
1196 
1197  QMutexLocker locker(m_infoLock);
1198  if (!m_downloadReplies.contains(reply))
1199  {
1200  reply->deleteLater();
1201  return;
1202  }
1203 
1204  MythDownloadInfo *dlInfo = m_downloadReplies[reply];
1205 
1206  if (!dlInfo || !dlInfo->m_reply)
1207  return;
1208 
1209  downloadFinished(dlInfo);
1210 }
1211 
1216 {
1217  if (!dlInfo)
1218  return;
1219 
1220  int statusCode = -1;
1221  static constexpr char kDateFormat[] = "ddd, dd MMM yyyy hh:mm:ss 'GMT'";
1222  QNetworkReply *reply = dlInfo->m_reply;
1223 
1224  if (reply)
1225  {
1226  QUrl possibleRedirectUrl =
1227  reply->attribute(QNetworkRequest::RedirectionTargetAttribute).toUrl();
1228 
1229  if (!possibleRedirectUrl.isEmpty() &&
1230  possibleRedirectUrl.isValid() &&
1231  possibleRedirectUrl.isRelative()) // Turn relative Url to absolute
1232  possibleRedirectUrl = reply->url().resolved(possibleRedirectUrl);
1233 
1234  if (!possibleRedirectUrl.isEmpty() && dlInfo->m_finalUrl != nullptr)
1235  *dlInfo->m_finalUrl = QString(possibleRedirectUrl.toString());
1236 
1237  dlInfo->m_redirectedTo =
1238  redirectUrl(possibleRedirectUrl, dlInfo->m_redirectedTo);
1239 
1240  QVariant status =
1241  reply->attribute(QNetworkRequest::HttpStatusCodeAttribute);
1242  if (status.isValid())
1243  statusCode = status.toInt();
1244  }
1245 
1246  if(reply && !dlInfo->m_redirectedTo.isEmpty() &&
1247  ((dlInfo->m_requestType != kRequestPost) ||
1248  (statusCode == 301 || statusCode == 302 ||
1249  statusCode == 303)))
1250  {
1251  LOG(VB_FILE, LOG_DEBUG, LOC +
1252  QString("downloadFinished(%1): Redirect: %2 -> %3")
1253  .arg((long long)dlInfo)
1254  .arg(reply->url().toString())
1255  .arg(dlInfo->m_redirectedTo.toString()));
1256 
1257  if (dlInfo->m_data)
1258  dlInfo->m_data->clear();
1259 
1260  dlInfo->m_bytesReceived = 0;
1261  dlInfo->m_bytesTotal = 0;
1262 
1263  QNetworkRequest request(dlInfo->m_redirectedTo);
1264 
1265  if (dlInfo->m_reload)
1266  {
1267  request.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
1268  QNetworkRequest::AlwaysNetwork);
1269  }
1270  else if (dlInfo->m_preferCache)
1271  {
1272  request.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
1273  QNetworkRequest::PreferCache);
1274  }
1275 
1276  request.setRawHeader("User-Agent",
1277  "MythDownloadManager v" MYTH_BINARY_VERSION);
1278 
1279  switch (dlInfo->m_requestType)
1280  {
1281  case kRequestHead :
1282  dlInfo->m_reply = m_manager->head(request);
1283  break;
1284  case kRequestGet :
1285  default:
1286  dlInfo->m_reply = m_manager->get(request);
1287  break;
1288  }
1289 
1290  m_downloadReplies[dlInfo->m_reply] = dlInfo;
1291 
1292  connect(dlInfo->m_reply, SIGNAL(error(QNetworkReply::NetworkError)),
1293  this, SLOT(downloadError(QNetworkReply::NetworkError)));
1294  connect(dlInfo->m_reply, SIGNAL(downloadProgress(qint64, qint64)),
1295  this, SLOT(downloadProgress(qint64, qint64)));
1296 
1297  m_downloadReplies.remove(reply);
1298  reply->deleteLater();
1299  }
1300  else
1301  {
1302  LOG(VB_FILE, LOG_DEBUG, QString("downloadFinished(%1): COMPLETE: %2")
1303  .arg((long long)dlInfo).arg(dlInfo->m_url));
1304 
1305  // HACK Insert a Date header into the cached metadata if one doesn't
1306  // already exist
1307  QUrl fileUrl = dlInfo->m_url;
1308  QString redirectLoc;
1309  int limit = 0;
1310  while (!(redirectLoc = getHeader(fileUrl, "Location")).isNull())
1311  {
1312  if (limit == CACHE_REDIRECTION_LIMIT)
1313  {
1314  LOG(VB_GENERAL, LOG_WARNING, QString("Cache Redirection limit "
1315  "reached for %1")
1316  .arg(fileUrl.toString()));
1317  return;
1318  }
1319  fileUrl.setUrl(redirectLoc);
1320  limit++;
1321  }
1322 
1323  m_infoLock->lock();
1324  QNetworkCacheMetaData urlData = m_manager->cache()->metaData(fileUrl);
1325  m_infoLock->unlock();
1326  if (getHeader(urlData, "Date").isNull())
1327  {
1328  QNetworkCacheMetaData::RawHeaderList headers = urlData.rawHeaders();
1329  QNetworkCacheMetaData::RawHeader newheader;
1330  QDateTime now = MythDate::current();
1331  newheader = QNetworkCacheMetaData::RawHeader("Date",
1332  now.toString(kDateFormat).toLatin1());
1333  headers.append(newheader);
1334  urlData.setRawHeaders(headers);
1335  m_infoLock->lock();
1336  m_manager->cache()->updateMetaData(urlData);
1337  m_infoLock->unlock();
1338  }
1339  // End HACK
1340 
1341  dlInfo->m_redirectedTo.clear();
1342 
1343  int dataSize = -1;
1344 
1345  // If we downloaded via the QNetworkAccessManager
1346  // AND the caller isn't handling the reply directly
1347  if (reply && dlInfo->m_processReply)
1348  {
1349  bool append = (!dlInfo->m_syncMode && dlInfo->m_caller);
1350  QByteArray data = reply->readAll();
1351  dataSize = data.size();
1352 
1353  if (append)
1354  dlInfo->m_bytesReceived += dataSize;
1355  else
1356  dlInfo->m_bytesReceived = dataSize;
1357 
1358  dlInfo->m_bytesTotal = dlInfo->m_bytesReceived;
1359 
1360  if (dlInfo->m_data)
1361  {
1362  if (append)
1363  dlInfo->m_data->append(data);
1364  else
1365  *dlInfo->m_data = data;
1366  }
1367  else if (!dlInfo->m_outFile.isEmpty())
1368  {
1369  saveFile(dlInfo->m_outFile, data, append);
1370  }
1371  }
1372  else if (!reply) // If we downloaded via RemoteFile
1373  {
1374  if (dlInfo->m_data)
1375  {
1376  (*dlInfo->m_data) = dlInfo->m_privData;
1377  }
1378  else if (!dlInfo->m_outFile.isEmpty())
1379  {
1380  saveFile(dlInfo->m_outFile, dlInfo->m_privData);
1381  }
1382  dlInfo->m_bytesReceived += dataSize;
1383  dlInfo->m_bytesTotal = dlInfo->m_bytesReceived;
1384  }
1385  // else we downloaded via QNetworkAccessManager
1386  // AND the caller is handling the reply
1387 
1388  m_infoLock->lock();
1389  if (!m_downloadInfos.remove(dlInfo->m_url))
1390  LOG(VB_GENERAL, LOG_ERR, LOC +
1391  QString("ERROR download finished but failed to remove url: %1")
1392  .arg(dlInfo->m_url));
1393 
1394  if (reply)
1395  m_downloadReplies.remove(reply);
1396  m_infoLock->unlock();
1397 
1398  dlInfo->SetDone(true);
1399 
1400  if (!dlInfo->m_syncMode)
1401  {
1402  if (dlInfo->m_caller)
1403  {
1404  LOG(VB_FILE, LOG_DEBUG, QString("downloadFinished(%1): "
1405  "COMPLETE: %2, sending event to caller")
1406  .arg((long long)dlInfo).arg(dlInfo->m_url));
1407 
1408  QStringList args;
1409  args << dlInfo->m_url;
1410  args << dlInfo->m_outFile;
1411  args << QString::number(dlInfo->m_bytesTotal);
1412  // placeholder for error string
1413  args << (reply ? reply->errorString() : QString());
1414  args << QString::number((int)(reply ? reply->error() :
1415  dlInfo->m_errorCode));
1416 
1417  QCoreApplication::postEvent(dlInfo->m_caller,
1418  new MythEvent("DOWNLOAD_FILE FINISHED", args));
1419  }
1420 
1421  delete dlInfo;
1422  }
1423 
1424  m_queueWaitCond.wakeAll();
1425  }
1426 }
1427 
1433 void MythDownloadManager::downloadProgress(qint64 bytesReceived,
1434  qint64 bytesTotal)
1435 {
1436  auto reply = dynamic_cast<QNetworkReply *>(sender());
1437  if (reply == nullptr)
1438  return;
1439 
1440  LOG(VB_FILE, LOG_DEBUG, LOC +
1441  QString("downloadProgress(%1, %2) (for reply %3)")
1442  .arg(bytesReceived).arg(bytesTotal).arg((long long)reply));
1443 
1444  QMutexLocker locker(m_infoLock);
1445  if (!m_downloadReplies.contains(reply))
1446  return;
1447 
1448  MythDownloadInfo *dlInfo = m_downloadReplies[reply];
1449 
1450  if (!dlInfo)
1451  return;
1452 
1453  dlInfo->m_lastStat = MythDate::current();
1454 
1455  LOG(VB_FILE, LOG_DEBUG, LOC +
1456  QString("downloadProgress: %1 to %2 is at %3 of %4 bytes downloaded")
1457  .arg(dlInfo->m_url).arg(dlInfo->m_outFile)
1458  .arg(bytesReceived).arg(bytesTotal));
1459 
1460  if (!dlInfo->m_syncMode && dlInfo->m_caller)
1461  {
1462  LOG(VB_FILE, LOG_DEBUG, QString("downloadProgress(%1): "
1463  "sending event to caller")
1464  .arg(reply->url().toString()));
1465 
1466  bool appendToFile = (dlInfo->m_bytesReceived != 0);
1467  QByteArray data = reply->readAll();
1468  if (!dlInfo->m_outFile.isEmpty())
1469  saveFile(dlInfo->m_outFile, data, appendToFile);
1470 
1471  if (dlInfo->m_data)
1472  dlInfo->m_data->append(data);
1473 
1474  dlInfo->m_bytesReceived = bytesReceived;
1475  dlInfo->m_bytesTotal = bytesTotal;
1476 
1477  QStringList args;
1478  args << dlInfo->m_url;
1479  args << dlInfo->m_outFile;
1480  args << QString::number(bytesReceived);
1481  args << QString::number(bytesTotal);
1482 
1483  QCoreApplication::postEvent(dlInfo->m_caller,
1484  new MythEvent("DOWNLOAD_FILE UPDATE", args));
1485  }
1486 }
1487 
1495 bool MythDownloadManager::saveFile(const QString &outFile,
1496  const QByteArray &data,
1497  const bool append)
1498 {
1499  if (outFile.isEmpty() || !data.size())
1500  return false;
1501 
1502  QFile file(outFile);
1503  QFileInfo fileInfo(outFile);
1504  QDir qdir(fileInfo.absolutePath());
1505 
1506  if (!qdir.exists() && !qdir.mkpath(fileInfo.absolutePath()))
1507  {
1508  LOG(VB_GENERAL, LOG_ERR, QString("Failed to create: '%1'")
1509  .arg(fileInfo.absolutePath()));
1510  return false;
1511  }
1512 
1513  QIODevice::OpenMode mode = QIODevice::Unbuffered|QIODevice::WriteOnly;
1514  if (append)
1515  mode |= QIODevice::Append;
1516 
1517  if (!file.open(mode))
1518  {
1519  LOG(VB_GENERAL, LOG_ERR, QString("Failed to open: '%1'") .arg(outFile));
1520  return false;
1521  }
1522 
1523  off_t offset = 0;
1524  size_t remaining = data.size();
1525  uint failure_cnt = 0;
1526  while ((remaining > 0) && (failure_cnt < 5))
1527  {
1528  ssize_t written = file.write(data.data() + offset, remaining);
1529  if (written < 0)
1530  {
1531  failure_cnt++;
1532  usleep(50000);
1533  continue;
1534  }
1535 
1536  failure_cnt = 0;
1537  offset += written;
1538  remaining -= written;
1539  }
1540 
1541  return remaining <= 0;
1542 }
1543 
1548 QDateTime MythDownloadManager::GetLastModified(const QString &url)
1549 {
1550  // If the header has not expired and
1551  // the last modification date is less than 1 hours old or if
1552  // the cache object is less than 20 minutes old,
1553  // then use the cached header otherwise redownload the header
1554 
1555  static constexpr char kDateFormat[] = "ddd, dd MMM yyyy hh:mm:ss 'GMT'";
1556  LOG(VB_FILE, LOG_DEBUG, LOC + QString("GetLastModified('%1')").arg(url));
1557  QDateTime result;
1558 
1559  QDateTime now = MythDate::current();
1560 
1561  QUrl cacheUrl = QUrl(url);
1562 
1563  // Deal with redirects, we want the cached data for the final url
1564  QString redirectLoc;
1565  int limit = 0;
1566  while (!(redirectLoc = getHeader(cacheUrl, "Location")).isNull())
1567  {
1568  if (limit == CACHE_REDIRECTION_LIMIT)
1569  {
1570  LOG(VB_GENERAL, LOG_WARNING, QString("Cache Redirection limit "
1571  "reached for %1")
1572  .arg(cacheUrl.toString()));
1573  return result;
1574  }
1575  cacheUrl.setUrl(redirectLoc);
1576  limit++;
1577  }
1578 
1579  m_infoLock->lock();
1580  QNetworkCacheMetaData urlData = m_manager->cache()->metaData(cacheUrl);
1581  m_infoLock->unlock();
1582 
1583  if (urlData.isValid() &&
1584  ((!urlData.expirationDate().isValid()) ||
1585  (urlData.expirationDate().secsTo(now) < 0)))
1586  {
1587  if (urlData.lastModified().toUTC().secsTo(now) <= 3600) // 1 Hour
1588  {
1589  result = urlData.lastModified().toUTC();
1590  }
1591  else
1592  {
1593  QString date = getHeader(urlData, "Date");
1594  if (!date.isNull())
1595  {
1596  QDateTime loadDate =
1597  MythDate::fromString(date, kDateFormat);
1598  loadDate.setTimeSpec(Qt::UTC);
1599  if (loadDate.secsTo(now) <= 1200) // 20 Minutes
1600  {
1601  result = urlData.lastModified().toUTC();
1602  }
1603  }
1604  }
1605  }
1606 
1607  if (!result.isValid())
1608  {
1609  auto *dlInfo = new MythDownloadInfo;
1610  dlInfo->m_url = url;
1611  dlInfo->m_syncMode = true;
1612  // Head request, we only want to inspect the headers
1613  dlInfo->m_requestType = kRequestHead;
1614 
1615  if (downloadNow(dlInfo, false))
1616  {
1617  if (dlInfo->m_reply)
1618  {
1619  QVariant lastMod =
1620  dlInfo->m_reply->header(
1621  QNetworkRequest::LastModifiedHeader);
1622  if (lastMod.isValid())
1623  result = lastMod.toDateTime().toUTC();
1624  }
1625 
1626  // downloadNow() will set a flag to trigger downloadFinished()
1627  // to delete the dlInfo if the download times out
1628  delete dlInfo;
1629  }
1630  }
1631 
1632  LOG(VB_FILE, LOG_DEBUG, LOC + QString("GetLastModified('%1'): Result %2")
1633  .arg(url).arg(result.toString()));
1634 
1635  return result;
1636 }
1637 
1638 
1643 {
1644  QMutexLocker locker(&m_cookieLock);
1645 
1646  auto *jar = new MythCookieJar;
1647  jar->load(filename);
1648  m_manager->setCookieJar(jar);
1649 }
1650 
1655 {
1656  QMutexLocker locker(&m_cookieLock);
1657 
1658  if (!m_manager->cookieJar())
1659  return;
1660 
1661  auto jar = dynamic_cast<MythCookieJar *>(m_manager->cookieJar());
1662  if (jar == nullptr)
1663  return;
1664  jar->save(filename);
1665 }
1666 
1667 void MythDownloadManager::setCookieJar(QNetworkCookieJar *cookieJar)
1668 {
1669  QMutexLocker locker(&m_cookieLock);
1670  m_manager->setCookieJar(cookieJar);
1671 }
1672 
1676 QNetworkCookieJar *MythDownloadManager::copyCookieJar(void)
1677 {
1678  QMutexLocker locker(&m_cookieLock);
1679 
1680  if (!m_manager->cookieJar())
1681  return nullptr;
1682 
1683  auto inJar = dynamic_cast<MythCookieJar *>(m_manager->cookieJar());
1684  if (inJar == nullptr)
1685  return nullptr;
1686  auto *outJar = new MythCookieJar;
1687  outJar->copyAllCookies(*inJar);
1688 
1689  return static_cast<QNetworkCookieJar *>(outJar);
1690 }
1691 
1695 void MythDownloadManager::refreshCookieJar(QNetworkCookieJar *jar)
1696 {
1697  QMutexLocker locker(&m_cookieLock);
1698  delete m_inCookieJar;
1699 
1700  auto inJar = dynamic_cast<MythCookieJar *>(jar);
1701  if (inJar == nullptr)
1702  return;
1703 
1704  auto *outJar = new MythCookieJar;
1705  outJar->copyAllCookies(*inJar);
1706  m_inCookieJar = static_cast<QNetworkCookieJar *>(outJar);
1707 
1708  QMutexLocker locker2(&m_queueWaitLock);
1709  m_queueWaitCond.wakeAll();
1710 }
1711 
1715 {
1716  QMutexLocker locker(&m_cookieLock);
1717 
1718  auto inJar = dynamic_cast<MythCookieJar *>(m_inCookieJar);
1719  if (inJar != nullptr)
1720  {
1721  auto *outJar = new MythCookieJar;
1722  outJar->copyAllCookies(*inJar);
1723  m_manager->setCookieJar(static_cast<QNetworkCookieJar *>(outJar));
1724  }
1725 
1726  delete m_inCookieJar;
1727  m_inCookieJar = nullptr;
1728 }
1729 
1730 QString MythDownloadManager::getHeader(const QUrl& url, const QString& header)
1731 {
1732  if (!m_manager || !m_manager->cache())
1733  return QString();
1734 
1735  m_infoLock->lock();
1736  QNetworkCacheMetaData metadata = m_manager->cache()->metaData(url);
1737  m_infoLock->unlock();
1738 
1739  return getHeader(metadata, header);
1740 }
1741 
1747 QString MythDownloadManager::getHeader(const QNetworkCacheMetaData &cacheData,
1748  const QString& header)
1749 {
1750  QNetworkCacheMetaData::RawHeaderList headers = cacheData.rawHeaders();
1751 
1752  for (auto it = headers.begin(); it != headers.end(); ++it)
1753  if (QString((*it).first) == header)
1754  return QString((*it).second);
1755  return QString();
1756 }
1757 
1758 
1763 {
1764  const QList<QNetworkCookie> cookieList = old.allCookies();
1765  setAllCookies(cookieList);
1766 }
1767 
1771 void MythCookieJar::load(const QString &filename)
1772 {
1773  LOG(VB_GENERAL, LOG_DEBUG, QString("MythCookieJar: loading cookies from: %1").arg(filename));
1774 
1775  QFile f(filename);
1776  if (!f.open(QIODevice::ReadOnly))
1777  {
1778  LOG(VB_GENERAL, LOG_WARNING, QString("MythCookieJar::load() failed to open file for reading: %1").arg(filename));
1779  return;
1780  }
1781 
1782  QList<QNetworkCookie> cookieList;
1783  QTextStream stream(&f);
1784  while (!stream.atEnd())
1785  {
1786  QString cookie = stream.readLine();
1787  cookieList << QNetworkCookie::parseCookies(cookie.toLocal8Bit());
1788  }
1789 
1790  setAllCookies(cookieList);
1791 }
1792 
1796 void MythCookieJar::save(const QString &filename)
1797 {
1798  LOG(VB_GENERAL, LOG_DEBUG, QString("MythCookieJar: saving cookies to: %1").arg(filename));
1799 
1800  QFile f(filename);
1801  if (!f.open(QIODevice::WriteOnly))
1802  {
1803  LOG(VB_GENERAL, LOG_ERR, QString("MythCookieJar::save() failed to open file for writing: %1").arg(filename));
1804  return;
1805  }
1806 
1807  QList<QNetworkCookie> cookieList = allCookies();
1808  QTextStream stream(&f);
1809 
1810  for (QList<QNetworkCookie>::iterator it = cookieList.begin();
1811  it != cookieList.end(); ++it)
1812  {
1813  stream << (*it).toRawForm() << endl;
1814  }
1815 }
1816 
1817 
1818 /* vim: set expandtab tabstop=4 shiftwidth=4: */
QNetworkCookieJar * copyCookieJar(void)
Copy from one cookie jar to another.
QNetworkReply::NetworkError m_errorCode
void save(const QString &filename)
Saves the cookie jar to a cookie file.
MRequestType
void queueItem(const QString &url, QNetworkRequest *req, const QString &dest, QByteArray *data, QObject *caller, const MRequestType reqType=kRequestGet, const bool reload=false)
Adds a request to the download queue.
MythDownloadManager * downloadManager
void removeListener(QObject *caller)
Disconnects the specified caller from any existing MythDownloadInfo instances.
void queueDownload(const QString &url, const QString &dest, QObject *caller, const bool reload=false)
Adds a url to the download queue.
~MythDownloadManager()
Destructor for MythDownloadManager.
static void error(const char *str,...)
Definition: vbi.c:42
void ShutdownMythDownloadManager(void)
Deletes the running MythDownloadManager at program exit.
#define CACHE_REDIRECTION_LIMIT
A subclassed QNetworkCookieJar that allows for reading and writing cookie files that contain raw form...
void downloadQNetworkRequest(MythDownloadInfo *dlInfo)
Downloads a QNetworkRequest via the QNetworkAccessManager.
AuthCallback m_authCallback
static bool saveFile(const QString &outFile, const QByteArray &data, const bool append=false)
Saves a QByteArray of data to a given filename.
QNetworkReply * m_reply
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
void downloadProgress(qint64 bytesReceived, qint64 bytesTotal)
Slot to process download update events.
const QHash< QByteArray, QByteArray > * m_headers
MRequestType m_requestType
bool postAuth(const QString &url, QByteArray *data, AuthCallback authCallback, void *authArg, const QHash< QByteArray, QByteArray > *headers=nullptr)
Posts data to a url via the QNetworkAccessManager.
void run(void) override
Runs a loop to process incoming download requests and triggers download events to be processed.
void downloadFinished(QNetworkReply *reply)
Slot to process download finished events.
#define off_t
static bool resolveLinkLocal(QString &host, int port, int timeLimit=30000)
Convenience method to resolve link-local address.
QString GetConfDir(void)
Definition: mythdirs.cpp:224
void authCallback(QNetworkReply *reply, QAuthenticator *authenticator)
Signal handler for authentication requests.
This class is used as a container for messages.
Definition: mythevent.h:16
void setCookieJar(QNetworkCookieJar *cookieJar)
static QString downloadRemoteFile(const QString &cmd, const QString &url, const QString &storageGroup, const QString &filename)
void copyAllCookies(MythCookieJar &old)
Copies all cookies from one MythCookieJar to another.
RemoteFileDownloadThread(MythDownloadManager *parent, MythDownloadInfo *dlInfo)
QDateTime current(bool stripped)
Returns current Date and Time in UTC.
Definition: mythdate.cpp:10
void queuePost(const QString &url, QByteArray *data, QObject *caller)
Queues a post to a URL via the QNetworkAccessManager.
void refreshCookieJar(QNetworkCookieJar *jar)
Refresh the temporary cookie jar from another cookie jar.
MythDownloadManager * GetMythDownloadManager(void)
Gets the pointer to the MythDownloadManager singleton.
bool download(const QString &url, const QString &dest, const bool reload=false)
Downloads a URL to a file in blocking mode.
QDateTime GetLastModified(const QString &url)
Gets the Last Modified timestamp for a URI.
unsigned int uint
Definition: compat.h:140
void downloadError(QNetworkReply::NetworkError errorCode)
Slot to process download error events.
#define LOC
QString getHeader(const QUrl &url, const QString &header)
void(*)(QNetworkReply *, QAuthenticator *, void *) AuthCallback
static QUrl redirectUrl(const QUrl &possibleRedirectUrl, const QUrl &oldRedirectUrl)
Checks whether we were redirected to the given URL.
static MThreadPool * globalInstance(void)
void downloadRemoteFile(MythDownloadInfo *dlInfo)
Triggers a myth:// URI download in the background via RemoteFile.
void SetDone(bool done)
#define LOG(_MASK_, _LEVEL_, _STRING_)
Definition: mythlogging.h:41
void start(QRunnable *runnable, const QString &debugName, int priority=0)
QNetworkRequest * m_request
bool downloadAuth(const QString &url, const QString &dest, const bool reload=false, AuthCallback authCallback=nullptr, void *authArg=nullptr, const QHash< QByteArray, QByteArray > *headers=nullptr)
Downloads a URL to a file in blocking mode.
void updateCookieJar(void)
Update the cookie jar from the temporary cookie jar.
bool downloadNow(MythDownloadInfo *dlInfo, bool deleteInfo=true)
Download helper for download() blocking methods.
#define MYTH_BINARY_VERSION
Update this whenever the plug-in ABI changes.
Definition: mythversion.h:16
void cancelDownload(const QString &url, bool block=true)
Cancel a queued or current download.
bool post(const QString &url, QByteArray *data)
Posts data to a url via the QNetworkAccessManager.
bool processItem(const QString &url, QNetworkRequest *req, const QString &dest, QByteArray *data, const MRequestType reqType=kRequestGet, const bool reload=false, AuthCallback authCallback=nullptr, void *authArg=nullptr, const QHash< QByteArray, QByteArray > *headers=nullptr, QString *finalUrl=nullptr)
Processes a network request immediately and waits for a response.
QMutex dmCreateLock
QString GetHostName(void)
void saveCookieJar(const QString &filename)
Saves the cookie jar to a cookie file.
QDateTime fromString(const QString &dtstr)
Converts kFilename && kISODate formats to QDateTime.
Definition: mythdate.cpp:30
void loadCookieJar(const QString &filename)
Loads the cookie jar from a cookie file.
void load(const QString &filename)
Loads the cookie jar from a cookie file.
void preCache(const QString &url)
Downloads a URL but doesn't store the resulting data anywhere.