MythTV  master
mythdownloadmanager.cpp
Go to the documentation of this file.
1 // qt
2 #include <QCoreApplication>
3 #include <QRunnable>
4 #include <QString>
5 #include <QByteArray>
6 #include <QFile>
7 #include <QDir>
8 #include <QNetworkCookie>
9 #include <QAuthenticator>
10 #include <QTextStream>
11 #include <QNetworkProxy>
12 #include <QMutexLocker>
13 #include <QUrl>
14 #include <QTcpSocket>
15 
16 #include <cstdlib>
17 #include <unistd.h> // for usleep()
18 
19 // libmythbase
20 #include "compat.h"
21 #include "mythcorecontext.h"
22 #include "mythcoreutil.h"
23 #include "mthreadpool.h"
24 #include "mythdirs.h"
25 #include "mythevent.h"
26 #include "mythversion.h"
27 #include "remotefile.h"
28 #include "mythdate.h"
29 
30 #include "mythdownloadmanager.h"
31 #include "mythlogging.h"
32 #include "portchecker.h"
33 
34 #define LOC QString("DownloadManager: ")
35 static constexpr int CACHE_REDIRECTION_LIMIT { 10 };
36 
38 QMutex dmCreateLock;
39 
44 {
45  public:
48  {
49  qRegisterMetaType<QNetworkReply::NetworkError>("QNetworkReply::NetworkError");
50  }
51 
53  {
54  delete m_request;
55  if (m_reply && m_processReply)
56  m_reply->deleteLater();
57  }
58 
59  bool IsDone(void)
60  {
61  QMutexLocker lock(&m_lock);
62  return m_done;
63  }
64 
65  void SetDone(bool done)
66  {
67  QMutexLocker lock(&m_lock);
68  m_done = done;
69  }
70 
71  QString m_url;
73  QString *m_finalUrl {nullptr};
74  QNetworkRequest *m_request {nullptr};
75  QNetworkReply *m_reply {nullptr};
76  QString m_outFile;
77  QByteArray *m_data {nullptr};
78  QByteArray m_privData;
79  QObject *m_caller {nullptr};
81  bool m_reload {false};
82  bool m_preferCache {false};
83  bool m_syncMode {false};
84  bool m_processReply {true};
85  bool m_done {false};
86  qint64 m_bytesReceived {0};
87  qint64 m_bytesTotal {0};
88  QDateTime m_lastStat;
90  void *m_authArg {nullptr};
91  const QHash<QByteArray, QByteArray> *m_headers {nullptr};
92 
93  QNetworkReply::NetworkError m_errorCode {QNetworkReply::NoError};
94  QMutex m_lock;
95 };
96 
97 
101 class RemoteFileDownloadThread : public QRunnable
102 {
103  public:
105  MythDownloadInfo *dlInfo) :
106  m_parent(parent),
107  m_dlInfo(dlInfo) {}
108 
109  void run() override // QRunnable
110  {
111  bool ok = false;
112 
113  auto *rf = new RemoteFile(m_dlInfo->m_url, false, false, 0ms);
114  ok = rf->SaveAs(m_dlInfo->m_privData);
115  delete rf;
116 
117  if (!ok)
118  m_dlInfo->m_errorCode = QNetworkReply::UnknownNetworkError;
119 
122 
124  }
125 
126  private:
129 };
130 
134 {
135  if (downloadManager)
136  {
137  delete downloadManager;
138  downloadManager = nullptr;
139  }
140 }
141 
146 {
147  if (downloadManager)
148  return downloadManager;
149 
150  QMutexLocker locker(&dmCreateLock);
151 
152  // Check once more in case the download manager was created
153  // while we were securing the lock.
154  if (downloadManager)
155  return downloadManager;
156 
157  auto *tmpDLM = new MythDownloadManager();
158  tmpDLM->start();
159  while (!tmpDLM->getQueueThread())
160  usleep(10000);
161 
162  tmpDLM->moveToThread(tmpDLM->getQueueThread());
163  tmpDLM->setRunThread();
164 
165  while (!tmpDLM->isRunning())
166  usleep(10000);
167 
168  downloadManager = tmpDLM;
169 
171 
172  return downloadManager;
173 }
174 
178 {
179  m_runThread = false;
180  m_queueWaitCond.wakeAll();
181 
182  wait();
183 
184  delete m_infoLock;
185  delete m_inCookieJar;
186 }
187 
192 {
193  RunProlog();
194 
195  bool downloading = false;
196  bool itemsInQueue = false;
197  bool itemsInCancellationQueue = false;
198  bool waitAnyway = false;
199 
200  m_queueThread = QThread::currentThread();
201 
202  while (!m_runThread)
203  usleep(50ms);
204 
205  m_manager = new QNetworkAccessManager(this);
206  m_diskCache = new QNetworkDiskCache(this);
207  m_proxy = new QNetworkProxy();
208  m_diskCache->setCacheDirectory(GetConfDir() + "/cache/" +
209  QCoreApplication::applicationName() + "-" +
211  m_manager->setCache(m_diskCache);
212 
213  // Set the proxy for the manager to be the application default proxy,
214  // which has already been setup
215  m_manager->setProxy(*m_proxy);
216 
217  // make sure the cookieJar is created in the same thread as the manager
218  // and set its parent to nullptr so it can be shared between managers
219  m_manager->cookieJar()->setParent(nullptr);
220 
221  QObject::connect(m_manager, &QNetworkAccessManager::finished,
222  this, qOverload<QNetworkReply*>(&MythDownloadManager::downloadFinished));
223 
224  m_isRunning = true;
225  while (m_runThread)
226  {
227  if (m_inCookieJar)
228  {
229  LOG(VB_GENERAL, LOG_DEBUG, "Updating DLManager's Cookie Jar");
230  updateCookieJar();
231  }
232  m_infoLock->lock();
233  LOG(VB_FILE, LOG_DEBUG, LOC + QString("items downloading %1").arg(m_downloadInfos.count()));
234  LOG(VB_FILE, LOG_DEBUG, LOC + QString("items queued %1").arg(m_downloadQueue.count()));
235  downloading = !m_downloadInfos.isEmpty();
236  itemsInCancellationQueue = !m_cancellationQueue.isEmpty();
237  m_infoLock->unlock();
238 
239  if (itemsInCancellationQueue)
240  {
242  }
243  if (downloading)
244  QCoreApplication::processEvents();
245 
246  m_infoLock->lock();
247  itemsInQueue = !m_downloadQueue.isEmpty();
248  m_infoLock->unlock();
249 
250  if (!itemsInQueue || waitAnyway)
251  {
252  waitAnyway = false;
253  m_queueWaitLock.lock();
254 
255  if (downloading)
256  {
257  LOG(VB_FILE, LOG_DEBUG, LOC + QString("waiting 200ms"));
258  m_queueWaitCond.wait(&m_queueWaitLock, 200);
259  }
260  else
261  {
262  LOG(VB_FILE, LOG_DEBUG, LOC + QString("waiting for more items to download"));
264  }
265 
266  m_queueWaitLock.unlock();
267  }
268 
269  m_infoLock->lock();
270  if (!m_downloadQueue.isEmpty())
271  {
272  MythDownloadInfo *dlInfo = m_downloadQueue.front();
273 
274  m_downloadQueue.pop_front();
275 
276  if (!dlInfo)
277  {
278  m_infoLock->unlock();
279  continue;
280  }
281 
282  if (m_downloadInfos.contains(dlInfo->m_url))
283  {
284  // Push request to the end of the queue to let others process.
285  // If this is the only item in the queue, force the loop to
286  // wait a little.
287  if (m_downloadQueue.isEmpty())
288  waitAnyway = true;
289  m_downloadQueue.push_back(dlInfo);
290  m_infoLock->unlock();
291  continue;
292  }
293 
294  if (dlInfo->m_url.startsWith("myth://"))
295  downloadRemoteFile(dlInfo);
296  else
297  {
298  QMutexLocker cLock(&m_cookieLock);
299  downloadQNetworkRequest(dlInfo);
300  }
301 
302  m_downloadInfos[dlInfo->m_url] = dlInfo;
303  }
304  m_infoLock->unlock();
305  }
306  m_isRunning = false;
307 
308  RunEpilog();
309 }
310 
321 void MythDownloadManager::queueItem(const QString &url, QNetworkRequest *req,
322  const QString &dest, QByteArray *data,
323  QObject *caller, const MRequestType reqType,
324  const bool reload)
325 {
326  auto *dlInfo = new MythDownloadInfo;
327 
328  dlInfo->m_url = url;
329  dlInfo->m_request = req;
330  dlInfo->m_outFile = dest;
331  dlInfo->m_data = data;
332  dlInfo->m_caller = caller;
333  dlInfo->m_requestType = reqType;
334  dlInfo->m_reload = reload;
335 
336  QMutexLocker locker(m_infoLock);
337  m_downloadQueue.push_back(dlInfo);
338  m_queueWaitCond.wakeAll();
339 }
340 
353 bool MythDownloadManager::processItem(const QString &url, QNetworkRequest *req,
354  const QString &dest, QByteArray *data,
355  const MRequestType reqType,
356  const bool reload,
357  AuthCallback authCallbackFn, void *authArg,
358  const QHash<QByteArray, QByteArray> *headers,
359  QString *finalUrl)
360 {
361  auto *dlInfo = new MythDownloadInfo;
362 
363  dlInfo->m_url = url;
364  dlInfo->m_request = req;
365  dlInfo->m_outFile = dest;
366  dlInfo->m_data = data;
367  dlInfo->m_requestType = reqType;
368  dlInfo->m_reload = reload;
369  dlInfo->m_syncMode = true;
370  dlInfo->m_authCallback = authCallbackFn;
371  dlInfo->m_authArg = authArg;
372  dlInfo->m_headers = headers;
373  dlInfo->m_finalUrl = finalUrl;
374 
375  return downloadNow(dlInfo, true);
376 }
377 
381 void MythDownloadManager::preCache(const QString &url)
382 {
383  LOG(VB_FILE, LOG_DEBUG, LOC + QString("preCache('%1')").arg(url));
384  queueItem(url, nullptr, QString(), nullptr, nullptr);
385 }
386 
393 void MythDownloadManager::queueDownload(const QString &url,
394  const QString &dest,
395  QObject *caller,
396  const bool reload)
397 {
398  LOG(VB_FILE, LOG_DEBUG, LOC + QString("queueDownload('%1', '%2', %3)")
399  .arg(url, dest, QString::number((long long)caller)));
400 
401  queueItem(url, nullptr, dest, nullptr, caller, kRequestGet, reload);
402 }
403 
409 void MythDownloadManager::queueDownload(QNetworkRequest *req,
410  QByteArray *data,
411  QObject *caller)
412 {
413  LOG(VB_FILE, LOG_DEBUG, LOC + QString("queueDownload('%1', '%2', %3)")
414  .arg(req->url().toString()).arg((long long)data)
415  .arg((long long)caller));
416 
417  queueItem(req->url().toString(), req, QString(), data, caller,
418  kRequestGet,
419  (QNetworkRequest::AlwaysNetwork == req->attribute(
420  QNetworkRequest::CacheLoadControlAttribute,
421  QNetworkRequest::PreferNetwork).toInt()));
422 }
423 
430 bool MythDownloadManager::download(const QString &url, const QString &dest,
431  const bool reload)
432 {
433  return processItem(url, nullptr, dest, nullptr, kRequestGet, reload);
434 }
435 
442 bool MythDownloadManager::download(const QString &url, QByteArray *data,
443  const bool reload, QString *finalUrl)
444 {
445  QString redirected;
446  if (!processItem(url, nullptr, QString(), data, kRequestGet, reload,
447  nullptr, nullptr, nullptr, &redirected))
448  return false;
449  if (!redirected.isEmpty() && finalUrl != nullptr)
450  *finalUrl = redirected;
451  return true;
452 }
453 
460 QNetworkReply *MythDownloadManager::download(const QString &url,
461  const bool reload)
462 {
463  auto *dlInfo = new MythDownloadInfo;
464  QNetworkReply *reply = nullptr;
465 
466  dlInfo->m_url = url;
467  dlInfo->m_reload = reload;
468  dlInfo->m_syncMode = true;
469  dlInfo->m_processReply = false;
470 
471  if (downloadNow(dlInfo, false))
472  {
473  if (dlInfo->m_reply)
474  {
475  reply = dlInfo->m_reply;
476  // prevent dlInfo dtor from deleting the reply
477  dlInfo->m_reply = nullptr;
478 
479  delete dlInfo;
480 
481  return reply;
482  }
483 
484  delete dlInfo;
485  }
486 
487  return nullptr;
488 }
489 
495 bool MythDownloadManager::download(QNetworkRequest *req, QByteArray *data)
496 {
497  LOG(VB_FILE, LOG_DEBUG, LOC + QString("download('%1', '%2')")
498  .arg(req->url().toString()).arg((long long)data));
499  return processItem(req->url().toString(), req, QString(), data,
500  kRequestGet,
501  (QNetworkRequest::AlwaysNetwork == req->attribute(
502  QNetworkRequest::CacheLoadControlAttribute,
503  QNetworkRequest::PreferNetwork).toInt()));
504 }
505 
515 bool MythDownloadManager::downloadAuth(const QString &url, const QString &dest,
516  const bool reload, AuthCallback authCallbackFn, void *authArg,
517  const QHash<QByteArray, QByteArray> *headers)
518 {
519  return processItem(url, nullptr, dest, nullptr, kRequestGet, reload, authCallbackFn,
520  authArg, headers);
521 }
522 
523 
529 void MythDownloadManager::queuePost(const QString &url,
530  QByteArray *data,
531  QObject *caller)
532 {
533  LOG(VB_FILE, LOG_DEBUG, LOC + QString("queuePost('%1', '%2')")
534  .arg(url).arg((long long)data));
535 
536  if (!data)
537  {
538  LOG(VB_GENERAL, LOG_ERR, LOC + "queuePost(), data is NULL!");
539  return;
540  }
541 
542  queueItem(url, nullptr, QString(), data, caller, kRequestPost);
543 }
544 
550 void MythDownloadManager::queuePost(QNetworkRequest *req,
551  QByteArray *data,
552  QObject *caller)
553 {
554  LOG(VB_FILE, LOG_DEBUG, LOC + QString("queuePost('%1', '%2')")
555  .arg(req->url().toString()).arg((long long)data));
556 
557  if (!data)
558  {
559  LOG(VB_GENERAL, LOG_ERR, LOC + "queuePost(), data is NULL!");
560  return;
561  }
562 
563  queueItem(req->url().toString(), req, QString(), data, caller,
564  kRequestPost,
565  (QNetworkRequest::AlwaysNetwork == req->attribute(
566  QNetworkRequest::CacheLoadControlAttribute,
567  QNetworkRequest::PreferNetwork).toInt()));
568 
569 }
570 
576 bool MythDownloadManager::post(const QString &url, QByteArray *data)
577 {
578  LOG(VB_FILE, LOG_DEBUG, LOC + QString("post('%1', '%2')")
579  .arg(url).arg((long long)data));
580 
581  if (!data)
582  {
583  LOG(VB_GENERAL, LOG_ERR, LOC + "post(), data is NULL!");
584  return false;
585  }
586 
587  return processItem(url, nullptr, QString(), data, kRequestPost);
588 }
589 
595 bool MythDownloadManager::post(QNetworkRequest *req, QByteArray *data)
596 {
597  LOG(VB_FILE, LOG_DEBUG, LOC + QString("post('%1', '%2')")
598  .arg(req->url().toString()).arg((long long)data));
599 
600  if (!data)
601  {
602  LOG(VB_GENERAL, LOG_ERR, LOC + "post(), data is NULL!");
603  return false;
604  }
605 
606  return processItem(req->url().toString(), req, QString(), data,
607  kRequestPost,
608  (QNetworkRequest::AlwaysNetwork == req->attribute(
609  QNetworkRequest::CacheLoadControlAttribute,
610  QNetworkRequest::PreferNetwork).toInt()));
611 
612 }
613 
622 bool MythDownloadManager::postAuth(const QString &url, QByteArray *data,
623  AuthCallback authCallbackFn, void *authArg,
624  const QHash<QByteArray, QByteArray> *headers)
625 {
626  LOG(VB_FILE, LOG_DEBUG, LOC + QString("postAuth('%1', '%2')")
627  .arg(url).arg((long long)data));
628 
629  if (!data)
630  {
631  LOG(VB_GENERAL, LOG_ERR, LOC + "postAuth(), data is NULL!");
632  return false;
633  }
634 
635  return processItem(url, nullptr, nullptr, data, kRequestPost, false, authCallbackFn,
636  authArg, headers);
637 }
638 
643 {
644  auto *dlThread = new RemoteFileDownloadThread(this, dlInfo);
645  MThreadPool::globalInstance()->start(dlThread, "RemoteFileDownload");
646 }
647 
652 {
653  if (!dlInfo)
654  return;
655 
656  static const QString kDateFormat = "ddd, dd MMM yyyy hh:mm:ss 'GMT'";
657  QUrl qurl(dlInfo->m_url);
658  QNetworkRequest request;
659 
660  if (dlInfo->m_request)
661  {
662  request = *dlInfo->m_request;
663  delete dlInfo->m_request;
664  dlInfo->m_request = nullptr;
665  }
666  else
667  {
668  request.setUrl(qurl);
669  }
670 
671  if (dlInfo->m_reload)
672  {
673  request.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
674  QNetworkRequest::AlwaysNetwork);
675  }
676  else
677  {
678  // Prefer the in-cache item if one exists and it is less than 5 minutes
679  // old and it will not expire in the next 10 seconds
680  QDateTime now = MythDate::current();
681 
682  // Handle redirects, we want the metadata of the file headers
683  QString redirectLoc;
684  int limit = 0;
685  while (!(redirectLoc = getHeader(qurl, "Location")).isNull())
686  {
687  if (limit == CACHE_REDIRECTION_LIMIT)
688  {
689  LOG(VB_GENERAL, LOG_WARNING, QString("Cache Redirection limit "
690  "reached for %1")
691  .arg(qurl.toString()));
692  return;
693  }
694  qurl.setUrl(redirectLoc);
695  limit++;
696  }
697 
698  LOG(VB_NETWORK, LOG_DEBUG, QString("Checking cache for %1")
699  .arg(qurl.toString()));
700 
701  m_infoLock->lock();
702  QNetworkCacheMetaData urlData = m_manager->cache()->metaData(qurl);
703  m_infoLock->unlock();
704  if ((urlData.isValid()) &&
705  ((!urlData.expirationDate().isValid()) ||
706  (urlData.expirationDate().toUTC().secsTo(now) < 10)))
707  {
708  QString dateString = getHeader(urlData, "Date");
709 
710  if (!dateString.isNull())
711  {
712  QDateTime loadDate =
713  MythDate::fromString(dateString, kDateFormat);
714  loadDate.setTimeSpec(Qt::UTC);
715  if (loadDate.secsTo(now) <= 720)
716  {
717  dlInfo->m_preferCache = true;
718  LOG(VB_NETWORK, LOG_DEBUG, QString("Preferring cache for %1")
719  .arg(qurl.toString()));
720  }
721  }
722  }
723  }
724 
725  if (dlInfo->m_preferCache)
726  request.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
727  QNetworkRequest::PreferCache);
728 
729  if (!request.hasRawHeader("User-Agent"))
730  {
731  request.setRawHeader("User-Agent",
732  QByteArray("MythTV v") + MYTH_BINARY_VERSION +
733  " MythDownloadManager");
734  }
735 
736  if (dlInfo->m_headers)
737  {
738  QHash<QByteArray, QByteArray>::const_iterator it =
739  dlInfo->m_headers->constBegin();
740  for ( ; it != dlInfo->m_headers->constEnd(); ++it )
741  {
742  if (!it.key().isEmpty() && !it.value().isEmpty())
743  {
744  request.setRawHeader(it.key(), it.value());
745  }
746  }
747  }
748 
749  switch (dlInfo->m_requestType)
750  {
751  case kRequestPost :
752  dlInfo->m_reply = m_manager->post(request, *dlInfo->m_data);
753  break;
754  case kRequestHead :
755  dlInfo->m_reply = m_manager->head(request);
756  break;
757  case kRequestGet :
758  default:
759  dlInfo->m_reply = m_manager->get(request);
760  break;
761  }
762 
763  m_downloadReplies[dlInfo->m_reply] = dlInfo;
764 
765  if (dlInfo->m_authCallback)
766  {
767  connect(m_manager, &QNetworkAccessManager::authenticationRequired,
769  }
770 
771  connect(dlInfo->m_reply, &QNetworkReply::errorOccurred,
773  connect(dlInfo->m_reply, &QNetworkReply::downloadProgress,
775 }
776 
781 void MythDownloadManager::authCallback(QNetworkReply *reply,
782  QAuthenticator *authenticator)
783 {
784  if (!reply)
785  return;
786 
787  MythDownloadInfo *dlInfo = m_downloadReplies[reply];
788 
789  if (!dlInfo)
790  return;
791 
792  if (dlInfo->m_authCallback)
793  {
794  LOG(VB_FILE, LOG_DEBUG, "Calling auth callback");
795  dlInfo->m_authCallback(reply, authenticator, dlInfo->m_authArg);
796  }
797 }
798 
806 {
807  if (!dlInfo)
808  return false;
809 
810  dlInfo->m_syncMode = true;
811 
812  // Special handling for link-local
813  // Not needed for Windows because windows does not need
814  // the scope id.
815 #ifndef _WIN32
816  if (dlInfo->m_url.startsWith("http://[fe80::",Qt::CaseInsensitive))
817  return downloadNowLinkLocal(dlInfo, deleteInfo);
818 #endif
819  m_infoLock->lock();
820  m_downloadQueue.push_back(dlInfo);
821  m_infoLock->unlock();
822  m_queueWaitCond.wakeAll();
823 
824  // timeout myth:// RemoteFile transfers 20 seconds from now
825  // timeout non-myth:// QNetworkAccessManager transfers 60 seconds after
826  // their last progress update
827  QDateTime startedAt = MythDate::current();
828  m_infoLock->lock();
829  while ((!dlInfo->IsDone()) &&
830  (dlInfo->m_errorCode == QNetworkReply::NoError) &&
831  (((!dlInfo->m_url.startsWith("myth://")) &&
832  (MythDate::secsInPast(dlInfo->m_lastStat) < 60s)) ||
833  ((dlInfo->m_url.startsWith("myth://")) &&
834  (MythDate::secsInPast(startedAt) < 20s))))
835  {
836  m_infoLock->unlock();
837  m_queueWaitLock.lock();
838  m_queueWaitCond.wait(&m_queueWaitLock, 200);
839  m_queueWaitLock.unlock();
840  m_infoLock->lock();
841  }
842  bool done = dlInfo->IsDone();
843  bool success =
844  done && (dlInfo->m_errorCode == QNetworkReply::NoError);
845 
846  if (!done)
847  {
848  dlInfo->m_data = nullptr; // Prevent downloadFinished() from updating
849  dlInfo->m_syncMode = false; // Let downloadFinished() cleanup for us
850  if ((dlInfo->m_reply) &&
851  (dlInfo->m_errorCode == QNetworkReply::NoError))
852  {
853  LOG(VB_FILE, LOG_DEBUG,
854  LOC + QString("Aborting download - lack of data transfer"));
855  dlInfo->m_reply->abort();
856  }
857  }
858  else if (deleteInfo)
859  {
860  delete dlInfo;
861  }
862 
863  m_infoLock->unlock();
864 
865  return success;
866 }
867 
868 #ifndef _WIN32
869 
889 bool MythDownloadManager::downloadNowLinkLocal(MythDownloadInfo *dlInfo, bool deleteInfo)
890 {
891  bool ok = true;
892 
893  // No buffer - no reply...
894  if (!dlInfo->m_data)
895  {
896  LOG(VB_GENERAL, LOG_ERR, LOC + QString("No data buffer provided for %1").arg(dlInfo->m_url));
897  ok = false;
898  }
899 
900  // Only certain features are supported here
901  if (dlInfo->m_authCallback || dlInfo->m_authArg)
902  {
903  LOG(VB_GENERAL, LOG_ERR, LOC + QString("Unsupported authentication for %1").arg(dlInfo->m_url));
904  ok = false;
905  }
906 
907  if (ok && !dlInfo->m_outFile.isEmpty())
908  {
909  LOG(VB_GENERAL, LOG_ERR, LOC + QString("Unsupported File output %1 for %2")
910  .arg(dlInfo->m_outFile, dlInfo->m_url));
911  ok = false;
912  }
913 
914  if (ok && (!deleteInfo || dlInfo->m_requestType == kRequestHead))
915  {
916  // We do not have the ability to return a network reply in dlInfo
917  // so if we are asked to do that, return an error.
918  LOG(VB_GENERAL, LOG_ERR, LOC + QString("Unsupported link-local operation %1")
919  .arg(dlInfo->m_url));
920  ok = false;
921  }
922 
923  QUrl url(dlInfo->m_url);
924  QString host(url.host());
925  int port(url.port(80));
926  if (ok && PortChecker::resolveLinkLocal(host, port))
927  {
928  QString reqType;
929  switch (dlInfo->m_requestType)
930  {
931  case kRequestPost :
932  reqType = "POST";
933  break;
934  case kRequestGet :
935  default:
936  reqType = "GET";
937  break;
938  }
939  QByteArray* buffer = dlInfo->m_data;
940  QHash<QByteArray, QByteArray> headers;
941  if (dlInfo->m_headers)
942  headers = *dlInfo->m_headers;
943  if (!headers.contains("User-Agent"))
944  headers.insert("User-Agent", QByteArray("MythDownloadManager v") +
945  MYTH_BINARY_VERSION);
946  headers.insert("Connection", "close");
947  headers.insert("Accept-Encoding", "identity");
948  if (!buffer->isEmpty())
949  headers.insert("Content-Length", QString::number(buffer->size()).toUtf8());
950  headers.insert("Host", (url.host() + ":" + QString::number(port)).toUtf8());
951 
952  QByteArray requestMessage;
953  QString path (url.path());
954  requestMessage.append("POST ");
955  requestMessage.append(path.toLatin1());
956  requestMessage.append(" HTTP/1.1\r\n");
957  QHashIterator<QByteArray, QByteArray> it(headers);
958  while (it.hasNext())
959  {
960  it.next();
961  requestMessage.append(it.key());
962  requestMessage.append(": ");
963  requestMessage.append(it.value());
964  requestMessage.append("\r\n");
965  }
966  requestMessage.append("\r\n");
967  if (!buffer->isEmpty())
968  requestMessage.append(*buffer);
969 
970  QTcpSocket socket;
971  socket.connectToHost(host, static_cast<uint16_t>(port));
972  // QT Warning - this may not work on Windows
973  if (!socket.waitForConnected(5000))
974  ok = false;
975  if (ok)
976  ok = socket.write(requestMessage) > 0;
977  if (ok)
978  // QT Warning - this may not work on Windows
979  ok = socket.waitForDisconnected(5000);
980  if (ok)
981  {
982  *buffer = socket.readAll();
983  // Find the start of the content
984  QByteArray delim("\r\n\r\n");
985  int delimLoc = buffer->indexOf(delim);
986  if (delimLoc > -1)
987  *buffer = buffer->right(buffer->size() - delimLoc - 4);
988  else
989  ok=false;
990  }
991  socket.close();
992  }
993  else
994  {
995  ok = false;
996  }
997 
998  if (deleteInfo)
999  delete dlInfo;
1000 
1001  if (ok)
1002  return true;
1003 
1004  LOG(VB_GENERAL, LOG_ERR, LOC + QString("Link Local request failed: %1").arg(url.toString()));
1005  return false;
1006 }
1007 #endif
1008 
1013 void MythDownloadManager::cancelDownload(const QString &url, bool block)
1014 {
1015  cancelDownload(QStringList(url), block);
1016 }
1017 
1022 void MythDownloadManager::cancelDownload(const QStringList &urls, bool block)
1023 {
1024  m_infoLock->lock();
1025  for (const auto& url : std::as_const(urls))
1026  {
1027  QMutableListIterator<MythDownloadInfo*> lit(m_downloadQueue);
1028  while (lit.hasNext())
1029  {
1030  lit.next();
1031  MythDownloadInfo *dlInfo = lit.value();
1032  if (dlInfo->m_url == url)
1033  {
1034  if (!m_cancellationQueue.contains(dlInfo))
1035  m_cancellationQueue.append(dlInfo);
1036  lit.remove();
1037  }
1038  }
1039 
1040  if (m_downloadInfos.contains(url))
1041  {
1042  MythDownloadInfo *dlInfo = m_downloadInfos[url];
1043 
1044  if (!m_cancellationQueue.contains(dlInfo))
1045  m_cancellationQueue.append(dlInfo);
1046 
1047  if (dlInfo->m_reply)
1048  m_downloadReplies.remove(dlInfo->m_reply);
1049 
1050  m_downloadInfos.remove(url);
1051  }
1052  }
1053  m_infoLock->unlock();
1054 
1055  if (QThread::currentThread() == this->thread())
1056  {
1057  downloadCanceled();
1058  return;
1059  }
1060 
1061  // wake-up running thread
1062  m_queueWaitCond.wakeAll();
1063 
1064  if (!block)
1065  return;
1066 
1067  while (!m_cancellationQueue.isEmpty())
1068  {
1069  usleep(50ms); // re-test in another 50ms
1070  }
1071 }
1072 
1074 {
1075  QMutexLocker locker(m_infoLock);
1076 
1077  QMutableListIterator<MythDownloadInfo*> lit(m_cancellationQueue);
1078  while (lit.hasNext())
1079  {
1080  lit.next();
1081  MythDownloadInfo *dlInfo = lit.value();
1082  dlInfo->m_lock.lock();
1083 
1084  if (dlInfo->m_reply)
1085  {
1086  LOG(VB_FILE, LOG_DEBUG,
1087  LOC + QString("Aborting download - user request"));
1088  dlInfo->m_reply->abort();
1089  }
1090  lit.remove();
1091  if (dlInfo->m_done)
1092  {
1093  dlInfo->m_lock.unlock();
1094  continue;
1095  }
1096  dlInfo->m_errorCode = QNetworkReply::OperationCanceledError;
1097  dlInfo->m_done = true;
1098  dlInfo->m_lock.unlock();
1099  }
1100 }
1101 
1107 {
1108  QMutexLocker locker(m_infoLock);
1109 
1110  QList <MythDownloadInfo*>::iterator lit = m_downloadQueue.begin();
1111  for (; lit != m_downloadQueue.end(); ++lit)
1112  {
1113  MythDownloadInfo *dlInfo = *lit;
1114  if (dlInfo->m_caller == caller)
1115  {
1116  dlInfo->m_caller = nullptr;
1117  dlInfo->m_outFile = QString();
1118  dlInfo->m_data = nullptr;
1119  }
1120  }
1121 
1122  QMap <QString, MythDownloadInfo*>::iterator mit = m_downloadInfos.begin();
1123  for (; mit != m_downloadInfos.end(); ++mit)
1124  {
1125  MythDownloadInfo *dlInfo = mit.value();
1126  if (dlInfo->m_caller == caller)
1127  {
1128  dlInfo->m_caller = nullptr;
1129  dlInfo->m_outFile = QString();
1130  dlInfo->m_data = nullptr;
1131  }
1132  }
1133 }
1134 
1138 void MythDownloadManager::downloadError(QNetworkReply::NetworkError errorCode)
1139 {
1140  auto *reply = qobject_cast<QNetworkReply *>(sender());
1141  if (reply == nullptr)
1142  return;
1143 
1144  LOG(VB_FILE, LOG_DEBUG, LOC + QString("downloadError %1 ")
1145  .arg(errorCode) + reply->errorString() );
1146 
1147  QMutexLocker locker(m_infoLock);
1148  if (!m_downloadReplies.contains(reply))
1149  {
1150  reply->deleteLater();
1151  return;
1152  }
1153 
1154  MythDownloadInfo *dlInfo = m_downloadReplies[reply];
1155 
1156  if (!dlInfo)
1157  return;
1158 
1159  dlInfo->m_errorCode = errorCode;
1160 }
1161 
1167 QUrl MythDownloadManager::redirectUrl(const QUrl& possibleRedirectUrl,
1168  const QUrl& oldRedirectUrl)
1169 {
1170  LOG(VB_FILE, LOG_DEBUG, LOC + QString("redirectUrl()"));
1171  QUrl redirectUrl;
1172 
1173  if(!possibleRedirectUrl.isEmpty() && possibleRedirectUrl != oldRedirectUrl)
1174  redirectUrl = possibleRedirectUrl;
1175 
1176  return redirectUrl;
1177 }
1178 
1182 void MythDownloadManager::downloadFinished(QNetworkReply* reply)
1183 {
1184  LOG(VB_FILE, LOG_DEBUG, LOC + QString("downloadFinished(%1)")
1185  .arg((long long)reply));
1186 
1187  QMutexLocker locker(m_infoLock);
1188  if (!m_downloadReplies.contains(reply))
1189  {
1190  reply->deleteLater();
1191  return;
1192  }
1193 
1194  MythDownloadInfo *dlInfo = m_downloadReplies[reply];
1195 
1196  if (!dlInfo || !dlInfo->m_reply)
1197  return;
1198 
1199  downloadFinished(dlInfo);
1200 }
1201 
1206 {
1207  if (!dlInfo)
1208  return;
1209 
1210  int statusCode = -1;
1211  static const QString kDateFormat = "ddd, dd MMM yyyy hh:mm:ss 'GMT'";
1212  QNetworkReply *reply = dlInfo->m_reply;
1213 
1214  if (reply)
1215  {
1216  QUrl possibleRedirectUrl =
1217  reply->attribute(QNetworkRequest::RedirectionTargetAttribute).toUrl();
1218 
1219  if (!possibleRedirectUrl.isEmpty() &&
1220  possibleRedirectUrl.isValid() &&
1221  possibleRedirectUrl.isRelative()) // Turn relative Url to absolute
1222  possibleRedirectUrl = reply->url().resolved(possibleRedirectUrl);
1223 
1224  if (!possibleRedirectUrl.isEmpty() && dlInfo->m_finalUrl != nullptr)
1225  *dlInfo->m_finalUrl = QString(possibleRedirectUrl.toString());
1226 
1227  dlInfo->m_redirectedTo =
1228  redirectUrl(possibleRedirectUrl, dlInfo->m_redirectedTo);
1229 
1230  QVariant status =
1231  reply->attribute(QNetworkRequest::HttpStatusCodeAttribute);
1232  if (status.isValid())
1233  statusCode = status.toInt();
1234  }
1235 
1236  if(reply && !dlInfo->m_redirectedTo.isEmpty() &&
1237  ((dlInfo->m_requestType != kRequestPost) ||
1238  (statusCode == 301 || statusCode == 302 ||
1239  statusCode == 303)))
1240  {
1241  LOG(VB_FILE, LOG_DEBUG, LOC +
1242  QString("downloadFinished(%1): Redirect: %2 -> %3")
1243  .arg(QString::number((long long)dlInfo),
1244  reply->url().toString(),
1245  dlInfo->m_redirectedTo.toString()));
1246 
1247  if (dlInfo->m_data)
1248  dlInfo->m_data->clear();
1249 
1250  dlInfo->m_bytesReceived = 0;
1251  dlInfo->m_bytesTotal = 0;
1252 
1253  QNetworkRequest request(dlInfo->m_redirectedTo);
1254 
1255  if (dlInfo->m_reload)
1256  {
1257  request.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
1258  QNetworkRequest::AlwaysNetwork);
1259  }
1260  else if (dlInfo->m_preferCache)
1261  {
1262  request.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
1263  QNetworkRequest::PreferCache);
1264  }
1265 
1266  request.setRawHeader("User-Agent",
1267  "MythDownloadManager v" +
1268  QByteArray(MYTH_BINARY_VERSION));
1269 
1270  switch (dlInfo->m_requestType)
1271  {
1272  case kRequestHead :
1273  dlInfo->m_reply = m_manager->head(request);
1274  break;
1275  case kRequestGet :
1276  default:
1277  dlInfo->m_reply = m_manager->get(request);
1278  break;
1279  }
1280 
1281  m_downloadReplies[dlInfo->m_reply] = dlInfo;
1282 
1283  connect(dlInfo->m_reply, &QNetworkReply::errorOccurred,
1285  connect(dlInfo->m_reply, &QNetworkReply::downloadProgress,
1287 
1288  m_downloadReplies.remove(reply);
1289  reply->deleteLater();
1290  }
1291  else
1292  {
1293  LOG(VB_FILE, LOG_DEBUG, QString("downloadFinished(%1): COMPLETE: %2")
1294  .arg((long long)dlInfo).arg(dlInfo->m_url));
1295 
1296  // HACK Insert a Date header into the cached metadata if one doesn't
1297  // already exist
1298  QUrl fileUrl = dlInfo->m_url;
1299  QString redirectLoc;
1300  int limit = 0;
1301  while (!(redirectLoc = getHeader(fileUrl, "Location")).isNull())
1302  {
1303  if (limit == CACHE_REDIRECTION_LIMIT)
1304  {
1305  LOG(VB_GENERAL, LOG_WARNING, QString("Cache Redirection limit "
1306  "reached for %1")
1307  .arg(fileUrl.toString()));
1308  return;
1309  }
1310  fileUrl.setUrl(redirectLoc);
1311  limit++;
1312  }
1313 
1314  m_infoLock->lock();
1315  QNetworkCacheMetaData urlData = m_manager->cache()->metaData(fileUrl);
1316  m_infoLock->unlock();
1317  if (getHeader(urlData, "Date").isNull())
1318  {
1319  QNetworkCacheMetaData::RawHeaderList headers = urlData.rawHeaders();
1320  QNetworkCacheMetaData::RawHeader newheader;
1321  QDateTime now = MythDate::current();
1322  newheader = QNetworkCacheMetaData::RawHeader("Date",
1323  now.toString(kDateFormat).toLatin1());
1324  headers.append(newheader);
1325  urlData.setRawHeaders(headers);
1326  m_infoLock->lock();
1327  m_manager->cache()->updateMetaData(urlData);
1328  m_infoLock->unlock();
1329  }
1330  // End HACK
1331 
1332  dlInfo->m_redirectedTo.clear();
1333 
1334  int dataSize = -1;
1335 
1336  // If we downloaded via the QNetworkAccessManager
1337  // AND the caller isn't handling the reply directly
1338  if (reply && dlInfo->m_processReply)
1339  {
1340  bool append = (!dlInfo->m_syncMode && dlInfo->m_caller);
1341  QByteArray data = reply->readAll();
1342  dataSize = data.size();
1343 
1344  if (append)
1345  dlInfo->m_bytesReceived += dataSize;
1346  else
1347  dlInfo->m_bytesReceived = dataSize;
1348 
1349  dlInfo->m_bytesTotal = dlInfo->m_bytesReceived;
1350 
1351  if (dlInfo->m_data)
1352  {
1353  if (append)
1354  dlInfo->m_data->append(data);
1355  else
1356  *dlInfo->m_data = data;
1357  }
1358  else if (!dlInfo->m_outFile.isEmpty())
1359  {
1360  saveFile(dlInfo->m_outFile, data, append);
1361  }
1362  }
1363  else if (!reply) // If we downloaded via RemoteFile
1364  {
1365  if (dlInfo->m_data)
1366  {
1367  (*dlInfo->m_data) = dlInfo->m_privData;
1368  }
1369  else if (!dlInfo->m_outFile.isEmpty())
1370  {
1371  saveFile(dlInfo->m_outFile, dlInfo->m_privData);
1372  }
1373  dlInfo->m_bytesReceived += dataSize;
1374  dlInfo->m_bytesTotal = dlInfo->m_bytesReceived;
1375  }
1376  // else we downloaded via QNetworkAccessManager
1377  // AND the caller is handling the reply
1378 
1379  m_infoLock->lock();
1380  if (!m_downloadInfos.remove(dlInfo->m_url))
1381  {
1382  LOG(VB_GENERAL, LOG_ERR, LOC +
1383  QString("ERROR download finished but failed to remove url: %1")
1384  .arg(dlInfo->m_url));
1385  }
1386 
1387  if (reply)
1388  m_downloadReplies.remove(reply);
1389  m_infoLock->unlock();
1390 
1391  dlInfo->SetDone(true);
1392 
1393  if (!dlInfo->m_syncMode)
1394  {
1395  if (dlInfo->m_caller)
1396  {
1397  LOG(VB_FILE, LOG_DEBUG, QString("downloadFinished(%1): "
1398  "COMPLETE: %2, sending event to caller")
1399  .arg((long long)dlInfo).arg(dlInfo->m_url));
1400 
1401  QStringList args;
1402  args << dlInfo->m_url;
1403  args << dlInfo->m_outFile;
1404  args << QString::number(dlInfo->m_bytesTotal);
1405  // placeholder for error string
1406  args << (reply ? reply->errorString() : QString());
1407  args << QString::number((int)(reply ? reply->error() :
1408  dlInfo->m_errorCode));
1409 
1410  QCoreApplication::postEvent(dlInfo->m_caller,
1411  new MythEvent("DOWNLOAD_FILE FINISHED", args));
1412  }
1413 
1414  delete dlInfo;
1415  }
1416 
1417  m_queueWaitCond.wakeAll();
1418  }
1419 }
1420 
1426 void MythDownloadManager::downloadProgress(qint64 bytesReceived,
1427  qint64 bytesTotal)
1428 {
1429  auto *reply = qobject_cast<QNetworkReply *>(sender());
1430  if (reply == nullptr)
1431  return;
1432 
1433  LOG(VB_FILE, LOG_DEBUG, LOC +
1434  QString("downloadProgress(%1, %2) (for reply %3)")
1435  .arg(bytesReceived).arg(bytesTotal).arg((long long)reply));
1436 
1437  QMutexLocker locker(m_infoLock);
1438  if (!m_downloadReplies.contains(reply))
1439  return;
1440 
1441  MythDownloadInfo *dlInfo = m_downloadReplies[reply];
1442 
1443  if (!dlInfo)
1444  return;
1445 
1446  dlInfo->m_lastStat = MythDate::current();
1447 
1448  LOG(VB_FILE, LOG_DEBUG, LOC +
1449  QString("downloadProgress: %1 to %2 is at %3 of %4 bytes downloaded")
1450  .arg(dlInfo->m_url, dlInfo->m_outFile)
1451  .arg(bytesReceived).arg(bytesTotal));
1452 
1453  if (!dlInfo->m_syncMode && dlInfo->m_caller)
1454  {
1455  LOG(VB_FILE, LOG_DEBUG, QString("downloadProgress(%1): "
1456  "sending event to caller")
1457  .arg(reply->url().toString()));
1458 
1459  bool appendToFile = (dlInfo->m_bytesReceived != 0);
1460  QByteArray data = reply->readAll();
1461  if (!dlInfo->m_outFile.isEmpty())
1462  saveFile(dlInfo->m_outFile, data, appendToFile);
1463 
1464  if (dlInfo->m_data)
1465  dlInfo->m_data->append(data);
1466 
1467  dlInfo->m_bytesReceived = bytesReceived;
1468  dlInfo->m_bytesTotal = bytesTotal;
1469 
1470  QStringList args;
1471  args << dlInfo->m_url;
1472  args << dlInfo->m_outFile;
1473  args << QString::number(bytesReceived);
1474  args << QString::number(bytesTotal);
1475 
1476  QCoreApplication::postEvent(dlInfo->m_caller,
1477  new MythEvent("DOWNLOAD_FILE UPDATE", args));
1478  }
1479 }
1480 
1488 bool MythDownloadManager::saveFile(const QString &outFile,
1489  const QByteArray &data,
1490  const bool append)
1491 {
1492  if (outFile.isEmpty() || data.isEmpty())
1493  return false;
1494 
1495  QFile file(outFile);
1496  QFileInfo fileInfo(outFile);
1497  QDir qdir(fileInfo.absolutePath());
1498 
1499  if (!qdir.exists() && !qdir.mkpath(fileInfo.absolutePath()))
1500  {
1501  LOG(VB_GENERAL, LOG_ERR, QString("Failed to create: '%1'")
1502  .arg(fileInfo.absolutePath()));
1503  return false;
1504  }
1505 
1506  QIODevice::OpenMode mode = QIODevice::Unbuffered|QIODevice::WriteOnly;
1507  if (append)
1508  mode |= QIODevice::Append;
1509 
1510  if (!file.open(mode))
1511  {
1512  LOG(VB_GENERAL, LOG_ERR, QString("Failed to open: '%1'") .arg(outFile));
1513  return false;
1514  }
1515 
1516  off_t offset = 0;
1517  size_t remaining = data.size();
1518  uint failure_cnt = 0;
1519  while ((remaining > 0) && (failure_cnt < 5))
1520  {
1521  ssize_t written = file.write(data.data() + offset, remaining);
1522  if (written < 0)
1523  {
1524  failure_cnt++;
1525  usleep(50ms);
1526  continue;
1527  }
1528 
1529  failure_cnt = 0;
1530  offset += written;
1531  remaining -= written;
1532  }
1533 
1534  return remaining <= 0;
1535 }
1536 
1541 QDateTime MythDownloadManager::GetLastModified(const QString &url)
1542 {
1543  // If the header has not expired and
1544  // the last modification date is less than 1 hours old or if
1545  // the cache object is less than 20 minutes old,
1546  // then use the cached header otherwise redownload the header
1547 
1548  static const QString kDateFormat = "ddd, dd MMM yyyy hh:mm:ss 'GMT'";
1549  LOG(VB_FILE, LOG_DEBUG, LOC + QString("GetLastModified('%1')").arg(url));
1550  QDateTime result;
1551 
1552  QDateTime now = MythDate::current();
1553 
1554  QUrl cacheUrl = QUrl(url);
1555 
1556  // Deal with redirects, we want the cached data for the final url
1557  QString redirectLoc;
1558  int limit = 0;
1559  while (!(redirectLoc = getHeader(cacheUrl, "Location")).isNull())
1560  {
1561  if (limit == CACHE_REDIRECTION_LIMIT)
1562  {
1563  LOG(VB_GENERAL, LOG_WARNING, QString("Cache Redirection limit "
1564  "reached for %1")
1565  .arg(cacheUrl.toString()));
1566  return result;
1567  }
1568  cacheUrl.setUrl(redirectLoc);
1569  limit++;
1570  }
1571 
1572  m_infoLock->lock();
1573  QNetworkCacheMetaData urlData = m_manager->cache()->metaData(cacheUrl);
1574  m_infoLock->unlock();
1575 
1576  if (urlData.isValid() &&
1577  ((!urlData.expirationDate().isValid()) ||
1578  (urlData.expirationDate().secsTo(now) < 0)))
1579  {
1580  if (urlData.lastModified().toUTC().secsTo(now) <= 3600) // 1 Hour
1581  {
1582  result = urlData.lastModified().toUTC();
1583  }
1584  else
1585  {
1586  QString date = getHeader(urlData, "Date");
1587  if (!date.isNull())
1588  {
1589  QDateTime loadDate =
1590  MythDate::fromString(date, kDateFormat);
1591  loadDate.setTimeSpec(Qt::UTC);
1592  if (loadDate.secsTo(now) <= 1200) // 20 Minutes
1593  {
1594  result = urlData.lastModified().toUTC();
1595  }
1596  }
1597  }
1598  }
1599 
1600  if (!result.isValid())
1601  {
1602  auto *dlInfo = new MythDownloadInfo;
1603  dlInfo->m_url = url;
1604  dlInfo->m_syncMode = true;
1605  // Head request, we only want to inspect the headers
1606  dlInfo->m_requestType = kRequestHead;
1607 
1608  if (downloadNow(dlInfo, false))
1609  {
1610  if (dlInfo->m_reply)
1611  {
1612  QVariant lastMod =
1613  dlInfo->m_reply->header(
1614  QNetworkRequest::LastModifiedHeader);
1615  if (lastMod.isValid())
1616  result = lastMod.toDateTime().toUTC();
1617  }
1618 
1619  // downloadNow() will set a flag to trigger downloadFinished()
1620  // to delete the dlInfo if the download times out
1621  delete dlInfo;
1622  }
1623  }
1624 
1625  LOG(VB_FILE, LOG_DEBUG, LOC + QString("GetLastModified('%1'): Result %2")
1626  .arg(url, result.toString()));
1627 
1628  return result;
1629 }
1630 
1631 
1636 {
1637  QMutexLocker locker(&m_cookieLock);
1638 
1639  auto *jar = new MythCookieJar;
1640  jar->load(filename);
1641  m_manager->setCookieJar(jar);
1642 }
1643 
1648 {
1649  QMutexLocker locker(&m_cookieLock);
1650 
1651  if (!m_manager->cookieJar())
1652  return;
1653 
1654  auto *jar = qobject_cast<MythCookieJar *>(m_manager->cookieJar());
1655  if (jar == nullptr)
1656  return;
1657  jar->save(filename);
1658 }
1659 
1660 void MythDownloadManager::setCookieJar(QNetworkCookieJar *cookieJar)
1661 {
1662  QMutexLocker locker(&m_cookieLock);
1663  m_manager->setCookieJar(cookieJar);
1664 }
1665 
1669 QNetworkCookieJar *MythDownloadManager::copyCookieJar(void)
1670 {
1671  QMutexLocker locker(&m_cookieLock);
1672 
1673  if (!m_manager->cookieJar())
1674  return nullptr;
1675 
1676  auto *inJar = qobject_cast<MythCookieJar *>(m_manager->cookieJar());
1677  if (inJar == nullptr)
1678  return nullptr;
1679  auto *outJar = new MythCookieJar;
1680  outJar->copyAllCookies(*inJar);
1681 
1682  return outJar;
1683 }
1684 
1688 void MythDownloadManager::refreshCookieJar(QNetworkCookieJar *jar)
1689 {
1690  QMutexLocker locker(&m_cookieLock);
1691  delete m_inCookieJar;
1692 
1693  auto *inJar = qobject_cast<MythCookieJar *>(jar);
1694  if (inJar == nullptr)
1695  return;
1696 
1697  auto *outJar = new MythCookieJar;
1698  outJar->copyAllCookies(*inJar);
1699  m_inCookieJar = outJar;
1700 
1701  QMutexLocker locker2(&m_queueWaitLock);
1702  m_queueWaitCond.wakeAll();
1703 }
1704 
1708 {
1709  QMutexLocker locker(&m_cookieLock);
1710 
1711  auto *inJar = qobject_cast<MythCookieJar *>(m_inCookieJar);
1712  if (inJar != nullptr)
1713  {
1714  auto *outJar = new MythCookieJar;
1715  outJar->copyAllCookies(*inJar);
1716  m_manager->setCookieJar(outJar);
1717  }
1718 
1719  delete m_inCookieJar;
1720  m_inCookieJar = nullptr;
1721 }
1722 
1723 QString MythDownloadManager::getHeader(const QUrl& url, const QString& header)
1724 {
1725  if (!m_manager || !m_manager->cache())
1726  return {};
1727 
1728  m_infoLock->lock();
1729  QNetworkCacheMetaData metadata = m_manager->cache()->metaData(url);
1730  m_infoLock->unlock();
1731 
1732  return getHeader(metadata, header);
1733 }
1734 
1740 QString MythDownloadManager::getHeader(const QNetworkCacheMetaData &cacheData,
1741  const QString& header)
1742 {
1743  auto headers = cacheData.rawHeaders();
1744  for (const auto& rh : std::as_const(headers))
1745  if (QString(rh.first) == header)
1746  return {rh.second};
1747  return {};
1748 }
1749 
1750 
1755 {
1756  const QList<QNetworkCookie> cookieList = old.allCookies();
1757  setAllCookies(cookieList);
1758 }
1759 
1763 void MythCookieJar::load(const QString &filename)
1764 {
1765  LOG(VB_GENERAL, LOG_DEBUG, QString("MythCookieJar: loading cookies from: %1").arg(filename));
1766 
1767  QFile f(filename);
1768  if (!f.open(QIODevice::ReadOnly))
1769  {
1770  LOG(VB_GENERAL, LOG_WARNING, QString("MythCookieJar::load() failed to open file for reading: %1").arg(filename));
1771  return;
1772  }
1773 
1774  QList<QNetworkCookie> cookieList;
1775  QTextStream stream(&f);
1776  while (!stream.atEnd())
1777  {
1778  QString cookie = stream.readLine();
1779  cookieList << QNetworkCookie::parseCookies(cookie.toLocal8Bit());
1780  }
1781 
1782  setAllCookies(cookieList);
1783 }
1784 
1788 void MythCookieJar::save(const QString &filename)
1789 {
1790  LOG(VB_GENERAL, LOG_DEBUG, QString("MythCookieJar: saving cookies to: %1").arg(filename));
1791 
1792  QFile f(filename);
1793  if (!f.open(QIODevice::WriteOnly))
1794  {
1795  LOG(VB_GENERAL, LOG_ERR, QString("MythCookieJar::save() failed to open file for writing: %1").arg(filename));
1796  return;
1797  }
1798 
1799  QList<QNetworkCookie> cookieList = allCookies();
1800  QTextStream stream(&f);
1801 
1802  for (const auto& cookie : std::as_const(cookieList))
1803  stream << cookie.toRawForm() << Qt::endl;
1804 }
1805 
1806 
1807 /* vim: set expandtab tabstop=4 shiftwidth=4: */
MythDownloadInfo::m_reload
bool m_reload
Definition: mythdownloadmanager.cpp:81
MythDownloadManager::m_cookieLock
QMutex m_cookieLock
Definition: mythdownloadmanager.h:178
build_compdb.args
args
Definition: build_compdb.py:11
mythevent.h
build_compdb.dest
dest
Definition: build_compdb.py:9
MythDownloadManager::processItem
bool processItem(const QString &url, QNetworkRequest *req, const QString &dest, QByteArray *data, MRequestType reqType=kRequestGet, bool reload=false, AuthCallback authCallback=nullptr, void *authArg=nullptr, const QHash< QByteArray, QByteArray > *headers=nullptr, QString *finalUrl=nullptr)
Processes a network request immediately and waits for a response.
Definition: mythdownloadmanager.cpp:353
downloadManager
MythDownloadManager * downloadManager
Definition: mythdownloadmanager.cpp:37
MythDownloadInfo::m_lastStat
QDateTime m_lastStat
Definition: mythdownloadmanager.cpp:88
MythDownloadInfo::m_data
QByteArray * m_data
Definition: mythdownloadmanager.cpp:77
MythCookieJar::save
void save(const QString &filename)
Saves the cookie jar to a cookie file.
Definition: mythdownloadmanager.cpp:1788
MythDownloadInfo::m_privData
QByteArray m_privData
Definition: mythdownloadmanager.cpp:78
MThread::wait
bool wait(std::chrono::milliseconds time=std::chrono::milliseconds::max())
Wait for the MThread to exit, with a maximum timeout.
Definition: mthread.cpp:300
MythDownloadManager::m_downloadInfos
QMap< QString, MythDownloadInfo * > m_downloadInfos
Definition: mythdownloadmanager.h:167
dmCreateLock
QMutex dmCreateLock
Definition: mythdownloadmanager.cpp:38
MythDownloadManager::removeListener
void removeListener(QObject *caller)
Disconnects the specified caller from any existing MythDownloadInfo instances.
Definition: mythdownloadmanager.cpp:1106
ShutdownMythDownloadManager
void ShutdownMythDownloadManager(void)
Deletes the running MythDownloadManager at program exit.
Definition: mythdownloadmanager.cpp:133
RemoteFileDownloadThread
Definition: mythdownloadmanager.cpp:101
RemoteFileDownloadThread::run
void run() override
Definition: mythdownloadmanager.cpp:109
MythEvent
This class is used as a container for messages.
Definition: mythevent.h:16
mythcoreutil.h
MThread::usleep
static void usleep(std::chrono::microseconds time)
Definition: mthread.cpp:335
MythDownloadInfo::m_lock
QMutex m_lock
Definition: mythdownloadmanager.cpp:94
MythDownloadManager::queueDownload
void queueDownload(const QString &url, const QString &dest, QObject *caller, bool reload=false)
Adds a url to the download queue.
Definition: mythdownloadmanager.cpp:393
MythDownloadManager::m_diskCache
QNetworkDiskCache * m_diskCache
Definition: mythdownloadmanager.h:160
MythDownloadManager::m_proxy
QNetworkProxy * m_proxy
Definition: mythdownloadmanager.h:161
RemoteFile
Definition: remotefile.h:17
MythDownloadManager::postAuth
bool postAuth(const QString &url, QByteArray *data, AuthCallback authCallback, void *authArg, const QHash< QByteArray, QByteArray > *headers=nullptr)
Posts data to a url via the QNetworkAccessManager.
Definition: mythdownloadmanager.cpp:622
LOG
#define LOG(_MASK_, _LEVEL_, _QSTRING_)
Definition: mythlogging.h:39
MThread::RunProlog
void RunProlog(void)
Sets up a thread, call this if you reimplement run().
Definition: mthread.cpp:196
MythDownloadManager::downloadNow
bool downloadNow(MythDownloadInfo *dlInfo, bool deleteInfo=true)
Download helper for download() blocking methods.
Definition: mythdownloadmanager.cpp:805
MythDownloadManager
Definition: mythdownloadmanager.h:48
MythDownloadInfo::SetDone
void SetDone(bool done)
Definition: mythdownloadmanager.cpp:65
kRequestHead
@ kRequestHead
Definition: mythdownloadmanager.h:42
MythDownloadManager::saveFile
static bool saveFile(const QString &outFile, const QByteArray &data, bool append=false)
Saves a QByteArray of data to a given filename.
Definition: mythdownloadmanager.cpp:1488
MythDownloadInfo::IsDone
bool IsDone(void)
Definition: mythdownloadmanager.cpp:59
build_compdb.file
file
Definition: build_compdb.py:55
MythDownloadInfo::m_request
QNetworkRequest * m_request
Definition: mythdownloadmanager.cpp:74
mythdirs.h
RemoteFileDownloadThread::RemoteFileDownloadThread
RemoteFileDownloadThread(MythDownloadManager *parent, MythDownloadInfo *dlInfo)
Definition: mythdownloadmanager.cpp:104
MythDownloadInfo::m_bytesTotal
qint64 m_bytesTotal
Definition: mythdownloadmanager.cpp:87
MythDownloadInfo::m_processReply
bool m_processReply
Definition: mythdownloadmanager.cpp:84
MythDownloadManager::authCallback
void authCallback(QNetworkReply *reply, QAuthenticator *authenticator)
Signal handler for authentication requests.
Definition: mythdownloadmanager.cpp:781
MythDate::current
QDateTime current(bool stripped)
Returns current Date and Time in UTC.
Definition: mythdate.cpp:14
MythDownloadInfo
Definition: mythdownloadmanager.cpp:43
MythDownloadInfo::m_bytesReceived
qint64 m_bytesReceived
Definition: mythdownloadmanager.cpp:86
kRequestPost
@ kRequestPost
Definition: mythdownloadmanager.h:43
mythdate.h
mythlogging.h
GetConfDir
QString GetConfDir(void)
Definition: mythdirs.cpp:256
MythDownloadInfo::m_authArg
void * m_authArg
Definition: mythdownloadmanager.cpp:90
MythDownloadManager::queuePost
void queuePost(const QString &url, QByteArray *data, QObject *caller)
Queues a post to a URL via the QNetworkAccessManager.
Definition: mythdownloadmanager.cpp:529
remotefile.h
MythDownloadManager::run
void run(void) override
Runs a loop to process incoming download requests and triggers download events to be processed.
Definition: mythdownloadmanager.cpp:191
LOC
#define LOC
Definition: mythdownloadmanager.cpp:34
MythDownloadManager::m_isRunning
bool m_isRunning
Definition: mythdownloadmanager.h:175
compat.h
MythDownloadInfo::m_finalUrl
QString * m_finalUrl
Definition: mythdownloadmanager.cpp:73
MythDownloadManager::downloadError
void downloadError(QNetworkReply::NetworkError errorCode)
Slot to process download error events.
Definition: mythdownloadmanager.cpp:1138
MythDownloadInfo::m_errorCode
QNetworkReply::NetworkError m_errorCode
Definition: mythdownloadmanager.cpp:93
MythDownloadManager::m_runThread
bool m_runThread
Definition: mythdownloadmanager.h:174
MThread::RunEpilog
void RunEpilog(void)
Cleans up a thread's resources, call this if you reimplement run().
Definition: mthread.cpp:209
MythDownloadManager::download
bool download(const QString &url, const QString &dest, bool reload=false)
Downloads a URL to a file in blocking mode.
Definition: mythdownloadmanager.cpp:430
MythDownloadManager::getHeader
QString getHeader(const QUrl &url, const QString &header)
Definition: mythdownloadmanager.cpp:1723
MythCookieJar::copyAllCookies
void copyAllCookies(MythCookieJar &old)
Copies all cookies from one MythCookieJar to another.
Definition: mythdownloadmanager.cpp:1754
MythDownloadManager::setCookieJar
void setCookieJar(QNetworkCookieJar *cookieJar)
Definition: mythdownloadmanager.cpp:1660
MythDownloadManager::downloadProgress
void downloadProgress(qint64 bytesReceived, qint64 bytesTotal)
Slot to process download update events.
Definition: mythdownloadmanager.cpp:1426
portchecker.h
MythDownloadManager::cancelDownload
void cancelDownload(const QString &url, bool block=true)
Cancel a queued or current download.
Definition: mythdownloadmanager.cpp:1013
PortChecker::resolveLinkLocal
static bool resolveLinkLocal(QString &host, int port, std::chrono::milliseconds timeLimit=30s)
Convenience method to resolve link-local address.
Definition: portchecker.cpp:230
MythDownloadInfo::m_syncMode
bool m_syncMode
Definition: mythdownloadmanager.cpp:83
uint
unsigned int uint
Definition: compat.h:81
MythDownloadManager::downloadRemoteFile
void downloadRemoteFile(MythDownloadInfo *dlInfo)
Triggers a myth:// URI download in the background via RemoteFile.
Definition: mythdownloadmanager.cpp:642
gCoreContext
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
Definition: mythcorecontext.cpp:55
MythDownloadManager::redirectUrl
static QUrl redirectUrl(const QUrl &possibleRedirectUrl, const QUrl &oldRedirectUrl)
Checks whether we were redirected to the given URL.
Definition: mythdownloadmanager.cpp:1167
MythDownloadManager::downloadFinished
void downloadFinished(QNetworkReply *reply)
Slot to process download finished events.
Definition: mythdownloadmanager.cpp:1182
MythDownloadInfo::~MythDownloadInfo
~MythDownloadInfo()
Definition: mythdownloadmanager.cpp:52
MythDate::fromString
QDateTime fromString(const QString &dtstr)
Converts kFilename && kISODate formats to QDateTime.
Definition: mythdate.cpp:34
MythDownloadManager::~MythDownloadManager
~MythDownloadManager() override
Destructor for MythDownloadManager.
Definition: mythdownloadmanager.cpp:177
off_t
#define off_t
Definition: mythiowrapper.cpp:241
mthreadpool.h
MythDownloadInfo::m_reply
QNetworkReply * m_reply
Definition: mythdownloadmanager.cpp:75
MythDownloadInfo::m_preferCache
bool m_preferCache
Definition: mythdownloadmanager.cpp:82
MythDownloadManager::preCache
void preCache(const QString &url)
Downloads a URL but doesn't store the resulting data anywhere.
Definition: mythdownloadmanager.cpp:381
MythDownloadManager::m_manager
QNetworkAccessManager * m_manager
Definition: mythdownloadmanager.h:159
MythDownloadManager::downloadAuth
bool downloadAuth(const QString &url, const QString &dest, bool reload=false, AuthCallback authCallback=nullptr, void *authArg=nullptr, const QHash< QByteArray, QByteArray > *headers=nullptr)
Downloads a URL to a file in blocking mode.
Definition: mythdownloadmanager.cpp:515
MythDownloadManager::loadCookieJar
void loadCookieJar(const QString &filename)
Loads the cookie jar from a cookie file.
Definition: mythdownloadmanager.cpp:1635
MythDownloadManager::m_inCookieJar
QNetworkCookieJar * m_inCookieJar
Definition: mythdownloadmanager.h:177
MythDate::secsInPast
std::chrono::seconds secsInPast(const QDateTime &past)
Definition: mythdate.cpp:203
mythcorecontext.h
MythDownloadManager::m_infoLock
QRecursiveMutex * m_infoLock
Definition: mythdownloadmanager.h:166
MythDownloadManager::copyCookieJar
QNetworkCookieJar * copyCookieJar(void)
Copy from one cookie jar to another.
Definition: mythdownloadmanager.cpp:1669
MythDownloadInfo::m_headers
const QHash< QByteArray, QByteArray > * m_headers
Definition: mythdownloadmanager.cpp:91
MythDownloadInfo::m_authCallback
AuthCallback m_authCallback
Definition: mythdownloadmanager.cpp:89
MythDate
Definition: mythdate.cpp:11
MythDownloadManager::m_queueThread
QThread * m_queueThread
Definition: mythdownloadmanager.h:172
MythDownloadManager::m_queueWaitCond
QWaitCondition m_queueWaitCond
Definition: mythdownloadmanager.h:163
MythDownloadInfo::m_done
bool m_done
Definition: mythdownloadmanager.cpp:85
MythDownloadManager::RemoteFileDownloadThread
friend class RemoteFileDownloadThread
Definition: mythdownloadmanager.h:180
MythDownloadManager::refreshCookieJar
void refreshCookieJar(QNetworkCookieJar *jar)
Refresh the temporary cookie jar from another cookie jar.
Definition: mythdownloadmanager.cpp:1688
AuthCallback
void(*)(QNetworkReply *, QAuthenticator *, void *) AuthCallback
Definition: mythdownloadmanager.h:46
MythCookieJar
A subclassed QNetworkCookieJar that allows for reading and writing cookie files that contain raw form...
Definition: mythdownloadmanager.h:29
MythDownloadManager::downloadCanceled
void downloadCanceled(void)
Definition: mythdownloadmanager.cpp:1073
MythDownloadInfo::m_requestType
MRequestType m_requestType
Definition: mythdownloadmanager.cpp:80
MythCoreContext::GetHostName
QString GetHostName(void)
Definition: mythcorecontext.cpp:842
uint16_t
unsigned short uint16_t
Definition: iso6937tables.h:3
MythDownloadManager::saveCookieJar
void saveCookieJar(const QString &filename)
Saves the cookie jar to a cookie file.
Definition: mythdownloadmanager.cpp:1647
MythDownloadManager::GetLastModified
QDateTime GetLastModified(const QString &url)
Gets the Last Modified timestamp for a URI.
Definition: mythdownloadmanager.cpp:1541
MythDownloadInfo::MythDownloadInfo
MythDownloadInfo()
Definition: mythdownloadmanager.cpp:46
MythDownloadManager::updateCookieJar
void updateCookieJar(void)
Update the cookie jar from the temporary cookie jar.
Definition: mythdownloadmanager.cpp:1707
MythDownloadManager::downloadQNetworkRequest
void downloadQNetworkRequest(MythDownloadInfo *dlInfo)
Downloads a QNetworkRequest via the QNetworkAccessManager.
Definition: mythdownloadmanager.cpp:651
MythCookieJar::load
void load(const QString &filename)
Loads the cookie jar from a cookie file.
Definition: mythdownloadmanager.cpp:1763
mythdownloadmanager.h
MythDownloadInfo::m_redirectedTo
QUrl m_redirectedTo
Definition: mythdownloadmanager.cpp:72
MythDownloadManager::post
bool post(const QString &url, QByteArray *data)
Posts data to a url via the QNetworkAccessManager.
Definition: mythdownloadmanager.cpp:576
build_compdb.filename
filename
Definition: build_compdb.py:21
MThreadPool::globalInstance
static MThreadPool * globalInstance(void)
Definition: mthreadpool.cpp:307
culrcscrapers.music163.lyricsScraper.headers
dictionary headers
Definition: lyricsScraper.py:19
MythDownloadInfo::m_outFile
QString m_outFile
Definition: mythdownloadmanager.cpp:76
MRequestType
MRequestType
Definition: mythdownloadmanager.h:40
MythDownloadInfo::m_caller
QObject * m_caller
Definition: mythdownloadmanager.cpp:79
MythDownloadManager::queueItem
void queueItem(const QString &url, QNetworkRequest *req, const QString &dest, QByteArray *data, QObject *caller, MRequestType reqType=kRequestGet, bool reload=false)
Adds a request to the download queue.
Definition: mythdownloadmanager.cpp:321
CACHE_REDIRECTION_LIMIT
static constexpr int CACHE_REDIRECTION_LIMIT
Definition: mythdownloadmanager.cpp:35
RemoteFileDownloadThread::m_dlInfo
MythDownloadInfo * m_dlInfo
Definition: mythdownloadmanager.cpp:128
kRequestGet
@ kRequestGet
Definition: mythdownloadmanager.h:41
MythDownloadManager::m_cancellationQueue
QList< MythDownloadInfo * > m_cancellationQueue
Definition: mythdownloadmanager.h:170
MythDownloadManager::m_queueWaitLock
QMutex m_queueWaitLock
Definition: mythdownloadmanager.h:164
RemoteFileDownloadThread::m_parent
MythDownloadManager * m_parent
Definition: mythdownloadmanager.cpp:127
MThreadPool::start
void start(QRunnable *runnable, const QString &debugName, int priority=0)
Definition: mthreadpool.cpp:342
MythDownloadInfo::m_url
QString m_url
Definition: mythdownloadmanager.cpp:71
MythDownloadManager::m_downloadQueue
QList< MythDownloadInfo * > m_downloadQueue
Definition: mythdownloadmanager.h:169
MythDownloadManager::m_downloadReplies
QMap< QNetworkReply *, MythDownloadInfo * > m_downloadReplies
Definition: mythdownloadmanager.h:168
GetMythDownloadManager
MythDownloadManager * GetMythDownloadManager(void)
Gets the pointer to the MythDownloadManager singleton.
Definition: mythdownloadmanager.cpp:145