MythTV  master
mythdownloadmanager.cpp
Go to the documentation of this file.
1 // qt
2 #include <QCoreApplication>
3 #include <QRunnable>
4 #include <QString>
5 #include <QByteArray>
6 #include <QFile>
7 #include <QDir>
8 #include <QNetworkCookieJar>
9 #include <QNetworkCookie>
10 #include <QAuthenticator>
11 #include <QTextStream>
12 #include <QNetworkProxy>
13 #include <QMutexLocker>
14 #include <QUrl>
15 #include <QTcpSocket>
16 
17 #include <cstdlib>
18 #include <unistd.h> // for usleep()
19 
20 // libmythbase
21 #include "compat.h"
22 #include "mythcorecontext.h"
23 #include "mythcoreutil.h"
24 #include "mthreadpool.h"
25 #include "mythdirs.h"
26 #include "mythevent.h"
27 #include "mythversion.h"
28 #include "remotefile.h"
29 #include "mythdate.h"
30 
31 #include "mythdownloadmanager.h"
32 #include "mythlogging.h"
33 #include "portchecker.h"
34 
35 using namespace std;
36 
37 #define LOC QString("DownloadManager: ")
38 #define CACHE_REDIRECTION_LIMIT 10
39 
41 QMutex dmCreateLock;
42 
47 {
48  public:
50  m_lastStat(MythDate::current())
51  {
52  qRegisterMetaType<QNetworkReply::NetworkError>("QNetworkReply::NetworkError");
53  }
54 
56  {
57  delete m_request;
58  if (m_reply && m_processReply)
59  m_reply->deleteLater();
60  }
61 
62  bool IsDone(void)
63  {
64  QMutexLocker lock(&m_lock);
65  return m_done;
66  }
67 
68  void SetDone(bool done)
69  {
70  QMutexLocker lock(&m_lock);
71  m_done = done;
72  }
73 
74  QString m_url;
76  QString *m_finalUrl {nullptr};
77  QNetworkRequest *m_request {nullptr};
78  QNetworkReply *m_reply {nullptr};
79  QString m_outFile;
80  QByteArray *m_data {nullptr};
81  QByteArray m_privData;
82  QObject *m_caller {nullptr};
83  MRequestType m_requestType {kRequestGet};
84  bool m_reload {false};
85  bool m_preferCache {false};
86  bool m_syncMode {false};
87  bool m_processReply {true};
88  bool m_done {false};
89  qint64 m_bytesReceived {0};
90  qint64 m_bytesTotal {0};
91  QDateTime m_lastStat;
92  AuthCallback m_authCallback {nullptr};
93  void *m_authArg {nullptr};
94  const QHash<QByteArray, QByteArray> *m_headers {nullptr};
95 
96  QNetworkReply::NetworkError m_errorCode {QNetworkReply::NoError};
97  QMutex m_lock;
98 };
99 
100 
105 class MythCookieJar : public QNetworkCookieJar
106 {
107  public:
108  MythCookieJar() = default;
109  void copyAllCookies(MythCookieJar &old);
110  void load(const QString &filename);
111  void save(const QString &filename);
112 };
113 
117 class RemoteFileDownloadThread : public QRunnable
118 {
119  public:
121  MythDownloadInfo *dlInfo) :
122  m_parent(parent),
123  m_dlInfo(dlInfo) {}
124 
125  void run() override // QRunnable
126  {
127  bool ok = false;
128 
129  RemoteFile *rf = new RemoteFile(m_dlInfo->m_url, false, false, 0);
130  ok = rf->SaveAs(m_dlInfo->m_privData);
131  delete rf;
132 
133  if (!ok)
134  m_dlInfo->m_errorCode = QNetworkReply::UnknownNetworkError;
135 
136  m_dlInfo->m_bytesReceived = m_dlInfo->m_privData.size();
137  m_dlInfo->m_bytesTotal = m_dlInfo->m_bytesReceived;
138 
139  m_parent->downloadFinished(m_dlInfo);
140  }
141 
142  private:
143  MythDownloadManager *m_parent {nullptr};
144  MythDownloadInfo *m_dlInfo {nullptr};
145 };
146 
150 {
151  if (downloadManager)
152  {
153  delete downloadManager;
154  downloadManager = nullptr;
155  }
156 }
157 
162 {
163  if (downloadManager)
164  return downloadManager;
165 
166  QMutexLocker locker(&dmCreateLock);
167 
168  // Check once more in case the download manager was created
169  // while we were securing the lock.
170  if (downloadManager)
171  return downloadManager;
172 
174  tmpDLM->start();
175  while (!tmpDLM->getQueueThread())
176  usleep(10000);
177 
178  tmpDLM->moveToThread(tmpDLM->getQueueThread());
179  tmpDLM->setRunThread();
180 
181  while (!tmpDLM->isRunning())
182  usleep(10000);
183 
184  downloadManager = tmpDLM;
185 
187 
188  return downloadManager;
189 }
190 
194 {
195  m_runThread = false;
196  m_queueWaitCond.wakeAll();
197 
198  wait();
199 
200  delete m_infoLock;
201  delete m_inCookieJar;
202 }
203 
208 {
209  RunProlog();
210 
211  bool downloading = false;
212  bool itemsInQueue = false;
213  bool itemsInCancellationQueue = false;
214  bool waitAnyway = false;
215 
216  m_queueThread = QThread::currentThread();
217 
218  while (!m_runThread)
219  usleep(50000);
220 
221  m_manager = new QNetworkAccessManager(this);
222  m_diskCache = new QNetworkDiskCache(this);
223  m_proxy = new QNetworkProxy();
224  m_diskCache->setCacheDirectory(GetConfDir() + "/cache/" +
225  QCoreApplication::applicationName() + "-" +
227  m_manager->setCache(m_diskCache);
228 
229  // Set the proxy for the manager to be the application default proxy,
230  // which has already been setup
231  m_manager->setProxy(*m_proxy);
232 
233  // make sure the cookieJar is created in the same thread as the manager
234  // and set its parent to nullptr so it can be shared between managers
235  m_manager->cookieJar()->setParent(nullptr);
236 
237  QObject::connect(m_manager, SIGNAL(finished(QNetworkReply*)), this,
238  SLOT(downloadFinished(QNetworkReply*)));
239 
240  m_isRunning = true;
241  while (m_runThread)
242  {
243  if (m_inCookieJar)
244  {
245  LOG(VB_GENERAL, LOG_DEBUG, "Updating DLManager's Cookie Jar");
246  updateCookieJar();
247  }
248  m_infoLock->lock();
249  LOG(VB_FILE, LOG_DEBUG, LOC + QString("items downloading %1").arg(m_downloadInfos.count()));
250  LOG(VB_FILE, LOG_DEBUG, LOC + QString("items queued %1").arg(m_downloadQueue.count()));
251  downloading = !m_downloadInfos.isEmpty();
252  itemsInCancellationQueue = !m_cancellationQueue.isEmpty();
253  m_infoLock->unlock();
254 
255  if (itemsInCancellationQueue)
256  {
257  downloadCanceled();
258  }
259  if (downloading)
260  QCoreApplication::processEvents();
261 
262  m_infoLock->lock();
263  itemsInQueue = !m_downloadQueue.isEmpty();
264  m_infoLock->unlock();
265 
266  if (!itemsInQueue || waitAnyway)
267  {
268  waitAnyway = false;
269  m_queueWaitLock.lock();
270 
271  if (downloading)
272  {
273  LOG(VB_FILE, LOG_DEBUG, LOC + QString("waiting 200ms"));
274  m_queueWaitCond.wait(&m_queueWaitLock, 200);
275  }
276  else
277  {
278  LOG(VB_FILE, LOG_DEBUG, LOC + QString("waiting for more items to download"));
279  m_queueWaitCond.wait(&m_queueWaitLock);
280  }
281 
282  m_queueWaitLock.unlock();
283  }
284 
285  m_infoLock->lock();
286  if (!m_downloadQueue.isEmpty())
287  {
288  MythDownloadInfo *dlInfo = m_downloadQueue.front();
289 
290  m_downloadQueue.pop_front();
291 
292  if (!dlInfo)
293  {
294  m_infoLock->unlock();
295  continue;
296  }
297 
298  if (m_downloadInfos.contains(dlInfo->m_url))
299  {
300  // Push request to the end of the queue to let others process.
301  // If this is the only item in the queue, force the loop to
302  // wait a little.
303  if (m_downloadQueue.isEmpty())
304  waitAnyway = true;
305  m_downloadQueue.push_back(dlInfo);
306  m_infoLock->unlock();
307  continue;
308  }
309 
310  if (dlInfo->m_url.startsWith("myth://"))
311  downloadRemoteFile(dlInfo);
312  else
313  {
314  QMutexLocker cLock(&m_cookieLock);
315  downloadQNetworkRequest(dlInfo);
316  }
317 
318  m_downloadInfos[dlInfo->m_url] = dlInfo;
319  }
320  m_infoLock->unlock();
321  }
322  m_isRunning = false;
323 
324  RunEpilog();
325 }
326 
337 void MythDownloadManager::queueItem(const QString &url, QNetworkRequest *req,
338  const QString &dest, QByteArray *data,
339  QObject *caller, const MRequestType reqType,
340  const bool reload)
341 {
342  MythDownloadInfo *dlInfo = new MythDownloadInfo;
343 
344  dlInfo->m_url = url;
345  dlInfo->m_request = req;
346  dlInfo->m_outFile = dest;
347  dlInfo->m_data = data;
348  dlInfo->m_caller = caller;
349  dlInfo->m_requestType = reqType;
350  dlInfo->m_reload = reload;
351 
352  QMutexLocker locker(m_infoLock);
353  m_downloadQueue.push_back(dlInfo);
354  m_queueWaitCond.wakeAll();
355 }
356 
369 bool MythDownloadManager::processItem(const QString &url, QNetworkRequest *req,
370  const QString &dest, QByteArray *data,
371  const MRequestType reqType,
372  const bool reload,
373  AuthCallback authCallbackFn, void *authArg,
374  const QHash<QByteArray, QByteArray> *headers,
375  QString *finalUrl)
376 {
377  MythDownloadInfo *dlInfo = new MythDownloadInfo;
378 
379  dlInfo->m_url = url;
380  dlInfo->m_request = req;
381  dlInfo->m_outFile = dest;
382  dlInfo->m_data = data;
383  dlInfo->m_requestType = reqType;
384  dlInfo->m_reload = reload;
385  dlInfo->m_syncMode = true;
386  dlInfo->m_authCallback = authCallbackFn;
387  dlInfo->m_authArg = authArg;
388  dlInfo->m_headers = headers;
389  dlInfo->m_finalUrl = finalUrl;
390 
391  return downloadNow(dlInfo, true);
392 }
393 
397 void MythDownloadManager::preCache(const QString &url)
398 {
399  LOG(VB_FILE, LOG_DEBUG, LOC + QString("preCache('%1')").arg(url));
400  queueItem(url, nullptr, QString(), nullptr, nullptr);
401 }
402 
409 void MythDownloadManager::queueDownload(const QString &url,
410  const QString &dest,
411  QObject *caller,
412  const bool reload)
413 {
414  LOG(VB_FILE, LOG_DEBUG, LOC + QString("queueDownload('%1', '%2', %3)")
415  .arg(url).arg(dest).arg((long long)caller));
416 
417  queueItem(url, nullptr, dest, nullptr, caller, kRequestGet, reload);
418 }
419 
425 void MythDownloadManager::queueDownload(QNetworkRequest *req,
426  QByteArray *data,
427  QObject *caller)
428 {
429  LOG(VB_FILE, LOG_DEBUG, LOC + QString("queueDownload('%1', '%2', %3)")
430  .arg(req->url().toString()).arg((long long)data)
431  .arg((long long)caller));
432 
433  queueItem(req->url().toString(), req, QString(), data, caller,
434  kRequestGet,
435  (QNetworkRequest::AlwaysNetwork == req->attribute(
436  QNetworkRequest::CacheLoadControlAttribute,
437  QNetworkRequest::PreferNetwork).toInt()));
438 }
439 
446 bool MythDownloadManager::download(const QString &url, const QString &dest,
447  const bool reload)
448 {
449  return processItem(url, nullptr, dest, nullptr, kRequestGet, reload);
450 }
451 
458 bool MythDownloadManager::download(const QString &url, QByteArray *data,
459  const bool reload, QString *finalUrl)
460 {
461  QString redirected;
462  if (!processItem(url, nullptr, QString(), data, kRequestGet, reload, NULL, NULL, NULL, &redirected))
463  return false;
464  if (!redirected.isEmpty() && finalUrl != nullptr)
465  *finalUrl = redirected;
466  return true;
467 }
468 
475 QNetworkReply *MythDownloadManager::download(const QString &url,
476  const bool reload)
477 {
478  MythDownloadInfo *dlInfo = new MythDownloadInfo;
479  QNetworkReply *reply = nullptr;
480 
481  dlInfo->m_url = url;
482  dlInfo->m_reload = reload;
483  dlInfo->m_syncMode = true;
484  dlInfo->m_processReply = false;
485 
486  if (downloadNow(dlInfo, false))
487  {
488  if (dlInfo->m_reply)
489  {
490  reply = dlInfo->m_reply;
491  // prevent dlInfo dtor from deleting the reply
492  dlInfo->m_reply = nullptr;
493 
494  delete dlInfo;
495 
496  return reply;
497  }
498 
499  delete dlInfo;
500  }
501 
502  return nullptr;
503 }
504 
510 bool MythDownloadManager::download(QNetworkRequest *req, QByteArray *data)
511 {
512  LOG(VB_FILE, LOG_DEBUG, LOC + QString("download('%1', '%2')")
513  .arg(req->url().toString()).arg((long long)data));
514  return processItem(req->url().toString(), req, QString(), data,
515  kRequestGet,
516  (QNetworkRequest::AlwaysNetwork == req->attribute(
517  QNetworkRequest::CacheLoadControlAttribute,
518  QNetworkRequest::PreferNetwork).toInt()));
519 }
520 
530 bool MythDownloadManager::downloadAuth(const QString &url, const QString &dest,
531  const bool reload, AuthCallback authCallbackFn, void *authArg,
532  const QHash<QByteArray, QByteArray> *headers)
533 {
534  return processItem(url, nullptr, dest, nullptr, kRequestGet, reload, authCallbackFn,
535  authArg, headers);
536 }
537 
538 
544 void MythDownloadManager::queuePost(const QString &url,
545  QByteArray *data,
546  QObject *caller)
547 {
548  LOG(VB_FILE, LOG_DEBUG, LOC + QString("queuePost('%1', '%2')")
549  .arg(url).arg((long long)data));
550 
551  if (!data)
552  {
553  LOG(VB_GENERAL, LOG_ERR, LOC + "queuePost(), data is NULL!");
554  return;
555  }
556 
557  queueItem(url, nullptr, QString(), data, caller, kRequestPost);
558 }
559 
565 void MythDownloadManager::queuePost(QNetworkRequest *req,
566  QByteArray *data,
567  QObject *caller)
568 {
569  LOG(VB_FILE, LOG_DEBUG, LOC + QString("queuePost('%1', '%2')")
570  .arg(req->url().toString()).arg((long long)data));
571 
572  if (!data)
573  {
574  LOG(VB_GENERAL, LOG_ERR, LOC + "queuePost(), data is NULL!");
575  return;
576  }
577 
578  queueItem(req->url().toString(), req, QString(), data, caller,
579  kRequestPost,
580  (QNetworkRequest::AlwaysNetwork == req->attribute(
581  QNetworkRequest::CacheLoadControlAttribute,
582  QNetworkRequest::PreferNetwork).toInt()));
583 
584 }
585 
591 bool MythDownloadManager::post(const QString &url, QByteArray *data)
592 {
593  LOG(VB_FILE, LOG_DEBUG, LOC + QString("post('%1', '%2')")
594  .arg(url).arg((long long)data));
595 
596  if (!data)
597  {
598  LOG(VB_GENERAL, LOG_ERR, LOC + "post(), data is NULL!");
599  return false;
600  }
601 
602  return processItem(url, nullptr, QString(), data, kRequestPost);
603 }
604 
610 bool MythDownloadManager::post(QNetworkRequest *req, QByteArray *data)
611 {
612  LOG(VB_FILE, LOG_DEBUG, LOC + QString("post('%1', '%2')")
613  .arg(req->url().toString()).arg((long long)data));
614 
615  if (!data)
616  {
617  LOG(VB_GENERAL, LOG_ERR, LOC + "post(), data is NULL!");
618  return false;
619  }
620 
621  return processItem(req->url().toString(), req, QString(), data,
622  kRequestPost,
623  (QNetworkRequest::AlwaysNetwork == req->attribute(
624  QNetworkRequest::CacheLoadControlAttribute,
625  QNetworkRequest::PreferNetwork).toInt()));
626 
627 }
628 
637 bool MythDownloadManager::postAuth(const QString &url, QByteArray *data,
638  AuthCallback authCallbackFn, void *authArg,
639  const QHash<QByteArray, QByteArray> *headers)
640 {
641  LOG(VB_FILE, LOG_DEBUG, LOC + QString("postAuth('%1', '%2')")
642  .arg(url).arg((long long)data));
643 
644  if (!data)
645  {
646  LOG(VB_GENERAL, LOG_ERR, LOC + "postAuth(), data is NULL!");
647  return false;
648  }
649 
650  return processItem(url, nullptr, nullptr, data, kRequestPost, false, authCallbackFn,
651  authArg, headers);
652 }
653 
658 {
659  RemoteFileDownloadThread *dlThread =
660  new RemoteFileDownloadThread(this, dlInfo);
661  MThreadPool::globalInstance()->start(dlThread, "RemoteFileDownload");
662 }
663 
668 {
669  if (!dlInfo)
670  return;
671 
672  static const char dateFormat[] = "ddd, dd MMM yyyy hh:mm:ss 'GMT'";
673  QUrl qurl(dlInfo->m_url);
674  QNetworkRequest request;
675 
676  if (dlInfo->m_request)
677  {
678  request = *dlInfo->m_request;
679  delete dlInfo->m_request;
680  dlInfo->m_request = nullptr;
681  }
682  else
683  request.setUrl(qurl);
684 
685  if (dlInfo->m_reload)
686  {
687  request.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
688  QNetworkRequest::AlwaysNetwork);
689  }
690  else
691  {
692  // Prefer the in-cache item if one exists and it is less than 5 minutes
693  // old and it will not expire in the next 10 seconds
694  QDateTime now = MythDate::current();
695 
696  // Handle redirects, we want the metadata of the file headers
697  QString redirectLoc;
698  int limit = 0;
699  while (!(redirectLoc = getHeader(qurl, "Location")).isNull())
700  {
701  if (limit == CACHE_REDIRECTION_LIMIT)
702  {
703  LOG(VB_GENERAL, LOG_WARNING, QString("Cache Redirection limit "
704  "reached for %1")
705  .arg(qurl.toString()));
706  return;
707  }
708  qurl.setUrl(redirectLoc);
709  limit++;
710  }
711 
712  LOG(VB_NETWORK, LOG_DEBUG, QString("Checking cache for %1")
713  .arg(qurl.toString()));
714 
715  m_infoLock->lock();
716  QNetworkCacheMetaData urlData = m_manager->cache()->metaData(qurl);
717  m_infoLock->unlock();
718  if ((urlData.isValid()) &&
719  ((!urlData.expirationDate().isValid()) ||
720  (urlData.expirationDate().toUTC().secsTo(now) < 10)))
721  {
722  QString dateString = getHeader(urlData, "Date");
723 
724  if (!dateString.isNull())
725  {
726  QDateTime loadDate =
727  MythDate::fromString(dateString, dateFormat);
728  loadDate.setTimeSpec(Qt::UTC);
729  if (loadDate.secsTo(now) <= 720)
730  {
731  dlInfo->m_preferCache = true;
732  LOG(VB_NETWORK, LOG_DEBUG, QString("Preferring cache for %1")
733  .arg(qurl.toString()));
734  }
735  }
736  }
737  }
738 
739  if (dlInfo->m_preferCache)
740  request.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
741  QNetworkRequest::PreferCache);
742 
743  if (!request.hasRawHeader("User-Agent"))
744  {
745  request.setRawHeader("User-Agent",
746  "MythTV v" MYTH_BINARY_VERSION
747  " MythDownloadManager");
748  }
749 
750  if (dlInfo->m_headers)
751  {
752  QHash<QByteArray, QByteArray>::const_iterator it =
753  dlInfo->m_headers->constBegin();
754  for ( ; it != dlInfo->m_headers->constEnd(); ++it )
755  {
756  if (!it.key().isEmpty() && !it.value().isEmpty())
757  {
758  request.setRawHeader(it.key(), it.value());
759  }
760  }
761  }
762 
763  switch (dlInfo->m_requestType)
764  {
765  case kRequestPost :
766  dlInfo->m_reply = m_manager->post(request, *dlInfo->m_data);
767  break;
768  case kRequestHead :
769  dlInfo->m_reply = m_manager->head(request);
770  break;
771  case kRequestGet :
772  default:
773  dlInfo->m_reply = m_manager->get(request);
774  break;
775  }
776 
777  m_downloadReplies[dlInfo->m_reply] = dlInfo;
778 
779  if (dlInfo->m_authCallback)
780  {
781  connect(m_manager, SIGNAL(authenticationRequired(QNetworkReply *,
782  QAuthenticator *)),
783  this, SLOT(authCallback(QNetworkReply *, QAuthenticator *)));
784  }
785 
786  connect(dlInfo->m_reply, SIGNAL(error(QNetworkReply::NetworkError)), this,
787  SLOT(downloadError(QNetworkReply::NetworkError)));
788  connect(dlInfo->m_reply, SIGNAL(downloadProgress(qint64, qint64)),
789  this, SLOT(downloadProgress(qint64, qint64)));
790 }
791 
796 void MythDownloadManager::authCallback(QNetworkReply *reply,
797  QAuthenticator *authenticator)
798 {
799  if (!reply)
800  return;
801 
802  MythDownloadInfo *dlInfo = m_downloadReplies[reply];
803 
804  if (!dlInfo)
805  return;
806 
807  if (dlInfo->m_authCallback)
808  {
809  LOG(VB_FILE, LOG_DEBUG, "Calling auth callback");
810  dlInfo->m_authCallback(reply, authenticator, dlInfo->m_authArg);
811  }
812 }
813 
821 {
822  if (!dlInfo)
823  return false;
824 
825  dlInfo->m_syncMode = true;
826 
827  // Special handling for link-local
828  // Not needed for Windows because windows does not need
829  // the scope id.
830 #ifndef _WIN32
831  if (dlInfo->m_url.startsWith("http://[fe80::",Qt::CaseInsensitive))
832  return downloadNowLinkLocal(dlInfo,deleteInfo);
833 #endif
834  m_infoLock->lock();
835  m_downloadQueue.push_back(dlInfo);
836  m_infoLock->unlock();
837  m_queueWaitCond.wakeAll();
838 
839  // timeout myth:// RemoteFile transfers 20 seconds from now
840  // timeout non-myth:// QNetworkAccessManager transfers 60 seconds after
841  // their last progress update
842  QDateTime startedAt = MythDate::current();
843  m_infoLock->lock();
844  while ((!dlInfo->IsDone()) &&
845  (dlInfo->m_errorCode == QNetworkReply::NoError) &&
846  (((!dlInfo->m_url.startsWith("myth://")) &&
847  (dlInfo->m_lastStat.secsTo(MythDate::current()) < 60)) ||
848  ((dlInfo->m_url.startsWith("myth://")) &&
849  (startedAt.secsTo(MythDate::current()) < 20))))
850  {
851  m_infoLock->unlock();
852  m_queueWaitLock.lock();
853  m_queueWaitCond.wait(&m_queueWaitLock, 200);
854  m_queueWaitLock.unlock();
855  m_infoLock->lock();
856  }
857  bool done = dlInfo->IsDone();
858  bool success =
859  done && (dlInfo->m_errorCode == QNetworkReply::NoError);
860 
861  if (!done)
862  {
863  dlInfo->m_data = nullptr; // Prevent downloadFinished() from updating
864  dlInfo->m_syncMode = false; // Let downloadFinished() cleanup for us
865  if ((dlInfo->m_reply) &&
866  (dlInfo->m_errorCode == QNetworkReply::NoError))
867  {
868  LOG(VB_FILE, LOG_DEBUG,
869  LOC + QString("Aborting download - lack of data transfer"));
870  dlInfo->m_reply->abort();
871  }
872  }
873  else if (deleteInfo)
874  delete dlInfo;
875 
876  m_infoLock->unlock();
877 
878  return success;
879 }
880 
881 #ifndef _WIN32
882 
902 bool MythDownloadManager::downloadNowLinkLocal(MythDownloadInfo *dlInfo, bool deleteInfo)
903 {
904  bool isOK = true;
905 
906  // Only certain features are supported here
907  if (dlInfo->m_authCallback || dlInfo->m_authArg)
908  {
909  LOG(VB_GENERAL, LOG_ERR, LOC +
910  QString("Unsupported authentication for %1").arg(dlInfo->m_url));
911  isOK = false;
912  }
913  if (!dlInfo->m_outFile.isEmpty())
914  {
915  LOG(VB_GENERAL, LOG_ERR, LOC +
916  QString("Unsupported File output %1 for %2")
917  .arg(dlInfo->m_outFile).arg(dlInfo->m_url));
918  isOK = false;
919  }
920 
921  if (!deleteInfo || dlInfo->m_requestType == kRequestHead)
922  {
923  // We do not have the ability to return a network reply in dlInfo
924  // so if we are asked to do that, return an error.
925  LOG(VB_GENERAL, LOG_ERR, LOC +
926  QString("Unsupported link-local operation %1")
927  .arg(dlInfo->m_url));
928  isOK = false;
929  }
930 
931  QUrl url(dlInfo->m_url);
932  QString host(url.host());
933  int port(url.port(80));
934  if (isOK && PortChecker::resolveLinkLocal(host, port))
935  {
936  QString reqType;
937  switch (dlInfo->m_requestType)
938  {
939  case kRequestPost :
940  reqType = "POST";
941  break;
942  case kRequestGet :
943  default:
944  reqType = "GET";
945  break;
946  }
947  QByteArray *aBuffer = dlInfo->m_data;
948  QHash<QByteArray, QByteArray> headers;
949  if (dlInfo->m_headers)
950  headers = *dlInfo->m_headers;
951  if (!headers.contains("User-Agent"))
952  headers.insert("User-Agent",
953  "MythDownloadManager v" MYTH_BINARY_VERSION);
954  headers.insert("Connection", "close");
955  headers.insert("Accept-Encoding", "identity");
956  if (aBuffer && !aBuffer->isEmpty())
957  headers.insert("Content-Length",
958  (QString::number(aBuffer->size())).toUtf8());
959  headers.insert("Host",
960  (url.host()+":"+QString::number(port)).toUtf8());
961 
962  QByteArray requestMessage;
963  QString path (url.path());
964  requestMessage.append("POST ");
965  requestMessage.append(path);
966  requestMessage.append(" HTTP/1.1\r\n");
967  QHashIterator<QByteArray, QByteArray> it(headers);
968  while (it.hasNext())
969  {
970  it.next();
971  requestMessage.append(it.key());
972  requestMessage.append(": ");
973  requestMessage.append(it.value());
974  requestMessage.append("\r\n");
975  }
976  requestMessage.append("\r\n");
977  if (aBuffer && !aBuffer->isEmpty())
978  {
979  requestMessage.append(*aBuffer);
980  }
981  QTcpSocket socket;
982  socket.connectToHost(host, port);
983  // QT Warning - this may not work on Windows
984  if (!socket.waitForConnected(5000))
985  isOK = false;
986  if (isOK)
987  isOK = (socket.write(requestMessage) > 0);
988  if (isOK)
989  // QT Warning - this may not work on Windows
990  isOK = socket.waitForDisconnected(5000);
991  if (isOK)
992  {
993  *aBuffer = socket.readAll();
994  // Find the start of the content
995  QByteArray delim("\r\n\r\n");
996  int delimLoc=aBuffer->indexOf(delim);
997  if (delimLoc > -1)
998  *aBuffer = aBuffer->right
999  (aBuffer->size()-delimLoc-4);
1000  else
1001  isOK=false;
1002  }
1003  socket.close();
1004  }
1005  else
1006  isOK = false;
1007 
1008  if (deleteInfo)
1009  delete dlInfo;
1010 
1011  if (isOK)
1012  return true;
1013  LOG(VB_GENERAL, LOG_ERR, LOC + QString("Link Local request failed: %1")
1014  .arg(url.toString()));
1015  return false;
1016 }
1017 #endif
1018 
1023 void MythDownloadManager::cancelDownload(const QString &url, bool block)
1024 {
1025  cancelDownload(QStringList(url), block);
1026 }
1027 
1032 void MythDownloadManager::cancelDownload(const QStringList &urls, bool block)
1033 {
1034  MythDownloadInfo *dlInfo;
1035 
1036  m_infoLock->lock();
1037  foreach (QString url, urls)
1038  {
1039  QMutableListIterator<MythDownloadInfo*> lit(m_downloadQueue);
1040  while (lit.hasNext())
1041  {
1042  lit.next();
1043  dlInfo = lit.value();
1044  if (dlInfo->m_url == url)
1045  {
1046  if (!m_cancellationQueue.contains(dlInfo))
1047  m_cancellationQueue.append(dlInfo);
1048  lit.remove();
1049  }
1050  }
1051 
1052  if (m_downloadInfos.contains(url))
1053  {
1054  dlInfo = m_downloadInfos[url];
1055 
1056  if (!m_cancellationQueue.contains(dlInfo))
1057  m_cancellationQueue.append(dlInfo);
1058 
1059  if (dlInfo->m_reply)
1060  m_downloadReplies.remove(dlInfo->m_reply);
1061 
1062  m_downloadInfos.remove(url);
1063  }
1064  }
1065  m_infoLock->unlock();
1066 
1067  if (QThread::currentThread() == this->thread())
1068  {
1069  downloadCanceled();
1070  return;
1071  }
1072 
1073  // wake-up running thread
1074  m_queueWaitCond.wakeAll();
1075 
1076  if (!block)
1077  return;
1078 
1079  while (!m_cancellationQueue.isEmpty())
1080  {
1081  usleep(50000); // re-test in another 50ms
1082  }
1083 }
1084 
1086 {
1087  QMutexLocker locker(m_infoLock);
1088  MythDownloadInfo *dlInfo;
1089 
1090  QMutableListIterator<MythDownloadInfo*> lit(m_cancellationQueue);
1091  while (lit.hasNext())
1092  {
1093  lit.next();
1094  dlInfo = lit.value();
1095  dlInfo->m_lock.lock();
1096 
1097  if (dlInfo->m_reply)
1098  {
1099  LOG(VB_FILE, LOG_DEBUG,
1100  LOC + QString("Aborting download - user request"));
1101  dlInfo->m_reply->abort();
1102  }
1103  lit.remove();
1104  if (dlInfo->m_done)
1105  {
1106  dlInfo->m_lock.unlock();
1107  continue;
1108  }
1109  dlInfo->m_errorCode = QNetworkReply::OperationCanceledError;
1110  dlInfo->m_done = true;
1111  dlInfo->m_lock.unlock();
1112  }
1113 }
1114 
1120 {
1121  QMutexLocker locker(m_infoLock);
1122  MythDownloadInfo *dlInfo;
1123 
1124  QList <MythDownloadInfo*>::iterator lit = m_downloadQueue.begin();
1125  for (; lit != m_downloadQueue.end(); ++lit)
1126  {
1127  dlInfo = *lit;
1128  if (dlInfo->m_caller == caller)
1129  {
1130  dlInfo->m_caller = nullptr;
1131  dlInfo->m_outFile = QString();
1132  dlInfo->m_data = nullptr;
1133  }
1134  }
1135 
1136  QMap <QString, MythDownloadInfo*>::iterator mit = m_downloadInfos.begin();
1137  for (; mit != m_downloadInfos.end(); ++mit)
1138  {
1139  dlInfo = mit.value();
1140  if (dlInfo->m_caller == caller)
1141  {
1142  dlInfo->m_caller = nullptr;
1143  dlInfo->m_outFile = QString();
1144  dlInfo->m_data = nullptr;
1145  }
1146  }
1147 }
1148 
1152 void MythDownloadManager::downloadError(QNetworkReply::NetworkError errorCode)
1153 {
1154  QNetworkReply *reply = (QNetworkReply*)sender();
1155 
1156  LOG(VB_FILE, LOG_DEBUG, LOC + QString("downloadError %1 ")
1157  .arg(errorCode) + reply->errorString() );
1158 
1159  QMutexLocker locker(m_infoLock);
1160  if (!m_downloadReplies.contains(reply))
1161  {
1162  reply->deleteLater();
1163  return;
1164  }
1165 
1166  MythDownloadInfo *dlInfo = m_downloadReplies[reply];
1167 
1168  if (!dlInfo)
1169  return;
1170 
1171  dlInfo->m_errorCode = errorCode;
1172 }
1173 
1179 QUrl MythDownloadManager::redirectUrl(const QUrl& possibleRedirectUrl,
1180  const QUrl& oldRedirectUrl) const
1181 {
1182  LOG(VB_FILE, LOG_DEBUG, LOC + QString("redirectUrl()"));
1183  QUrl redirectUrl;
1184 
1185  if(!possibleRedirectUrl.isEmpty() && possibleRedirectUrl != oldRedirectUrl)
1186  redirectUrl = possibleRedirectUrl;
1187 
1188  return redirectUrl;
1189 }
1190 
1194 void MythDownloadManager::downloadFinished(QNetworkReply* reply)
1195 {
1196  LOG(VB_FILE, LOG_DEBUG, LOC + QString("downloadFinished(%1)")
1197  .arg((long long)reply));
1198 
1199  QMutexLocker locker(m_infoLock);
1200  if (!m_downloadReplies.contains(reply))
1201  {
1202  reply->deleteLater();
1203  return;
1204  }
1205 
1206  MythDownloadInfo *dlInfo = m_downloadReplies[reply];
1207 
1208  if (!dlInfo || !dlInfo->m_reply)
1209  return;
1210 
1211  downloadFinished(dlInfo);
1212 }
1213 
1218 {
1219  if (!dlInfo)
1220  return;
1221 
1222  int statusCode = -1;
1223  static const char dateFormat[] = "ddd, dd MMM yyyy hh:mm:ss 'GMT'";
1224  QNetworkReply *reply = dlInfo->m_reply;
1225 
1226  if (reply)
1227  {
1228  QUrl possibleRedirectUrl =
1229  reply->attribute(QNetworkRequest::RedirectionTargetAttribute).toUrl();
1230 
1231  if (!possibleRedirectUrl.isEmpty() &&
1232  possibleRedirectUrl.isValid() &&
1233  possibleRedirectUrl.isRelative()) // Turn relative Url to absolute
1234  possibleRedirectUrl = reply->url().resolved(possibleRedirectUrl);
1235 
1236  if (!possibleRedirectUrl.isEmpty() && dlInfo->m_finalUrl != nullptr)
1237  *dlInfo->m_finalUrl = QString(possibleRedirectUrl.toString());
1238 
1239  dlInfo->m_redirectedTo =
1240  redirectUrl(possibleRedirectUrl, dlInfo->m_redirectedTo);
1241 
1242  QVariant status =
1243  reply->attribute(QNetworkRequest::HttpStatusCodeAttribute);
1244  if (status.isValid())
1245  statusCode = status.toInt();
1246  }
1247 
1248  if(reply && !dlInfo->m_redirectedTo.isEmpty() &&
1249  ((dlInfo->m_requestType != kRequestPost) ||
1250  (statusCode == 301 || statusCode == 302 ||
1251  statusCode == 303)))
1252  {
1253  LOG(VB_FILE, LOG_DEBUG, LOC +
1254  QString("downloadFinished(%1): Redirect: %2 -> %3")
1255  .arg((long long)dlInfo)
1256  .arg(reply->url().toString())
1257  .arg(dlInfo->m_redirectedTo.toString()));
1258 
1259  if (dlInfo->m_data)
1260  dlInfo->m_data->clear();
1261 
1262  dlInfo->m_bytesReceived = 0;
1263  dlInfo->m_bytesTotal = 0;
1264 
1265  QNetworkRequest request(dlInfo->m_redirectedTo);
1266 
1267  if (dlInfo->m_reload)
1268  {
1269  request.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
1270  QNetworkRequest::AlwaysNetwork);
1271  }
1272  else if (dlInfo->m_preferCache)
1273  {
1274  request.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
1275  QNetworkRequest::PreferCache);
1276  }
1277 
1278  request.setRawHeader("User-Agent",
1279  "MythDownloadManager v" MYTH_BINARY_VERSION);
1280 
1281  switch (dlInfo->m_requestType)
1282  {
1283  case kRequestHead :
1284  dlInfo->m_reply = m_manager->head(request);
1285  break;
1286  case kRequestGet :
1287  default:
1288  dlInfo->m_reply = m_manager->get(request);
1289  break;
1290  }
1291 
1292  m_downloadReplies[dlInfo->m_reply] = dlInfo;
1293 
1294  connect(dlInfo->m_reply, SIGNAL(error(QNetworkReply::NetworkError)),
1295  this, SLOT(downloadError(QNetworkReply::NetworkError)));
1296  connect(dlInfo->m_reply, SIGNAL(downloadProgress(qint64, qint64)),
1297  this, SLOT(downloadProgress(qint64, qint64)));
1298 
1299  m_downloadReplies.remove(reply);
1300  reply->deleteLater();
1301  }
1302  else
1303  {
1304  LOG(VB_FILE, LOG_DEBUG, QString("downloadFinished(%1): COMPLETE: %2")
1305  .arg((long long)dlInfo).arg(dlInfo->m_url));
1306 
1307  // HACK Insert a Date header into the cached metadata if one doesn't
1308  // already exist
1309  QUrl fileUrl = dlInfo->m_url;
1310  QString redirectLoc;
1311  int limit = 0;
1312  while (!(redirectLoc = getHeader(fileUrl, "Location")).isNull())
1313  {
1314  if (limit == CACHE_REDIRECTION_LIMIT)
1315  {
1316  LOG(VB_GENERAL, LOG_WARNING, QString("Cache Redirection limit "
1317  "reached for %1")
1318  .arg(fileUrl.toString()));
1319  return;
1320  }
1321  fileUrl.setUrl(redirectLoc);
1322  limit++;
1323  }
1324 
1325  m_infoLock->lock();
1326  QNetworkCacheMetaData urlData = m_manager->cache()->metaData(fileUrl);
1327  m_infoLock->unlock();
1328  if (getHeader(urlData, "Date").isNull())
1329  {
1330  QNetworkCacheMetaData::RawHeaderList headers = urlData.rawHeaders();
1331  QNetworkCacheMetaData::RawHeader newheader;
1332  QDateTime now = MythDate::current();
1333  newheader = QNetworkCacheMetaData::RawHeader("Date",
1334  now.toString(dateFormat).toLatin1());
1335  headers.append(newheader);
1336  urlData.setRawHeaders(headers);
1337  m_infoLock->lock();
1338  m_manager->cache()->updateMetaData(urlData);
1339  m_infoLock->unlock();
1340  }
1341  // End HACK
1342 
1343  dlInfo->m_redirectedTo.clear();
1344 
1345  int dataSize = -1;
1346 
1347  // If we downloaded via the QNetworkAccessManager
1348  // AND the caller isn't handling the reply directly
1349  if (reply && dlInfo->m_processReply)
1350  {
1351  bool append = (!dlInfo->m_syncMode && dlInfo->m_caller);
1352  QByteArray data = reply->readAll();
1353  dataSize = data.size();
1354 
1355  if (append)
1356  dlInfo->m_bytesReceived += dataSize;
1357  else
1358  dlInfo->m_bytesReceived = dataSize;
1359 
1360  dlInfo->m_bytesTotal = dlInfo->m_bytesReceived;
1361 
1362  if (dlInfo->m_data)
1363  {
1364  if (append)
1365  dlInfo->m_data->append(data);
1366  else
1367  *dlInfo->m_data = data;
1368  }
1369  else if (!dlInfo->m_outFile.isEmpty())
1370  {
1371  saveFile(dlInfo->m_outFile, data, append);
1372  }
1373  }
1374  else if (!reply) // If we downloaded via RemoteFile
1375  {
1376  if (dlInfo->m_data)
1377  {
1378  (*dlInfo->m_data) = dlInfo->m_privData;
1379  }
1380  else if (!dlInfo->m_outFile.isEmpty())
1381  {
1382  saveFile(dlInfo->m_outFile, dlInfo->m_privData);
1383  }
1384  dlInfo->m_bytesReceived += dataSize;
1385  dlInfo->m_bytesTotal = dlInfo->m_bytesReceived;
1386  }
1387  // else we downloaded via QNetworkAccessManager
1388  // AND the caller is handling the reply
1389 
1390  m_infoLock->lock();
1391  if (!m_downloadInfos.remove(dlInfo->m_url))
1392  LOG(VB_GENERAL, LOG_ERR, LOC +
1393  QString("ERROR download finished but failed to remove url: %1")
1394  .arg(dlInfo->m_url));
1395 
1396  if (reply)
1397  m_downloadReplies.remove(reply);
1398  m_infoLock->unlock();
1399 
1400  dlInfo->SetDone(true);
1401 
1402  if (!dlInfo->m_syncMode)
1403  {
1404  if (dlInfo->m_caller)
1405  {
1406  LOG(VB_FILE, LOG_DEBUG, QString("downloadFinished(%1): "
1407  "COMPLETE: %2, sending event to caller")
1408  .arg((long long)dlInfo).arg(dlInfo->m_url));
1409 
1410  QStringList args;
1411  args << dlInfo->m_url;
1412  args << dlInfo->m_outFile;
1413  args << QString::number(dlInfo->m_bytesTotal);
1414  // placeholder for error string
1415  args << (reply ? reply->errorString() : QString());
1416  args << QString::number((int)(reply ? reply->error() :
1417  dlInfo->m_errorCode));
1418 
1419  QCoreApplication::postEvent(dlInfo->m_caller,
1420  new MythEvent("DOWNLOAD_FILE FINISHED", args));
1421  }
1422 
1423  delete dlInfo;
1424  }
1425 
1426  m_queueWaitCond.wakeAll();
1427  }
1428 }
1429 
1435 void MythDownloadManager::downloadProgress(qint64 bytesReceived,
1436  qint64 bytesTotal)
1437 {
1438  QNetworkReply *reply = (QNetworkReply*)sender();
1439 
1440  LOG(VB_FILE, LOG_DEBUG, LOC +
1441  QString("downloadProgress(%1, %2) (for reply %3)")
1442  .arg(bytesReceived).arg(bytesTotal).arg((long long)reply));
1443 
1444  QMutexLocker locker(m_infoLock);
1445  if (!m_downloadReplies.contains(reply))
1446  return;
1447 
1448  MythDownloadInfo *dlInfo = m_downloadReplies[reply];
1449 
1450  if (!dlInfo)
1451  return;
1452 
1453  dlInfo->m_lastStat = MythDate::current();
1454 
1455  LOG(VB_FILE, LOG_DEBUG, LOC +
1456  QString("downloadProgress: %1 to %2 is at %3 of %4 bytes downloaded")
1457  .arg(dlInfo->m_url).arg(dlInfo->m_outFile)
1458  .arg(bytesReceived).arg(bytesTotal));
1459 
1460  if (!dlInfo->m_syncMode && dlInfo->m_caller)
1461  {
1462  LOG(VB_FILE, LOG_DEBUG, QString("downloadProgress(%1): "
1463  "sending event to caller")
1464  .arg(reply->url().toString()));
1465 
1466  bool appendToFile = (dlInfo->m_bytesReceived != 0);
1467  QByteArray data = reply->readAll();
1468  if (!dlInfo->m_outFile.isEmpty())
1469  saveFile(dlInfo->m_outFile, data, appendToFile);
1470 
1471  if (dlInfo->m_data)
1472  dlInfo->m_data->append(data);
1473 
1474  dlInfo->m_bytesReceived = bytesReceived;
1475  dlInfo->m_bytesTotal = bytesTotal;
1476 
1477  QStringList args;
1478  args << dlInfo->m_url;
1479  args << dlInfo->m_outFile;
1480  args << QString::number(bytesReceived);
1481  args << QString::number(bytesTotal);
1482 
1483  QCoreApplication::postEvent(dlInfo->m_caller,
1484  new MythEvent("DOWNLOAD_FILE UPDATE", args));
1485  }
1486 }
1487 
1495 bool MythDownloadManager::saveFile(const QString &outFile,
1496  const QByteArray &data,
1497  const bool append)
1498 {
1499  if (outFile.isEmpty() || !data.size())
1500  return false;
1501 
1502  QFile file(outFile);
1503  QFileInfo fileInfo(outFile);
1504  QDir qdir(fileInfo.absolutePath());
1505 
1506  if (!qdir.exists() && !qdir.mkpath(fileInfo.absolutePath()))
1507  {
1508  LOG(VB_GENERAL, LOG_ERR, QString("Failed to create: '%1'")
1509  .arg(fileInfo.absolutePath()));
1510  return false;
1511  }
1512 
1513  QIODevice::OpenMode mode = QIODevice::Unbuffered|QIODevice::WriteOnly;
1514  if (append)
1515  mode |= QIODevice::Append;
1516 
1517  if (!file.open(mode))
1518  {
1519  LOG(VB_GENERAL, LOG_ERR, QString("Failed to open: '%1'") .arg(outFile));
1520  return false;
1521  }
1522 
1523  off_t offset = 0;
1524  size_t remaining = data.size();
1525  uint failure_cnt = 0;
1526  while ((remaining > 0) && (failure_cnt < 5))
1527  {
1528  ssize_t written = file.write(data.data() + offset, remaining);
1529  if (written < 0)
1530  {
1531  failure_cnt++;
1532  usleep(50000);
1533  continue;
1534  }
1535 
1536  failure_cnt = 0;
1537  offset += written;
1538  remaining -= written;
1539  }
1540 
1541  return remaining <= 0;
1542 }
1543 
1548 QDateTime MythDownloadManager::GetLastModified(const QString &url)
1549 {
1550  // If the header has not expired and
1551  // the last modification date is less than 1 hours old or if
1552  // the cache object is less than 20 minutes old,
1553  // then use the cached header otherwise redownload the header
1554 
1555  static const char dateFormat[] = "ddd, dd MMM yyyy hh:mm:ss 'GMT'";
1556  LOG(VB_FILE, LOG_DEBUG, LOC + QString("GetLastModified('%1')").arg(url));
1557  QDateTime result;
1558 
1559  QDateTime now = MythDate::current();
1560 
1561  QUrl cacheUrl = QUrl(url);
1562 
1563  // Deal with redirects, we want the cached data for the final url
1564  QString redirectLoc;
1565  int limit = 0;
1566  while (!(redirectLoc = getHeader(cacheUrl, "Location")).isNull())
1567  {
1568  if (limit == CACHE_REDIRECTION_LIMIT)
1569  {
1570  LOG(VB_GENERAL, LOG_WARNING, QString("Cache Redirection limit "
1571  "reached for %1")
1572  .arg(cacheUrl.toString()));
1573  return result;
1574  }
1575  cacheUrl.setUrl(redirectLoc);
1576  limit++;
1577  }
1578 
1579  m_infoLock->lock();
1580  QNetworkCacheMetaData urlData = m_manager->cache()->metaData(cacheUrl);
1581  m_infoLock->unlock();
1582 
1583  if (urlData.isValid() &&
1584  ((!urlData.expirationDate().isValid()) ||
1585  (urlData.expirationDate().secsTo(now) < 0)))
1586  {
1587  if (urlData.lastModified().toUTC().secsTo(now) <= 3600) // 1 Hour
1588  {
1589  result = urlData.lastModified().toUTC();
1590  }
1591  else
1592  {
1593  QString date = getHeader(urlData, "Date");
1594  if (!date.isNull())
1595  {
1596  QDateTime loadDate =
1597  MythDate::fromString(date, dateFormat);
1598  loadDate.setTimeSpec(Qt::UTC);
1599  if (loadDate.secsTo(now) <= 1200) // 20 Minutes
1600  {
1601  result = urlData.lastModified().toUTC();
1602  }
1603  }
1604  }
1605  }
1606 
1607  if (!result.isValid())
1608  {
1609  MythDownloadInfo *dlInfo = new MythDownloadInfo;
1610  dlInfo->m_url = url;
1611  dlInfo->m_syncMode = true;
1612  // Head request, we only want to inspect the headers
1613  dlInfo->m_requestType = kRequestHead;
1614 
1615  if (downloadNow(dlInfo, false))
1616  {
1617  if (dlInfo->m_reply)
1618  {
1619  QVariant lastMod =
1620  dlInfo->m_reply->header(
1621  QNetworkRequest::LastModifiedHeader);
1622  if (lastMod.isValid())
1623  result = lastMod.toDateTime().toUTC();
1624  }
1625 
1626  // downloadNow() will set a flag to trigger downloadFinished()
1627  // to delete the dlInfo if the download times out
1628  delete dlInfo;
1629  }
1630  }
1631 
1632  LOG(VB_FILE, LOG_DEBUG, LOC + QString("GetLastModified('%1'): Result %2")
1633  .arg(url).arg(result.toString()));
1634 
1635  return result;
1636 }
1637 
1638 
1642 void MythDownloadManager::loadCookieJar(const QString &filename)
1643 {
1644  QMutexLocker locker(&m_cookieLock);
1645 
1646  MythCookieJar *jar = new MythCookieJar;
1647  jar->load(filename);
1648  m_manager->setCookieJar(jar);
1649 }
1650 
1654 void MythDownloadManager::saveCookieJar(const QString &filename)
1655 {
1656  QMutexLocker locker(&m_cookieLock);
1657 
1658  if (!m_manager->cookieJar())
1659  return;
1660 
1661  MythCookieJar *jar = static_cast<MythCookieJar *>(m_manager->cookieJar());
1662  jar->save(filename);
1663 }
1664 
1665 void MythDownloadManager::setCookieJar(QNetworkCookieJar *cookieJar)
1666 {
1667  QMutexLocker locker(&m_cookieLock);
1668  m_manager->setCookieJar(cookieJar);
1669 }
1670 
1674 QNetworkCookieJar *MythDownloadManager::copyCookieJar(void)
1675 {
1676  QMutexLocker locker(&m_cookieLock);
1677 
1678  if (!m_manager->cookieJar())
1679  return nullptr;
1680 
1681  MythCookieJar *inJar = static_cast<MythCookieJar *>(m_manager->cookieJar());
1682  MythCookieJar *outJar = new MythCookieJar;
1683  outJar->copyAllCookies(*inJar);
1684 
1685  return static_cast<QNetworkCookieJar *>(outJar);
1686 }
1687 
1691 void MythDownloadManager::refreshCookieJar(QNetworkCookieJar *jar)
1692 {
1693  QMutexLocker locker(&m_cookieLock);
1694  delete m_inCookieJar;
1695 
1696  MythCookieJar *inJar = static_cast<MythCookieJar *>(jar);
1697  MythCookieJar *outJar = new MythCookieJar;
1698  outJar->copyAllCookies(*inJar);
1699  m_inCookieJar = static_cast<QNetworkCookieJar *>(outJar);
1700 
1701  QMutexLocker locker2(&m_queueWaitLock);
1702  m_queueWaitCond.wakeAll();
1703 }
1704 
1708 {
1709  QMutexLocker locker(&m_cookieLock);
1710 
1711  MythCookieJar *inJar = static_cast<MythCookieJar *>(m_inCookieJar);
1712  MythCookieJar *outJar = new MythCookieJar;
1713  outJar->copyAllCookies(*inJar);
1714  m_manager->setCookieJar(static_cast<QNetworkCookieJar *>(outJar));
1715 
1716  delete m_inCookieJar;
1717  m_inCookieJar = nullptr;
1718 }
1719 
1720 QString MythDownloadManager::getHeader(const QUrl& url, const QString& header)
1721 {
1722  if (!m_manager || !m_manager->cache())
1723  return QString();
1724 
1725  m_infoLock->lock();
1726  QNetworkCacheMetaData metadata = m_manager->cache()->metaData(url);
1727  m_infoLock->unlock();
1728 
1729  return getHeader(metadata, header);
1730 }
1731 
1737 QString MythDownloadManager::getHeader(const QNetworkCacheMetaData &cacheData,
1738  const QString& header)
1739 {
1740  QNetworkCacheMetaData::RawHeaderList headers = cacheData.rawHeaders();
1741 
1742  for (auto it = headers.begin(); it != headers.end(); ++it)
1743  if (QString((*it).first) == header)
1744  return QString((*it).second);
1745  return QString();
1746 }
1747 
1748 
1753 {
1754  const QList<QNetworkCookie> cookieList = old.allCookies();
1755  setAllCookies(cookieList);
1756 }
1757 
1761 void MythCookieJar::load(const QString &filename)
1762 {
1763  LOG(VB_GENERAL, LOG_DEBUG, QString("MythCookieJar: loading cookies from: %1").arg(filename));
1764 
1765  QFile f(filename);
1766  if (!f.open(QIODevice::ReadOnly))
1767  {
1768  LOG(VB_GENERAL, LOG_WARNING, QString("MythCookieJar::load() failed to open file for reading: %1").arg(filename));
1769  return;
1770  }
1771 
1772  QList<QNetworkCookie> cookieList;
1773  QTextStream stream(&f);
1774  while (!stream.atEnd())
1775  {
1776  QString cookie = stream.readLine();
1777  cookieList << QNetworkCookie::parseCookies(cookie.toLocal8Bit());
1778  }
1779 
1780  setAllCookies(cookieList);
1781 }
1782 
1786 void MythCookieJar::save(const QString &filename)
1787 {
1788  LOG(VB_GENERAL, LOG_DEBUG, QString("MythCookieJar: saving cookies to: %1").arg(filename));
1789 
1790  QFile f(filename);
1791  if (!f.open(QIODevice::WriteOnly))
1792  {
1793  LOG(VB_GENERAL, LOG_ERR, QString("MythCookieJar::save() failed to open file for writing: %1").arg(filename));
1794  return;
1795  }
1796 
1797  QList<QNetworkCookie> cookieList = allCookies();
1798  QTextStream stream(&f);
1799 
1800  for (QList<QNetworkCookie>::iterator it = cookieList.begin();
1801  it != cookieList.end(); ++it)
1802  {
1803  stream << (*it).toRawForm() << endl;
1804  }
1805 }
1806 
1807 
1808 /* vim: set expandtab tabstop=4 shiftwidth=4: */
QNetworkCookieJar * copyCookieJar(void)
Copy from one cookie jar to another.
QNetworkReply::NetworkError m_errorCode
void start(QThread::Priority=QThread::InheritPriority)
Tell MThread to start running the thread in the near future.
Definition: mthread.cpp:294
void save(const QString &filename)
Saves the cookie jar to a cookie file.
MRequestType
void queueItem(const QString &url, QNetworkRequest *req, const QString &dest, QByteArray *data, QObject *caller, const MRequestType reqType=kRequestGet, const bool reload=false)
Adds a request to the download queue.
MythDownloadManager * downloadManager
void removeListener(QObject *caller)
Disconnects the specified caller from any existing MythDownloadInfo instances.
#define NULL
Definition: H264Parser.h:62
void queueDownload(const QString &url, const QString &dest, QObject *caller, const bool reload=false)
Adds a url to the download queue.
~MythDownloadManager()
Destructor for MythDownloadManager.
static void error(const char *str,...)
Definition: vbi.c:42
void ShutdownMythDownloadManager(void)
Deletes the running MythDownloadManager at program exit.
#define CACHE_REDIRECTION_LIMIT
A subclassed QNetworkCookieJar that allows for reading and writing cookie files that contain raw form...
void downloadQNetworkRequest(MythDownloadInfo *dlInfo)
Downloads a QNetworkRequest via the QNetworkAccessManager.
AuthCallback m_authCallback
bool saveFile(const QString &outFile, const QByteArray &data, const bool append=false)
Saves a QByteArray of data to a given filename.
QNetworkReply * m_reply
unsigned int uint
Definition: compat.h:140
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
void downloadProgress(qint64 bytesReceived, qint64 bytesTotal)
Slot to process download update events.
const QHash< QByteArray, QByteArray > * m_headers
bool postAuth(const QString &url, QByteArray *data, AuthCallback authCallback, void *authArg, const QHash< QByteArray, QByteArray > *headers=nullptr)
Posts data to a url via the QNetworkAccessManager.
MRequestType m_requestType
void run(void) override
Runs a loop to process incoming download requests and triggers download events to be processed.
void downloadFinished(QNetworkReply *reply)
Slot to process download finished events.
#define off_t
static bool resolveLinkLocal(QString &host, int port, int timeLimit=30000)
Convenience method to resolve link-local address.
QString GetConfDir(void)
Definition: mythdirs.cpp:224
void authCallback(QNetworkReply *reply, QAuthenticator *authenticator)
Signal handler for authentication requests.
This class is used as a container for messages.
Definition: mythevent.h:16
QUrl redirectUrl(const QUrl &possibleRedirectUrl, const QUrl &oldRedirectUrl) const
Checks whether we were redirected to the given URL.
void setCookieJar(QNetworkCookieJar *cookieJar)
static QString downloadRemoteFile(const QString &cmd, const QString &url, const QString &storageGroup, const QString &filename)
void copyAllCookies(MythCookieJar &old)
Copies all cookies from one MythCookieJar to another.
RemoteFileDownloadThread(MythDownloadManager *parent, MythDownloadInfo *dlInfo)
QDateTime current(bool stripped)
Returns current Date and Time in UTC.
Definition: mythdate.cpp:10
void queuePost(const QString &url, QByteArray *data, QObject *caller)
Queues a post to a URL via the QNetworkAccessManager.
void refreshCookieJar(QNetworkCookieJar *jar)
Refresh the temporary cookie jar from another cookie jar.
MythDownloadManager * GetMythDownloadManager(void)
Gets the pointer to the MythDownloadManager singleton.
bool download(const QString &url, const QString &dest, const bool reload=false)
Downloads a URL to a file in blocking mode.
QDateTime GetLastModified(const QString &url)
Gets the Last Modified timestamp for a URI.
void downloadError(QNetworkReply::NetworkError errorCode)
Slot to process download error events.
#define LOC
void(* AuthCallback)(QNetworkReply *, QAuthenticator *, void *)
QString getHeader(const QUrl &url, const QString &header)
static MThreadPool * globalInstance(void)
bool SaveAs(QByteArray &data)
void downloadRemoteFile(MythDownloadInfo *dlInfo)
Triggers a myth:// URI download in the background via RemoteFile.
void SetDone(bool done)
#define LOG(_MASK_, _LEVEL_, _STRING_)
Definition: mythlogging.h:41
void start(QRunnable *runnable, const QString &debugName, int priority=0)
QNetworkRequest * m_request
bool downloadAuth(const QString &url, const QString &dest, const bool reload=false, AuthCallback authCallback=nullptr, void *authArg=nullptr, const QHash< QByteArray, QByteArray > *headers=nullptr)
Downloads a URL to a file in blocking mode.
void updateCookieJar(void)
Update the cookie jar from the temporary cookie jar.
bool downloadNow(MythDownloadInfo *dlInfo, bool deleteInfo=true)
Download helper for download() blocking methods.
#define MYTH_BINARY_VERSION
Update this whenever the plug-in ABI changes.
Definition: mythversion.h:16
void cancelDownload(const QString &url, bool block=true)
Cancel a queued or current download.
bool post(const QString &url, QByteArray *data)
Posts data to a url via the QNetworkAccessManager.
bool processItem(const QString &url, QNetworkRequest *req, const QString &dest, QByteArray *data, const MRequestType reqType=kRequestGet, const bool reload=false, AuthCallback authCallback=nullptr, void *authArg=nullptr, const QHash< QByteArray, QByteArray > *headers=nullptr, QString *finalUrl=nullptr)
Processes a network request immediately and waits for a response.
QThread * getQueueThread(void)
QMutex dmCreateLock
QString GetHostName(void)
void saveCookieJar(const QString &filename)
Saves the cookie jar to a cookie file.
QDateTime fromString(const QString &dtstr)
Converts kFilename && kISODate formats to QDateTime.
Definition: mythdate.cpp:30
void loadCookieJar(const QString &filename)
Loads the cookie jar from a cookie file.
void load(const QString &filename)
Loads the cookie jar from a cookie file.
void preCache(const QString &url)
Downloads a URL but doesn't store the resulting data anywhere.