MythTV  master
mythdownloadmanager.cpp
Go to the documentation of this file.
1 // qt
2 #include <QCoreApplication>
3 #include <QRunnable>
4 #include <QString>
5 #include <QByteArray>
6 #include <QFile>
7 #include <QDir>
8 #include <QNetworkCookieJar>
9 #include <QNetworkCookie>
10 #include <QAuthenticator>
11 #include <QTextStream>
12 #include <QNetworkProxy>
13 #include <QMutexLocker>
14 #include <QUrl>
15 #include <QTcpSocket>
16 
17 #include <cstdlib>
18 #include <unistd.h> // for usleep()
19 
20 // libmythbase
21 #include "compat.h"
22 #include "mythcorecontext.h"
23 #include "mythcoreutil.h"
24 #include "mthreadpool.h"
25 #include "mythdirs.h"
26 #include "mythevent.h"
27 #include "mythversion.h"
28 #include "remotefile.h"
29 #include "mythdate.h"
30 
31 #include "mythdownloadmanager.h"
32 #include "mythlogging.h"
33 #include "portchecker.h"
34 
35 using namespace std;
36 
37 #define LOC QString("DownloadManager: ")
38 #define CACHE_REDIRECTION_LIMIT 10
39 
41 QMutex dmCreateLock;
42 
47 {
48  public:
50  m_lastStat(MythDate::current())
51  {
52  qRegisterMetaType<QNetworkReply::NetworkError>("QNetworkReply::NetworkError");
53  }
54 
56  {
57  delete m_request;
58  if (m_reply && m_processReply)
59  m_reply->deleteLater();
60  }
61 
62  bool IsDone(void)
63  {
64  QMutexLocker lock(&m_lock);
65  return m_done;
66  }
67 
68  void SetDone(bool done)
69  {
70  QMutexLocker lock(&m_lock);
71  m_done = done;
72  }
73 
74  QString m_url;
76  QNetworkRequest *m_request {nullptr};
77  QNetworkReply *m_reply {nullptr};
78  QString m_outFile;
79  QByteArray *m_data {nullptr};
80  QByteArray m_privData;
81  QObject *m_caller {nullptr};
82  MRequestType m_requestType {kRequestGet};
83  bool m_reload {false};
84  bool m_preferCache {false};
85  bool m_syncMode {false};
86  bool m_processReply {true};
87  bool m_done {false};
88  qint64 m_bytesReceived {0};
89  qint64 m_bytesTotal {0};
90  QDateTime m_lastStat;
91  AuthCallback m_authCallback {nullptr};
92  void *m_authArg {nullptr};
93  const QHash<QByteArray, QByteArray> *m_headers {nullptr};
94 
95  QNetworkReply::NetworkError m_errorCode {QNetworkReply::NoError};
96  QMutex m_lock;
97 };
98 
99 
104 class MythCookieJar : public QNetworkCookieJar
105 {
106  public:
107  MythCookieJar() = default;
108  void copyAllCookies(MythCookieJar &old);
109  void load(const QString &filename);
110  void save(const QString &filename);
111 };
112 
116 class RemoteFileDownloadThread : public QRunnable
117 {
118  public:
120  MythDownloadInfo *dlInfo) :
121  m_parent(parent),
122  m_dlInfo(dlInfo) {}
123 
124  void run() override // QRunnable
125  {
126  bool ok = false;
127 
128  RemoteFile *rf = new RemoteFile(m_dlInfo->m_url, false, false, 0);
129  ok = rf->SaveAs(m_dlInfo->m_privData);
130  delete rf;
131 
132  if (!ok)
133  m_dlInfo->m_errorCode = QNetworkReply::UnknownNetworkError;
134 
135  m_dlInfo->m_bytesReceived = m_dlInfo->m_privData.size();
136  m_dlInfo->m_bytesTotal = m_dlInfo->m_bytesReceived;
137 
138  m_parent->downloadFinished(m_dlInfo);
139  }
140 
141  private:
142  MythDownloadManager *m_parent {nullptr};
143  MythDownloadInfo *m_dlInfo {nullptr};
144 };
145 
149 {
150  if (downloadManager)
151  {
152  delete downloadManager;
153  downloadManager = nullptr;
154  }
155 }
156 
161 {
162  if (downloadManager)
163  return downloadManager;
164 
165  QMutexLocker locker(&dmCreateLock);
166 
167  // Check once more in case the download manager was created
168  // while we were securing the lock.
169  if (downloadManager)
170  return downloadManager;
171 
173  tmpDLM->start();
174  while (!tmpDLM->getQueueThread())
175  usleep(10000);
176 
177  tmpDLM->moveToThread(tmpDLM->getQueueThread());
178  tmpDLM->setRunThread();
179 
180  while (!tmpDLM->isRunning())
181  usleep(10000);
182 
183  downloadManager = tmpDLM;
184 
186 
187  return downloadManager;
188 }
189 
193 {
194  m_runThread = false;
195  m_queueWaitCond.wakeAll();
196 
197  wait();
198 
199  delete m_infoLock;
200  delete m_inCookieJar;
201 }
202 
207 {
208  RunProlog();
209 
210  bool downloading = false;
211  bool itemsInQueue = false;
212  bool itemsInCancellationQueue = false;
213  bool waitAnyway = false;
214 
215  m_queueThread = QThread::currentThread();
216 
217  while (!m_runThread)
218  usleep(50000);
219 
220  m_manager = new QNetworkAccessManager(this);
221  m_diskCache = new QNetworkDiskCache(this);
222  m_proxy = new QNetworkProxy();
223  m_diskCache->setCacheDirectory(GetConfDir() + "/cache/" +
224  QCoreApplication::applicationName() + "-" +
226  m_manager->setCache(m_diskCache);
227 
228  // Set the proxy for the manager to be the application default proxy,
229  // which has already been setup
230  m_manager->setProxy(*m_proxy);
231 
232  // make sure the cookieJar is created in the same thread as the manager
233  // and set its parent to nullptr so it can be shared between managers
234  m_manager->cookieJar()->setParent(nullptr);
235 
236  QObject::connect(m_manager, SIGNAL(finished(QNetworkReply*)), this,
237  SLOT(downloadFinished(QNetworkReply*)));
238 
239  m_isRunning = true;
240  while (m_runThread)
241  {
242  if (m_inCookieJar)
243  {
244  LOG(VB_GENERAL, LOG_DEBUG, "Updating DLManager's Cookie Jar");
245  updateCookieJar();
246  }
247  m_infoLock->lock();
248  LOG(VB_FILE, LOG_DEBUG, LOC + QString("items downloading %1").arg(m_downloadInfos.count()));
249  LOG(VB_FILE, LOG_DEBUG, LOC + QString("items queued %1").arg(m_downloadQueue.count()));
250  downloading = !m_downloadInfos.isEmpty();
251  itemsInCancellationQueue = !m_cancellationQueue.isEmpty();
252  m_infoLock->unlock();
253 
254  if (itemsInCancellationQueue)
255  {
256  downloadCanceled();
257  }
258  if (downloading)
259  QCoreApplication::processEvents();
260 
261  m_infoLock->lock();
262  itemsInQueue = !m_downloadQueue.isEmpty();
263  m_infoLock->unlock();
264 
265  if (!itemsInQueue || waitAnyway)
266  {
267  waitAnyway = false;
268  m_queueWaitLock.lock();
269 
270  if (downloading)
271  {
272  LOG(VB_FILE, LOG_DEBUG, LOC + QString("waiting 200ms"));
273  m_queueWaitCond.wait(&m_queueWaitLock, 200);
274  }
275  else
276  {
277  LOG(VB_FILE, LOG_DEBUG, LOC + QString("waiting for more items to download"));
278  m_queueWaitCond.wait(&m_queueWaitLock);
279  }
280 
281  m_queueWaitLock.unlock();
282  }
283 
284  m_infoLock->lock();
285  if (!m_downloadQueue.isEmpty())
286  {
287  MythDownloadInfo *dlInfo = m_downloadQueue.front();
288 
289  m_downloadQueue.pop_front();
290 
291  if (!dlInfo)
292  {
293  m_infoLock->unlock();
294  continue;
295  }
296 
297  if (m_downloadInfos.contains(dlInfo->m_url))
298  {
299  // Push request to the end of the queue to let others process.
300  // If this is the only item in the queue, force the loop to
301  // wait a little.
302  if (m_downloadQueue.isEmpty())
303  waitAnyway = true;
304  m_downloadQueue.push_back(dlInfo);
305  m_infoLock->unlock();
306  continue;
307  }
308 
309  if (dlInfo->m_url.startsWith("myth://"))
310  downloadRemoteFile(dlInfo);
311  else
312  {
313  QMutexLocker cLock(&m_cookieLock);
314  downloadQNetworkRequest(dlInfo);
315  }
316 
317  m_downloadInfos[dlInfo->m_url] = dlInfo;
318  }
319  m_infoLock->unlock();
320  }
321  m_isRunning = false;
322 
323  RunEpilog();
324 }
325 
336 void MythDownloadManager::queueItem(const QString &url, QNetworkRequest *req,
337  const QString &dest, QByteArray *data,
338  QObject *caller, const MRequestType reqType,
339  const bool reload)
340 {
341  MythDownloadInfo *dlInfo = new MythDownloadInfo;
342 
343  dlInfo->m_url = url;
344  dlInfo->m_request = req;
345  dlInfo->m_outFile = dest;
346  dlInfo->m_data = data;
347  dlInfo->m_caller = caller;
348  dlInfo->m_requestType = reqType;
349  dlInfo->m_reload = reload;
350 
351  QMutexLocker locker(m_infoLock);
352  m_downloadQueue.push_back(dlInfo);
353  m_queueWaitCond.wakeAll();
354 }
355 
368 bool MythDownloadManager::processItem(const QString &url, QNetworkRequest *req,
369  const QString &dest, QByteArray *data,
370  const MRequestType reqType,
371  const bool reload,
372  AuthCallback authCallbackFn, void *authArg,
373  const QHash<QByteArray, QByteArray> *headers)
374 {
375  MythDownloadInfo *dlInfo = new MythDownloadInfo;
376 
377  dlInfo->m_url = url;
378  dlInfo->m_request = req;
379  dlInfo->m_outFile = dest;
380  dlInfo->m_data = data;
381  dlInfo->m_requestType = reqType;
382  dlInfo->m_reload = reload;
383  dlInfo->m_syncMode = true;
384  dlInfo->m_authCallback = authCallbackFn;
385  dlInfo->m_authArg = authArg;
386  dlInfo->m_headers = headers;
387 
388  return downloadNow(dlInfo, true);
389 }
390 
394 void MythDownloadManager::preCache(const QString &url)
395 {
396  LOG(VB_FILE, LOG_DEBUG, LOC + QString("preCache('%1')").arg(url));
397  queueItem(url, nullptr, QString(), nullptr, nullptr);
398 }
399 
406 void MythDownloadManager::queueDownload(const QString &url,
407  const QString &dest,
408  QObject *caller,
409  const bool reload)
410 {
411  LOG(VB_FILE, LOG_DEBUG, LOC + QString("queueDownload('%1', '%2', %3)")
412  .arg(url).arg(dest).arg((long long)caller));
413 
414  queueItem(url, nullptr, dest, nullptr, caller, kRequestGet, reload);
415 }
416 
422 void MythDownloadManager::queueDownload(QNetworkRequest *req,
423  QByteArray *data,
424  QObject *caller)
425 {
426  LOG(VB_FILE, LOG_DEBUG, LOC + QString("queueDownload('%1', '%2', %3)")
427  .arg(req->url().toString()).arg((long long)data)
428  .arg((long long)caller));
429 
430  queueItem(req->url().toString(), req, QString(), data, caller,
431  kRequestGet,
432  (QNetworkRequest::AlwaysNetwork == req->attribute(
433  QNetworkRequest::CacheLoadControlAttribute,
434  QNetworkRequest::PreferNetwork).toInt()));
435 }
436 
443 bool MythDownloadManager::download(const QString &url, const QString &dest,
444  const bool reload)
445 {
446  return processItem(url, nullptr, dest, nullptr, kRequestGet, reload);
447 }
448 
455 bool MythDownloadManager::download(const QString &url, QByteArray *data,
456  const bool reload)
457 {
458  return processItem(url, nullptr, QString(), data, kRequestGet, reload);
459 }
460 
467 QNetworkReply *MythDownloadManager::download(const QString &url,
468  const bool reload)
469 {
470  MythDownloadInfo *dlInfo = new MythDownloadInfo;
471  QNetworkReply *reply = nullptr;
472 
473  dlInfo->m_url = url;
474  dlInfo->m_reload = reload;
475  dlInfo->m_syncMode = true;
476  dlInfo->m_processReply = false;
477 
478  if (downloadNow(dlInfo, false))
479  {
480  if (dlInfo->m_reply)
481  {
482  reply = dlInfo->m_reply;
483  // prevent dlInfo dtor from deleting the reply
484  dlInfo->m_reply = nullptr;
485 
486  delete dlInfo;
487 
488  return reply;
489  }
490 
491  delete dlInfo;
492  }
493 
494  return nullptr;
495 }
496 
502 bool MythDownloadManager::download(QNetworkRequest *req, QByteArray *data)
503 {
504  LOG(VB_FILE, LOG_DEBUG, LOC + QString("download('%1', '%2')")
505  .arg(req->url().toString()).arg((long long)data));
506  return processItem(req->url().toString(), req, QString(), data,
507  kRequestGet,
508  (QNetworkRequest::AlwaysNetwork == req->attribute(
509  QNetworkRequest::CacheLoadControlAttribute,
510  QNetworkRequest::PreferNetwork).toInt()));
511 }
512 
522 bool MythDownloadManager::downloadAuth(const QString &url, const QString &dest,
523  const bool reload, AuthCallback authCallbackFn, void *authArg,
524  const QHash<QByteArray, QByteArray> *headers)
525 {
526  return processItem(url, nullptr, dest, nullptr, kRequestGet, reload, authCallbackFn,
527  authArg, headers);
528 }
529 
530 
536 void MythDownloadManager::queuePost(const QString &url,
537  QByteArray *data,
538  QObject *caller)
539 {
540  LOG(VB_FILE, LOG_DEBUG, LOC + QString("queuePost('%1', '%2')")
541  .arg(url).arg((long long)data));
542 
543  if (!data)
544  {
545  LOG(VB_GENERAL, LOG_ERR, LOC + "queuePost(), data is NULL!");
546  return;
547  }
548 
549  queueItem(url, nullptr, QString(), data, caller, kRequestPost);
550 }
551 
557 void MythDownloadManager::queuePost(QNetworkRequest *req,
558  QByteArray *data,
559  QObject *caller)
560 {
561  LOG(VB_FILE, LOG_DEBUG, LOC + QString("queuePost('%1', '%2')")
562  .arg(req->url().toString()).arg((long long)data));
563 
564  if (!data)
565  {
566  LOG(VB_GENERAL, LOG_ERR, LOC + "queuePost(), data is NULL!");
567  return;
568  }
569 
570  queueItem(req->url().toString(), req, QString(), data, caller,
571  kRequestPost,
572  (QNetworkRequest::AlwaysNetwork == req->attribute(
573  QNetworkRequest::CacheLoadControlAttribute,
574  QNetworkRequest::PreferNetwork).toInt()));
575 
576 }
577 
583 bool MythDownloadManager::post(const QString &url, QByteArray *data)
584 {
585  LOG(VB_FILE, LOG_DEBUG, LOC + QString("post('%1', '%2')")
586  .arg(url).arg((long long)data));
587 
588  if (!data)
589  {
590  LOG(VB_GENERAL, LOG_ERR, LOC + "post(), data is NULL!");
591  return false;
592  }
593 
594  return processItem(url, nullptr, QString(), data, kRequestPost);
595 }
596 
602 bool MythDownloadManager::post(QNetworkRequest *req, QByteArray *data)
603 {
604  LOG(VB_FILE, LOG_DEBUG, LOC + QString("post('%1', '%2')")
605  .arg(req->url().toString()).arg((long long)data));
606 
607  if (!data)
608  {
609  LOG(VB_GENERAL, LOG_ERR, LOC + "post(), data is NULL!");
610  return false;
611  }
612 
613  return processItem(req->url().toString(), req, QString(), data,
614  kRequestPost,
615  (QNetworkRequest::AlwaysNetwork == req->attribute(
616  QNetworkRequest::CacheLoadControlAttribute,
617  QNetworkRequest::PreferNetwork).toInt()));
618 
619 }
620 
629 bool MythDownloadManager::postAuth(const QString &url, QByteArray *data,
630  AuthCallback authCallbackFn, void *authArg,
631  const QHash<QByteArray, QByteArray> *headers)
632 {
633  LOG(VB_FILE, LOG_DEBUG, LOC + QString("postAuth('%1', '%2')")
634  .arg(url).arg((long long)data));
635 
636  if (!data)
637  {
638  LOG(VB_GENERAL, LOG_ERR, LOC + "postAuth(), data is NULL!");
639  return false;
640  }
641 
642  return processItem(url, nullptr, nullptr, data, kRequestPost, false, authCallbackFn,
643  authArg, headers);
644 }
645 
650 {
651  RemoteFileDownloadThread *dlThread =
652  new RemoteFileDownloadThread(this, dlInfo);
653  MThreadPool::globalInstance()->start(dlThread, "RemoteFileDownload");
654 }
655 
660 {
661  if (!dlInfo)
662  return;
663 
664  static const char dateFormat[] = "ddd, dd MMM yyyy hh:mm:ss 'GMT'";
665  QUrl qurl(dlInfo->m_url);
666  QNetworkRequest request;
667 
668  if (dlInfo->m_request)
669  {
670  request = *dlInfo->m_request;
671  delete dlInfo->m_request;
672  dlInfo->m_request = nullptr;
673  }
674  else
675  request.setUrl(qurl);
676 
677  if (dlInfo->m_reload)
678  {
679  request.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
680  QNetworkRequest::AlwaysNetwork);
681  }
682  else
683  {
684  // Prefer the in-cache item if one exists and it is less than 5 minutes
685  // old and it will not expire in the next 10 seconds
686  QDateTime now = MythDate::current();
687 
688  // Handle redirects, we want the metadata of the file headers
689  QString redirectLoc;
690  int limit = 0;
691  while (!(redirectLoc = getHeader(qurl, "Location")).isNull())
692  {
693  if (limit == CACHE_REDIRECTION_LIMIT)
694  {
695  LOG(VB_GENERAL, LOG_WARNING, QString("Cache Redirection limit "
696  "reached for %1")
697  .arg(qurl.toString()));
698  return;
699  }
700  qurl.setUrl(redirectLoc);
701  limit++;
702  }
703 
704  LOG(VB_NETWORK, LOG_DEBUG, QString("Checking cache for %1")
705  .arg(qurl.toString()));
706 
707  m_infoLock->lock();
708  QNetworkCacheMetaData urlData = m_manager->cache()->metaData(qurl);
709  m_infoLock->unlock();
710  if ((urlData.isValid()) &&
711  ((!urlData.expirationDate().isValid()) ||
712  (urlData.expirationDate().toUTC().secsTo(now) < 10)))
713  {
714  QString dateString = getHeader(urlData, "Date");
715 
716  if (!dateString.isNull())
717  {
718  QDateTime loadDate =
719  MythDate::fromString(dateString, dateFormat);
720  loadDate.setTimeSpec(Qt::UTC);
721  if (loadDate.secsTo(now) <= 720)
722  {
723  dlInfo->m_preferCache = true;
724  LOG(VB_NETWORK, LOG_DEBUG, QString("Preferring cache for %1")
725  .arg(qurl.toString()));
726  }
727  }
728  }
729  }
730 
731  if (dlInfo->m_preferCache)
732  request.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
733  QNetworkRequest::PreferCache);
734 
735  if (!request.hasRawHeader("User-Agent"))
736  {
737  request.setRawHeader("User-Agent",
738  "MythTV v" MYTH_BINARY_VERSION
739  " MythDownloadManager");
740  }
741 
742  if (dlInfo->m_headers)
743  {
744  QHash<QByteArray, QByteArray>::const_iterator it =
745  dlInfo->m_headers->constBegin();
746  for ( ; it != dlInfo->m_headers->constEnd(); ++it )
747  {
748  if (!it.key().isEmpty() && !it.value().isEmpty())
749  {
750  request.setRawHeader(it.key(), it.value());
751  }
752  }
753  }
754 
755  switch (dlInfo->m_requestType)
756  {
757  case kRequestPost :
758  dlInfo->m_reply = m_manager->post(request, *dlInfo->m_data);
759  break;
760  case kRequestHead :
761  dlInfo->m_reply = m_manager->head(request);
762  break;
763  case kRequestGet :
764  default:
765  dlInfo->m_reply = m_manager->get(request);
766  break;
767  }
768 
769  m_downloadReplies[dlInfo->m_reply] = dlInfo;
770 
771  if (dlInfo->m_authCallback)
772  {
773  connect(m_manager, SIGNAL(authenticationRequired(QNetworkReply *,
774  QAuthenticator *)),
775  this, SLOT(authCallback(QNetworkReply *, QAuthenticator *)));
776  }
777 
778  connect(dlInfo->m_reply, SIGNAL(error(QNetworkReply::NetworkError)), this,
779  SLOT(downloadError(QNetworkReply::NetworkError)));
780  connect(dlInfo->m_reply, SIGNAL(downloadProgress(qint64, qint64)),
781  this, SLOT(downloadProgress(qint64, qint64)));
782 }
783 
788 void MythDownloadManager::authCallback(QNetworkReply *reply,
789  QAuthenticator *authenticator)
790 {
791  if (!reply)
792  return;
793 
794  MythDownloadInfo *dlInfo = m_downloadReplies[reply];
795 
796  if (!dlInfo)
797  return;
798 
799  if (dlInfo->m_authCallback)
800  {
801  LOG(VB_FILE, LOG_DEBUG, "Calling auth callback");
802  dlInfo->m_authCallback(reply, authenticator, dlInfo->m_authArg);
803  }
804 }
805 
813 {
814  if (!dlInfo)
815  return false;
816 
817  dlInfo->m_syncMode = true;
818 
819  // Special handling for link-local
820  // Not needed for Windows because windows does not need
821  // the scope id.
822 #ifndef _WIN32
823  if (dlInfo->m_url.startsWith("http://[fe80::",Qt::CaseInsensitive))
824  return downloadNowLinkLocal(dlInfo,deleteInfo);
825 #endif
826  m_infoLock->lock();
827  m_downloadQueue.push_back(dlInfo);
828  m_infoLock->unlock();
829  m_queueWaitCond.wakeAll();
830 
831  // timeout myth:// RemoteFile transfers 20 seconds from now
832  // timeout non-myth:// QNetworkAccessManager transfers 60 seconds after
833  // their last progress update
834  QDateTime startedAt = MythDate::current();
835  m_infoLock->lock();
836  while ((!dlInfo->IsDone()) &&
837  (dlInfo->m_errorCode == QNetworkReply::NoError) &&
838  (((!dlInfo->m_url.startsWith("myth://")) &&
839  (dlInfo->m_lastStat.secsTo(MythDate::current()) < 60)) ||
840  ((dlInfo->m_url.startsWith("myth://")) &&
841  (startedAt.secsTo(MythDate::current()) < 20))))
842  {
843  m_infoLock->unlock();
844  m_queueWaitLock.lock();
845  m_queueWaitCond.wait(&m_queueWaitLock, 200);
846  m_queueWaitLock.unlock();
847  m_infoLock->lock();
848  }
849  bool done = dlInfo->IsDone();
850  bool success =
851  done && (dlInfo->m_errorCode == QNetworkReply::NoError);
852 
853  if (!done)
854  {
855  dlInfo->m_data = nullptr; // Prevent downloadFinished() from updating
856  dlInfo->m_syncMode = false; // Let downloadFinished() cleanup for us
857  if ((dlInfo->m_reply) &&
858  (dlInfo->m_errorCode == QNetworkReply::NoError))
859  {
860  LOG(VB_FILE, LOG_DEBUG,
861  LOC + QString("Aborting download - lack of data transfer"));
862  dlInfo->m_reply->abort();
863  }
864  }
865  else if (deleteInfo)
866  delete dlInfo;
867 
868  m_infoLock->unlock();
869 
870  return success;
871 }
872 
873 #ifndef _WIN32
874 
894 bool MythDownloadManager::downloadNowLinkLocal(MythDownloadInfo *dlInfo, bool deleteInfo)
895 {
896  bool isOK = true;
897 
898  // Only certain features are supported here
899  if (dlInfo->m_authCallback || dlInfo->m_authArg)
900  {
901  LOG(VB_GENERAL, LOG_ERR, LOC +
902  QString("Unsupported authentication for %1").arg(dlInfo->m_url));
903  isOK = false;
904  }
905  if (!dlInfo->m_outFile.isEmpty())
906  {
907  LOG(VB_GENERAL, LOG_ERR, LOC +
908  QString("Unsupported File output %1 for %2")
909  .arg(dlInfo->m_outFile).arg(dlInfo->m_url));
910  isOK = false;
911  }
912 
913  if (!deleteInfo || dlInfo->m_requestType == kRequestHead)
914  {
915  // We do not have the ability to return a network reply in dlInfo
916  // so if we are asked to do that, return an error.
917  LOG(VB_GENERAL, LOG_ERR, LOC +
918  QString("Unsupported link-local operation %1")
919  .arg(dlInfo->m_url));
920  isOK = false;
921  }
922 
923  QUrl url(dlInfo->m_url);
924  QString host(url.host());
925  int port(url.port(80));
926  if (isOK && PortChecker::resolveLinkLocal(host, port))
927  {
928  QString reqType;
929  switch (dlInfo->m_requestType)
930  {
931  case kRequestPost :
932  reqType = "POST";
933  break;
934  case kRequestGet :
935  default:
936  reqType = "GET";
937  break;
938  }
939  QByteArray *aBuffer = dlInfo->m_data;
940  QHash<QByteArray, QByteArray> headers;
941  if (dlInfo->m_headers)
942  headers = *dlInfo->m_headers;
943  if (!headers.contains("User-Agent"))
944  headers.insert("User-Agent",
945  "MythDownloadManager v" MYTH_BINARY_VERSION);
946  headers.insert("Connection", "close");
947  headers.insert("Accept-Encoding", "identity");
948  if (aBuffer && !aBuffer->isEmpty())
949  headers.insert("Content-Length",
950  (QString::number(aBuffer->size())).toUtf8());
951  headers.insert("Host",
952  (url.host()+":"+QString::number(port)).toUtf8());
953 
954  QByteArray requestMessage;
955  QString path (url.path());
956  requestMessage.append("POST ");
957  requestMessage.append(path);
958  requestMessage.append(" HTTP/1.1\r\n");
959  QHashIterator<QByteArray, QByteArray> it(headers);
960  while (it.hasNext())
961  {
962  it.next();
963  requestMessage.append(it.key());
964  requestMessage.append(": ");
965  requestMessage.append(it.value());
966  requestMessage.append("\r\n");
967  }
968  requestMessage.append("\r\n");
969  if (aBuffer && !aBuffer->isEmpty())
970  {
971  requestMessage.append(*aBuffer);
972  }
973  QTcpSocket socket;
974  socket.connectToHost(host, port);
975  // QT Warning - this may not work on Windows
976  if (!socket.waitForConnected(5000))
977  isOK = false;
978  if (isOK)
979  isOK = (socket.write(requestMessage) > 0);
980  if (isOK)
981  // QT Warning - this may not work on Windows
982  isOK = socket.waitForDisconnected(5000);
983  if (isOK)
984  {
985  *aBuffer = socket.readAll();
986  // Find the start of the content
987  QByteArray delim("\r\n\r\n");
988  int delimLoc=aBuffer->indexOf(delim);
989  if (delimLoc > -1)
990  *aBuffer = aBuffer->right
991  (aBuffer->size()-delimLoc-4);
992  else
993  isOK=false;
994  }
995  socket.close();
996  }
997  else
998  isOK = false;
999 
1000  if (deleteInfo)
1001  delete dlInfo;
1002 
1003  if (isOK)
1004  return true;
1005  LOG(VB_GENERAL, LOG_ERR, LOC + QString("Link Local request failed: %1")
1006  .arg(url.toString()));
1007  return false;
1008 }
1009 #endif
1010 
1015 void MythDownloadManager::cancelDownload(const QString &url, bool block)
1016 {
1017  cancelDownload(QStringList(url), block);
1018 }
1019 
1024 void MythDownloadManager::cancelDownload(const QStringList &urls, bool block)
1025 {
1026  MythDownloadInfo *dlInfo;
1027 
1028  m_infoLock->lock();
1029  foreach (QString url, urls)
1030  {
1031  QMutableListIterator<MythDownloadInfo*> lit(m_downloadQueue);
1032  while (lit.hasNext())
1033  {
1034  lit.next();
1035  dlInfo = lit.value();
1036  if (dlInfo->m_url == url)
1037  {
1038  if (!m_cancellationQueue.contains(dlInfo))
1039  m_cancellationQueue.append(dlInfo);
1040  lit.remove();
1041  }
1042  }
1043 
1044  if (m_downloadInfos.contains(url))
1045  {
1046  dlInfo = m_downloadInfos[url];
1047 
1048  if (!m_cancellationQueue.contains(dlInfo))
1049  m_cancellationQueue.append(dlInfo);
1050 
1051  if (dlInfo->m_reply)
1052  m_downloadReplies.remove(dlInfo->m_reply);
1053 
1054  m_downloadInfos.remove(url);
1055  }
1056  }
1057  m_infoLock->unlock();
1058 
1059  if (QThread::currentThread() == this->thread())
1060  {
1061  downloadCanceled();
1062  return;
1063  }
1064 
1065  // wake-up running thread
1066  m_queueWaitCond.wakeAll();
1067 
1068  if (!block)
1069  return;
1070 
1071  while (!m_cancellationQueue.isEmpty())
1072  {
1073  usleep(50000); // re-test in another 50ms
1074  }
1075 }
1076 
1078 {
1079  QMutexLocker locker(m_infoLock);
1080  MythDownloadInfo *dlInfo;
1081 
1082  QMutableListIterator<MythDownloadInfo*> lit(m_cancellationQueue);
1083  while (lit.hasNext())
1084  {
1085  lit.next();
1086  dlInfo = lit.value();
1087  dlInfo->m_lock.lock();
1088 
1089  if (dlInfo->m_reply)
1090  {
1091  LOG(VB_FILE, LOG_DEBUG,
1092  LOC + QString("Aborting download - user request"));
1093  dlInfo->m_reply->abort();
1094  }
1095  lit.remove();
1096  if (dlInfo->m_done)
1097  {
1098  dlInfo->m_lock.unlock();
1099  continue;
1100  }
1101  dlInfo->m_errorCode = QNetworkReply::OperationCanceledError;
1102  dlInfo->m_done = true;
1103  dlInfo->m_lock.unlock();
1104  }
1105 }
1106 
1112 {
1113  QMutexLocker locker(m_infoLock);
1114  MythDownloadInfo *dlInfo;
1115 
1116  QList <MythDownloadInfo*>::iterator lit = m_downloadQueue.begin();
1117  for (; lit != m_downloadQueue.end(); ++lit)
1118  {
1119  dlInfo = *lit;
1120  if (dlInfo->m_caller == caller)
1121  {
1122  dlInfo->m_caller = nullptr;
1123  dlInfo->m_outFile = QString();
1124  dlInfo->m_data = nullptr;
1125  }
1126  }
1127 
1128  QMap <QString, MythDownloadInfo*>::iterator mit = m_downloadInfos.begin();
1129  for (; mit != m_downloadInfos.end(); ++mit)
1130  {
1131  dlInfo = mit.value();
1132  if (dlInfo->m_caller == caller)
1133  {
1134  dlInfo->m_caller = nullptr;
1135  dlInfo->m_outFile = QString();
1136  dlInfo->m_data = nullptr;
1137  }
1138  }
1139 }
1140 
1144 void MythDownloadManager::downloadError(QNetworkReply::NetworkError errorCode)
1145 {
1146  QNetworkReply *reply = (QNetworkReply*)sender();
1147 
1148  LOG(VB_FILE, LOG_DEBUG, LOC + QString("downloadError %1 ")
1149  .arg(errorCode) + reply->errorString() );
1150 
1151  QMutexLocker locker(m_infoLock);
1152  if (!m_downloadReplies.contains(reply))
1153  {
1154  reply->deleteLater();
1155  return;
1156  }
1157 
1158  MythDownloadInfo *dlInfo = m_downloadReplies[reply];
1159 
1160  if (!dlInfo)
1161  return;
1162 
1163  dlInfo->m_errorCode = errorCode;
1164 }
1165 
1171 QUrl MythDownloadManager::redirectUrl(const QUrl& possibleRedirectUrl,
1172  const QUrl& oldRedirectUrl) const
1173 {
1174  LOG(VB_FILE, LOG_DEBUG, LOC + QString("redirectUrl()"));
1175  QUrl redirectUrl;
1176 
1177  if(!possibleRedirectUrl.isEmpty() && possibleRedirectUrl != oldRedirectUrl)
1178  redirectUrl = possibleRedirectUrl;
1179 
1180  return redirectUrl;
1181 }
1182 
1186 void MythDownloadManager::downloadFinished(QNetworkReply* reply)
1187 {
1188  LOG(VB_FILE, LOG_DEBUG, LOC + QString("downloadFinished(%1)")
1189  .arg((long long)reply));
1190 
1191  QMutexLocker locker(m_infoLock);
1192  if (!m_downloadReplies.contains(reply))
1193  {
1194  reply->deleteLater();
1195  return;
1196  }
1197 
1198  MythDownloadInfo *dlInfo = m_downloadReplies[reply];
1199 
1200  if (!dlInfo || !dlInfo->m_reply)
1201  return;
1202 
1203  downloadFinished(dlInfo);
1204 }
1205 
1210 {
1211  if (!dlInfo)
1212  return;
1213 
1214  int statusCode = -1;
1215  static const char dateFormat[] = "ddd, dd MMM yyyy hh:mm:ss 'GMT'";
1216  QNetworkReply *reply = dlInfo->m_reply;
1217 
1218  if (reply)
1219  {
1220  QUrl possibleRedirectUrl =
1221  reply->attribute(QNetworkRequest::RedirectionTargetAttribute).toUrl();
1222 
1223  if (!possibleRedirectUrl.isEmpty() &&
1224  possibleRedirectUrl.isValid() &&
1225  possibleRedirectUrl.isRelative()) // Turn relative Url to absolute
1226  possibleRedirectUrl = reply->url().resolved(possibleRedirectUrl);
1227 
1228  dlInfo->m_redirectedTo =
1229  redirectUrl(possibleRedirectUrl, dlInfo->m_redirectedTo);
1230 
1231  QVariant status =
1232  reply->attribute(QNetworkRequest::HttpStatusCodeAttribute);
1233  if (status.isValid())
1234  statusCode = status.toInt();
1235  }
1236 
1237  if(reply && !dlInfo->m_redirectedTo.isEmpty() &&
1238  ((dlInfo->m_requestType != kRequestPost) ||
1239  (statusCode == 301 || statusCode == 302 ||
1240  statusCode == 303)))
1241  {
1242  LOG(VB_FILE, LOG_DEBUG, LOC +
1243  QString("downloadFinished(%1): Redirect: %2 -> %3")
1244  .arg((long long)dlInfo)
1245  .arg(reply->url().toString())
1246  .arg(dlInfo->m_redirectedTo.toString()));
1247 
1248  if (dlInfo->m_data)
1249  dlInfo->m_data->clear();
1250 
1251  dlInfo->m_bytesReceived = 0;
1252  dlInfo->m_bytesTotal = 0;
1253 
1254  QNetworkRequest request(dlInfo->m_redirectedTo);
1255 
1256  if (dlInfo->m_reload)
1257  {
1258  request.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
1259  QNetworkRequest::AlwaysNetwork);
1260  }
1261  else if (dlInfo->m_preferCache)
1262  {
1263  request.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
1264  QNetworkRequest::PreferCache);
1265  }
1266 
1267  request.setRawHeader("User-Agent",
1268  "MythDownloadManager v" MYTH_BINARY_VERSION);
1269 
1270  switch (dlInfo->m_requestType)
1271  {
1272  case kRequestHead :
1273  dlInfo->m_reply = m_manager->head(request);
1274  break;
1275  case kRequestGet :
1276  default:
1277  dlInfo->m_reply = m_manager->get(request);
1278  break;
1279  }
1280 
1281  m_downloadReplies[dlInfo->m_reply] = dlInfo;
1282 
1283  connect(dlInfo->m_reply, SIGNAL(error(QNetworkReply::NetworkError)),
1284  this, SLOT(downloadError(QNetworkReply::NetworkError)));
1285  connect(dlInfo->m_reply, SIGNAL(downloadProgress(qint64, qint64)),
1286  this, SLOT(downloadProgress(qint64, qint64)));
1287 
1288  m_downloadReplies.remove(reply);
1289  reply->deleteLater();
1290  }
1291  else
1292  {
1293  LOG(VB_FILE, LOG_DEBUG, QString("downloadFinished(%1): COMPLETE: %2")
1294  .arg((long long)dlInfo).arg(dlInfo->m_url));
1295 
1296  // HACK Insert a Date header into the cached metadata if one doesn't
1297  // already exist
1298  QUrl fileUrl = dlInfo->m_url;
1299  QString redirectLoc;
1300  int limit = 0;
1301  while (!(redirectLoc = getHeader(fileUrl, "Location")).isNull())
1302  {
1303  if (limit == CACHE_REDIRECTION_LIMIT)
1304  {
1305  LOG(VB_GENERAL, LOG_WARNING, QString("Cache Redirection limit "
1306  "reached for %1")
1307  .arg(fileUrl.toString()));
1308  return;
1309  }
1310  fileUrl.setUrl(redirectLoc);
1311  limit++;
1312  }
1313 
1314  m_infoLock->lock();
1315  QNetworkCacheMetaData urlData = m_manager->cache()->metaData(fileUrl);
1316  m_infoLock->unlock();
1317  if (getHeader(urlData, "Date").isNull())
1318  {
1319  QNetworkCacheMetaData::RawHeaderList headers = urlData.rawHeaders();
1320  QNetworkCacheMetaData::RawHeader newheader;
1321  QDateTime now = MythDate::current();
1322  newheader = QNetworkCacheMetaData::RawHeader("Date",
1323  now.toString(dateFormat).toLatin1());
1324  headers.append(newheader);
1325  urlData.setRawHeaders(headers);
1326  m_infoLock->lock();
1327  m_manager->cache()->updateMetaData(urlData);
1328  m_infoLock->unlock();
1329  }
1330  // End HACK
1331 
1332  dlInfo->m_redirectedTo.clear();
1333 
1334  int dataSize = -1;
1335 
1336  // If we downloaded via the QNetworkAccessManager
1337  // AND the caller isn't handling the reply directly
1338  if (reply && dlInfo->m_processReply)
1339  {
1340  bool append = (!dlInfo->m_syncMode && dlInfo->m_caller);
1341  QByteArray data = reply->readAll();
1342  dataSize = data.size();
1343 
1344  if (append)
1345  dlInfo->m_bytesReceived += dataSize;
1346  else
1347  dlInfo->m_bytesReceived = dataSize;
1348 
1349  dlInfo->m_bytesTotal = dlInfo->m_bytesReceived;
1350 
1351  if (dlInfo->m_data)
1352  {
1353  if (append)
1354  dlInfo->m_data->append(data);
1355  else
1356  *dlInfo->m_data = data;
1357  }
1358  else if (!dlInfo->m_outFile.isEmpty())
1359  {
1360  saveFile(dlInfo->m_outFile, data, append);
1361  }
1362  }
1363  else if (!reply) // If we downloaded via RemoteFile
1364  {
1365  if (dlInfo->m_data)
1366  {
1367  (*dlInfo->m_data) = dlInfo->m_privData;
1368  }
1369  else if (!dlInfo->m_outFile.isEmpty())
1370  {
1371  saveFile(dlInfo->m_outFile, dlInfo->m_privData);
1372  }
1373  dlInfo->m_bytesReceived += dataSize;
1374  dlInfo->m_bytesTotal = dlInfo->m_bytesReceived;
1375  }
1376  // else we downloaded via QNetworkAccessManager
1377  // AND the caller is handling the reply
1378 
1379  m_infoLock->lock();
1380  if (!m_downloadInfos.remove(dlInfo->m_url))
1381  LOG(VB_GENERAL, LOG_ERR, LOC +
1382  QString("ERROR download finished but failed to remove url: %1")
1383  .arg(dlInfo->m_url));
1384 
1385  if (reply)
1386  m_downloadReplies.remove(reply);
1387  m_infoLock->unlock();
1388 
1389  dlInfo->SetDone(true);
1390 
1391  if (!dlInfo->m_syncMode)
1392  {
1393  if (dlInfo->m_caller)
1394  {
1395  LOG(VB_FILE, LOG_DEBUG, QString("downloadFinished(%1): "
1396  "COMPLETE: %2, sending event to caller")
1397  .arg((long long)dlInfo).arg(dlInfo->m_url));
1398 
1399  QStringList args;
1400  args << dlInfo->m_url;
1401  args << dlInfo->m_outFile;
1402  args << QString::number(dlInfo->m_bytesTotal);
1403  // placeholder for error string
1404  args << (reply ? reply->errorString() : QString());
1405  args << QString::number((int)(reply ? reply->error() :
1406  dlInfo->m_errorCode));
1407 
1408  QCoreApplication::postEvent(dlInfo->m_caller,
1409  new MythEvent("DOWNLOAD_FILE FINISHED", args));
1410  }
1411 
1412  delete dlInfo;
1413  }
1414 
1415  m_queueWaitCond.wakeAll();
1416  }
1417 }
1418 
1424 void MythDownloadManager::downloadProgress(qint64 bytesReceived,
1425  qint64 bytesTotal)
1426 {
1427  QNetworkReply *reply = (QNetworkReply*)sender();
1428 
1429  LOG(VB_FILE, LOG_DEBUG, LOC +
1430  QString("downloadProgress(%1, %2) (for reply %3)")
1431  .arg(bytesReceived).arg(bytesTotal).arg((long long)reply));
1432 
1433  QMutexLocker locker(m_infoLock);
1434  if (!m_downloadReplies.contains(reply))
1435  return;
1436 
1437  MythDownloadInfo *dlInfo = m_downloadReplies[reply];
1438 
1439  if (!dlInfo)
1440  return;
1441 
1442  dlInfo->m_lastStat = MythDate::current();
1443 
1444  LOG(VB_FILE, LOG_DEBUG, LOC +
1445  QString("downloadProgress: %1 to %2 is at %3 of %4 bytes downloaded")
1446  .arg(dlInfo->m_url).arg(dlInfo->m_outFile)
1447  .arg(bytesReceived).arg(bytesTotal));
1448 
1449  if (!dlInfo->m_syncMode && dlInfo->m_caller)
1450  {
1451  LOG(VB_FILE, LOG_DEBUG, QString("downloadProgress(%1): "
1452  "sending event to caller")
1453  .arg(reply->url().toString()));
1454 
1455  bool appendToFile = (dlInfo->m_bytesReceived != 0);
1456  QByteArray data = reply->readAll();
1457  if (!dlInfo->m_outFile.isEmpty())
1458  saveFile(dlInfo->m_outFile, data, appendToFile);
1459 
1460  if (dlInfo->m_data)
1461  dlInfo->m_data->append(data);
1462 
1463  dlInfo->m_bytesReceived = bytesReceived;
1464  dlInfo->m_bytesTotal = bytesTotal;
1465 
1466  QStringList args;
1467  args << dlInfo->m_url;
1468  args << dlInfo->m_outFile;
1469  args << QString::number(bytesReceived);
1470  args << QString::number(bytesTotal);
1471 
1472  QCoreApplication::postEvent(dlInfo->m_caller,
1473  new MythEvent("DOWNLOAD_FILE UPDATE", args));
1474  }
1475 }
1476 
1484 bool MythDownloadManager::saveFile(const QString &outFile,
1485  const QByteArray &data,
1486  const bool append)
1487 {
1488  if (outFile.isEmpty() || !data.size())
1489  return false;
1490 
1491  QFile file(outFile);
1492  QFileInfo fileInfo(outFile);
1493  QDir qdir(fileInfo.absolutePath());
1494 
1495  if (!qdir.exists() && !qdir.mkpath(fileInfo.absolutePath()))
1496  {
1497  LOG(VB_GENERAL, LOG_ERR, QString("Failed to create: '%1'")
1498  .arg(fileInfo.absolutePath()));
1499  return false;
1500  }
1501 
1502  QIODevice::OpenMode mode = QIODevice::Unbuffered|QIODevice::WriteOnly;
1503  if (append)
1504  mode |= QIODevice::Append;
1505 
1506  if (!file.open(mode))
1507  {
1508  LOG(VB_GENERAL, LOG_ERR, QString("Failed to open: '%1'") .arg(outFile));
1509  return false;
1510  }
1511 
1512  off_t offset = 0;
1513  size_t remaining = data.size();
1514  uint failure_cnt = 0;
1515  while ((remaining > 0) && (failure_cnt < 5))
1516  {
1517  ssize_t written = file.write(data.data() + offset, remaining);
1518  if (written < 0)
1519  {
1520  failure_cnt++;
1521  usleep(50000);
1522  continue;
1523  }
1524 
1525  failure_cnt = 0;
1526  offset += written;
1527  remaining -= written;
1528  }
1529 
1530  return remaining <= 0;
1531 }
1532 
1537 QDateTime MythDownloadManager::GetLastModified(const QString &url)
1538 {
1539  // If the header has not expired and
1540  // the last modification date is less than 1 hours old or if
1541  // the cache object is less than 20 minutes old,
1542  // then use the cached header otherwise redownload the header
1543 
1544  static const char dateFormat[] = "ddd, dd MMM yyyy hh:mm:ss 'GMT'";
1545  LOG(VB_FILE, LOG_DEBUG, LOC + QString("GetLastModified('%1')").arg(url));
1546  QDateTime result;
1547 
1548  QDateTime now = MythDate::current();
1549 
1550  QUrl cacheUrl = QUrl(url);
1551 
1552  // Deal with redirects, we want the cached data for the final url
1553  QString redirectLoc;
1554  int limit = 0;
1555  while (!(redirectLoc = getHeader(cacheUrl, "Location")).isNull())
1556  {
1557  if (limit == CACHE_REDIRECTION_LIMIT)
1558  {
1559  LOG(VB_GENERAL, LOG_WARNING, QString("Cache Redirection limit "
1560  "reached for %1")
1561  .arg(cacheUrl.toString()));
1562  return result;
1563  }
1564  cacheUrl.setUrl(redirectLoc);
1565  limit++;
1566  }
1567 
1568  m_infoLock->lock();
1569  QNetworkCacheMetaData urlData = m_manager->cache()->metaData(cacheUrl);
1570  m_infoLock->unlock();
1571 
1572  if (urlData.isValid() &&
1573  ((!urlData.expirationDate().isValid()) ||
1574  (urlData.expirationDate().secsTo(now) < 0)))
1575  {
1576  if (urlData.lastModified().toUTC().secsTo(now) <= 3600) // 1 Hour
1577  {
1578  result = urlData.lastModified().toUTC();
1579  }
1580  else
1581  {
1582  QString date = getHeader(urlData, "Date");
1583  if (!date.isNull())
1584  {
1585  QDateTime loadDate =
1586  MythDate::fromString(date, dateFormat);
1587  loadDate.setTimeSpec(Qt::UTC);
1588  if (loadDate.secsTo(now) <= 1200) // 20 Minutes
1589  {
1590  result = urlData.lastModified().toUTC();
1591  }
1592  }
1593  }
1594  }
1595 
1596  if (!result.isValid())
1597  {
1598  MythDownloadInfo *dlInfo = new MythDownloadInfo;
1599  dlInfo->m_url = url;
1600  dlInfo->m_syncMode = true;
1601  // Head request, we only want to inspect the headers
1602  dlInfo->m_requestType = kRequestHead;
1603 
1604  if (downloadNow(dlInfo, false))
1605  {
1606  if (dlInfo->m_reply)
1607  {
1608  QVariant lastMod =
1609  dlInfo->m_reply->header(
1610  QNetworkRequest::LastModifiedHeader);
1611  if (lastMod.isValid())
1612  result = lastMod.toDateTime().toUTC();
1613  }
1614 
1615  // downloadNow() will set a flag to trigger downloadFinished()
1616  // to delete the dlInfo if the download times out
1617  delete dlInfo;
1618  }
1619  }
1620 
1621  LOG(VB_FILE, LOG_DEBUG, LOC + QString("GetLastModified('%1'): Result %2")
1622  .arg(url).arg(result.toString()));
1623 
1624  return result;
1625 }
1626 
1627 
1631 void MythDownloadManager::loadCookieJar(const QString &filename)
1632 {
1633  QMutexLocker locker(&m_cookieLock);
1634 
1635  MythCookieJar *jar = new MythCookieJar;
1636  jar->load(filename);
1637  m_manager->setCookieJar(jar);
1638 }
1639 
1643 void MythDownloadManager::saveCookieJar(const QString &filename)
1644 {
1645  QMutexLocker locker(&m_cookieLock);
1646 
1647  if (!m_manager->cookieJar())
1648  return;
1649 
1650  MythCookieJar *jar = static_cast<MythCookieJar *>(m_manager->cookieJar());
1651  jar->save(filename);
1652 }
1653 
1654 void MythDownloadManager::setCookieJar(QNetworkCookieJar *cookieJar)
1655 {
1656  QMutexLocker locker(&m_cookieLock);
1657  m_manager->setCookieJar(cookieJar);
1658 }
1659 
1663 QNetworkCookieJar *MythDownloadManager::copyCookieJar(void)
1664 {
1665  QMutexLocker locker(&m_cookieLock);
1666 
1667  if (!m_manager->cookieJar())
1668  return nullptr;
1669 
1670  MythCookieJar *inJar = static_cast<MythCookieJar *>(m_manager->cookieJar());
1671  MythCookieJar *outJar = new MythCookieJar;
1672  outJar->copyAllCookies(*inJar);
1673 
1674  return static_cast<QNetworkCookieJar *>(outJar);
1675 }
1676 
1680 void MythDownloadManager::refreshCookieJar(QNetworkCookieJar *jar)
1681 {
1682  QMutexLocker locker(&m_cookieLock);
1683  delete m_inCookieJar;
1684 
1685  MythCookieJar *inJar = static_cast<MythCookieJar *>(jar);
1686  MythCookieJar *outJar = new MythCookieJar;
1687  outJar->copyAllCookies(*inJar);
1688  m_inCookieJar = static_cast<QNetworkCookieJar *>(outJar);
1689 
1690  QMutexLocker locker2(&m_queueWaitLock);
1691  m_queueWaitCond.wakeAll();
1692 }
1693 
1697 {
1698  QMutexLocker locker(&m_cookieLock);
1699 
1700  MythCookieJar *inJar = static_cast<MythCookieJar *>(m_inCookieJar);
1701  MythCookieJar *outJar = new MythCookieJar;
1702  outJar->copyAllCookies(*inJar);
1703  m_manager->setCookieJar(static_cast<QNetworkCookieJar *>(outJar));
1704 
1705  delete m_inCookieJar;
1706  m_inCookieJar = nullptr;
1707 }
1708 
1709 QString MythDownloadManager::getHeader(const QUrl& url, const QString& header)
1710 {
1711  if (!m_manager || !m_manager->cache())
1712  return QString();
1713 
1714  m_infoLock->lock();
1715  QNetworkCacheMetaData metadata = m_manager->cache()->metaData(url);
1716  m_infoLock->unlock();
1717 
1718  return getHeader(metadata, header);
1719 }
1720 
1726 QString MythDownloadManager::getHeader(const QNetworkCacheMetaData &cacheData,
1727  const QString& header)
1728 {
1729  QNetworkCacheMetaData::RawHeaderList headers = cacheData.rawHeaders();
1730 
1731  for (auto it = headers.begin(); it != headers.end(); ++it)
1732  if (QString((*it).first) == header)
1733  return QString((*it).second);
1734  return QString();
1735 }
1736 
1737 
1742 {
1743  const QList<QNetworkCookie> cookieList = old.allCookies();
1744  setAllCookies(cookieList);
1745 }
1746 
1750 void MythCookieJar::load(const QString &filename)
1751 {
1752  LOG(VB_GENERAL, LOG_DEBUG, QString("MythCookieJar: loading cookies from: %1").arg(filename));
1753 
1754  QFile f(filename);
1755  if (!f.open(QIODevice::ReadOnly))
1756  {
1757  LOG(VB_GENERAL, LOG_WARNING, QString("MythCookieJar::load() failed to open file for reading: %1").arg(filename));
1758  return;
1759  }
1760 
1761  QList<QNetworkCookie> cookieList;
1762  QTextStream stream(&f);
1763  while (!stream.atEnd())
1764  {
1765  QString cookie = stream.readLine();
1766  cookieList << QNetworkCookie::parseCookies(cookie.toLocal8Bit());
1767  }
1768 
1769  setAllCookies(cookieList);
1770 }
1771 
1775 void MythCookieJar::save(const QString &filename)
1776 {
1777  LOG(VB_GENERAL, LOG_DEBUG, QString("MythCookieJar: saving cookies to: %1").arg(filename));
1778 
1779  QFile f(filename);
1780  if (!f.open(QIODevice::WriteOnly))
1781  {
1782  LOG(VB_GENERAL, LOG_ERR, QString("MythCookieJar::save() failed to open file for writing: %1").arg(filename));
1783  return;
1784  }
1785 
1786  QList<QNetworkCookie> cookieList = allCookies();
1787  QTextStream stream(&f);
1788 
1789  for (QList<QNetworkCookie>::iterator it = cookieList.begin();
1790  it != cookieList.end(); ++it)
1791  {
1792  stream << (*it).toRawForm() << endl;
1793  }
1794 }
1795 
1796 
1797 /* vim: set expandtab tabstop=4 shiftwidth=4: */
QNetworkCookieJar * copyCookieJar(void)
Copy from one cookie jar to another.
QNetworkReply::NetworkError m_errorCode
void start(QThread::Priority=QThread::InheritPriority)
Tell MThread to start running the thread in the near future.
Definition: mthread.cpp:294
void save(const QString &filename)
Saves the cookie jar to a cookie file.
MRequestType
void queueItem(const QString &url, QNetworkRequest *req, const QString &dest, QByteArray *data, QObject *caller, const MRequestType reqType=kRequestGet, const bool reload=false)
Adds a request to the download queue.
MythDownloadManager * downloadManager
void removeListener(QObject *caller)
Disconnects the specified caller from any existing MythDownloadInfo instances.
void queueDownload(const QString &url, const QString &dest, QObject *caller, const bool reload=false)
Adds a url to the download queue.
~MythDownloadManager()
Destructor for MythDownloadManager.
static void error(const char *str,...)
Definition: vbi.c:42
void ShutdownMythDownloadManager(void)
Deletes the running MythDownloadManager at program exit.
#define CACHE_REDIRECTION_LIMIT
A subclassed QNetworkCookieJar that allows for reading and writing cookie files that contain raw form...
void downloadQNetworkRequest(MythDownloadInfo *dlInfo)
Downloads a QNetworkRequest via the QNetworkAccessManager.
AuthCallback m_authCallback
bool saveFile(const QString &outFile, const QByteArray &data, const bool append=false)
Saves a QByteArray of data to a given filename.
QNetworkReply * m_reply
unsigned int uint
Definition: compat.h:140
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
void downloadProgress(qint64 bytesReceived, qint64 bytesTotal)
Slot to process download update events.
const QHash< QByteArray, QByteArray > * m_headers
bool postAuth(const QString &url, QByteArray *data, AuthCallback authCallback, void *authArg, const QHash< QByteArray, QByteArray > *headers=nullptr)
Posts data to a url via the QNetworkAccessManager.
MRequestType m_requestType
void run(void) override
Runs a loop to process incoming download requests and triggers download events to be processed.
void downloadFinished(QNetworkReply *reply)
Slot to process download finished events.
#define off_t
static bool resolveLinkLocal(QString &host, int port, int timeLimit=30000)
Convenience method to resolve link-local address.
QString GetConfDir(void)
Definition: mythdirs.cpp:224
void authCallback(QNetworkReply *reply, QAuthenticator *authenticator)
Signal handler for authentication requests.
This class is used as a container for messages.
Definition: mythevent.h:16
QUrl redirectUrl(const QUrl &possibleRedirectUrl, const QUrl &oldRedirectUrl) const
Checks whether we were redirected to the given URL.
void setCookieJar(QNetworkCookieJar *cookieJar)
static QString downloadRemoteFile(const QString &cmd, const QString &url, const QString &storageGroup, const QString &filename)
void copyAllCookies(MythCookieJar &old)
Copies all cookies from one MythCookieJar to another.
RemoteFileDownloadThread(MythDownloadManager *parent, MythDownloadInfo *dlInfo)
QDateTime current(bool stripped)
Returns current Date and Time in UTC.
Definition: mythdate.cpp:10
void queuePost(const QString &url, QByteArray *data, QObject *caller)
Queues a post to a URL via the QNetworkAccessManager.
void refreshCookieJar(QNetworkCookieJar *jar)
Refresh the temporary cookie jar from another cookie jar.
MythDownloadManager * GetMythDownloadManager(void)
Gets the pointer to the MythDownloadManager singleton.
bool download(const QString &url, const QString &dest, const bool reload=false)
Downloads a URL to a file in blocking mode.
QDateTime GetLastModified(const QString &url)
Gets the Last Modified timestamp for a URI.
void downloadError(QNetworkReply::NetworkError errorCode)
Slot to process download error events.
#define LOC
void(* AuthCallback)(QNetworkReply *, QAuthenticator *, void *)
QString getHeader(const QUrl &url, const QString &header)
static MThreadPool * globalInstance(void)
bool SaveAs(QByteArray &data)
void downloadRemoteFile(MythDownloadInfo *dlInfo)
Triggers a myth:// URI download in the background via RemoteFile.
void SetDone(bool done)
#define LOG(_MASK_, _LEVEL_, _STRING_)
Definition: mythlogging.h:41
void start(QRunnable *runnable, const QString &debugName, int priority=0)
QNetworkRequest * m_request
bool downloadAuth(const QString &url, const QString &dest, const bool reload=false, AuthCallback authCallback=nullptr, void *authArg=nullptr, const QHash< QByteArray, QByteArray > *headers=nullptr)
Downloads a URL to a file in blocking mode.
void updateCookieJar(void)
Update the cookie jar from the temporary cookie jar.
bool downloadNow(MythDownloadInfo *dlInfo, bool deleteInfo=true)
Download helper for download() blocking methods.
#define MYTH_BINARY_VERSION
Update this whenever the plug-in ABI changes.
Definition: mythversion.h:16
void cancelDownload(const QString &url, bool block=true)
Cancel a queued or current download.
bool post(const QString &url, QByteArray *data)
Posts data to a url via the QNetworkAccessManager.
bool processItem(const QString &url, QNetworkRequest *req, const QString &dest, QByteArray *data, const MRequestType reqType=kRequestGet, const bool reload=false, AuthCallback authCallback=nullptr, void *authArg=nullptr, const QHash< QByteArray, QByteArray > *headers=nullptr)
Processes a network request immediately and waits for a response.
QThread * getQueueThread(void)
QMutex dmCreateLock
QString GetHostName(void)
void saveCookieJar(const QString &filename)
Saves the cookie jar to a cookie file.
QDateTime fromString(const QString &dtstr)
Converts kFilename && kISODate formats to QDateTime.
Definition: mythdate.cpp:30
void loadCookieJar(const QString &filename)
Loads the cookie jar from a cookie file.
void load(const QString &filename)
Loads the cookie jar from a cookie file.
void preCache(const QString &url)
Downloads a URL but doesn't store the resulting data anywhere.