MythTV  master
metadataimagedownload.cpp
Go to the documentation of this file.
1 // qt
2 #include <QCoreApplication>
3 #include <QDir>
4 #include <QEvent>
5 #include <QFileInfo>
6 #include <QImage>
7 #include <utility>
8 
9 // myth
10 #include "mythcorecontext.h"
11 #include "mythuihelper.h"
12 #include "mythdirs.h"
13 #include "storagegroup.h"
14 #include "metadataimagedownload.h"
15 #include "remotefile.h"
16 #include "mythdownloadmanager.h"
17 #include "mythlogging.h"
18 #include "mythdate.h"
19 
20 QEvent::Type ImageDLEvent::kEventType =
21  (QEvent::Type) QEvent::registerEventType();
22 
24  (QEvent::Type) QEvent::registerEventType();
25 
26 QEvent::Type ThumbnailDLEvent::kEventType =
27  (QEvent::Type) QEvent::registerEventType();
28 
30 {
31  cancel();
32  wait();
33 }
34 
36  QString url, QVariant data)
37 {
38  QMutexLocker lock(&m_mutex);
39 
40  auto *id = new ThumbnailData();
41  id->title = std::move(title);
42  id->data = std::move(data);
43  id->url = std::move(url);
44  m_thumbnailList.append(id);
45  if (!isRunning())
46  start();
47 }
48 
54 {
55  QMutexLocker lock(&m_mutex);
56 
57  m_downloadList.append(lookup);
58  lookup->DecrRef();
59  if (!isRunning())
60  start();
61 }
62 
64 {
65  QMutexLocker lock(&m_mutex);
66 
67  qDeleteAll(m_thumbnailList);
68  m_thumbnailList.clear();
69  // clearing m_downloadList automatically delete all its content
70  m_downloadList.clear();
71 }
72 
74 {
75  RunProlog();
76 
77  // Always handle thumbnails first, they're higher priority.
78  ThumbnailData *thumb = nullptr;
79  while ((thumb = moreThumbs()) != nullptr)
80  {
81  QString sFilename = getDownloadFilename(thumb->title, thumb->url);
82 
83  bool exists = QFile::exists(sFilename);
84  if (!exists && !thumb->url.isEmpty())
85  {
86  if (!GetMythDownloadManager()->download(thumb->url, sFilename))
87  {
88  LOG(VB_GENERAL, LOG_ERR,
89  QString("MetadataImageDownload: failed to download thumbnail from: %1")
90  .arg(thumb->url));
91 
92  delete thumb;
93  continue;
94  }
95  }
96 
97  // inform parent we have thumbnail ready for it
98  if (QFile::exists(sFilename) && m_parent)
99  {
100  LOG(VB_GENERAL, LOG_DEBUG,
101  QString("Threaded Image Thumbnail Download: %1")
102  .arg(sFilename));
103  thumb->url = sFilename;
104  QCoreApplication::postEvent(m_parent,
105  new ThumbnailDLEvent(thumb));
106  }
107  else
108  delete thumb;
109  }
110 
111  while (true)
112  {
113  m_mutex.lock();
114  if (m_downloadList.isEmpty())
115  {
116  // no more to process, we're done
117  m_mutex.unlock();
118  break;
119  }
120  // Ref owns the MetadataLookup object for the duration of the loop
121  // and it will be deleted automatically when the loop completes
123  m_mutex.unlock();
124  MetadataLookup *lookup = ref;
125  DownloadMap downloads = lookup->GetDownloads();
126  DownloadMap downloaded;
127 
128  bool errored = false;
129  for (DownloadMap::iterator i = downloads.begin();
130  i != downloads.end(); ++i)
131  {
132  VideoArtworkType type = i.key();
133  ArtworkInfo info = i.value();
134  QString filename = getDownloadFilename( type, lookup,
135  info.url );
136  if (lookup->GetHost().isEmpty())
137  {
138  QString path = getLocalWritePath(lookup->GetType(), type);
139  QDir dirPath(path);
140  if (!dirPath.exists())
141  {
142  if (!dirPath.mkpath(path))
143  {
144  LOG(VB_GENERAL, LOG_ERR,
145  QString("Metadata Image Download: Unable to create "
146  "path %1, aborting download.").arg(path));
147  errored = true;
148  break;
149  }
150  }
151  QString finalfile = path + "/" + filename;
152  QString oldurl = info.url;
153  info.url = finalfile;
154  if (!QFile::exists(finalfile) || lookup->GetAllowOverwrites())
155  {
156  QFile dest_file(finalfile);
157  if (dest_file.exists())
158  {
159  QFileInfo fi(finalfile);
160  GetMythUI()->RemoveFromCacheByFile(fi.fileName());
161  dest_file.remove();
162  }
163 
164  LOG(VB_GENERAL, LOG_INFO,
165  QString("Metadata Image Download: %1 ->%2")
166  .arg(oldurl).arg(finalfile));
167  QByteArray download;
168  GetMythDownloadManager()->download(oldurl, &download);
169 
170  QImage testImage;
171  bool didLoad = testImage.loadFromData(download);
172  if (!didLoad)
173  {
174  LOG(VB_GENERAL, LOG_ERR,
175  QString("Tried to write %1, but it appears to be "
176  "an HTML redirect (filesize %2).")
177  .arg(oldurl).arg(download.size()));
178  errored = true;
179  break;
180  }
181 
182  if (dest_file.open(QIODevice::WriteOnly))
183  {
184  off_t size = dest_file.write(download,
185  download.size());
186  dest_file.close();
187  if (size != download.size())
188  {
189  // File creation failed for some reason, delete it
190  RemoteFile::DeleteFile(finalfile);
191  LOG(VB_GENERAL, LOG_ERR,
192  QString("Image Download: Error Writing Image "
193  "to file: %1").arg(finalfile));
194  errored = true;
195  break;
196  }
197  }
198  }
199  }
200  else
201  {
202  QString path = getStorageGroupURL(type, lookup->GetHost());
203  QString finalfile = path + filename;
204  QString oldurl = info.url;
205  info.url = finalfile;
206  bool exists = false;
207  bool onMaster = false;
208  QString resolvedFN;
209  if (gCoreContext->IsMasterBackend() &&
210  gCoreContext->IsThisHost(lookup->GetHost()))
211  {
213  resolvedFN = sg.FindFile(filename);
214  exists = !resolvedFN.isEmpty() && QFile::exists(resolvedFN);
215  if (!exists)
216  {
217  resolvedFN = getLocalStorageGroupPath(type,
218  lookup->GetHost()) + "/" + filename;
219  }
220  onMaster = true;
221  }
222  else
223  exists = RemoteFile::Exists(finalfile);
224 
225  if (!exists || lookup->GetAllowOverwrites())
226  {
227  if (exists && !onMaster)
228  {
229  QFileInfo fi(finalfile);
230  GetMythUI()->RemoveFromCacheByFile(fi.fileName());
231  RemoteFile::DeleteFile(finalfile);
232  }
233  else if (exists)
234  QFile::remove(resolvedFN);
235 
236  LOG(VB_GENERAL, LOG_INFO,
237  QString("Metadata Image Download: %1 -> %2")
238  .arg(oldurl).arg(finalfile));
239  QByteArray download;
240  GetMythDownloadManager()->download(oldurl, &download);
241 
242  QImage testImage;
243  bool didLoad = testImage.loadFromData(download);
244  if (!didLoad)
245  {
246  LOG(VB_GENERAL, LOG_ERR,
247  QString("Tried to write %1, but it appears to be "
248  "an HTML redirect or corrupt file "
249  "(filesize %2).")
250  .arg(oldurl).arg(download.size()));
251  errored = true;
252  break;
253  }
254 
255  if (!onMaster)
256  {
257  RemoteFile outFile(finalfile, true);
258 
259  if (!outFile.isOpen())
260  {
261  LOG(VB_GENERAL, LOG_ERR,
262  QString("Image Download: Failed to open "
263  "remote file (%1) for write. Does "
264  "Storage Group Exist?")
265  .arg(finalfile));
266  errored = true;
267  break;
268  }
269  off_t written = outFile.Write(download,
270  download.size());
271  if (written != download.size())
272  {
273  // File creation failed for some reason, delete it
274  RemoteFile::DeleteFile(finalfile);
275 
276  LOG(VB_GENERAL, LOG_ERR,
277  QString("Image Download: Error Writing Image "
278  "to file: %1").arg(finalfile));
279  errored = true;
280  break;
281  }
282  }
283  else
284  {
285  QFile dest_file(resolvedFN);
286  if (dest_file.open(QIODevice::WriteOnly))
287  {
288  off_t size = dest_file.write(download,
289  download.size());
290  dest_file.close();
291  if (size != download.size())
292  {
293  // File creation failed for some reason, delete it
294  RemoteFile::DeleteFile(resolvedFN);
295  LOG(VB_GENERAL, LOG_ERR,
296  QString("Image Download: Error Writing Image "
297  "to file: %1").arg(finalfile));
298  errored = true;
299  break;
300  }
301  }
302  }
303  }
304  }
305  if (!errored)
306  {
307  // update future Artwork Map with what we've successfully
308  // retrieved (either downloaded or already existing
309  downloaded.insert(type, info);
310  }
311  }
312  if (errored)
313  {
314  QCoreApplication::postEvent(m_parent,
315  new ImageDLFailureEvent(lookup));
316  }
317  lookup->SetDownloads(downloaded);
318  QCoreApplication::postEvent(m_parent, new ImageDLEvent(lookup));
319  }
320 
321  RunEpilog();
322 }
323 
325 {
326  QMutexLocker lock(&m_mutex);
327  ThumbnailData *ret = nullptr;
328 
329  if (!m_thumbnailList.isEmpty())
330  ret = m_thumbnailList.takeFirst();
331  return ret;
332 }
333 
334 QString getDownloadFilename(const QString& title, const QString& url)
335 {
336  QString fileprefix = GetConfDir();
337 
338  QDir dir(fileprefix);
339  if (!dir.exists())
340  dir.mkdir(fileprefix);
341 
342  fileprefix += "/cache/metadata-thumbcache";
343 
344  dir = QDir(fileprefix);
345  if (!dir.exists())
346  dir.mkdir(fileprefix);
347 
348  QByteArray titlearr(title.toLatin1());
349  quint16 titleChecksum = qChecksum(titlearr.data(), titlearr.length());
350  QByteArray urlarr(url.toLatin1());
351  quint16 urlChecksum = qChecksum(urlarr.data(), urlarr.length());
352  QUrl qurl(url);
353  QString ext = QFileInfo(qurl.path()).suffix();
354  QString basefilename = QString("thumbnail_%1_%2.%3")
355  .arg(QString::number(urlChecksum))
356  .arg(QString::number(titleChecksum)).arg(ext);
357 
358  QString outputfile = QString("%1/%2").arg(fileprefix).arg(basefilename);
359 
360  return outputfile;
361 }
362 
364  const QString& url)
365 {
366  QString basefilename;
367  QString title;
368  QString inter;
369  uint tracknum = lookup->GetTrackNumber();
370  uint season = lookup->GetSeason();
371  uint episode = lookup->GetEpisode();
372  QString system = lookup->GetSystem();
373  if (!lookup->GetIsCollection() && (season > 0 || episode > 0))
374  {
375  title = lookup->GetTitle();
376  if (title.contains("/"))
377  title.replace("/", "-");
378  if (title.contains("?"))
379  title.replace("?", "");
380  if (title.contains("*"))
381  title.replace("*", "");
382  inter = QString(" Season %1").arg(QString::number(season));
383  if (type == kArtworkScreenshot)
384  inter += QString("x%1").arg(QString::number(episode));
385  }
386  else if (lookup->GetType() == kMetadataVideo ||
387  lookup->GetType() == kMetadataRecording)
388  title = lookup->GetInetref();
389  else if (lookup->GetType() == kMetadataGame)
390  title = QString("%1 (%2)").arg(lookup->GetTitle())
391  .arg(lookup->GetSystem());
392 
393  if (tracknum > 0)
394  inter = QString(" Track %1").arg(QString::number(tracknum));
395  else if (!system.isEmpty())
396  inter = QString(" (%1)").arg(system);
397 
398  QString suffix;
399  QUrl qurl(url);
400  QString ext = QFileInfo(qurl.path()).suffix();
401 
402  if (type == kArtworkCoverart)
403  suffix = "_coverart";
404  else if (type == kArtworkFanart)
405  suffix = "_fanart";
406  else if (type == kArtworkBanner)
407  suffix = "_banner";
408  else if (type == kArtworkScreenshot)
409  suffix = "_screenshot";
410  else if (type == kArtworkPoster)
411  suffix = "_poster";
412  else if (type == kArtworkBackCover)
413  suffix = "_backcover";
414  else if (type == kArtworkInsideCover)
415  suffix = "_insidecover";
416  else if (type == kArtworkCDImage)
417  suffix = "_cdimage";
418 
419  basefilename = title + inter + suffix + "." + ext;
420 
421  return basefilename;
422 }
423 
425 {
426  QString ret;
427 
428  if (metadatatype == kMetadataVideo)
429  {
430  if (type == kArtworkCoverart)
431  ret = gCoreContext->GetSetting("VideoArtworkDir");
432  else if (type == kArtworkFanart)
433  ret = gCoreContext->GetSetting("mythvideo.fanartDir");
434  else if (type == kArtworkBanner)
435  ret = gCoreContext->GetSetting("mythvideo.bannerDir");
436  else if (type == kArtworkScreenshot)
437  ret = gCoreContext->GetSetting("mythvideo.screenshotDir");
438  }
439  else if (metadatatype == kMetadataMusic)
440  {
441  }
442  else if (metadatatype == kMetadataGame)
443  {
444  if (type == kArtworkCoverart)
445  ret = gCoreContext->GetSetting("mythgame.boxartdir");
446  else if (type == kArtworkFanart)
447  ret = gCoreContext->GetSetting("mythgame.fanartdir");
448  else if (type == kArtworkScreenshot)
449  ret = gCoreContext->GetSetting("mythgame.screenshotdir");
450  }
451 
452  return ret;
453 }
454 
455 QString getStorageGroupURL(VideoArtworkType type, const QString& host)
456 {
457  QString sgroup = getStorageGroupName(type);
458  uint port = gCoreContext->GetBackendServerPort(host);
459 
460  return MythCoreContext::GenMythURL(host, port, "", sgroup);
461 }
462 
463 QString getLocalStorageGroupPath(VideoArtworkType type, const QString& host)
464 {
465  QString path;
466 
468 
469  path = sg.FindNextDirMostFree();
470 
471  return path;
472 }
473 
475 {
476  switch (type)
477  {
478  case kArtworkCoverart:
479  return "Coverart";
480  case kArtworkFanart:
481  return "Fanart";
482  case kArtworkBanner:
483  return "Banners";
484  case kArtworkScreenshot:
485  return "Screenshots";
486  default:
487  return "Default";
488  }
489 }
490 
492 {
493  QString cache = QString("%1/cache/metadata-thumbcache")
494  .arg(GetConfDir());
495  QDir cacheDir(cache);
496  QStringList thumbs = cacheDir.entryList(QDir::Files);
497 
498  for (QStringList::const_iterator i = thumbs.end() - 1;
499  i != thumbs.begin() - 1; --i)
500  {
501  QString filename = QString("%1/%2").arg(cache).arg(*i);
502  QFileInfo fi(filename);
503  QDateTime lastmod = fi.lastModified();
504  if (lastmod.addDays(2) < MythDate::current())
505  {
506  LOG(VB_GENERAL, LOG_DEBUG, QString("Deleting file %1")
507  .arg(filename));
508  QFile::remove(filename);
509  }
510  }
511 }
512 
void RunEpilog(void)
Cleans up a thread's resources, call this if you reimplement run().
Definition: mthread.cpp:215
void start(QThread::Priority=QThread::InheritPriority)
Tell MThread to start running the thread in the near future.
Definition: mthread.cpp:294
static bool DeleteFile(const QString &url)
Definition: remotefile.cpp:418
MetadataLookupList m_downloadList
MetadataType
QList< ThumbnailData * > m_thumbnailList
bool GetIsCollection() const
bool wait(unsigned long time=ULONG_MAX)
Wait for the MThread to exit, with a maximum timeout.
Definition: mthread.cpp:311
VideoArtworkType
QString GetSystem() const
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
void cleanThumbnailCacheDir()
QString getStorageGroupName(VideoArtworkType type)
RefCountHandler< T > takeFirstAndDecr(void)
Removes the first item in the list and returns it.
uint GetTrackNumber() const
#define off_t
QString getLocalWritePath(MetadataType metadatatype, VideoArtworkType type)
bool IsThisHost(const QString &addr)
is this address mapped to this host
int GetBackendServerPort(void)
Returns the locally defined backend control port.
MetadataType GetType() const
uint GetSeason() const
QString GetTitle() const
QString GetConfDir(void)
Definition: mythdirs.cpp:224
uint GetEpisode() const
QString getStorageGroupURL(VideoArtworkType type, const QString &host)
static QString GenMythURL(const QString &host=QString(), int port=0, QString path=QString(), const QString &storageGroup=QString())
QDateTime current(bool stripped)
Returns current Date and Time in UTC.
Definition: mythdate.cpp:10
QString GetSetting(const QString &key, const QString &defaultval="")
void addDownloads(MetadataLookup *lookup)
addLookup: Add lookup to bottom of the queue MetadataDownload::m_downloadList takes ownership of the ...
virtual int DecrRef(void)
Decrements reference count and deletes on 0.
QString FindNextDirMostFree(void)
bool isRunning(void) const
Definition: mthread.cpp:274
MythDownloadManager * GetMythDownloadManager(void)
Gets the pointer to the MythDownloadManager singleton.
bool download(const QString &url, const QString &dest, const bool reload=false)
Downloads a URL to a file in blocking mode.
bool GetAllowOverwrites() const
unsigned int uint
Definition: compat.h:140
void SetDownloads(ArtworkMap map)
QString GetInetref() const
MythUIHelper * GetMythUI()
static bool Exists(const QString &url, struct stat *fileinfo)
Definition: remotefile.cpp:461
#define LOG(_MASK_, _LEVEL_, _STRING_)
Definition: mythlogging.h:41
QMap< VideoArtworkType, ArtworkInfo > DownloadMap
static Type kEventType
bool isOpen(void) const
Definition: remotefile.cpp:246
QString getLocalStorageGroupPath(VideoArtworkType type, const QString &host)
QString FindFile(const QString &filename)
void addThumb(QString title, QString url, QVariant data)
void RunProlog(void)
Sets up a thread, call this if you reimplement run().
Definition: mthread.cpp:202
void run() override
Runs the Qt event loop unless we have a QRunnable, in which case we run the runnable run instead.
DownloadMap GetDownloads() const
bool IsMasterBackend(void)
is this the actual MBE process
QString GetHost() const
int Write(const void *data, int size)
Definition: remotefile.cpp:834
QString getDownloadFilename(const QString &title, const QString &url)
void RemoveFromCacheByFile(const QString &fname)