MythTV  master
videoscan.cpp
Go to the documentation of this file.
1 
2 #include "videoscan.h"
3 
4 #include <QApplication>
5 #include <QImageReader>
6 #include <QUrl>
7 #include <utility>
8 
9 // libmythbase
10 #include "mythevent.h"
11 #include "mythlogging.h"
12 #include "mythdate.h"
13 
14 // libmyth
15 #include "mythcontext.h"
16 #include "remoteutil.h"
17 
18 // libmythui
19 #include "mythscreenstack.h"
20 #include "mythprogressdialog.h"
21 #include "mythdialogbox.h"
22 #include "mythmainwindow.h"
23 
24 // libmythmetadata
26 #include "videoutils.h"
27 #include "globals.h"
28 #include "dbaccess.h"
29 #include "dirscan.h"
30 
31 QEvent::Type VideoScanChanges::kEventType =
32  (QEvent::Type) QEvent::registerEventType();
33 
34 namespace
35 {
36  template <typename DirListType>
37  class dirhandler : public DirectoryHandler
38  {
39  public:
40  dirhandler(DirListType &video_files,
41  const QStringList &image_extensions) :
42  m_videoFiles(video_files)
43  {
44  foreach (const auto & ext, image_extensions)
45  m_imageExt.insert(ext.toLower());
46  }
47 
48  DirectoryHandler *newDir(const QString &dir_name,
49  const QString &fq_dir_name) override // DirectoryHandler
50  {
51  (void) dir_name;
52  (void) fq_dir_name;
53  return this;
54  }
55 
56  void handleFile(const QString &file_name,
57  const QString &fq_file_name,
58  const QString &extension,
59  const QString &host) override // DirectoryHandler
60 
61  {
62 #if 0
63  LOG(VB_GENERAL, LOG_DEBUG,
64  QString("handleFile: %1 :: %2").arg(fq_file_name).arg(host));
65 #endif
66  (void) file_name;
67  if (m_imageExt.find(extension.toLower()) == m_imageExt.end())
68  {
69  m_videoFiles[fq_file_name].check = false;
70  m_videoFiles[fq_file_name].host = host;
71  }
72  }
73 
74  private:
75  using image_ext = std::set<QString>;
76  image_ext m_imageExt;
77  DirListType &m_videoFiles;
78  };
79 }
80 
83 
85  MThread("VideoScanner")
86 {
87  m_parent = parent;
90  m_listUnknown = gCoreContext->GetBoolSetting("VideoListUnknownFiletypes", false);
91 }
92 
94 {
95  delete m_dbMetadata;
96 }
97 
98 void VideoScannerThread::SetHosts(const QStringList &hosts)
99 {
100  m_liveSGHosts.clear();
101  foreach (const auto & host, hosts)
102  m_liveSGHosts << host.toLower();
103 }
104 
105 void VideoScannerThread::SetDirs(QStringList dirs)
106 {
107  QString master = gCoreContext->GetMasterHostName().toLower();
108  QStringList searchhosts;
109  QStringList mdirs;
110  m_offlineSGHosts.clear();
111 
112  QStringList::iterator iter = dirs.begin();
113  QStringList::iterator iter2;
114  while ( iter != dirs.end() )
115  {
116  if (iter->startsWith("myth://"))
117  {
118  QUrl sgurl = *iter;
119  QString host = sgurl.host().toLower();
120  QString path = sgurl.path();
121 
122  if (!m_liveSGHosts.contains(host))
123  {
124  // mark host as offline to warn user
125  if (!m_offlineSGHosts.contains(host))
126  m_offlineSGHosts.append(host);
127  // erase from directory list to skip scanning
128  iter = dirs.erase(iter);
129  continue;
130  }
131  if ((host == master) && (!mdirs.contains(path)))
132  {
133  // collect paths defined on master backend so other
134  // online backends can be set to fall through to them
135  mdirs.append(path);
136  }
137  else if (!searchhosts.contains(host))
138  {
139  // mark host as having directories defined so it
140  // does not fall through to those on the master
141  searchhosts.append(host);
142  }
143  }
144 
145  ++iter;
146  }
147 
148  for (iter = m_liveSGHosts.begin(); iter != m_liveSGHosts.end(); ++iter)
149  {
150  if ((!searchhosts.contains(*iter)) && (master != *iter))
151  {
152  for (iter2 = mdirs.begin(); iter2 != mdirs.end(); ++iter2)
153  {
154  // backend is online, but has no directories listed
155  // fall back to those on the master backend
156  dirs.append(MythCoreContext::GenMythURL(*iter,
157  0, *iter2, "Videos"));
158  }
159  }
160  }
161 
162  m_directories = dirs;
163 }
164 
166 {
167  RunProlog();
168 
171  m_dbMetadata->setList(ml);
172 
173  QList<QByteArray> image_types = QImageReader::supportedImageFormats();
174  QStringList imageExtensions;
175  for (QList<QByteArray>::const_iterator p = image_types.begin();
176  p != image_types.end(); ++p)
177  {
178  imageExtensions.push_back(QString(*p));
179  }
180 
181  LOG(VB_GENERAL, LOG_INFO, QString("Beginning Video Scan."));
182 
183  uint counter = 0;
184  FileCheckList fs_files;
185 
186  if (m_hasGUI)
187  SendProgressEvent(counter, (uint)m_directories.size(),
188  tr("Searching for video files"));
189  for (QStringList::const_iterator iter = m_directories.begin();
190  iter != m_directories.end(); ++iter)
191  {
192  if (!buildFileList(*iter, imageExtensions, fs_files))
193  {
194  if (iter->startsWith("myth://"))
195  {
196  QUrl sgurl = *iter;
197  QString host = sgurl.host().toLower();
198  QString path = sgurl.path();
199 
200  m_liveSGHosts.removeAll(host);
201 
202  LOG(VB_GENERAL, LOG_ERR,
203  QString("Failed to scan :%1:").arg(*iter));
204  }
205  }
206  if (m_hasGUI)
207  SendProgressEvent(++counter);
208  }
209 
210  PurgeList db_remove;
211  verifyFiles(fs_files, db_remove);
212  m_dbDataChanged = updateDB(fs_files, db_remove);
213 
214  if (m_dbDataChanged)
215  {
216  QCoreApplication::postEvent(m_parent,
218  m_delList));
219 
220  QStringList slist;
221 
222  QList<int>::const_iterator i;
223  for (i = m_addList.begin(); i != m_addList.end(); ++i)
224  slist << QString("added::%1").arg(*i);
225  for (i = m_movList.begin(); i != m_movList.end(); ++i)
226  slist << QString("moved::%1").arg(*i);
227  for (i = m_delList.begin(); i != m_delList.end(); ++i)
228  slist << QString("deleted::%1").arg(*i);
229 
230  MythEvent me("VIDEO_LIST_CHANGE", slist);
231 
232  gCoreContext->SendEvent(me);
233  }
234  else
235  gCoreContext->SendMessage("VIDEO_LIST_NO_CHANGE");
236 
237  RunEpilog();
238 }
239 
240 
242  const QString &filename)
243 {
244  (void) filename;
245 
246  // TODO: use single DB connection for all calls
247  if (m_removeAll)
248  m_dbMetadata->purgeByID(id);
249 
250  if (!m_keepAll && !m_removeAll)
251  {
252  m_removeAll = true;
253  m_dbMetadata->purgeByID(id);
254  }
255 }
256 
258  PurgeList &remove)
259 {
260  int counter = 0;
261  FileCheckList::iterator iter;
262 
263  if (m_hasGUI)
264  SendProgressEvent(counter, (uint)m_dbMetadata->getList().size(),
265  tr("Verifying video files"));
266 
267  // For every file we know about, check to see if it still exists.
268  for (const auto & file : m_dbMetadata->getList())
269  {
270  QString lname = file->GetFilename();
271  QString lhost = file->GetHost().toLower();
272  if (!lname.isEmpty())
273  {
274  iter = files.find(lname);
275  if (iter != files.end())
276  {
277  if (lhost != iter->second.host)
278  {
279  // file has changed hosts
280  // add to delete list for further processing
281  remove.push_back(std::make_pair(file->GetID(), lname));
282  }
283  else
284  {
285  // file is on disk on the proper host and in the database
286  // we're done with it
287  iter->second.check = true;
288  }
289  }
290  else if (lhost.isEmpty())
291  {
292  // If it's only in the database, and not on a host we
293  // cannot reach, mark it as for removal later.
294  remove.push_back(std::make_pair(file->GetID(), lname));
295  }
296  else if (m_liveSGHosts.contains(lhost))
297  {
298  LOG(VB_GENERAL, LOG_INFO,
299  QString("Removing file SG(%1) :%2:")
300  .arg(lhost).arg(lname));
301  remove.push_back(std::make_pair(file->GetID(), lname));
302  }
303  else
304  {
305  LOG(VB_GENERAL, LOG_WARNING,
306  QString("SG(%1) not available. Not removing file :%2:")
307  .arg(lhost).arg(lname));
308  if (!m_offlineSGHosts.contains(lhost))
309  m_offlineSGHosts.append(lhost);
310  }
311  }
312  if (m_hasGUI)
313  SendProgressEvent(++counter);
314  }
315 }
316 
317 bool VideoScannerThread::updateDB(const FileCheckList &add, const PurgeList &remove)
318 {
319  int ret = 0;
320  uint counter = 0;
321  if (m_hasGUI)
322  SendProgressEvent(counter, (uint)(add.size() + remove.size()),
323  tr("Updating video database"));
324 
325  for (auto p = add.cbegin(); p != add.cend(); ++p)
326  {
327  // add files not already in the DB
328  if (!p->second.check)
329  {
330  int id = -1;
331 
332  // Are we sure this needs adding? Let's check our Hash list.
333  QString hash = VideoMetadata::VideoFileHash(p->first, p->second.host);
334  if (hash != "NULL" && !hash.isEmpty())
335  {
336  id = VideoMetadata::UpdateHashedDBRecord(hash, p->first, p->second.host);
337  if (id != -1)
338  {
339  // Whew, that was close. Let's remove that thing from
340  // our purge list, too.
341  LOG(VB_GENERAL, LOG_ERR,
342  QString("Hash %1 already exists in the "
343  "database, updating record %2 "
344  "with new filename %3")
345  .arg(hash).arg(id).arg(p->first));
346  m_movList.append(id);
347  }
348  }
349  if (id == -1)
350  {
351  VideoMetadata newFile(
352  p->first, QString(), hash,
358  QString(), QString(), QString(), QString(),
359  QString(),
361  QDate::fromString("0000-00-00","YYYY-MM-DD"),
362  VIDEO_INETREF_DEFAULT, 0, QString(),
364  0.0, VIDEO_RATING_DEFAULT, 0, 0,
365  0, 0,
366  MythDate::current().date(),
368 
369  LOG(VB_GENERAL, LOG_INFO, QString("Adding : %1 : %2 : %3")
370  .arg(newFile.GetHost()).arg(newFile.GetFilename())
371  .arg(hash));
372  newFile.SetHost(p->second.host);
373  newFile.SaveToDatabase();
374  m_addList << newFile.GetID();
375  }
376  ret += 1;
377  }
378  if (m_hasGUI)
379  SendProgressEvent(++counter);
380  }
381 
382  // When prompting is restored, account for the answer here.
383  ret += remove.size();
384  for (const auto & item : remove)
385  {
386  if (!m_movList.contains(item.first))
387  {
388  removeOrphans(item.first, item.second);
389  m_delList << item.first;
390  }
391  if (m_hasGUI)
392  SendProgressEvent(++counter);
393  }
394 
395  return ret > 0;
396 }
397 
398 bool VideoScannerThread::buildFileList(const QString &directory,
399  const QStringList &imageExtensions,
400  FileCheckList &filelist)
401 {
402  // TODO: FileCheckList is a std::map, keyed off the filename. In the event
403  // multiple backends have access to shared storage, the potential exists
404  // for files to be scanned onto the wrong host. Add in some logic to prefer
405  // the backend with the content stored in a storage group determined to be
406  // local.
407 
408  LOG(VB_GENERAL,LOG_INFO, QString("buildFileList directory = %1")
409  .arg(directory));
412 
413  dirhandler<FileCheckList> dh(filelist, imageExtensions);
414  return ScanVideoDirectory(directory, &dh, ext_list, m_listUnknown);
415 }
416 
418  QString messsage)
419 {
420  if (!m_dialog)
421  return;
422 
423  auto *pue = new ProgressUpdateEvent(progress, total, std::move(messsage));
424  QApplication::postEvent(m_dialog, pue);
425 }
426 
428 {
429  m_scanThread = new VideoScannerThread(this);
430 }
431 
433 {
434  if (m_scanThread && m_scanThread->wait())
435  delete m_scanThread;
436 }
437 
438 void VideoScanner::doScan(const QStringList &dirs)
439 {
440  if (m_scanThread->isRunning())
441  return;
442 
443  if (gCoreContext->HasGUI())
444  {
445  MythScreenStack *popupStack = GetMythMainWindow()->GetStack("popup stack");
446 
447  auto *progressDlg = new MythUIProgressDialog("", popupStack,
448  "videoscanprogressdialog");
449 
450  if (progressDlg->Create())
451  {
452  popupStack->AddScreen(progressDlg, false);
453  connect(m_scanThread->qthread(), SIGNAL(finished()),
454  progressDlg, SLOT(Close()));
455  connect(m_scanThread->qthread(), SIGNAL(finished()),
456  SLOT(finishedScan()));
457  }
458  else
459  {
460  delete progressDlg;
461  progressDlg = nullptr;
462  }
463  m_scanThread->SetProgressDialog(progressDlg);
464  }
465 
466  QStringList hosts;
467  if (!RemoteGetActiveBackends(&hosts))
468  {
469  LOG(VB_GENERAL, LOG_WARNING, "Could not retrieve list of "
470  "available backends.");
471  hosts.clear();
472  }
473  m_scanThread->SetHosts(hosts);
474  m_scanThread->SetDirs(dirs);
475  m_scanThread->start();
476 }
477 
479 {
480  doScan(GetVideoDirs());
481 }
482 
484 {
485  QStringList failedHosts = m_scanThread->GetOfflineSGHosts();
486  if (!failedHosts.empty())
487  {
488  QString hosts = failedHosts.join(" ");
489  QString msg = tr("Failed to Scan SG Video Hosts:\n\n%1\n\n"
490  "If they no longer exist please remove them")
491  .arg(hosts);
492 
493  ShowOkPopup(msg);
494  }
495 
497 }
498 
void RunEpilog(void)
Cleans up a thread's resources, call this if you reimplement run().
Definition: mthread.cpp:215
bool updateDB(const FileCheckList &add, const PurgeList &remove)
Definition: videoscan.cpp:317
void verifyFiles(FileCheckList &files, PurgeList &remove)
Definition: videoscan.cpp:257
This is a wrapper around QThread that does several additional things.
Definition: mthread.h:46
QList< int > m_addList
Definition: videoscan.h:112
QStringList GetVideoDirs()
Definition: videoutils.cpp:120
const QString VIDEO_SCREENSHOT_DEFAULT
Definition: globals.cpp:27
const metadata_list & getList() const
MythConfirmationDialog * ShowOkPopup(const QString &message, QObject *parent, const char *slot, bool showCancel)
Non-blocking version of MythPopupBox::showOkPopup()
static void loadAllFromDatabase(metadata_list &items, const QString &sql="")
Load videometadata database into memory.
bool HasGUI(void) const
QDateTime fromString(const QString &dtstr)
Converts kFilename && kISODate formats to QDateTime.
Definition: mythdate.cpp:30
bool getDataChanged()
Definition: videoscan.h:73
VideoMetadataListManager * m_dbMetadata
Definition: videoscan.h:109
class VideoScannerThread * m_scanThread
Definition: videoscan.h:40
bool wait(unsigned long time=ULONG_MAX)
Wait for the MThread to exit, with a maximum timeout.
Definition: mthread.cpp:311
bool ScanVideoDirectory(const QString &start_path, DirectoryHandler *handler, const FileAssociations::ext_ignore_list &ext_disposition, bool list_unknown_extensions)
Definition: dirscan.cpp:226
void doScanAll(void)
Definition: videoscan.cpp:478
void SetHosts(const QStringList &hosts)
Definition: videoscan.cpp:98
void run() override
Runs the Qt event loop unless we have a QRunnable, in which case we run the runnable run instead.
Definition: videoscan.cpp:165
bool buildFileList(const QString &directory, const QStringList &imageExtensions, FileCheckList &filelist)
Definition: videoscan.cpp:398
void finished(bool)
MythScreenStack * GetStack(const QString &stackname)
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
~VideoScannerThread() override
Definition: videoscan.cpp:93
unsigned int GetID() const
virtual void handleFile(const QString &file_name, const QString &fq_file_name, const QString &extension, const QString &host)=0
MythUIProgressDialog * m_dialog
Definition: videoscan.h:110
void SendMessage(const QString &message)
void SendProgressEvent(uint progress, uint total=0, QString messsage=QString())
Definition: videoscan.cpp:417
~VideoScanner() override
Definition: videoscan.cpp:432
void SetDirs(QStringList dirs)
Definition: videoscan.cpp:105
virtual void Close()
bool RemoteGetActiveBackends(QStringList *list)
return list of backends currently connected to the master
Definition: remoteutil.cpp:577
QList< int > m_delList
Definition: videoscan.h:114
virtual void AddScreen(MythScreenType *screen, bool allowFade=true)
virtual DirectoryHandler * newDir(const QString &dir_name, const QString &fq_dir_name)=0
void setList(metadata_list &list)
This class is used as a container for messages.
Definition: mythevent.h:16
const QString VIDEO_PLOT_DEFAULT
Definition: globals.cpp:32
dirhandler(smart_dir_node &directory, const QString &prefix, VideoMetadataListManager::metadata_list &metalist, free_list &dh_free_list, bool infer_title)
Definition: videolist.cpp:1142
std::map< QString, CheckStruct > FileCheckList
Definition: videoscan.h:86
const QString VIDEO_TRAILER_DEFAULT
Definition: globals.cpp:26
static QString GenMythURL(const QString &host=QString(), int port=0, QString path=QString(), const QString &storageGroup=QString())
QDateTime current(bool stripped)
Returns current Date and Time in UTC.
Definition: mythdate.cpp:10
bool isRunning(void) const
Definition: mthread.cpp:274
QStringList m_liveSGHosts
Definition: videoscan.h:106
QString GetMasterHostName(void)
unsigned int uint
Definition: compat.h:140
static int UpdateHashedDBRecord(const QString &hash, const QString &file_name, const QString &host)
static QString VideoFileHash(const QString &file_name, const QString &host)
const QString & GetHost() const
std::vector< std::pair< unsigned int, QString > > PurgeList
Definition: videoscan.h:85
static FileAssociations & getFileAssociation()
Definition: dbaccess.cpp:831
MythMainWindow * GetMythMainWindow(void)
void removeOrphans(unsigned int id, const QString &filename)
Definition: videoscan.cpp:241
void SetHost(const QString &host)
std::vector< std::pair< QString, bool > > ext_ignore_list
Definition: dbaccess.h:155
#define LOG(_MASK_, _LEVEL_, _STRING_)
Definition: mythlogging.h:41
const QString & GetFilename() const
const QString VIDEO_INETREF_DEFAULT
Definition: globals.cpp:24
bool GetBoolSetting(const QString &key, bool defaultval=false)
void RunProlog(void)
Sets up a thread, call this if you reimplement run().
Definition: mthread.cpp:202
VideoScannerThread(QObject *parent)
Definition: videoscan.cpp:84
void start(QThread::Priority p=QThread::InheritPriority)
Tell MThread to start running the thread in the near future.
Definition: mthread.cpp:294
const QString VIDEO_COVERFILE_DEFAULT
Definition: globals.cpp:25
QThread * qthread(void)
Returns the thread, this will always return the same pointer no matter how often you restart the thre...
Definition: mthread.cpp:244
void finishedScan()
Definition: videoscan.cpp:483
const QString VIDEO_RATING_DEFAULT
Definition: globals.cpp:30
QStringList GetOfflineSGHosts(void)
Definition: videoscan.h:72
QStringList m_directories
Definition: videoscan.h:105
QStringList m_offlineSGHosts
Definition: videoscan.h:107
const QString VIDEO_FANART_DEFAULT
Definition: globals.cpp:29
static Type kEventType
Definition: videoscan.h:57
bool purgeByID(unsigned int db_id)
QObject * m_parent
Definition: videoscan.h:99
const QString VIDEO_DIRECTOR_DEFAULT
Definition: globals.cpp:23
void SendEvent(const MythEvent &event)
void SetProgressDialog(MythUIProgressDialog *dialog)
Definition: videoscan.h:71
void getExtensionIgnoreList(ext_ignore_list &ext_ignore) const
Definition: dbaccess.cpp:811
void doScan(const QStringList &dirs)
Definition: videoscan.cpp:438
const QString VIDEO_BANNER_DEFAULT
Definition: globals.cpp:28
MythUIProgressDialog(QString message, MythScreenStack *parent, const char *name)
std::list< VideoMetadataPtr > metadata_list
QList< int > m_movList
Definition: videoscan.h:113