MythTV  master
videoscan.cpp
Go to the documentation of this file.
1 
2 #include "videoscan.h"
3 
4 #include <QApplication>
5 #include <QImageReader>
6 #include <QUrl>
7 #include <utility>
8 
9 // libmythbase
10 #include "mythevent.h"
11 #include "mythlogging.h"
12 #include "mythdate.h"
13 
14 // libmyth
15 #include "mythcontext.h"
16 #include "remoteutil.h"
17 
18 // libmythui
19 #include "mythscreenstack.h"
20 #include "mythprogressdialog.h"
21 #include "mythdialogbox.h"
22 #include "mythmainwindow.h"
23 
24 // libmythmetadata
26 #include "videoutils.h"
27 #include "globals.h"
28 #include "dbaccess.h"
29 #include "dirscan.h"
30 
31 QEvent::Type VideoScanChanges::kEventType =
32  (QEvent::Type) QEvent::registerEventType();
33 
34 namespace
35 {
36  template <typename DirListType>
37  class dirhandler : public DirectoryHandler
38  {
39  public:
40  dirhandler(DirListType &video_files,
41  const QStringList &image_extensions) :
42  m_videoFiles(video_files)
43  {
44  for (QStringList::const_iterator p = image_extensions.begin();
45  p != image_extensions.end(); ++p)
46  {
47  m_imageExt.insert((*p).toLower());
48  }
49  }
50 
51  DirectoryHandler *newDir(const QString &dir_name,
52  const QString &fq_dir_name) override // DirectoryHandler
53  {
54  (void) dir_name;
55  (void) fq_dir_name;
56  return this;
57  }
58 
59  void handleFile(const QString &file_name,
60  const QString &fq_file_name,
61  const QString &extension,
62  const QString &host) override // DirectoryHandler
63 
64  {
65 #if 0
66  LOG(VB_GENERAL, LOG_DEBUG,
67  QString("handleFile: %1 :: %2").arg(fq_file_name).arg(host));
68 #endif
69  (void) file_name;
70  if (m_imageExt.find(extension.toLower()) == m_imageExt.end())
71  {
72  m_videoFiles[fq_file_name].check = false;
73  m_videoFiles[fq_file_name].host = host;
74  }
75  }
76 
77  private:
78  using image_ext = std::set<QString>;
79  image_ext m_imageExt;
80  DirListType &m_videoFiles;
81  };
82 }
83 
86 
88  MThread("VideoScanner")
89 {
90  m_parent = parent;
93  m_ListUnknown = gCoreContext->GetBoolSetting("VideoListUnknownFiletypes", false);
94 }
95 
97 {
98  delete m_dbmetadata;
99 }
100 
101 void VideoScannerThread::SetHosts(const QStringList &hosts)
102 {
103  m_liveSGHosts.clear();
104  QStringList::const_iterator iter = hosts.begin();
105  for (; iter != hosts.end(); ++iter)
106  m_liveSGHosts << iter->toLower();
107 }
108 
109 void VideoScannerThread::SetDirs(QStringList dirs)
110 {
111  QString master = gCoreContext->GetMasterHostName().toLower();
112  QStringList searchhosts, mdirs;
113  m_offlineSGHosts.clear();
114 
115  QStringList::iterator iter = dirs.begin(), iter2;
116  while ( iter != dirs.end() )
117  {
118  if (iter->startsWith("myth://"))
119  {
120  QUrl sgurl = *iter;
121  QString host = sgurl.host().toLower();
122  QString path = sgurl.path();
123 
124  if (!m_liveSGHosts.contains(host))
125  {
126  // mark host as offline to warn user
127  if (!m_offlineSGHosts.contains(host))
128  m_offlineSGHosts.append(host);
129  // erase from directory list to skip scanning
130  iter = dirs.erase(iter);
131  continue;
132  }
133  if ((host == master) && (!mdirs.contains(path)))
134  // collect paths defined on master backend so other
135  // online backends can be set to fall through to them
136  mdirs.append(path);
137  else if (!searchhosts.contains(host))
138  // mark host as having directories defined so it
139  // does not fall through to those on the master
140  searchhosts.append(host);
141  }
142 
143  ++iter;
144  }
145 
146  for (iter = m_liveSGHosts.begin(); iter != m_liveSGHosts.end(); ++iter)
147  if ((!searchhosts.contains(*iter)) && (master != *iter))
148  for (iter2 = mdirs.begin(); iter2 != mdirs.end(); ++iter2)
149  // backend is online, but has no directories listed
150  // fall back to those on the master backend
151  dirs.append(MythCoreContext::GenMythURL(*iter,
152  0, *iter2, "Videos"));
153 
154  m_directories = dirs;
155 }
156 
158 {
159  RunProlog();
160 
163  m_dbmetadata->setList(ml);
164 
165  QList<QByteArray> image_types = QImageReader::supportedImageFormats();
166  QStringList imageExtensions;
167  for (QList<QByteArray>::const_iterator p = image_types.begin();
168  p != image_types.end(); ++p)
169  {
170  imageExtensions.push_back(QString(*p));
171  }
172 
173  LOG(VB_GENERAL, LOG_INFO, QString("Beginning Video Scan."));
174 
175  uint counter = 0;
176  FileCheckList fs_files;
177 
178  if (m_HasGUI)
179  SendProgressEvent(counter, (uint)m_directories.size(),
180  tr("Searching for video files"));
181  for (QStringList::const_iterator iter = m_directories.begin();
182  iter != m_directories.end(); ++iter)
183  {
184  if (!buildFileList(*iter, imageExtensions, fs_files))
185  {
186  if (iter->startsWith("myth://"))
187  {
188  QUrl sgurl = *iter;
189  QString host = sgurl.host().toLower();
190  QString path = sgurl.path();
191 
192  m_liveSGHosts.removeAll(host);
193 
194  LOG(VB_GENERAL, LOG_ERR,
195  QString("Failed to scan :%1:").arg(*iter));
196  }
197  }
198  if (m_HasGUI)
199  SendProgressEvent(++counter);
200  }
201 
202  PurgeList db_remove;
203  verifyFiles(fs_files, db_remove);
204  m_DBDataChanged = updateDB(fs_files, db_remove);
205 
206  if (m_DBDataChanged)
207  {
208  QCoreApplication::postEvent(m_parent,
210  m_delList));
211 
212  QStringList slist;
213 
214  QList<int>::const_iterator i;
215  for (i = m_addList.begin(); i != m_addList.end(); ++i)
216  slist << QString("added::%1").arg(*i);
217  for (i = m_movList.begin(); i != m_movList.end(); ++i)
218  slist << QString("moved::%1").arg(*i);
219  for (i = m_delList.begin(); i != m_delList.end(); ++i)
220  slist << QString("deleted::%1").arg(*i);
221 
222  MythEvent me("VIDEO_LIST_CHANGE", slist);
223 
224  gCoreContext->SendEvent(me);
225  }
226  else
227  gCoreContext->SendMessage("VIDEO_LIST_NO_CHANGE");
228 
229  RunEpilog();
230 }
231 
232 
234  const QString &filename)
235 {
236  (void) filename;
237 
238  // TODO: use single DB connection for all calls
239  if (m_RemoveAll)
240  m_dbmetadata->purgeByID(id);
241 
242  if (!m_KeepAll && !m_RemoveAll)
243  {
244  m_RemoveAll = true;
245  m_dbmetadata->purgeByID(id);
246  }
247 }
248 
250  PurgeList &remove)
251 {
252  int counter = 0;
253  FileCheckList::iterator iter;
254 
255  if (m_HasGUI)
256  SendProgressEvent(counter, (uint)m_dbmetadata->getList().size(),
257  tr("Verifying video files"));
258 
259  // For every file we know about, check to see if it still exists.
260  for (auto p = m_dbmetadata->getList().cbegin();
261  p != m_dbmetadata->getList().cend(); ++p)
262  {
263  QString lname = (*p)->GetFilename();
264  QString lhost = (*p)->GetHost().toLower();
265  if (!lname.isEmpty())
266  {
267  iter = files.find(lname);
268  if (iter != files.end())
269  {
270  if (lhost != iter->second.host)
271  // file has changed hosts
272  // add to delete list for further processing
273  remove.push_back(std::make_pair((*p)->GetID(), lname));
274  else
275  // file is on disk on the proper host and in the database
276  // we're done with it
277  iter->second.check = true;
278  }
279  else if (lhost.isEmpty())
280  {
281  // If it's only in the database, and not on a host we
282  // cannot reach, mark it as for removal later.
283  remove.push_back(std::make_pair((*p)->GetID(), lname));
284  }
285  else if (m_liveSGHosts.contains(lhost))
286  {
287  LOG(VB_GENERAL, LOG_INFO,
288  QString("Removing file SG(%1) :%2:")
289  .arg(lhost).arg(lname));
290  remove.push_back(std::make_pair((*p)->GetID(), lname));
291  }
292  else
293  {
294  LOG(VB_GENERAL, LOG_WARNING,
295  QString("SG(%1) not available. Not removing file :%2:")
296  .arg(lhost).arg(lname));
297  if (!m_offlineSGHosts.contains(lhost))
298  m_offlineSGHosts.append(lhost);
299  }
300  }
301  if (m_HasGUI)
302  SendProgressEvent(++counter);
303  }
304 }
305 
306 bool VideoScannerThread::updateDB(const FileCheckList &add, const PurgeList &remove)
307 {
308  int ret = 0;
309  uint counter = 0;
310  if (m_HasGUI)
311  SendProgressEvent(counter, (uint)(add.size() + remove.size()),
312  tr("Updating video database"));
313 
314  for (auto p = add.cbegin(); p != add.cend(); ++p)
315  {
316  // add files not already in the DB
317  if (!p->second.check)
318  {
319  int id = -1;
320 
321  // Are we sure this needs adding? Let's check our Hash list.
322  QString hash = VideoMetadata::VideoFileHash(p->first, p->second.host);
323  if (hash != "NULL" && !hash.isEmpty())
324  {
325  id = VideoMetadata::UpdateHashedDBRecord(hash, p->first, p->second.host);
326  if (id != -1)
327  {
328  // Whew, that was close. Let's remove that thing from
329  // our purge list, too.
330  LOG(VB_GENERAL, LOG_ERR,
331  QString("Hash %1 already exists in the "
332  "database, updating record %2 "
333  "with new filename %3")
334  .arg(hash).arg(id).arg(p->first));
335  m_movList.append(id);
336  }
337  }
338  if (id == -1)
339  {
340  VideoMetadata newFile(
341  p->first, QString(), hash,
347  QString(), QString(), QString(), QString(),
348  QString(),
350  QDate::fromString("0000-00-00","YYYY-MM-DD"),
351  VIDEO_INETREF_DEFAULT, 0, QString(),
353  0.0, VIDEO_RATING_DEFAULT, 0, 0,
354  0, 0,
355  MythDate::current().date(),
357 
358  LOG(VB_GENERAL, LOG_INFO, QString("Adding : %1 : %2 : %3")
359  .arg(newFile.GetHost()).arg(newFile.GetFilename())
360  .arg(hash));
361  newFile.SetHost(p->second.host);
362  newFile.SaveToDatabase();
363  m_addList << newFile.GetID();
364  }
365  ret += 1;
366  }
367  if (m_HasGUI)
368  SendProgressEvent(++counter);
369  }
370 
371  // When prompting is restored, account for the answer here.
372  ret += remove.size();
373  for (auto p = remove.cbegin(); p != remove.cend(); ++p)
374  {
375  if (!m_movList.contains(p->first))
376  {
377  removeOrphans(p->first, p->second);
378  m_delList << p->first;
379  }
380  if (m_HasGUI)
381  SendProgressEvent(++counter);
382  }
383 
384  return ret > 0;
385 }
386 
387 bool VideoScannerThread::buildFileList(const QString &directory,
388  const QStringList &imageExtensions,
389  FileCheckList &filelist)
390 {
391  // TODO: FileCheckList is a std::map, keyed off the filename. In the event
392  // multiple backends have access to shared storage, the potential exists
393  // for files to be scanned onto the wrong host. Add in some logic to prefer
394  // the backend with the content stored in a storage group determined to be
395  // local.
396 
397  LOG(VB_GENERAL,LOG_INFO, QString("buildFileList directory = %1")
398  .arg(directory));
401 
402  dirhandler<FileCheckList> dh(filelist, imageExtensions);
403  return ScanVideoDirectory(directory, &dh, ext_list, m_ListUnknown);
404 }
405 
407  QString messsage)
408 {
409  if (!m_dialog)
410  return;
411 
412  auto *pue = new ProgressUpdateEvent(progress, total, std::move(messsage));
413  QApplication::postEvent(m_dialog, pue);
414 }
415 
417 {
418  m_scanThread = new VideoScannerThread(this);
419 }
420 
422 {
423  if (m_scanThread && m_scanThread->wait())
424  delete m_scanThread;
425 }
426 
427 void VideoScanner::doScan(const QStringList &dirs)
428 {
429  if (m_scanThread->isRunning())
430  return;
431 
432  if (gCoreContext->HasGUI())
433  {
434  MythScreenStack *popupStack = GetMythMainWindow()->GetStack("popup stack");
435 
436  auto *progressDlg = new MythUIProgressDialog("", popupStack,
437  "videoscanprogressdialog");
438 
439  if (progressDlg->Create())
440  {
441  popupStack->AddScreen(progressDlg, false);
442  connect(m_scanThread->qthread(), SIGNAL(finished()),
443  progressDlg, SLOT(Close()));
444  connect(m_scanThread->qthread(), SIGNAL(finished()),
445  SLOT(finishedScan()));
446  }
447  else
448  {
449  delete progressDlg;
450  progressDlg = nullptr;
451  }
452  m_scanThread->SetProgressDialog(progressDlg);
453  }
454 
455  QStringList hosts;
456  if (!RemoteGetActiveBackends(&hosts))
457  {
458  LOG(VB_GENERAL, LOG_WARNING, "Could not retrieve list of "
459  "available backends.");
460  hosts.clear();
461  }
462  m_scanThread->SetHosts(hosts);
463  m_scanThread->SetDirs(dirs);
464  m_scanThread->start();
465 }
466 
468 {
469  doScan(GetVideoDirs());
470 }
471 
473 {
474  QStringList failedHosts = m_scanThread->GetOfflineSGHosts();
475  if (!failedHosts.empty())
476  {
477  QString hosts = failedHosts.join(" ");
478  QString msg = tr("Failed to Scan SG Video Hosts:\n\n%1\n\n"
479  "If they no longer exist please remove them")
480  .arg(hosts);
481 
482  ShowOkPopup(msg);
483  }
484 
486 }
487 
void RunEpilog(void)
Cleans up a thread's resources, call this if you reimplement run().
Definition: mthread.cpp:215
bool updateDB(const FileCheckList &add, const PurgeList &remove)
Definition: videoscan.cpp:306
void start(QThread::Priority=QThread::InheritPriority)
Tell MThread to start running the thread in the near future.
Definition: mthread.cpp:294
void verifyFiles(FileCheckList &files, PurgeList &remove)
Definition: videoscan.cpp:249
This is a wrapper around QThread that does several additional things.
Definition: mthread.h:46
QList< int > m_addList
Definition: videoscan.h:109
QStringList GetVideoDirs()
Definition: videoutils.cpp:122
const QString VIDEO_SCREENSHOT_DEFAULT
Definition: globals.cpp:27
const metadata_list & getList() const
MythConfirmationDialog * ShowOkPopup(const QString &message, QObject *parent, const char *slot, bool showCancel)
Non-blocking version of MythPopupBox::showOkPopup()
static void loadAllFromDatabase(metadata_list &items, const QString &sql="")
Load videometadata database into memory.
bool HasGUI(void) const
MythUIProgressDialog(const QString &message, MythScreenStack *parent, const char *name)
bool getDataChanged()
Definition: videoscan.h:70
class VideoScannerThread * m_scanThread
Definition: videoscan.h:37
bool wait(unsigned long time=ULONG_MAX)
Wait for the MThread to exit, with a maximum timeout.
Definition: mthread.cpp:311
bool ScanVideoDirectory(const QString &start_path, DirectoryHandler *handler, const FileAssociations::ext_ignore_list &ext_disposition, bool list_unknown_extensions)
Definition: dirscan.cpp:227
void doScanAll(void)
Definition: videoscan.cpp:467
void SetHosts(const QStringList &hosts)
Definition: videoscan.cpp:101
void run() override
Runs the Qt event loop unless we have a QRunnable, in which case we run the runnable run instead.
Definition: videoscan.cpp:157
bool buildFileList(const QString &directory, const QStringList &imageExtensions, FileCheckList &filelist)
Definition: videoscan.cpp:387
void finished(bool)
MythScreenStack * GetStack(const QString &stackname)
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
unsigned int GetID() const
virtual void handleFile(const QString &file_name, const QString &fq_file_name, const QString &extension, const QString &host)=0
MythUIProgressDialog * m_dialog
Definition: videoscan.h:107
void SendMessage(const QString &message)
void SendProgressEvent(uint progress, uint total=0, QString messsage=QString())
Definition: videoscan.cpp:406
void SetDirs(QStringList dirs)
Definition: videoscan.cpp:109
virtual void Close()
bool RemoteGetActiveBackends(QStringList *list)
return list of backends currently connected to the master
Definition: remoteutil.cpp:578
QList< int > m_delList
Definition: videoscan.h:111
virtual void AddScreen(MythScreenType *screen, bool allowFade=true)
virtual DirectoryHandler * newDir(const QString &dir_name, const QString &fq_dir_name)=0
void setList(metadata_list &list)
This class is used as a container for messages.
Definition: mythevent.h:16
const QString VIDEO_PLOT_DEFAULT
Definition: globals.cpp:32
dirhandler(smart_dir_node &directory, const QString &prefix, VideoMetadataListManager::metadata_list &metalist, free_list &dh_free_list, bool infer_title)
Definition: videolist.cpp:1163
std::map< QString, CheckStruct > FileCheckList
Definition: videoscan.h:83
const QString VIDEO_TRAILER_DEFAULT
Definition: globals.cpp:26
static QString GenMythURL(const QString &host=QString(), int port=0, QString path=QString(), const QString &storageGroup=QString())
QDateTime current(bool stripped)
Returns current Date and Time in UTC.
Definition: mythdate.cpp:10
bool isRunning(void) const
Definition: mthread.cpp:274
QStringList m_liveSGHosts
Definition: videoscan.h:103
QString GetMasterHostName(void)
unsigned int uint
Definition: compat.h:140
static int UpdateHashedDBRecord(const QString &hash, const QString &file_name, const QString &host)
static QString VideoFileHash(const QString &file_name, const QString &host)
const QString & GetHost() const
std::vector< std::pair< unsigned int, QString > > PurgeList
Definition: videoscan.h:82
static FileAssociations & getFileAssociation()
Definition: dbaccess.cpp:832
MythMainWindow * GetMythMainWindow(void)
void removeOrphans(unsigned int id, const QString &filename)
Definition: videoscan.cpp:233
void SetHost(const QString &host)
std::vector< std::pair< QString, bool > > ext_ignore_list
Definition: dbaccess.h:154
#define LOG(_MASK_, _LEVEL_, _STRING_)
Definition: mythlogging.h:41
const QString & GetFilename() const
const QString VIDEO_INETREF_DEFAULT
Definition: globals.cpp:24
bool GetBoolSetting(const QString &key, bool defaultval=false)
void RunProlog(void)
Sets up a thread, call this if you reimplement run().
Definition: mthread.cpp:202
VideoScannerThread(QObject *parent)
Definition: videoscan.cpp:87
const QString VIDEO_COVERFILE_DEFAULT
Definition: globals.cpp:25
QThread * qthread(void)
Returns the thread, this will always return the same pointer no matter how often you restart the thre...
Definition: mthread.cpp:244
void finishedScan()
Definition: videoscan.cpp:472
const QString VIDEO_RATING_DEFAULT
Definition: globals.cpp:30
QStringList GetOfflineSGHosts(void)
Definition: videoscan.h:69
VideoMetadataListManager * m_dbmetadata
Definition: videoscan.h:106
QStringList m_directories
Definition: videoscan.h:102
QStringList m_offlineSGHosts
Definition: videoscan.h:104
const QString VIDEO_FANART_DEFAULT
Definition: globals.cpp:29
static Type kEventType
Definition: videoscan.h:54
bool purgeByID(unsigned int db_id)
QObject * m_parent
Definition: videoscan.h:96
const QString VIDEO_DIRECTOR_DEFAULT
Definition: globals.cpp:23
QDateTime fromString(const QString &dtstr)
Converts kFilename && kISODate formats to QDateTime.
Definition: mythdate.cpp:30
void SendEvent(const MythEvent &event)
void SetProgressDialog(MythUIProgressDialog *dialog)
Definition: videoscan.h:68
void getExtensionIgnoreList(ext_ignore_list &ext_ignore) const
Definition: dbaccess.cpp:812
void doScan(const QStringList &dirs)
Definition: videoscan.cpp:427
const QString VIDEO_BANNER_DEFAULT
Definition: globals.cpp:28
std::list< VideoMetadataPtr > metadata_list
QList< int > m_movList
Definition: videoscan.h:110