MythTV  master
videoscan.cpp
Go to the documentation of this file.
1 
2 #include "videoscan.h"
3 
4 #include <QApplication>
5 #include <QImageReader>
6 #include <QUrl>
7 #include <utility>
8 
9 // libmythbase
10 #include "mythevent.h"
11 #include "mythlogging.h"
12 #include "mythdate.h"
13 
14 // libmyth
15 #include "mythcontext.h"
16 #include "remoteutil.h"
17 
18 // libmythui
19 #include "mythscreenstack.h"
20 #include "mythprogressdialog.h"
21 #include "mythdialogbox.h"
22 #include "mythmainwindow.h"
23 
24 // libmythmetadata
26 #include "videoutils.h"
27 #include "globals.h"
28 #include "dbaccess.h"
29 #include "dirscan.h"
30 
31 QEvent::Type VideoScanChanges::kEventType =
32  (QEvent::Type) QEvent::registerEventType();
33 
34 namespace
35 {
36  template <typename DirListType>
37  class dirhandler : public DirectoryHandler
38  {
39  public:
40  dirhandler(DirListType &video_files,
41  const QStringList &image_extensions) :
42  m_video_files(video_files)
43  {
44  for (QStringList::const_iterator p = image_extensions.begin();
45  p != image_extensions.end(); ++p)
46  {
47  m_image_ext.insert((*p).toLower());
48  }
49  }
50 
51  DirectoryHandler *newDir(const QString &dir_name,
52  const QString &fq_dir_name) override // DirectoryHandler
53  {
54  (void) dir_name;
55  (void) fq_dir_name;
56  return this;
57  }
58 
59  void handleFile(const QString &file_name,
60  const QString &fq_file_name,
61  const QString &extension,
62  const QString &host) override // DirectoryHandler
63 
64  {
65 #if 0
66  LOG(VB_GENERAL, LOG_DEBUG,
67  QString("handleFile: %1 :: %2").arg(fq_file_name).arg(host));
68 #endif
69  (void) file_name;
70  if (m_image_ext.find(extension.toLower()) == m_image_ext.end())
71  {
72  m_video_files[fq_file_name].check = false;
73  m_video_files[fq_file_name].host = host;
74  }
75  }
76 
77  private:
78  typedef std::set<QString> image_ext;
79  image_ext m_image_ext;
80  DirListType &m_video_files;
81  };
82 }
83 
86 
88  MThread("VideoScanner")
89 {
90  m_parent = parent;
93  m_ListUnknown = gCoreContext->GetBoolSetting("VideoListUnknownFiletypes", false);
94 }
95 
97 {
98  delete m_dbmetadata;
99 }
100 
101 void VideoScannerThread::SetHosts(const QStringList &hosts)
102 {
103  m_liveSGHosts.clear();
104  QStringList::const_iterator iter = hosts.begin();
105  for (; iter != hosts.end(); ++iter)
106  m_liveSGHosts << iter->toLower();
107 }
108 
109 void VideoScannerThread::SetDirs(QStringList dirs)
110 {
111  QString master = gCoreContext->GetMasterHostName().toLower();
112  QStringList searchhosts, mdirs;
113  m_offlineSGHosts.clear();
114 
115  QStringList::iterator iter = dirs.begin(), iter2;
116  while ( iter != dirs.end() )
117  {
118  if (iter->startsWith("myth://"))
119  {
120  QUrl sgurl = *iter;
121  QString host = sgurl.host().toLower();
122  QString path = sgurl.path();
123 
124  if (!m_liveSGHosts.contains(host))
125  {
126  // mark host as offline to warn user
127  if (!m_offlineSGHosts.contains(host))
128  m_offlineSGHosts.append(host);
129  // erase from directory list to skip scanning
130  iter = dirs.erase(iter);
131  continue;
132  }
133  if ((host == master) && (!mdirs.contains(path)))
134  // collect paths defined on master backend so other
135  // online backends can be set to fall through to them
136  mdirs.append(path);
137  else if (!searchhosts.contains(host))
138  // mark host as having directories defined so it
139  // does not fall through to those on the master
140  searchhosts.append(host);
141  }
142 
143  ++iter;
144  }
145 
146  for (iter = m_liveSGHosts.begin(); iter != m_liveSGHosts.end(); ++iter)
147  if ((!searchhosts.contains(*iter)) && (master != *iter))
148  for (iter2 = mdirs.begin(); iter2 != mdirs.end(); ++iter2)
149  // backend is online, but has no directories listed
150  // fall back to those on the master backend
151  dirs.append(gCoreContext->GenMythURL(*iter,
152  0, *iter2, "Videos"));
153 
154  m_directories = dirs;
155 }
156 
158 {
159  RunProlog();
160 
163  m_dbmetadata->setList(ml);
164 
165  QList<QByteArray> image_types = QImageReader::supportedImageFormats();
166  QStringList imageExtensions;
167  for (QList<QByteArray>::const_iterator p = image_types.begin();
168  p != image_types.end(); ++p)
169  {
170  imageExtensions.push_back(QString(*p));
171  }
172 
173  LOG(VB_GENERAL, LOG_INFO, QString("Beginning Video Scan."));
174 
175  uint counter = 0;
176  FileCheckList fs_files;
177 
178  if (m_HasGUI)
179  SendProgressEvent(counter, (uint)m_directories.size(),
180  tr("Searching for video files"));
181  for (QStringList::const_iterator iter = m_directories.begin();
182  iter != m_directories.end(); ++iter)
183  {
184  if (!buildFileList(*iter, imageExtensions, fs_files))
185  {
186  if (iter->startsWith("myth://"))
187  {
188  QUrl sgurl = *iter;
189  QString host = sgurl.host().toLower();
190  QString path = sgurl.path();
191 
192  m_liveSGHosts.removeAll(host);
193 
194  LOG(VB_GENERAL, LOG_ERR,
195  QString("Failed to scan :%1:").arg(*iter));
196  }
197  }
198  if (m_HasGUI)
199  SendProgressEvent(++counter);
200  }
201 
202  PurgeList db_remove;
203  verifyFiles(fs_files, db_remove);
204  m_DBDataChanged = updateDB(fs_files, db_remove);
205 
206  if (m_DBDataChanged)
207  {
208  QCoreApplication::postEvent(m_parent,
210  m_delList));
211 
212  QStringList slist;
213 
214  QList<int>::const_iterator i;
215  for (i = m_addList.begin(); i != m_addList.end(); ++i)
216  slist << QString("added::%1").arg(*i);
217  for (i = m_movList.begin(); i != m_movList.end(); ++i)
218  slist << QString("moved::%1").arg(*i);
219  for (i = m_delList.begin(); i != m_delList.end(); ++i)
220  slist << QString("deleted::%1").arg(*i);
221 
222  MythEvent me("VIDEO_LIST_CHANGE", slist);
223 
224  gCoreContext->SendEvent(me);
225  }
226  else
227  gCoreContext->SendMessage("VIDEO_LIST_NO_CHANGE");
228 
229  RunEpilog();
230 }
231 
232 
234  const QString &filename)
235 {
236  (void) filename;
237 
238  // TODO: use single DB connection for all calls
239  if (m_RemoveAll)
240  m_dbmetadata->purgeByID(id);
241 
242  if (!m_KeepAll && !m_RemoveAll)
243  {
244  m_RemoveAll = true;
245  m_dbmetadata->purgeByID(id);
246  }
247 }
248 
250  PurgeList &remove)
251 {
252  int counter = 0;
253  FileCheckList::iterator iter;
254 
255  if (m_HasGUI)
256  SendProgressEvent(counter, (uint)m_dbmetadata->getList().size(),
257  tr("Verifying video files"));
258 
259  // For every file we know about, check to see if it still exists.
260  for (VideoMetadataListManager::metadata_list::const_iterator p =
261  m_dbmetadata->getList().begin();
262  p != m_dbmetadata->getList().end(); ++p)
263  {
264  QString lname = (*p)->GetFilename();
265  QString lhost = (*p)->GetHost().toLower();
266  if (!lname.isEmpty())
267  {
268  iter = files.find(lname);
269  if (iter != files.end())
270  {
271  if (lhost != iter->second.host)
272  // file has changed hosts
273  // add to delete list for further processing
274  remove.push_back(std::make_pair((*p)->GetID(), lname));
275  else
276  // file is on disk on the proper host and in the database
277  // we're done with it
278  iter->second.check = true;
279  }
280  else if (lhost.isEmpty())
281  {
282  // If it's only in the database, and not on a host we
283  // cannot reach, mark it as for removal later.
284  remove.push_back(std::make_pair((*p)->GetID(), lname));
285  }
286  else if (m_liveSGHosts.contains(lhost))
287  {
288  LOG(VB_GENERAL, LOG_INFO,
289  QString("Removing file SG(%1) :%2:")
290  .arg(lhost).arg(lname));
291  remove.push_back(std::make_pair((*p)->GetID(), lname));
292  }
293  else
294  {
295  LOG(VB_GENERAL, LOG_WARNING,
296  QString("SG(%1) not available. Not removing file :%2:")
297  .arg(lhost).arg(lname));
298  if (!m_offlineSGHosts.contains(lhost))
299  m_offlineSGHosts.append(lhost);
300  }
301  }
302  if (m_HasGUI)
303  SendProgressEvent(++counter);
304  }
305 }
306 
307 bool VideoScannerThread::updateDB(const FileCheckList &add, const PurgeList &remove)
308 {
309  int ret = 0;
310  uint counter = 0;
311  if (m_HasGUI)
312  SendProgressEvent(counter, (uint)(add.size() + remove.size()),
313  tr("Updating video database"));
314 
315  for (FileCheckList::const_iterator p = add.begin(); p != add.end(); ++p)
316  {
317  // add files not already in the DB
318  if (!p->second.check)
319  {
320  int id = -1;
321 
322  // Are we sure this needs adding? Let's check our Hash list.
323  QString hash = VideoMetadata::VideoFileHash(p->first, p->second.host);
324  if (hash != "NULL" && !hash.isEmpty())
325  {
326  id = VideoMetadata::UpdateHashedDBRecord(hash, p->first, p->second.host);
327  if (id != -1)
328  {
329  // Whew, that was close. Let's remove that thing from
330  // our purge list, too.
331  LOG(VB_GENERAL, LOG_ERR,
332  QString("Hash %1 already exists in the "
333  "database, updating record %2 "
334  "with new filename %3")
335  .arg(hash).arg(id).arg(p->first));
336  m_movList.append(id);
337  }
338  }
339  if (id == -1)
340  {
341  VideoMetadata newFile(
342  p->first, QString(), hash,
348  QString(), QString(), QString(), QString(),
349  QString(),
351  QDate::fromString("0000-00-00","YYYY-MM-DD"),
352  VIDEO_INETREF_DEFAULT, 0, QString(),
354  0.0, VIDEO_RATING_DEFAULT, 0, 0,
355  0, 0,
356  MythDate::current().date(),
358 
359  LOG(VB_GENERAL, LOG_INFO, QString("Adding : %1 : %2 : %3")
360  .arg(newFile.GetHost()).arg(newFile.GetFilename())
361  .arg(hash));
362  newFile.SetHost(p->second.host);
363  newFile.SaveToDatabase();
364  m_addList << newFile.GetID();
365  }
366  ret += 1;
367  }
368  if (m_HasGUI)
369  SendProgressEvent(++counter);
370  }
371 
372  // When prompting is restored, account for the answer here.
373  ret += remove.size();
374  for (PurgeList::const_iterator p = remove.begin(); p != remove.end();
375  ++p)
376  {
377  if (!m_movList.contains(p->first))
378  {
379  removeOrphans(p->first, p->second);
380  m_delList << p->first;
381  }
382  if (m_HasGUI)
383  SendProgressEvent(++counter);
384  }
385 
386  return ret > 0;
387 }
388 
389 bool VideoScannerThread::buildFileList(const QString &directory,
390  const QStringList &imageExtensions,
391  FileCheckList &filelist)
392 {
393  // TODO: FileCheckList is a std::map, keyed off the filename. In the event
394  // multiple backends have access to shared storage, the potential exists
395  // for files to be scanned onto the wrong host. Add in some logic to prefer
396  // the backend with the content stored in a storage group determined to be
397  // local.
398 
399  LOG(VB_GENERAL,LOG_INFO, QString("buildFileList directory = %1")
400  .arg(directory));
403 
404  dirhandler<FileCheckList> dh(filelist, imageExtensions);
405  return ScanVideoDirectory(directory, &dh, ext_list, m_ListUnknown);
406 }
407 
409  QString messsage)
410 {
411  if (!m_dialog)
412  return;
413 
415  std::move(messsage));
416  QApplication::postEvent(m_dialog, pue);
417 }
418 
420 {
421  m_scanThread = new VideoScannerThread(this);
422 }
423 
425 {
426  if (m_scanThread && m_scanThread->wait())
427  delete m_scanThread;
428 }
429 
430 void VideoScanner::doScan(const QStringList &dirs)
431 {
432  if (m_scanThread->isRunning())
433  return;
434 
435  if (gCoreContext->HasGUI())
436  {
437  MythScreenStack *popupStack = GetMythMainWindow()->GetStack("popup stack");
438 
439  MythUIProgressDialog *progressDlg = new MythUIProgressDialog("",
440  popupStack, "videoscanprogressdialog");
441 
442  if (progressDlg->Create())
443  {
444  popupStack->AddScreen(progressDlg, false);
445  connect(m_scanThread->qthread(), SIGNAL(finished()),
446  progressDlg, SLOT(Close()));
447  connect(m_scanThread->qthread(), SIGNAL(finished()),
448  SLOT(finishedScan()));
449  }
450  else
451  {
452  delete progressDlg;
453  progressDlg = nullptr;
454  }
455  m_scanThread->SetProgressDialog(progressDlg);
456  }
457 
458  QStringList hosts;
459  if (!RemoteGetActiveBackends(&hosts))
460  {
461  LOG(VB_GENERAL, LOG_WARNING, "Could not retrieve list of "
462  "available backends.");
463  hosts.clear();
464  }
465  m_scanThread->SetHosts(hosts);
466  m_scanThread->SetDirs(dirs);
467  m_scanThread->start();
468 }
469 
471 {
472  doScan(GetVideoDirs());
473 }
474 
476 {
477  QStringList failedHosts = m_scanThread->GetOfflineSGHosts();
478  if (!failedHosts.empty())
479  {
480  QString hosts = failedHosts.join(" ");
481  QString msg = tr("Failed to Scan SG Video Hosts:\n\n%1\n\n"
482  "If they no longer exist please remove them")
483  .arg(hosts);
484 
485  ShowOkPopup(msg);
486  }
487 
489 }
490 
void RunEpilog(void)
Cleans up a thread's resources, call this if you reimplement run().
Definition: mthread.cpp:215
bool updateDB(const FileCheckList &add, const PurgeList &remove)
Definition: videoscan.cpp:307
void start(QThread::Priority=QThread::InheritPriority)
Tell MThread to start running the thread in the near future.
Definition: mthread.cpp:294
void verifyFiles(FileCheckList &files, PurgeList &remove)
Definition: videoscan.cpp:249
This is a wrapper around QThread that does several additional things.
Definition: mthread.h:46
QList< int > m_addList
Definition: videoscan.h:109
QStringList GetVideoDirs()
Definition: videoutils.cpp:122
std::vector< std::pair< QString, bool > > ext_ignore_list
Definition: dbaccess.h:154
const QString VIDEO_SCREENSHOT_DEFAULT
Definition: globals.cpp:27
const metadata_list & getList() const
MythConfirmationDialog * ShowOkPopup(const QString &message, QObject *parent, const char *slot, bool showCancel)
Non-blocking version of MythPopupBox::showOkPopup()
bool Create(void) override
static void loadAllFromDatabase(metadata_list &items, const QString &sql="")
Load videometadata database into memory.
bool HasGUI(void) const
MythUIProgressDialog(const QString &message, MythScreenStack *parent, const char *name)
bool getDataChanged()
Definition: videoscan.h:70
class VideoScannerThread * m_scanThread
Definition: videoscan.h:37
bool wait(unsigned long time=ULONG_MAX)
Wait for the MThread to exit, with a maximum timeout.
Definition: mthread.cpp:311
bool ScanVideoDirectory(const QString &start_path, DirectoryHandler *handler, const FileAssociations::ext_ignore_list &ext_disposition, bool list_unknown_extensions)
Definition: dirscan.cpp:226
void doScanAll(void)
Definition: videoscan.cpp:470
void SetHosts(const QStringList &hosts)
Definition: videoscan.cpp:101
void run() override
Runs the Qt event loop unless we have a QRunnable, in which case we run the runnable run instead.
Definition: videoscan.cpp:157
bool buildFileList(const QString &directory, const QStringList &imageExtensions, FileCheckList &filelist)
Definition: videoscan.cpp:389
void finished(bool)
MythScreenStack * GetStack(const QString &stackname)
unsigned int uint
Definition: compat.h:140
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
unsigned int GetID() const
virtual void handleFile(const QString &file_name, const QString &fq_file_name, const QString &extension, const QString &host)=0
MythUIProgressDialog * m_dialog
Definition: videoscan.h:107
void SendMessage(const QString &message)
void SendProgressEvent(uint progress, uint total=0, QString messsage=QString())
Definition: videoscan.cpp:408
void SetDirs(QStringList dirs)
Definition: videoscan.cpp:109
virtual void Close()
bool RemoteGetActiveBackends(QStringList *list)
return list of backends currently connected to the master
Definition: remoteutil.cpp:579
QList< int > m_delList
Definition: videoscan.h:111
virtual void AddScreen(MythScreenType *screen, bool allowFade=true)
virtual DirectoryHandler * newDir(const QString &dir_name, const QString &fq_dir_name)=0
void setList(metadata_list &list)
This class is used as a container for messages.
Definition: mythevent.h:16
const QString VIDEO_PLOT_DEFAULT
Definition: globals.cpp:32
dirhandler(smart_dir_node &directory, const QString &prefix, VideoMetadataListManager::metadata_list &metalist, free_list &dh_free_list, bool infer_title)
Definition: videolist.cpp:1169
const QString VIDEO_TRAILER_DEFAULT
Definition: globals.cpp:26
QString GenMythURL(const QString &host=QString(), int port=0, QString path=QString(), const QString &storageGroup=QString())
QDateTime current(bool stripped)
Returns current Date and Time in UTC.
Definition: mythdate.cpp:10
bool isRunning(void) const
Definition: mthread.cpp:274
QStringList m_liveSGHosts
Definition: videoscan.h:103
QString GetMasterHostName(void)
static int UpdateHashedDBRecord(const QString &hash, const QString &file_name, const QString &host)
static QString VideoFileHash(const QString &file_name, const QString &host)
std::vector< std::pair< unsigned int, QString > > PurgeList
Definition: videoscan.h:82
const QString & GetHost() const
static FileAssociations & getFileAssociation()
Definition: dbaccess.cpp:836
MythMainWindow * GetMythMainWindow(void)
void removeOrphans(unsigned int id, const QString &filename)
Definition: videoscan.cpp:233
void SetHost(const QString &host)
std::list< VideoMetadataPtr > metadata_list
#define LOG(_MASK_, _LEVEL_, _STRING_)
Definition: mythlogging.h:41
const QString & GetFilename() const
const QString VIDEO_INETREF_DEFAULT
Definition: globals.cpp:24
bool GetBoolSetting(const QString &key, bool defaultval=false)
void RunProlog(void)
Sets up a thread, call this if you reimplement run().
Definition: mthread.cpp:202
VideoScannerThread(QObject *parent)
Definition: videoscan.cpp:87
const QString VIDEO_COVERFILE_DEFAULT
Definition: globals.cpp:25
QThread * qthread(void)
Returns the thread, this will always return the same pointer no matter how often you restart the thre...
Definition: mthread.cpp:244
void finishedScan()
Definition: videoscan.cpp:475
std::map< QString, CheckStruct > FileCheckList
Definition: videoscan.h:83
const QString VIDEO_RATING_DEFAULT
Definition: globals.cpp:30
QStringList GetOfflineSGHosts(void)
Definition: videoscan.h:69
VideoMetadataListManager * m_dbmetadata
Definition: videoscan.h:106
QStringList m_directories
Definition: videoscan.h:102
QStringList m_offlineSGHosts
Definition: videoscan.h:104
const QString VIDEO_FANART_DEFAULT
Definition: globals.cpp:29
static Type kEventType
Definition: videoscan.h:54
bool purgeByID(unsigned int db_id)
QObject * m_parent
Definition: videoscan.h:96
const QString VIDEO_DIRECTOR_DEFAULT
Definition: globals.cpp:23
QDateTime fromString(const QString &dtstr)
Converts kFilename && kISODate formats to QDateTime.
Definition: mythdate.cpp:30
void SendEvent(const MythEvent &event)
void SetProgressDialog(MythUIProgressDialog *dialog)
Definition: videoscan.h:68
void getExtensionIgnoreList(ext_ignore_list &ext_ignore) const
Definition: dbaccess.cpp:816
void doScan(const QStringList &dirs)
Definition: videoscan.cpp:430
const QString VIDEO_BANNER_DEFAULT
Definition: globals.cpp:28
QList< int > m_movList
Definition: videoscan.h:110