MythTV  master
imagescanner.cpp
Go to the documentation of this file.
1 #include "imagescanner.h"
2 
3 #include "mythlogging.h"
4 #include "mythcorecontext.h" // for events
5 
6 #include "imagemetadata.h"
7 
13 template <class DBFS>
15  : MThread("ImageScanner"),
16  m_dbfs(*dbfs),
17  m_thumb(*thumbGen),
18  m_dir(m_dbfs.GetImageFilters())
19 { }
20 
21 
22 template <class DBFS>
24 {
25  cancel();
26  wait();
27 }
28 
29 
33 template <class DBFS>
35 {
36  m_mutexQueue.lock();
37  m_clearQueue.clear();
38  m_mutexQueue.unlock();
39 
40  QMutexLocker locker(&m_mutexState);
41  m_scanning = false;
42 }
43 
44 
49 template <class DBFS>
51 {
52  QMutexLocker locker(&m_mutexState);
53  return m_scanning;
54 }
55 
56 
61 template <class DBFS>
63 {
64  QMutexLocker locker(&m_mutexQueue);
65  return !m_clearQueue.isEmpty();
66 }
67 
68 
76 template <class DBFS>
78 {
79  QMutexLocker locker(&m_mutexState);
80  m_scanning = scan;
81 
82  // Restart thread if not already running
83  if (!isRunning())
84  start();
85 }
86 
87 
95 template <class DBFS>
96 void ImageScanThread<DBFS>::EnqueueClear(int devId, const QString &action)
97 {
98  m_mutexQueue.lock();
99  m_clearQueue << qMakePair(devId, action);
100  m_mutexQueue.unlock();
101 
102  ChangeState(false);
103 }
104 
105 
110 template <class DBFS>
112 {
113  QMutexLocker locker(&m_mutexProgress);
114  return QStringList() << QString::number(gCoreContext->IsBackend())
115  << QString::number(m_progressCount)
116  << QString::number(m_progressTotalCount);
117 }
118 
125 template <class DBFS>
127 {
128  RunProlog();
129 
130  setPriority(QThread::LowPriority);
131 
132  do
133  {
134  // Process all clears before scanning
135  while (ClearsPending())
136  {
137  m_mutexQueue.lock();
138  if (m_clearQueue.isEmpty())
139  break;
140  ClearTask task = m_clearQueue.takeFirst();
141  m_mutexQueue.unlock();
142 
143  int devId = task.first;
144  QString action = task.second;
145 
146  LOG(VB_GENERAL, LOG_INFO,
147  QString("Clearing Filesystem: %1 %2").arg(action).arg(devId));
148 
149  // Clear Db
150  m_dbfs.ClearDb(devId, action);
151 
152  // Pass on to thumb generator now scanning has stopped
153  m_thumb.ClearThumbs(devId, action);
154  }
155 
156  // Scan requested ?
157  if (IsScanning())
158  {
159  LOG(VB_GENERAL, LOG_INFO, "Starting scan");
160 
161  // Load known directories and files from the database
162  if (!m_dbfs.ReadAllImages(m_dbFileMap, m_dbDirMap))
163  // Abort on any Db error
164  break;
165 
166  bool firstScan = m_dbFileMap.isEmpty();
167 
168  // Pause thumb generator so that scans are fast as possible
169  m_thumb.PauseBackground(true);
170 
171  // Adapter determines list of dirs to scan
172  StringMap paths = m_dbfs.GetScanDirs();
173 
174  CountFiles(paths.values());
175 
176  // Now start the actual syncronization
177  m_seenFile.clear();
178  m_changedImages.clear();
179  StringMap::const_iterator i = paths.constBegin();
180  while (i != paths.constEnd() && IsScanning())
181  {
182  SyncSubTree(QFileInfo(i.value()), GALLERY_DB_ID, i.key(), i.value());
183  ++i;
184  }
185 
186  // Release thumb generator asap
187  m_thumb.PauseBackground(false);
188 
189  // Adding or updating directories has been completed.
190  // The maps now only contain old directories & files that are not
191  // in the filesystem anymore. Remove them from the database
192  m_dbfs.RemoveFromDB(m_dbDirMap.values());
193  m_dbfs.RemoveFromDB(m_dbFileMap.values());
194 
195  // Cleanup thumbnails
196  QStringList mesg(m_thumb.DeleteThumbs(m_dbFileMap.values()));
197  mesg << m_changedImages.join(",");
198 
199  // Cleanup dirs
200  m_dbFileMap.clear();
201  m_dbDirMap.clear();
202  m_seenDir.clear();
203 
204  m_mutexProgress.lock();
205  // (count == total) signals scan end
206  Broadcast(m_progressTotalCount);
207  // Must reset counts for scan queries
208  m_progressCount = m_progressTotalCount = 0;
209  m_mutexProgress.unlock();
210 
211  LOG(VB_GENERAL, LOG_INFO, "Finished scan");
212 
213  // For initial scans pause briefly to give thumb generator a headstart
214  // before being deluged by client requests
215  if (firstScan)
216  msleep(1000);
217 
218  // Notify clients of completion with removed & changed images
219  m_dbfs.Notify("IMAGE_DB_CHANGED", mesg);
220 
221  ChangeState(false);
222  }
223  }
224  while (ClearsPending());
225 
226  RunEpilog();
227 }
228 
229 
239 template <class DBFS>
240 void ImageScanThread<DBFS>::SyncSubTree(const QFileInfo &dirInfo, int parentId,
241  int devId, const QString &base)
242 {
243  // Ignore excluded dirs
244  if (MATCHES(m_exclusions, dirInfo.fileName()))
245  {
246  LOG(VB_FILE, LOG_INFO,
247  QString("Excluding dir %1").arg(dirInfo.absoluteFilePath()));
248  return;
249  }
250 
251  // Use global image filters
252  QDir dir = m_dir;
253  if (!dir.cd(dirInfo.absoluteFilePath()))
254  {
255  LOG(VB_FILE, LOG_INFO,
256  QString("Failed to open dir %1").arg(dirInfo.absoluteFilePath()));
257  return;
258  }
259 
260  // Create directory node
261  int id = SyncDirectory(dirInfo, devId, base, parentId);
262 
263  // Sync its contents
264  QFileInfoList list = dir.entryInfoList();
265  foreach(const QFileInfo &fileInfo, list)
266  {
267  if (!IsScanning())
268  {
269  LOG(VB_GENERAL, LOG_INFO,
270  QString("Scan interrupted in %2").arg(dirInfo.absoluteFilePath()));
271  return;
272  }
273 
274  if (fileInfo.isDir())
275  {
276  // Scan this directory
277  SyncSubTree(fileInfo, id, devId, base);
278  }
279  else
280  {
281  SyncFile(fileInfo, devId, base, id);
282 
283  QMutexLocker locker(&m_mutexProgress);
284  ++m_progressCount;
285 
286  // Throttle updates
287  if (m_bcastTimer.elapsed() > 250)
288  Broadcast(m_progressCount);
289  }
290  }
291 }
292 
293 
308 template <class DBFS>
309  int ImageScanThread<DBFS>::SyncDirectory(const QFileInfo &dirInfo, int devId, const QString &base, int parentId)
310 {
311  QString absFilePath = dirInfo.absoluteFilePath();
312 
313  LOG(VB_FILE, LOG_DEBUG, QString("Syncing directory %1").arg(absFilePath));
314 
315  ImagePtr dir(m_dbfs.CreateItem(dirInfo, parentId, devId, base));
316 
317  // Is dir already in Db ?
318  if (m_dbDirMap.contains(dir->m_filePath))
319  {
320  ImagePtr dbDir = m_dbDirMap.value(dir->m_filePath);
321 
322  // The directory already exists in the db. Retain its id
323  dir->m_id = dbDir->m_id;
324 
325  // Parent may have changed due to a move
326  if (dir->m_modTime != dbDir->m_modTime
327  || dir->m_parentId != dbDir->m_parentId)
328  {
329  LOG(VB_FILE, LOG_INFO,
330  QString("Changed directory %1").arg(absFilePath));
331 
332  // Retain existing id & settings
333  dir->m_isHidden = dbDir->m_isHidden;
334  dir->m_userThumbnail = dbDir->m_userThumbnail;
335 
336  m_dbfs.UpdateDbImage(*dir);
337  // Note modified images
338  m_changedImages << QString::number(dir->m_id);
339  }
340 
341  // Remove the entry from the dbList
342  m_dbDirMap.remove(dir->m_filePath);
343  }
344  // Detect clones (same path in different SG dir)
345  else if (m_seenDir.contains(dir->m_filePath))
346  {
347  ImagePtr cloneDir = m_seenDir.value(dir->m_filePath);
348 
349  // All clones point to same Db dir. Use latest
350  if (cloneDir->m_modTime >= dir->m_modTime )
351  {
352  LOG(VB_FILE, LOG_INFO, QString("Directory %1 is an older clone of %2")
353  .arg(absFilePath, cloneDir->m_filePath));
354 
355  // Use previous version
356  dir = cloneDir;
357  }
358  else
359  {
360  LOG(VB_FILE, LOG_INFO,
361  QString("Directory %1 is a more recent clone of %2")
362  .arg(absFilePath, cloneDir->m_filePath));
363 
364  // Use new version
365  dir->m_id = cloneDir->m_id;
366  // Note modified time
367  m_changedImages << QString::number(dir->m_id);
368  }
369 
370  // Mark non-devices as cloned (for info display only)
371  if (!dir->IsDevice())
372  {
373  dir->m_type = kCloneDir;
374  m_dbfs.UpdateDbImage(*dir);
375  }
376  }
377  else
378  {
379  LOG(VB_FILE, LOG_INFO, QString("New directory %1").arg(absFilePath));
380 
381  // Create new Db dir with new id
382  dir->m_id = m_dbfs.InsertDbImage(*dir);
383  }
384 
385  // Note it for clone detection
386  m_seenDir.insert(dir->m_filePath, dir);
387 
388  return dir->m_id;
389 }
390 
391 
400 template <class DBFS>
402  const QString &path, int type, QString &comment,
403 #if QT_VERSION < QT_VERSION_CHECK(5,8,0)
404  uint &time,
405 #else
406  qint64 &time,
407 #endif
408  int &orientation)
409 {
410  // Set orientation, date, comment from file meta data
411  ImageMetaData *metadata = (type == kImageFile)
413  : ImageMetaData::FromVideo(path);
414 
415  orientation = metadata->GetOrientation();
416  comment = metadata->GetComment().simplified();
417  QDateTime dt = metadata->GetOriginalDateTime();
418 #if QT_VERSION < QT_VERSION_CHECK(5,8,0)
419  time = (dt.isValid()) ? dt.toTime_t() : 0;
420 #else
421  time = (dt.isValid()) ? dt.toSecsSinceEpoch() : 0;
422 #endif
423 
424  delete metadata;
425 }
426 
427 
442 template <class DBFS>
443 void ImageScanThread<DBFS>::SyncFile(const QFileInfo &fileInfo, int devId,
444  const QString &base, int parentId)
445 {
446  // Ignore excluded files
447  if (MATCHES(m_exclusions, fileInfo.fileName()))
448  {
449  LOG(VB_FILE, LOG_INFO,
450  QString("Excluding file %1").arg(fileInfo.absoluteFilePath()));
451  return;
452  }
453 
454  QString absFilePath = fileInfo.absoluteFilePath();
455 
456  ImagePtr im(m_dbfs.CreateItem(fileInfo, parentId, devId, base));
457  if (!im)
458  // Ignore unknown file type
459  return;
460 
461  if (m_dbFileMap.contains(im->m_filePath))
462  {
463  ImagePtrK dbIm = m_dbFileMap.value(im->m_filePath);
464 
465  // Parent may have changed due to a move
466  if (im->m_modTime == dbIm->m_modTime && im->m_parentId == dbIm->m_parentId)
467  {
468  // File already known & hasn't changed
469  // Remove it from removed list
470  m_dbFileMap.remove(im->m_filePath);
471  // Detect duplicates
472  m_seenFile.insert(im->m_filePath, absFilePath);
473  return;
474  }
475 
476  LOG(VB_FILE, LOG_INFO, QString("Modified file %1").arg(absFilePath));
477 
478  // Retain existing id & settings
479  im->m_id = dbIm->m_id;
480  im->m_isHidden = dbIm->m_isHidden;
481 
482  // Set date, comment from file meta data
483  int fileOrient;
484  PopulateMetadata(absFilePath, im->m_type,
485  im->m_comment, im->m_date, fileOrient);
486 
487  // Reset file orientation, retaining existing setting
488  int currentOrient = Orientation(dbIm->m_orientation).GetCurrent(false);
489  im->m_orientation = Orientation(currentOrient, fileOrient).Composite();
490 
491  // Remove it from removed list
492  m_dbFileMap.remove(im->m_filePath);
493  // Note modified images
494  m_changedImages << QString::number(im->m_id);
495 
496  // Update db
497  m_dbfs.UpdateDbImage(*im);
498  }
499  else if (m_seenFile.contains(im->m_filePath))
500  {
501  LOG(VB_GENERAL, LOG_WARNING, QString("Ignoring %1 (Duplicate of %2)")
502  .arg(absFilePath, m_seenFile.value(im->m_filePath)));
503  return;
504  }
505  else
506  {
507  // New images will be assigned an id by the db AUTO-INCREMENT
508  LOG(VB_FILE, LOG_INFO, QString("New file %1").arg(absFilePath));
509 
510  // Set date, comment from file meta data
511  int fileOrient;
512  PopulateMetadata(absFilePath, im->m_type,
513  im->m_comment, im->m_date, fileOrient);
514 
515  // Set file orientation
516  im->m_orientation = Orientation(fileOrient, fileOrient).Composite();
517 
518  // Update db (Set id for thumb generator)
519  im->m_id = m_dbfs.InsertDbImage(*im);
520  }
521 
522  // Detect duplicate filepaths in SG
523  m_seenFile.insert(im->m_filePath, absFilePath);
524 
525  // Populate absolute filename so that thumbgen doesn't need to locate file
526  im->m_filePath = absFilePath;
527 
528  // Ensure thumbnail exists.
529  m_thumb.CreateThumbnail(im);
530 }
531 
532 
538 template <class DBFS>
540 {
541  QFileInfoList files = dir.entryInfoList();
542 
543  foreach(const QFileInfo &fileInfo, files)
544  {
545  // Ignore excluded dirs/files
546  if (MATCHES(m_exclusions, fileInfo.fileName()))
547  continue;
548 
549  if (fileInfo.isFile())
550  ++m_progressTotalCount;
551  // Ignore missing dirs
552  else if (dir.cd(fileInfo.fileName()))
553  {
554  CountTree(dir);
555  dir.cdUp();
556  }
557  }
558 }
559 
560 
565 template <class DBFS>
566 void ImageScanThread<DBFS>::CountFiles(const QStringList &paths)
567 {
568  // Get exclusions as comma-seperated list using glob chars * and ?
569  QString excPattern = gCoreContext->GetSetting("GalleryIgnoreFilter", "");
570 
571  // Combine into a single regexp
572  excPattern.replace(".", "\\."); // Preserve "."
573  excPattern.replace("*", ".*"); // Convert glob wildcard "*"
574  excPattern.replace("?", "."); // Convert glob wildcard "?"
575  excPattern.replace(",", "|"); // Convert list to OR's
576 
577  QString pattern = QString("^(%1)$").arg(excPattern);
578  m_exclusions = REGEXP(pattern);
579 
580  LOG(VB_FILE, LOG_DEBUG, QString("Exclude regexp is \"%1\"").arg(pattern));
581 
582  // Lock counts until counting complete
583  QMutexLocker locker(&m_mutexProgress);
584  m_progressCount = 0;
585  m_progressTotalCount = 0;
586 
587  // Use global image filters
588  QDir dir = m_dir;
589  foreach(const QString &sgDir, paths)
590  {
591  // Ignore missing dirs
592  if (dir.cd(sgDir))
593  CountTree(dir);
594  }
595  // 0 signifies a scan start
596  Broadcast(0);
597 }
598 
599 
606 template <class DBFS>
608 {
609  // Only 2 scanners are ever visible (FE & BE) so use bool as scanner id
610  QStringList status;
611  status << QString::number(gCoreContext->IsBackend())
612  << QString::number(progress)
613  << QString::number(m_progressTotalCount);
614 
615  m_dbfs.Notify("IMAGE_SCAN_STATUS", status);
616 
617  // Reset broadcast throttle
618  m_bcastTimer.start();
619 }
620 
621 
622 // Must define the valid template implementations to generate code for the
623 // instantiations (as they are defined in the cpp rather than header).
624 // Otherwise the linker will fail with undefined references...
625 #include "imagemanager.h"
626 template class ImageScanThread<ImageDbLocal>;
627 template class ImageScanThread<ImageDbSg>;
This is a wrapper around QThread that does several additional things.
Definition: mthread.h:46
int Composite()
Encode original & current orientation to a single Db field.
Definition: imagemetadata.h:67
def scan(profile, smoonURL, gate)
Definition: scan.py:43
Abstract class for image metadata.
Definition: imagemetadata.h:95
Encapsulates Exif orientation processing.
Definition: imagemetadata.h:58
void SyncFile(const QFileInfo &fileInfo, int devId, const QString &base, int parentId)
Updates/populates db for an image/video file.
QPair< int, QString > ClearTask
Definition: imagescanner.h:64
Manages a collection of images.
virtual QDateTime GetOriginalDateTime(bool *exists=nullptr)=0
unsigned int uint
Definition: compat.h:140
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
bool ClearsPending()
Get status of 'clear device' queue.
QMap< int, QString > StringMap
Definition: imagetypes.h:62
void PopulateMetadata(const QString &path, int type, QString &comment, qint64 &time, int &orientation)
Read image date, orientation, comment from metadata.
Handles Exif/FFMpeg metadata tags for images.
static ImageMetaData * FromPicture(const QString &filePath)
Factory to retrieve metadata from pictures.
void run() override
Synchronises database to the storage group.
void cancel()
Clears queued items so that the thread can exit.
ImageScanThread(DBFS *const dbfs, ImageThumb< DBFS > *thumbGen)
Constructor.
void CountFiles(const QStringList &paths)
Counts images in a list of subtrees.
static bool isRunning(const char *program)
Returns true if a program containing the specified string is running on this machine.
int GetCurrent(bool compensate)
Determines orientation required for an image.
QString GetSetting(const QString &key, const QString &defaultval="")
QSharedPointer< ImageItemK > ImagePtrK
Definition: imagetypes.h:179
void EnqueueClear(int devId, const QString &action)
Queues a 'Clear Device' request, which will be actioned immediately.
void Broadcast(int progress)
Notify listeners of scan progress.
Synchronises image database to filesystem.
A picture.
Definition: imagetypes.h:38
bool IsBackend(void) const
is this process a backend process
#define LOG(_MASK_, _LEVEL_, _STRING_)
Definition: mythlogging.h:41
int SyncDirectory(const QFileInfo &dirInfo, int devId, const QString &base, int parentId)
Updates/populates db for a dir.
virtual QString GetComment(bool *exists=nullptr)=0
QSharedPointer< ImageItem > ImagePtr
Definition: imagetypes.h:173
#define MATCHES(RE, SUBJECT)
Definition: imagescanner.h:21
virtual int GetOrientation(bool *exists=nullptr)=0
#define REGEXP
Definition: imagescanner.h:20
void ChangeState(bool scan)
Run or interrupt scanner.
void SyncSubTree(const QFileInfo &dirInfo, int parentId, int devId, const QString &base)
Scans a dir subtree.
void CountTree(QDir &dir)
Counts images in a dir subtree.
bool IsScanning()
Return current scanner status.
#define GALLERY_DB_ID
Definition: imagetypes.h:26
static ImageMetaData * FromVideo(const QString &filePath)
Factory to retrieve metadata from videos.
QStringList GetProgress()
Returns number of images scanned & total number to scan.
A device sub dir comprised from multiple SG dirs.
Definition: imagetypes.h:36