MythTV  master
imagescanner.cpp
Go to the documentation of this file.
1 #include "imagescanner.h"
2 
3 #include "mythlogging.h"
4 #include "mythcorecontext.h" // for events
5 
6 #include "imagemetadata.h"
7 
13 template <class DBFS>
15  : MThread("ImageScanner"),
16  m_dbfs(*dbfs),
17  m_thumb(*thumbGen),
18  m_dir(m_dbfs.GetImageFilters())
19 { }
20 
21 
22 template <class DBFS>
24 {
25  cancel();
26  wait();
27 }
28 
29 
33 template <class DBFS>
35 {
36  m_mutexQueue.lock();
37  m_clearQueue.clear();
38  m_mutexQueue.unlock();
39 
40  QMutexLocker locker(&m_mutexState);
41  m_scanning = false;
42 }
43 
44 
49 template <class DBFS>
51 {
52  QMutexLocker locker(&m_mutexState);
53  return m_scanning;
54 }
55 
56 
61 template <class DBFS>
63 {
64  QMutexLocker locker(&m_mutexQueue);
65  return !m_clearQueue.isEmpty();
66 }
67 
68 
76 template <class DBFS>
78 {
79  QMutexLocker locker(&m_mutexState);
80  m_scanning = scan;
81 
82  // Restart thread if not already running
83  if (!isRunning())
84  start();
85 }
86 
87 
95 template <class DBFS>
96 void ImageScanThread<DBFS>::EnqueueClear(int devId, const QString &action)
97 {
98  m_mutexQueue.lock();
99  m_clearQueue << qMakePair(devId, action);
100  m_mutexQueue.unlock();
101 
102  ChangeState(false);
103 }
104 
105 
110 template <class DBFS>
112 {
113  QMutexLocker locker(&m_mutexProgress);
114  return QStringList() << QString::number(static_cast<int>(gCoreContext->IsBackend()))
115  << QString::number(m_progressCount)
116  << QString::number(m_progressTotalCount);
117 }
118 
125 template <class DBFS>
127 {
128  RunProlog();
129 
130  setPriority(QThread::LowPriority);
131 
132  do
133  {
134  // Process all clears before scanning
135  while (ClearsPending())
136  {
137  m_mutexQueue.lock();
138  if (m_clearQueue.isEmpty())
139  break;
140  ClearTask task = m_clearQueue.takeFirst();
141  m_mutexQueue.unlock();
142 
143  int devId = task.first;
144  QString action = task.second;
145 
146  LOG(VB_GENERAL, LOG_INFO,
147  QString("Clearing Filesystem: %1 %2").arg(action).arg(devId));
148 
149  // Clear Db
150  m_dbfs.ClearDb(devId, action);
151 
152  // Pass on to thumb generator now scanning has stopped
153  m_thumb.ClearThumbs(devId, action);
154  }
155 
156  // Scan requested ?
157  if (IsScanning())
158  {
159  LOG(VB_GENERAL, LOG_INFO, "Starting scan");
160 
161  // Load known directories and files from the database
162  if (!m_dbfs.ReadAllImages(m_dbFileMap, m_dbDirMap))
163  // Abort on any Db error
164  break;
165 
166  bool firstScan = m_dbFileMap.isEmpty();
167 
168  // Pause thumb generator so that scans are fast as possible
169  m_thumb.PauseBackground(true);
170 
171  // Adapter determines list of dirs to scan
172  StringMap paths = m_dbfs.GetScanDirs();
173 
174  CountFiles(paths.values());
175 
176  // Now start the actual syncronization
177  m_seenFile.clear();
178  m_changedImages.clear();
179  StringMap::const_iterator i = paths.constBegin();
180  while (i != paths.constEnd() && IsScanning())
181  {
182  SyncSubTree(QFileInfo(i.value()), GALLERY_DB_ID, i.key(), i.value());
183  ++i;
184  }
185 
186  // Release thumb generator asap
187  m_thumb.PauseBackground(false);
188 
189  // Adding or updating directories has been completed.
190  // The maps now only contain old directories & files that are not
191  // in the filesystem anymore. Remove them from the database
192  m_dbfs.RemoveFromDB(m_dbDirMap.values());
193  m_dbfs.RemoveFromDB(m_dbFileMap.values());
194 
195  // Cleanup thumbnails
196  QStringList mesg(m_thumb.DeleteThumbs(m_dbFileMap.values()));
197  mesg << m_changedImages.join(",");
198 
199  // Cleanup dirs
200  m_dbFileMap.clear();
201  m_dbDirMap.clear();
202  m_seenDir.clear();
203 
204  m_mutexProgress.lock();
205  // (count == total) signals scan end
206  Broadcast(m_progressTotalCount);
207  // Must reset counts for scan queries
208  m_progressCount = m_progressTotalCount = 0;
209  m_mutexProgress.unlock();
210 
211  LOG(VB_GENERAL, LOG_INFO, "Finished scan");
212 
213  // For initial scans pause briefly to give thumb generator a headstart
214  // before being deluged by client requests
215  if (firstScan)
216  msleep(1000);
217 
218  // Notify clients of completion with removed & changed images
219  m_dbfs.Notify("IMAGE_DB_CHANGED", mesg);
220 
221  ChangeState(false);
222  }
223  }
224  while (ClearsPending());
225 
226  RunEpilog();
227 }
228 
229 
239 template <class DBFS>
240 void ImageScanThread<DBFS>::SyncSubTree(const QFileInfo &dirInfo, int parentId,
241  int devId, const QString &base)
242 {
243  // Ignore excluded dirs
244  if (MATCHES(m_exclusions, dirInfo.fileName()))
245  {
246  LOG(VB_FILE, LOG_INFO,
247  QString("Excluding dir %1").arg(dirInfo.absoluteFilePath()));
248  return;
249  }
250 
251  // Use global image filters
252  QDir dir = m_dir;
253  if (!dir.cd(dirInfo.absoluteFilePath()))
254  {
255  LOG(VB_FILE, LOG_INFO,
256  QString("Failed to open dir %1").arg(dirInfo.absoluteFilePath()));
257  return;
258  }
259 
260  // Create directory node
261  int id = SyncDirectory(dirInfo, devId, base, parentId);
262 
263  // Sync its contents
264  for (const auto& fileInfo : dir.entryInfoList())
265  {
266  if (!IsScanning())
267  {
268  LOG(VB_GENERAL, LOG_INFO,
269  QString("Scan interrupted in %2").arg(dirInfo.absoluteFilePath()));
270  return;
271  }
272 
273  if (fileInfo.isDir())
274  {
275  // Scan this directory
276  SyncSubTree(fileInfo, id, devId, base);
277  }
278  else
279  {
280  SyncFile(fileInfo, devId, base, id);
281 
282  QMutexLocker locker(&m_mutexProgress);
283  ++m_progressCount;
284 
285  // Throttle updates
286  if (m_bcastTimer.elapsed() > 250)
287  Broadcast(m_progressCount);
288  }
289  }
290 }
291 
292 
307 template <class DBFS>
308  int ImageScanThread<DBFS>::SyncDirectory(const QFileInfo &dirInfo, int devId, const QString &base, int parentId)
309 {
310  QString absFilePath = dirInfo.absoluteFilePath();
311 
312  LOG(VB_FILE, LOG_DEBUG, QString("Syncing directory %1").arg(absFilePath));
313 
314  ImagePtr dir(m_dbfs.CreateItem(dirInfo, parentId, devId, base));
315 
316  // Is dir already in Db ?
317  if (m_dbDirMap.contains(dir->m_filePath))
318  {
319  ImagePtr dbDir = m_dbDirMap.value(dir->m_filePath);
320 
321  // The directory already exists in the db. Retain its id
322  dir->m_id = dbDir->m_id;
323 
324  // Parent may have changed due to a move
325  if (dir->m_modTime != dbDir->m_modTime
326  || dir->m_parentId != dbDir->m_parentId)
327  {
328  LOG(VB_FILE, LOG_INFO,
329  QString("Changed directory %1").arg(absFilePath));
330 
331  // Retain existing id & settings
332  dir->m_isHidden = dbDir->m_isHidden;
333  dir->m_userThumbnail = dbDir->m_userThumbnail;
334 
335  m_dbfs.UpdateDbImage(*dir);
336  // Note modified images
337  m_changedImages << QString::number(dir->m_id);
338  }
339 
340  // Remove the entry from the dbList
341  m_dbDirMap.remove(dir->m_filePath);
342  }
343  // Detect clones (same path in different SG dir)
344  else if (m_seenDir.contains(dir->m_filePath))
345  {
346  ImagePtr cloneDir = m_seenDir.value(dir->m_filePath);
347 
348  // All clones point to same Db dir. Use latest
349  if (cloneDir->m_modTime >= dir->m_modTime )
350  {
351  LOG(VB_FILE, LOG_INFO, QString("Directory %1 is an older clone of %2")
352  .arg(absFilePath, cloneDir->m_filePath));
353 
354  // Use previous version
355  dir = cloneDir;
356  }
357  else
358  {
359  LOG(VB_FILE, LOG_INFO,
360  QString("Directory %1 is a more recent clone of %2")
361  .arg(absFilePath, cloneDir->m_filePath));
362 
363  // Use new version
364  dir->m_id = cloneDir->m_id;
365  // Note modified time
366  m_changedImages << QString::number(dir->m_id);
367  }
368 
369  // Mark non-devices as cloned (for info display only)
370  if (!dir->IsDevice())
371  {
372  dir->m_type = kCloneDir;
373  m_dbfs.UpdateDbImage(*dir);
374  }
375  }
376  else
377  {
378  LOG(VB_FILE, LOG_INFO, QString("New directory %1").arg(absFilePath));
379 
380  // Create new Db dir with new id
381  dir->m_id = m_dbfs.InsertDbImage(*dir);
382  }
383 
384  // Note it for clone detection
385  m_seenDir.insert(dir->m_filePath, dir);
386 
387  return dir->m_id;
388 }
389 
390 
399 template <class DBFS>
401  const QString &path, int type, QString &comment,
402 #if QT_VERSION < QT_VERSION_CHECK(5,8,0)
403  uint &time,
404 #else
405  qint64 &time,
406 #endif
407  int &orientation)
408 {
409  // Set orientation, date, comment from file meta data
410  ImageMetaData *metadata = (type == kImageFile)
412  : ImageMetaData::FromVideo(path);
413 
414  orientation = metadata->GetOrientation();
415  comment = metadata->GetComment().simplified();
416  QDateTime dt = metadata->GetOriginalDateTime();
417 #if QT_VERSION < QT_VERSION_CHECK(5,8,0)
418  time = (dt.isValid()) ? dt.toTime_t() : 0;
419 #else
420  time = (dt.isValid()) ? dt.toSecsSinceEpoch() : 0;
421 #endif
422 
423  delete metadata;
424 }
425 
426 
441 template <class DBFS>
442 void ImageScanThread<DBFS>::SyncFile(const QFileInfo &fileInfo, int devId,
443  const QString &base, int parentId)
444 {
445  // Ignore excluded files
446  if (MATCHES(m_exclusions, fileInfo.fileName()))
447  {
448  LOG(VB_FILE, LOG_INFO,
449  QString("Excluding file %1").arg(fileInfo.absoluteFilePath()));
450  return;
451  }
452 
453  QString absFilePath = fileInfo.absoluteFilePath();
454 
455  ImagePtr im(m_dbfs.CreateItem(fileInfo, parentId, devId, base));
456  if (!im)
457  // Ignore unknown file type
458  return;
459 
460  if (m_dbFileMap.contains(im->m_filePath))
461  {
462  ImagePtrK dbIm = m_dbFileMap.value(im->m_filePath);
463 
464  // Parent may have changed due to a move
465  if (im->m_modTime == dbIm->m_modTime && im->m_parentId == dbIm->m_parentId)
466  {
467  // File already known & hasn't changed
468  // Remove it from removed list
469  m_dbFileMap.remove(im->m_filePath);
470  // Detect duplicates
471  m_seenFile.insert(im->m_filePath, absFilePath);
472  return;
473  }
474 
475  LOG(VB_FILE, LOG_INFO, QString("Modified file %1").arg(absFilePath));
476 
477  // Retain existing id & settings
478  im->m_id = dbIm->m_id;
479  im->m_isHidden = dbIm->m_isHidden;
480 
481  // Set date, comment from file meta data
482  int fileOrient = 0;
483  PopulateMetadata(absFilePath, im->m_type,
484  im->m_comment, im->m_date, fileOrient);
485 
486  // Reset file orientation, retaining existing setting
487  int currentOrient = Orientation(dbIm->m_orientation).GetCurrent(false);
488  im->m_orientation = Orientation(currentOrient, fileOrient).Composite();
489 
490  // Remove it from removed list
491  m_dbFileMap.remove(im->m_filePath);
492  // Note modified images
493  m_changedImages << QString::number(im->m_id);
494 
495  // Update db
496  m_dbfs.UpdateDbImage(*im);
497  }
498  else if (m_seenFile.contains(im->m_filePath))
499  {
500  LOG(VB_GENERAL, LOG_WARNING, QString("Ignoring %1 (Duplicate of %2)")
501  .arg(absFilePath, m_seenFile.value(im->m_filePath)));
502  return;
503  }
504  else
505  {
506  // New images will be assigned an id by the db AUTO-INCREMENT
507  LOG(VB_FILE, LOG_INFO, QString("New file %1").arg(absFilePath));
508 
509  // Set date, comment from file meta data
510  int fileOrient = 0;
511  PopulateMetadata(absFilePath, im->m_type,
512  im->m_comment, im->m_date, fileOrient);
513 
514  // Set file orientation
515  im->m_orientation = Orientation(fileOrient, fileOrient).Composite();
516 
517  // Update db (Set id for thumb generator)
518  im->m_id = m_dbfs.InsertDbImage(*im);
519  }
520 
521  // Detect duplicate filepaths in SG
522  m_seenFile.insert(im->m_filePath, absFilePath);
523 
524  // Populate absolute filename so that thumbgen doesn't need to locate file
525  im->m_filePath = absFilePath;
526 
527  // Ensure thumbnail exists.
528  m_thumb.CreateThumbnail(im);
529 }
530 
531 
537 template <class DBFS>
539 {
540  for (const auto& fileInfo : dir.entryInfoList())
541  {
542  // Ignore excluded dirs/files
543  if (MATCHES(m_exclusions, fileInfo.fileName()))
544  continue;
545 
546  if (fileInfo.isFile())
547  {
548  ++m_progressTotalCount;
549  }
550  // Ignore missing dirs
551  else if (dir.cd(fileInfo.fileName()))
552  {
553  CountTree(dir);
554  dir.cdUp();
555  }
556  }
557 }
558 
559 
564 template <class DBFS>
565 void ImageScanThread<DBFS>::CountFiles(const QStringList &paths)
566 {
567  // Get exclusions as comma-seperated list using glob chars * and ?
568  QString excPattern = gCoreContext->GetSetting("GalleryIgnoreFilter", "");
569 
570  // Combine into a single regexp
571  excPattern.replace(".", "\\."); // Preserve "."
572  excPattern.replace("*", ".*"); // Convert glob wildcard "*"
573  excPattern.replace("?", "."); // Convert glob wildcard "?"
574  excPattern.replace(",", "|"); // Convert list to OR's
575 
576  QString pattern = QString("^(%1)$").arg(excPattern);
577  m_exclusions = REGEXP(pattern);
578 
579  LOG(VB_FILE, LOG_DEBUG, QString("Exclude regexp is \"%1\"").arg(pattern));
580 
581  // Lock counts until counting complete
582  QMutexLocker locker(&m_mutexProgress);
583  m_progressCount = 0;
584  m_progressTotalCount = 0;
585 
586  // Use global image filters
587  QDir dir = m_dir;
588  for (const auto& sgDir : qAsConst(paths))
589  {
590  // Ignore missing dirs
591  if (dir.cd(sgDir))
592  CountTree(dir);
593  }
594  // 0 signifies a scan start
595  Broadcast(0);
596 }
597 
598 
605 template <class DBFS>
607 {
608  // Only 2 scanners are ever visible (FE & BE) so use bool as scanner id
609  QStringList status;
610  status << QString::number(static_cast<int>(gCoreContext->IsBackend()))
611  << QString::number(progress)
612  << QString::number(m_progressTotalCount);
613 
614  m_dbfs.Notify("IMAGE_SCAN_STATUS", status);
615 
616  // Reset broadcast throttle
617  m_bcastTimer.start();
618 }
619 
620 
621 // Must define the valid template implementations to generate code for the
622 // instantiations (as they are defined in the cpp rather than header).
623 // Otherwise the linker will fail with undefined references...
624 #include "imagemanager.h"
625 template class ImageScanThread<ImageDbLocal>;
626 template class ImageScanThread<ImageDbSg>;
QSharedPointer< ImageItem > ImagePtr
Definition: imagetypes.h:166
This is a wrapper around QThread that does several additional things.
Definition: mthread.h:46
def scan(profile, smoonURL, gate)
Definition: scan.py:57
Abstract class for image metadata.
Definition: imagemetadata.h:99
Encapsulates Exif orientation processing.
Definition: imagemetadata.h:62
void SyncFile(const QFileInfo &fileInfo, int devId, const QString &base, int parentId)
Updates/populates db for an image/video file.
ImageScanThread(DBFS *dbfs, ImageThumb< DBFS > *thumbGen)
Constructor.
Manages a collection of images.
int Composite() const
Encode original & current orientation to a single Db field.
Definition: imagemetadata.h:71
virtual QDateTime GetOriginalDateTime(bool *exists=nullptr)=0
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
bool ClearsPending()
Get status of 'clear device' queue.
void PopulateMetadata(const QString &path, int type, QString &comment, qint64 &time, int &orientation)
Read image date, orientation, comment from metadata.
Handles Exif/FFMpeg metadata tags for images.
QSharedPointer< ImageItemK > ImagePtrK
Definition: imagetypes.h:172
static ImageMetaData * FromPicture(const QString &filePath)
Factory to retrieve metadata from pictures.
void run() override
Synchronises database to the storage group.
void cancel()
Clears queued items so that the thread can exit.
void CountFiles(const QStringList &paths)
Counts images in a list of subtrees.
static bool isRunning(const char *program)
Returns true if a program containing the specified string is running on this machine.
QPair< int, QString > ClearTask
Definition: imagescanner.h:64
int GetCurrent(bool compensate)
Determines orientation required for an image.
QMap< int, QString > StringMap
Definition: imagetypes.h:62
QString GetSetting(const QString &key, const QString &defaultval="")
unsigned int uint
Definition: compat.h:140
~ImageScanThread() override
void EnqueueClear(int devId, const QString &action)
Queues a 'Clear Device' request, which will be actioned immediately.
void Broadcast(int progress)
Notify listeners of scan progress.
Synchronises image database to filesystem.
A picture.
Definition: imagetypes.h:38
bool IsBackend(void) const
is this process a backend process
int SyncDirectory(const QFileInfo &dirInfo, int devId, const QString &base, int parentId)
Updates/populates db for a dir.
QFileInfo fileInfo(filename)
virtual QString GetComment(bool *exists=nullptr)=0
#define MATCHES(RE, SUBJECT)
Definition: imagescanner.h:21
virtual int GetOrientation(bool *exists=nullptr)=0
#define REGEXP
Definition: imagescanner.h:20
void ChangeState(bool scan)
Run or interrupt scanner.
void SyncSubTree(const QFileInfo &dirInfo, int parentId, int devId, const QString &base)
Scans a dir subtree.
void CountTree(QDir &dir)
Counts images in a dir subtree.
bool IsScanning()
Return current scanner status.
#define GALLERY_DB_ID
Definition: imagetypes.h:26
static ImageMetaData * FromVideo(const QString &filePath)
Factory to retrieve metadata from videos.
QStringList GetProgress()
Returns number of images scanned & total number to scan.
A device sub dir comprised from multiple SG dirs.
Definition: imagetypes.h:36
#define LOG(_MASK_, _LEVEL_, _QSTRING_)
Definition: mythlogging.h:23