MythTV  master
imagescanner.cpp
Go to the documentation of this file.
1 #include "imagescanner.h"
2 
3 #include "mythlogging.h"
4 #include "mythcorecontext.h" // for events
5 
6 #include "imagemetadata.h"
7 
13 template <class DBFS>
15  : MThread("ImageScanner"),
16  m_dbfs(*dbfs),
17  m_thumb(*thumbGen),
18  m_dir(m_dbfs.GetImageFilters())
19 { }
20 
21 
22 template <class DBFS>
24 {
25  cancel();
26  wait();
27 }
28 
29 
33 template <class DBFS>
35 {
36  m_mutexQueue.lock();
37  m_clearQueue.clear();
38  m_mutexQueue.unlock();
39 
40  QMutexLocker locker(&m_mutexState);
41  m_scanning = false;
42 }
43 
44 
49 template <class DBFS>
51 {
52  QMutexLocker locker(&m_mutexState);
53  return m_scanning;
54 }
55 
56 
61 template <class DBFS>
63 {
64  QMutexLocker locker(&m_mutexQueue);
65  return !m_clearQueue.isEmpty();
66 }
67 
68 
76 template <class DBFS>
78 {
79  QMutexLocker locker(&m_mutexState);
80  m_scanning = scan;
81 
82  // Restart thread if not already running
83  if (!isRunning())
84  start();
85 }
86 
87 
95 template <class DBFS>
96 void ImageScanThread<DBFS>::EnqueueClear(int devId, const QString &action)
97 {
98  m_mutexQueue.lock();
99  m_clearQueue << qMakePair(devId, action);
100  m_mutexQueue.unlock();
101 
102  ChangeState(false);
103 }
104 
105 
110 template <class DBFS>
112 {
113  QMutexLocker locker(&m_mutexProgress);
114  return QStringList() << QString::number(static_cast<int>(gCoreContext->IsBackend()))
115  << QString::number(m_progressCount)
116  << QString::number(m_progressTotalCount);
117 }
118 
125 template <class DBFS>
127 {
128  RunProlog();
129 
130  setPriority(QThread::LowPriority);
131 
132  do
133  {
134  // Process all clears before scanning
135  while (ClearsPending())
136  {
137  m_mutexQueue.lock();
138  if (m_clearQueue.isEmpty())
139  break;
140  ClearTask task = m_clearQueue.takeFirst();
141  m_mutexQueue.unlock();
142 
143  int devId = task.first;
144  QString action = task.second;
145 
146  LOG(VB_GENERAL, LOG_INFO,
147  QString("Clearing Filesystem: %1 %2").arg(action).arg(devId));
148 
149  // Clear Db
150  m_dbfs.ClearDb(devId, action);
151 
152  // Pass on to thumb generator now scanning has stopped
153  m_thumb.ClearThumbs(devId, action);
154  }
155 
156  // Scan requested ?
157  if (IsScanning())
158  {
159  LOG(VB_GENERAL, LOG_INFO, "Starting scan");
160 
161  // Load known directories and files from the database
162  if (!m_dbfs.ReadAllImages(m_dbFileMap, m_dbDirMap))
163  // Abort on any Db error
164  break;
165 
166  bool firstScan = m_dbFileMap.isEmpty();
167 
168  // Pause thumb generator so that scans are fast as possible
169  m_thumb.PauseBackground(true);
170 
171  // Adapter determines list of dirs to scan
172  StringMap paths = m_dbfs.GetScanDirs();
173 
174  CountFiles(paths.values());
175 
176  // Now start the actual syncronization
177  m_seenFile.clear();
178  m_changedImages.clear();
179  StringMap::const_iterator i = paths.constBegin();
180  while (i != paths.constEnd() && IsScanning())
181  {
182  SyncSubTree(QFileInfo(i.value()), GALLERY_DB_ID, i.key(), i.value());
183  ++i;
184  }
185 
186  // Release thumb generator asap
187  m_thumb.PauseBackground(false);
188 
189  // Adding or updating directories has been completed.
190  // The maps now only contain old directories & files that are not
191  // in the filesystem anymore. Remove them from the database
192  m_dbfs.RemoveFromDB(QVector<ImagePtr>::fromList(m_dbDirMap.values()));
193  m_dbfs.RemoveFromDB(QVector<ImagePtr>::fromList(m_dbFileMap.values()));
194 
195  // Cleanup thumbnails
196  QStringList mesg(m_thumb.DeleteThumbs(QVector<ImagePtr>::fromList(m_dbFileMap.values())));
197  mesg << m_changedImages.join(",");
198 
199  // Cleanup dirs
200  m_dbFileMap.clear();
201  m_dbDirMap.clear();
202  m_seenDir.clear();
203 
204  m_mutexProgress.lock();
205  // (count == total) signals scan end
206  Broadcast(m_progressTotalCount);
207  // Must reset counts for scan queries
208  m_progressCount = m_progressTotalCount = 0;
209  m_mutexProgress.unlock();
210 
211  LOG(VB_GENERAL, LOG_INFO, "Finished scan");
212 
213  // For initial scans pause briefly to give thumb generator a headstart
214  // before being deluged by client requests
215  if (firstScan)
216  msleep(1000);
217 
218  // Notify clients of completion with removed & changed images
219  m_dbfs.Notify("IMAGE_DB_CHANGED", mesg);
220 
221  ChangeState(false);
222  }
223  }
224  while (ClearsPending());
225 
226  RunEpilog();
227 }
228 
229 
239 template <class DBFS>
240 void ImageScanThread<DBFS>::SyncSubTree(const QFileInfo &dirInfo, int parentId,
241  int devId, const QString &base)
242 {
243  // Ignore excluded dirs
244  if (MATCHES(m_exclusions, dirInfo.fileName()))
245  {
246  LOG(VB_FILE, LOG_INFO,
247  QString("Excluding dir %1").arg(dirInfo.absoluteFilePath()));
248  return;
249  }
250 
251  // Use global image filters
252  QDir dir = m_dir;
253  if (!dir.cd(dirInfo.absoluteFilePath()))
254  {
255  LOG(VB_FILE, LOG_INFO,
256  QString("Failed to open dir %1").arg(dirInfo.absoluteFilePath()));
257  return;
258  }
259 
260  // Create directory node
261  int id = SyncDirectory(dirInfo, devId, base, parentId);
262  if (id == -1)
263  {
264  LOG(VB_FILE, LOG_INFO,
265  QString("Failed to sync dir %1").arg(dirInfo.absoluteFilePath()));
266  return;
267  }
268 
269  // Sync its contents
270  QFileInfoList entries = dir.entryInfoList();
271  for (const auto & fileInfo : qAsConst(entries))
272  {
273  if (!IsScanning())
274  {
275  LOG(VB_GENERAL, LOG_INFO,
276  QString("Scan interrupted in %2").arg(dirInfo.absoluteFilePath()));
277  return;
278  }
279 
280  if (fileInfo.isDir())
281  {
282  // Scan this directory
283  SyncSubTree(fileInfo, id, devId, base);
284  }
285  else
286  {
287  SyncFile(fileInfo, devId, base, id);
288 
289  QMutexLocker locker(&m_mutexProgress);
290  ++m_progressCount;
291 
292  // Throttle updates
293  if (m_bcastTimer.elapsed() > 250)
294  Broadcast(m_progressCount);
295  }
296  }
297 }
298 
299 
314 template <class DBFS>
315  int ImageScanThread<DBFS>::SyncDirectory(const QFileInfo &dirInfo, int devId, const QString &base, int parentId)
316 {
317  QString absFilePath = dirInfo.absoluteFilePath();
318 
319  LOG(VB_FILE, LOG_DEBUG, QString("Syncing directory %1").arg(absFilePath));
320 
321  ImagePtr dir(m_dbfs.CreateItem(dirInfo, parentId, devId, base));
322 
323  // Is dir already in Db ?
324  if (m_dbDirMap.contains(dir->m_filePath))
325  {
326  ImagePtr dbDir = m_dbDirMap.value(dir->m_filePath);
327  if (dbDir == nullptr)
328  return -1;
329 
330  // The directory already exists in the db. Retain its id
331  dir->m_id = dbDir->m_id;
332 
333  // Parent may have changed due to a move
334  if (dir->m_modTime != dbDir->m_modTime
335  || dir->m_parentId != dbDir->m_parentId)
336  {
337  LOG(VB_FILE, LOG_INFO,
338  QString("Changed directory %1").arg(absFilePath));
339 
340  // Retain existing id & settings
341  dir->m_isHidden = dbDir->m_isHidden;
342  dir->m_userThumbnail = dbDir->m_userThumbnail;
343 
344  m_dbfs.UpdateDbImage(*dir);
345  // Note modified images
346  m_changedImages << QString::number(dir->m_id);
347  }
348 
349  // Remove the entry from the dbList
350  m_dbDirMap.remove(dir->m_filePath);
351  }
352  // Detect clones (same path in different SG dir)
353  else if (m_seenDir.contains(dir->m_filePath))
354  {
355  ImagePtr cloneDir = m_seenDir.value(dir->m_filePath);
356  if (cloneDir == nullptr)
357  return -1;
358 
359  // All clones point to same Db dir. Use latest
360  if (cloneDir->m_modTime >= dir->m_modTime )
361  {
362  LOG(VB_FILE, LOG_INFO, QString("Directory %1 is an older clone of %2")
363  .arg(absFilePath, cloneDir->m_filePath));
364 
365  // Use previous version
366  dir = cloneDir;
367  }
368  else
369  {
370  LOG(VB_FILE, LOG_INFO,
371  QString("Directory %1 is a more recent clone of %2")
372  .arg(absFilePath, cloneDir->m_filePath));
373 
374  // Use new version
375  dir->m_id = cloneDir->m_id;
376  // Note modified time
377  m_changedImages << QString::number(dir->m_id);
378  }
379 
380  // Mark non-devices as cloned (for info display only)
381  if (!dir->IsDevice())
382  {
383  dir->m_type = kCloneDir;
384  m_dbfs.UpdateDbImage(*dir);
385  }
386  }
387  else
388  {
389  LOG(VB_FILE, LOG_INFO, QString("New directory %1").arg(absFilePath));
390 
391  // Create new Db dir with new id
392  dir->m_id = m_dbfs.InsertDbImage(*dir);
393  }
394 
395  // Note it for clone detection
396  m_seenDir.insert(dir->m_filePath, dir);
397 
398  return dir->m_id;
399 }
400 
401 
410 template <class DBFS>
412  const QString &path, int type, QString &comment,
413  qint64 &time,
414  int &orientation)
415 {
416  // Set orientation, date, comment from file meta data
417  ImageMetaData *metadata = (type == kImageFile)
419  : ImageMetaData::FromVideo(path);
420 
421  orientation = metadata->GetOrientation();
422  comment = metadata->GetComment().simplified();
423  QDateTime dt = metadata->GetOriginalDateTime();
424  time = (dt.isValid()) ? dt.toSecsSinceEpoch() : 0;
425 
426  delete metadata;
427 }
428 
429 
444 template <class DBFS>
445 void ImageScanThread<DBFS>::SyncFile(const QFileInfo &fileInfo, int devId,
446  const QString &base, int parentId)
447 {
448  // Ignore excluded files
449  if (MATCHES(m_exclusions, fileInfo.fileName()))
450  {
451  LOG(VB_FILE, LOG_INFO,
452  QString("Excluding file %1").arg(fileInfo.absoluteFilePath()));
453  return;
454  }
455 
456  QString absFilePath = fileInfo.absoluteFilePath();
457 
458  ImagePtr im(m_dbfs.CreateItem(fileInfo, parentId, devId, base));
459  if (!im)
460  // Ignore unknown file type
461  return;
462 
463  if (m_dbFileMap.contains(im->m_filePath))
464  {
465  ImagePtrK dbIm = m_dbFileMap.value(im->m_filePath);
466 
467  // Parent may have changed due to a move
468  if (im->m_modTime == dbIm->m_modTime && im->m_parentId == dbIm->m_parentId)
469  {
470  // File already known & hasn't changed
471  // Remove it from removed list
472  m_dbFileMap.remove(im->m_filePath);
473  // Detect duplicates
474  m_seenFile.insert(im->m_filePath, absFilePath);
475  return;
476  }
477 
478  LOG(VB_FILE, LOG_INFO, QString("Modified file %1").arg(absFilePath));
479 
480  // Retain existing id & settings
481  im->m_id = dbIm->m_id;
482  im->m_isHidden = dbIm->m_isHidden;
483 
484  // Set date, comment from file meta data
485  int fileOrient = 0;
486  PopulateMetadata(absFilePath, im->m_type,
487  im->m_comment, im->m_date, fileOrient);
488 
489  // Reset file orientation, retaining existing setting
490  int currentOrient = Orientation(dbIm->m_orientation).GetCurrent(false);
491  im->m_orientation = Orientation(currentOrient, fileOrient).Composite();
492 
493  // Remove it from removed list
494  m_dbFileMap.remove(im->m_filePath);
495  // Note modified images
496  m_changedImages << QString::number(im->m_id);
497 
498  // Update db
499  m_dbfs.UpdateDbImage(*im);
500  }
501  else if (m_seenFile.contains(im->m_filePath))
502  {
503  LOG(VB_GENERAL, LOG_WARNING, QString("Ignoring %1 (Duplicate of %2)")
504  .arg(absFilePath, m_seenFile.value(im->m_filePath)));
505  return;
506  }
507  else
508  {
509  // New images will be assigned an id by the db AUTO-INCREMENT
510  LOG(VB_FILE, LOG_INFO, QString("New file %1").arg(absFilePath));
511 
512  // Set date, comment from file meta data
513  int fileOrient = 0;
514  PopulateMetadata(absFilePath, im->m_type,
515  im->m_comment, im->m_date, fileOrient);
516 
517  // Set file orientation
518  im->m_orientation = Orientation(fileOrient, fileOrient).Composite();
519 
520  // Update db (Set id for thumb generator)
521  im->m_id = m_dbfs.InsertDbImage(*im);
522  }
523 
524  // Detect duplicate filepaths in SG
525  m_seenFile.insert(im->m_filePath, absFilePath);
526 
527  // Populate absolute filename so that thumbgen doesn't need to locate file
528  im->m_filePath = absFilePath;
529 
530  // Ensure thumbnail exists.
531  m_thumb.CreateThumbnail(im);
532 }
533 
534 
540 template <class DBFS>
542 {
543  QFileInfoList entries = dir.entryInfoList();
544  for (const auto & fileInfo : qAsConst(entries))
545  {
546  // Ignore excluded dirs/files
547  if (MATCHES(m_exclusions, fileInfo.fileName()))
548  continue;
549 
550  if (fileInfo.isFile())
551  {
552  ++m_progressTotalCount;
553  }
554  // Ignore missing dirs
555  else if (dir.cd(fileInfo.fileName()))
556  {
557  CountTree(dir);
558  dir.cdUp();
559  }
560  }
561 }
562 
563 
568 template <class DBFS>
569 void ImageScanThread<DBFS>::CountFiles(const QStringList &paths)
570 {
571  // Get exclusions as comma-seperated list using glob chars * and ?
572  QString excPattern = gCoreContext->GetSetting("GalleryIgnoreFilter", "");
573 
574  // Combine into a single regexp
575  excPattern.replace(".", "\\."); // Preserve "."
576  excPattern.replace("*", ".*"); // Convert glob wildcard "*"
577  excPattern.replace("?", "."); // Convert glob wildcard "?"
578  excPattern.replace(",", "|"); // Convert list to OR's
579 
580  QString pattern = QString("^(%1)$").arg(excPattern);
581  m_exclusions = REGEXP(pattern);
582 
583  LOG(VB_FILE, LOG_DEBUG, QString("Exclude regexp is \"%1\"").arg(pattern));
584 
585  // Lock counts until counting complete
586  QMutexLocker locker(&m_mutexProgress);
587  m_progressCount = 0;
588  m_progressTotalCount = 0;
589 
590  // Use global image filters
591  QDir dir = m_dir;
592  for (const auto& sgDir : qAsConst(paths))
593  {
594  // Ignore missing dirs
595  if (dir.cd(sgDir))
596  CountTree(dir);
597  }
598  // 0 signifies a scan start
599  Broadcast(0);
600 }
601 
602 
609 template <class DBFS>
611 {
612  // Only 2 scanners are ever visible (FE & BE) so use bool as scanner id
613  QStringList status;
614  status << QString::number(static_cast<int>(gCoreContext->IsBackend()))
615  << QString::number(progress)
616  << QString::number(m_progressTotalCount);
617 
618  m_dbfs.Notify("IMAGE_SCAN_STATUS", status);
619 
620  // Reset broadcast throttle
621  m_bcastTimer.start();
622 }
623 
624 
625 // Must define the valid template implementations to generate code for the
626 // instantiations (as they are defined in the cpp rather than header).
627 // Otherwise the linker will fail with undefined references...
628 #include "imagemanager.h"
629 template class ImageScanThread<ImageDbLocal>;
630 template class ImageScanThread<ImageDbSg>;
ImageScanThread::Broadcast
void Broadcast(int progress)
Notify listeners of scan progress.
Definition: imagescanner.cpp:610
ImagePtrK
QSharedPointer< ImageItemK > ImagePtrK
Definition: imagetypes.h:164
fileInfo
QFileInfo fileInfo(filename)
ImageScanThread::ChangeState
void ChangeState(bool scan)
Run or interrupt scanner.
Definition: imagescanner.cpp:77
ImageMetaData
Abstract class for image metadata.
Definition: imagemetadata.h:100
ImageScanThread::SyncDirectory
int SyncDirectory(const QFileInfo &dirInfo, int devId, const QString &base, int parentId)
Updates/populates db for a dir.
Definition: imagescanner.cpp:315
ImageScanThread::CountTree
void CountTree(QDir &dir)
Counts images in a dir subtree.
Definition: imagescanner.cpp:541
progress
bool progress
Definition: mythtv/programs/mythcommflag/main.cpp:73
imagemanager.h
Manages a collection of images.
arg
arg(title).arg(filename).arg(doDelete))
LOG
#define LOG(_MASK_, _LEVEL_, _QSTRING_)
Definition: mythlogging.h:23
ImageScanThread::ClearTask
QPair< int, QString > ClearTask
Definition: imagescanner.h:60
ImageMetaData::FromVideo
static ImageMetaData * FromVideo(const QString &filePath)
Factory to retrieve metadata from videos.
Definition: imagemetadata.cpp:709
hardwareprofile.scan.scan
def scan(profile, smoonURL, gate)
Definition: scan.py:57
ImageMetaData::GetOrientation
virtual int GetOrientation(bool *exists=nullptr)=0
Orientation::GetCurrent
int GetCurrent(bool compensate)
Determines orientation required for an image.
Definition: imagemetadata.cpp:85
ImageThumb
Definition: imagethumbs.h:128
ImageScanThread::run
void run() override
Synchronises database to the storage group.
Definition: imagescanner.cpp:126
ImageScanThread::cancel
void cancel()
Clears queued items so that the thread can exit.
Definition: imagescanner.cpp:34
REGEXP
#define REGEXP
Definition: imagescanner.h:20
mythlogging.h
ImageScanThread::EnqueueClear
void EnqueueClear(int devId, const QString &action)
Queues a 'Clear Device' request, which will be actioned immediately.
Definition: imagescanner.cpp:96
StringMap
QMap< int, QString > StringMap
Definition: imagetypes.h:62
Orientation::Composite
int Composite() const
Encode original & current orientation to a single Db field.
Definition: imagemetadata.h:71
imagescanner.h
Synchronises image database to filesystem.
MATCHES
#define MATCHES(RE, SUBJECT)
Definition: imagescanner.h:21
ImageScanThread::PopulateMetadata
void PopulateMetadata(const QString &path, int type, QString &comment, qint64 &time, int &orientation)
Read image date, orientation, comment from metadata.
Definition: imagescanner.cpp:411
ImageMetaData::GetOriginalDateTime
virtual QDateTime GetOriginalDateTime(bool *exists=nullptr)=0
ImageMetaData::FromPicture
static ImageMetaData * FromPicture(const QString &filePath)
Factory to retrieve metadata from pictures.
Definition: imagemetadata.cpp:700
gCoreContext
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
Definition: mythcorecontext.cpp:56
ImageScanThread::IsScanning
bool IsScanning()
Return current scanner status.
Definition: imagescanner.cpp:50
ImageScanThread::ClearsPending
bool ClearsPending()
Get status of 'clear device' queue.
Definition: imagescanner.cpp:62
isRunning
static bool isRunning(const char *program)
Returns true if a program containing the specified string is running on this machine.
Definition: mythtv/programs/mythshutdown/main.cpp:202
kImageFile
@ kImageFile
A picture.
Definition: imagetypes.h:38
ImagePtr
QSharedPointer< ImageItem > ImagePtr
Definition: imagetypes.h:158
mythcorecontext.h
ImageScanThread::SyncSubTree
void SyncSubTree(const QFileInfo &dirInfo, int parentId, int devId, const QString &base)
Scans a dir subtree.
Definition: imagescanner.cpp:240
ImageScanThread< ImageDbLocal >
MythCoreContext::IsBackend
bool IsBackend(void) const
is this process a backend process
Definition: mythcorecontext.cpp:661
dir
QDir dir
Definition: mythplugins/mytharchive/mytharchivehelper/main.cpp:1174
imagemetadata.h
Handles Exif/FFMpeg metadata tags for images.
ImageScanThread::SyncFile
void SyncFile(const QFileInfo &fileInfo, int devId, const QString &base, int parentId)
Updates/populates db for an image/video file.
Definition: imagescanner.cpp:445
MThread
This is a wrapper around QThread that does several additional things.
Definition: mthread.h:49
build_compdb.action
action
Definition: build_compdb.py:9
ImageMetaData::GetComment
virtual QString GetComment(bool *exists=nullptr)=0
ImageScanThread::ImageScanThread
ImageScanThread(DBFS *dbfs, ImageThumb< DBFS > *thumbGen)
Constructor.
Definition: imagescanner.cpp:14
Orientation
Encapsulates Exif orientation processing.
Definition: imagemetadata.h:63
ImageScanThread::CountFiles
void CountFiles(const QStringList &paths)
Counts images in a list of subtrees.
Definition: imagescanner.cpp:569
kCloneDir
@ kCloneDir
A device sub dir comprised from multiple SG dirs.
Definition: imagetypes.h:36
GALLERY_DB_ID
#define GALLERY_DB_ID
Definition: imagetypes.h:26
ImageScanThread::~ImageScanThread
~ImageScanThread() override
Definition: imagescanner.cpp:23
MythCoreContext::GetSetting
QString GetSetting(const QString &key, const QString &defaultval="")
Definition: mythcorecontext.cpp:915
build_compdb.paths
paths
Definition: build_compdb.py:13
ImageScanThread::GetProgress
QStringList GetProgress()
Returns number of images scanned & total number to scan.
Definition: imagescanner.cpp:111