MythTV master
imagescanner.cpp
Go to the documentation of this file.
1#include "imagescanner.h"
2
3#include "libmythbase/mythcorecontext.h" // for gCoreContext
5
6#include "imagemetadata.h"
7
13template <class DBFS>
15 : MThread("ImageScanner"),
16 m_dbfs(*dbfs),
17 m_thumb(*thumbGen),
18 m_dir(m_dbfs.GetImageFilters())
19{ }
20
21
22template <class DBFS>
24{
25 cancel();
26 wait();
27}
33template <class DBFS>
36 m_mutexQueue.lock();
37 m_clearQueue.clear();
38 m_mutexQueue.unlock();
40 QMutexLocker locker(&m_mutexState);
41 m_scanning = false;
42}
43
49template <class DBFS>
52 QMutexLocker locker(&m_mutexState);
53 return m_scanning;
56
61template <class DBFS>
63{
64 QMutexLocker locker(&m_mutexQueue);
65 return !m_clearQueue.isEmpty();
66}
67
68
76template <class DBFS>
78{
79 QMutexLocker locker(&m_mutexState);
80 m_scanning = scan;
81
82 // Restart thread if not already running
83 if (!isRunning())
84 start();
85}
86
87
95template <class DBFS>
96void ImageScanThread<DBFS>::EnqueueClear(int devId, const QString &action)
97{
98 m_mutexQueue.lock();
99 m_clearQueue << qMakePair(devId, action);
100 m_mutexQueue.unlock();
101
102 ChangeState(false);
103}
104
105
110template <class DBFS>
112{
113 QMutexLocker locker(&m_mutexProgress);
114 return QStringList() << QString::number(static_cast<int>(gCoreContext->IsBackend()))
115 << QString::number(m_progressCount)
116 << QString::number(m_progressTotalCount);
117}
118
125template <class DBFS>
127{
128 RunProlog();
129
130 setPriority(QThread::LowPriority);
131
132 bool at_least_once { true };
133 while (ClearsPending() || at_least_once)
134 {
135 at_least_once = false;
136
137 // Process all clears before scanning
138 while (ClearsPending())
139 {
140 m_mutexQueue.lock();
141 if (m_clearQueue.isEmpty())
142 break;
143 ClearTask task = m_clearQueue.takeFirst();
144 m_mutexQueue.unlock();
145
146 int devId = task.first;
147 QString action = task.second;
148
149 LOG(VB_GENERAL, LOG_INFO,
150 QString("Clearing Filesystem: %1 %2").arg(action).arg(devId));
151
152 // Clear Db
153 m_dbfs.ClearDb(devId, action);
154
155 // Pass on to thumb generator now scanning has stopped
156 m_thumb.ClearThumbs(devId, action);
157 }
158
159 // Scan requested ?
160 if (IsScanning())
161 {
162 LOG(VB_GENERAL, LOG_INFO, "Starting scan");
163
164 // Load known directories and files from the database
165 if (!m_dbfs.ReadAllImages(m_dbFileMap, m_dbDirMap))
166 // Abort on any Db error
167 break;
168
169 bool firstScan = m_dbFileMap.isEmpty();
170
171 // Pause thumb generator so that scans are fast as possible
172 m_thumb.PauseBackground(true);
173
174 // Adapter determines list of dirs to scan
175 StringMap paths = m_dbfs.GetScanDirs();
176
177 CountFiles(paths.values());
178
179 // Now start the actual syncronization
180 m_seenFile.clear();
181 m_changedImages.clear();
182 StringMap::const_iterator i = paths.constBegin();
183 while (i != paths.constEnd() && IsScanning())
184 {
185 SyncSubTree(QFileInfo(i.value()), GALLERY_DB_ID, i.key(), i.value());
186 ++i;
187 }
188
189 // Release thumb generator asap
190 m_thumb.PauseBackground(false);
191
192 // Adding or updating directories has been completed.
193 // The maps now only contain old directories & files that are not
194 // in the filesystem anymore. Remove them from the database
195 m_dbfs.RemoveFromDB(QVector<ImagePtr>::fromList(m_dbDirMap.values()));
196 m_dbfs.RemoveFromDB(QVector<ImagePtr>::fromList(m_dbFileMap.values()));
197
198 // Cleanup thumbnails
199 QStringList mesg(m_thumb.DeleteThumbs(QVector<ImagePtr>::fromList(m_dbFileMap.values())));
200 mesg << m_changedImages.join(",");
201
202 // Cleanup dirs
203 m_dbFileMap.clear();
204 m_dbDirMap.clear();
205 m_seenDir.clear();
206
207 m_mutexProgress.lock();
208 // (count == total) signals scan end
209 Broadcast(m_progressTotalCount);
210 // Must reset counts for scan queries
211 m_progressCount = m_progressTotalCount = 0;
212 m_mutexProgress.unlock();
213
214 LOG(VB_GENERAL, LOG_INFO, "Finished scan");
215
216 // For initial scans pause briefly to give thumb generator a headstart
217 // before being deluged by client requests
218 if (firstScan)
219 usleep(1s);
220
221 // Notify clients of completion with removed & changed images
222 m_dbfs.Notify("IMAGE_DB_CHANGED", mesg);
223
224 ChangeState(false);
225 }
226 }
227
228 RunEpilog();
229}
230
231
241template <class DBFS>
242void ImageScanThread<DBFS>::SyncSubTree(const QFileInfo &dirInfo, int parentId,
243 int devId, const QString &base)
244{
245 // Ignore excluded dirs
246 if (m_exclusions.match(dirInfo.fileName()).hasMatch())
247 {
248 LOG(VB_FILE, LOG_INFO,
249 QString("Excluding dir %1").arg(dirInfo.absoluteFilePath()));
250 return;
251 }
252
253 // Use global image filters
254 QDir dir = m_dir;
255 if (!dir.cd(dirInfo.absoluteFilePath()))
256 {
257 LOG(VB_FILE, LOG_INFO,
258 QString("Failed to open dir %1").arg(dirInfo.absoluteFilePath()));
259 return;
260 }
261
262 // Create directory node
263 int id = SyncDirectory(dirInfo, devId, base, parentId);
264 if (id == -1)
265 {
266 LOG(VB_FILE, LOG_INFO,
267 QString("Failed to sync dir %1").arg(dirInfo.absoluteFilePath()));
268 return;
269 }
270
271 // Sync its contents
272 QFileInfoList entries = dir.entryInfoList();
273 for (const auto & fileInfo : std::as_const(entries))
274 {
275 if (!IsScanning())
276 {
277 LOG(VB_GENERAL, LOG_INFO,
278 QString("Scan interrupted in %2").arg(dirInfo.absoluteFilePath()));
279 return;
280 }
281
282 if (fileInfo.isDir())
283 {
284 // Scan this directory
285 SyncSubTree(fileInfo, id, devId, base);
286 }
287 else
288 {
289 SyncFile(fileInfo, devId, base, id);
290
291 QMutexLocker locker(&m_mutexProgress);
292 ++m_progressCount;
293
294 // Throttle updates
295 if (m_bcastTimer.elapsed() > 250)
296 Broadcast(m_progressCount);
297 }
298 }
299}
300
301
316template <class DBFS>
317 int ImageScanThread<DBFS>::SyncDirectory(const QFileInfo &dirInfo, int devId, const QString &base, int parentId)
318{
319 QString absFilePath = dirInfo.absoluteFilePath();
320
321 LOG(VB_FILE, LOG_DEBUG, QString("Syncing directory %1").arg(absFilePath));
322
323 ImagePtr dir(m_dbfs.CreateItem(dirInfo, parentId, devId, base));
324
325 // Is dir already in Db ?
326 if (m_dbDirMap.contains(dir->m_filePath))
327 {
328 ImagePtr dbDir = m_dbDirMap.value(dir->m_filePath);
329 if (dbDir == nullptr)
330 return -1;
331
332 // The directory already exists in the db. Retain its id
333 dir->m_id = dbDir->m_id;
334
335 // Parent may have changed due to a move
336 if (dir->m_modTime != dbDir->m_modTime
337 || dir->m_parentId != dbDir->m_parentId)
338 {
339 LOG(VB_FILE, LOG_INFO,
340 QString("Changed directory %1").arg(absFilePath));
341
342 // Retain existing id & settings
343 dir->m_isHidden = dbDir->m_isHidden;
344 dir->m_userThumbnail = dbDir->m_userThumbnail;
345
346 m_dbfs.UpdateDbImage(*dir);
347 // Note modified images
348 m_changedImages << QString::number(dir->m_id);
349 }
350
351 // Remove the entry from the dbList
352 m_dbDirMap.remove(dir->m_filePath);
353 }
354 // Detect clones (same path in different SG dir)
355 else if (m_seenDir.contains(dir->m_filePath))
356 {
357 ImagePtr cloneDir = m_seenDir.value(dir->m_filePath);
358 if (cloneDir == nullptr)
359 return -1;
360
361 // All clones point to same Db dir. Use latest
362 if (cloneDir->m_modTime >= dir->m_modTime )
363 {
364 LOG(VB_FILE, LOG_INFO, QString("Directory %1 is an older clone of %2")
365 .arg(absFilePath, cloneDir->m_filePath));
366
367 // Use previous version
368 dir = cloneDir;
369 }
370 else
371 {
372 LOG(VB_FILE, LOG_INFO,
373 QString("Directory %1 is a more recent clone of %2")
374 .arg(absFilePath, cloneDir->m_filePath));
375
376 // Use new version
377 dir->m_id = cloneDir->m_id;
378 // Note modified time
379 m_changedImages << QString::number(dir->m_id);
380 }
381
382 // Mark non-devices as cloned (for info display only)
383 if (!dir->IsDevice())
384 {
385 dir->m_type = kCloneDir;
386 m_dbfs.UpdateDbImage(*dir);
387 }
388 }
389 else
390 {
391 LOG(VB_FILE, LOG_INFO, QString("New directory %1").arg(absFilePath));
392
393 // Create new Db dir with new id
394 dir->m_id = m_dbfs.InsertDbImage(*dir);
395 }
396
397 // Note it for clone detection
398 m_seenDir.insert(dir->m_filePath, dir);
399
400 return dir->m_id;
401}
402
403
412template <class DBFS>
414 const QString &path, int type, QString &comment,
415 std::chrono::seconds &time,
416 int &orientation)
417{
418 // Set orientation, date, comment from file meta data
419 ImageMetaData *metadata = (type == kImageFile)
422
423 orientation = metadata->GetOrientation();
424 comment = metadata->GetComment().simplified();
425 QDateTime dt = metadata->GetOriginalDateTime();
426 time = (dt.isValid()) ? std::chrono::seconds(dt.toSecsSinceEpoch()) : 0s;
427
428 delete metadata;
429}
430
431
446template <class DBFS>
447void ImageScanThread<DBFS>::SyncFile(const QFileInfo &fileInfo, int devId,
448 const QString &base, int parentId)
449{
450 // Ignore excluded files
451 if (m_exclusions.match(fileInfo.fileName()).hasMatch())
452 {
453 LOG(VB_FILE, LOG_INFO,
454 QString("Excluding file %1").arg(fileInfo.absoluteFilePath()));
455 return;
456 }
457
458 QString absFilePath = fileInfo.absoluteFilePath();
459
460 ImagePtr im(m_dbfs.CreateItem(fileInfo, parentId, devId, base));
461 if (!im)
462 // Ignore unknown file type
463 return;
464
465 if (m_dbFileMap.contains(im->m_filePath))
466 {
467 ImagePtrK dbIm = m_dbFileMap.value(im->m_filePath);
468
469 // Parent may have changed due to a move
470 if (im->m_modTime == dbIm->m_modTime && im->m_parentId == dbIm->m_parentId)
471 {
472 // File already known & hasn't changed
473 // Remove it from removed list
474 m_dbFileMap.remove(im->m_filePath);
475 // Detect duplicates
476 m_seenFile.insert(im->m_filePath, absFilePath);
477 return;
478 }
479
480 LOG(VB_FILE, LOG_INFO, QString("Modified file %1").arg(absFilePath));
481
482 // Retain existing id & settings
483 im->m_id = dbIm->m_id;
484 im->m_isHidden = dbIm->m_isHidden;
485
486 // Set date, comment from file meta data
487 int fileOrient = 0;
488 PopulateMetadata(absFilePath, im->m_type,
489 im->m_comment, im->m_date, fileOrient);
490
491 // Reset file orientation, retaining existing setting
492 int currentOrient = Orientation(dbIm->m_orientation).GetCurrent();
493 im->m_orientation = Orientation(currentOrient, fileOrient).Composite();
494
495 // Remove it from removed list
496 m_dbFileMap.remove(im->m_filePath);
497 // Note modified images
498 m_changedImages << QString::number(im->m_id);
499
500 // Update db
501 m_dbfs.UpdateDbImage(*im);
502 }
503 else if (m_seenFile.contains(im->m_filePath))
504 {
505 LOG(VB_GENERAL, LOG_WARNING, QString("Ignoring %1 (Duplicate of %2)")
506 .arg(absFilePath, m_seenFile.value(im->m_filePath)));
507 return;
508 }
509 else
510 {
511 // New images will be assigned an id by the db AUTO-INCREMENT
512 LOG(VB_FILE, LOG_INFO, QString("New file %1").arg(absFilePath));
513
514 // Set date, comment from file meta data
515 int fileOrient = 0;
516 PopulateMetadata(absFilePath, im->m_type,
517 im->m_comment, im->m_date, fileOrient);
518
519 // Set file orientation
520 im->m_orientation = Orientation(fileOrient, fileOrient).Composite();
521
522 // Update db (Set id for thumb generator)
523 im->m_id = m_dbfs.InsertDbImage(*im);
524 }
525
526 // Detect duplicate filepaths in SG
527 m_seenFile.insert(im->m_filePath, absFilePath);
528
529 // Populate absolute filename so that thumbgen doesn't need to locate file
530 im->m_filePath = absFilePath;
531
532 // Ensure thumbnail exists.
533 m_thumb.CreateThumbnail(im);
534}
535
536
542template <class DBFS>
544{
545 QFileInfoList entries = dir.entryInfoList();
546 for (const auto & fileInfo : std::as_const(entries))
547 {
548 // Ignore excluded dirs/files
549 if (m_exclusions.match(fileInfo.fileName()).hasMatch())
550 continue;
551
552 if (fileInfo.isFile())
553 {
554 ++m_progressTotalCount;
555 }
556 // Ignore missing dirs
557 else if (dir.cd(fileInfo.fileName()))
558 {
559 CountTree(dir);
560 dir.cdUp();
561 }
562 }
563}
564
565
570template <class DBFS>
572{
573 // Get exclusions as comma-seperated list using glob chars * and ?
574 QString excPattern = gCoreContext->GetSetting("GalleryIgnoreFilter", "");
575
576 // Combine into a single regexp
577 excPattern.replace(".", "\\."); // Preserve "."
578 excPattern.replace("*", ".*"); // Convert glob wildcard "*"
579 excPattern.replace("?", "."); // Convert glob wildcard "?"
580 excPattern.replace(",", "|"); // Convert list to OR's
581
582 QString pattern = QString("^(%1)$").arg(excPattern);
583 m_exclusions = QRegularExpression(pattern);
584
585 LOG(VB_FILE, LOG_DEBUG, QString("Exclude regexp is \"%1\"").arg(pattern));
586
587 // Lock counts until counting complete
588 QMutexLocker locker(&m_mutexProgress);
589 m_progressCount = 0;
590 m_progressTotalCount = 0;
591
592 // Use global image filters
593 QDir dir = m_dir;
594 for (const auto& sgDir : std::as_const(paths))
595 {
596 // Ignore missing dirs
597 if (dir.cd(sgDir))
598 CountTree(dir);
599 }
600 // 0 signifies a scan start
601 Broadcast(0);
602}
603
604
611template <class DBFS>
613{
614 // Only 2 scanners are ever visible (FE & BE) so use bool as scanner id
615 QStringList status;
616 status << QString::number(static_cast<int>(gCoreContext->IsBackend()))
617 << QString::number(progress)
618 << QString::number(m_progressTotalCount);
619
620 m_dbfs.Notify("IMAGE_SCAN_STATUS", status);
621
622 // Reset broadcast throttle
623 m_bcastTimer.start();
624}
625
626
627// Must define the valid template implementations to generate code for the
628// instantiations (as they are defined in the cpp rather than header).
629// Otherwise the linker will fail with undefined references...
630#include "imagemanager.h"
631template class ImageScanThread<ImageDbLocal>;
632template class ImageScanThread<ImageDbSg>;
Abstract class for image metadata.
Definition: imagemetadata.h:92
virtual QDateTime GetOriginalDateTime(bool *exists=nullptr)=0
static ImageMetaData * FromPicture(const QString &filePath)
Factory to retrieve metadata from pictures.
virtual QString GetComment(bool *exists=nullptr)=0
static ImageMetaData * FromVideo(const QString &filePath)
Factory to retrieve metadata from videos.
virtual int GetOrientation(bool *exists=nullptr)=0
~ImageScanThread() override
bool ClearsPending()
Get status of 'clear device' queue.
void CountTree(QDir &dir)
Counts images in a dir subtree.
void EnqueueClear(int devId, const QString &action)
Queues a 'Clear Device' request, which will be actioned immediately.
void PopulateMetadata(const QString &path, int type, QString &comment, std::chrono::seconds &time, int &orientation)
Read image date, orientation, comment from metadata.
bool IsScanning()
Return current scanner status.
ImageScanThread(DBFS *dbfs, ImageThumb< DBFS > *thumbGen)
Constructor.
int SyncDirectory(const QFileInfo &dirInfo, int devId, const QString &base, int parentId)
Updates/populates db for a dir.
void ChangeState(bool scan)
Run or interrupt scanner.
void SyncFile(const QFileInfo &fileInfo, int devId, const QString &base, int parentId)
Updates/populates db for an image/video file.
QStringList GetProgress()
Returns number of images scanned & total number to scan.
void CountFiles(const QStringList &paths)
Counts images in a list of subtrees.
void SyncSubTree(const QFileInfo &dirInfo, int parentId, int devId, const QString &base)
Scans a dir subtree.
void cancel()
Clears queued items so that the thread can exit.
QPair< int, QString > ClearTask
Definition: imagescanner.h:57
void run() override
Synchronises database to the storage group.
void Broadcast(int progress)
Notify listeners of scan progress.
This is a wrapper around QThread that does several additional things.
Definition: mthread.h:49
QString GetSetting(const QString &key, const QString &defaultval="")
bool IsBackend(void) const
is this process a backend process
Encapsulates Exif orientation processing.
Definition: imagemetadata.h:63
int Composite() const
Encode original & current orientation to a single Db field.
Definition: imagemetadata.h:71
int GetCurrent() const
Determines orientation required for an image.
Manages a collection of images.
Handles Exif/FFMpeg metadata tags for images.
Synchronises image database to filesystem.
QSharedPointer< ImageItemK > ImagePtrK
Definition: imagetypes.h:165
QMap< int, QString > StringMap
Definition: imagetypes.h:63
static constexpr int GALLERY_DB_ID
Definition: imagetypes.h:27
@ kCloneDir
A device sub dir comprised from multiple SG dirs.
Definition: imagetypes.h:37
@ kImageFile
A picture.
Definition: imagetypes.h:39
QSharedPointer< ImageItem > ImagePtr
Definition: imagetypes.h:159
bool progress
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
#define LOG(_MASK_, _LEVEL_, _QSTRING_)
Definition: mythlogging.h:39
static bool isRunning(const char *program)
Returns true if a program containing the specified string is running on this machine.
def scan(profile, smoonURL, gate)
Definition: scan.py:54