MythTV  master
autoexpire.cpp
Go to the documentation of this file.
1 // System headers
2 #include <sys/stat.h>
3 #ifdef __linux__
4 # include <sys/vfs.h>
5 #else // if !__linux__
6 # include <sys/param.h>
7 # ifndef _WIN32
8 # include <sys/mount.h>
9 # endif // _WIN32
10 #endif // !__linux__
11 
12 // POSIX headers
13 #include <unistd.h>
14 
15 // C headers
16 #include <cstdlib>
17 
18 // C++ headers
19 #include <iostream>
20 #include <algorithm>
21 
22 // Qt headers
23 #include <QDateTime>
24 #include <QFileInfo>
25 #include <QList>
26 
27 // MythTV headers
28 #include "filesysteminfo.h"
29 #include "autoexpire.h"
30 #include "programinfo.h"
31 #include "mythcorecontext.h"
32 #include "mythdb.h"
33 #include "mythdate.h"
34 #include "storagegroup.h"
35 #include "remoteutil.h"
36 #include "remoteencoder.h"
37 #include "encoderlink.h"
39 #include "mainserver.h"
40 #include "compat.h"
41 #include "mythlogging.h"
42 #include "tv_rec.h"
43 
44 #define LOC QString("AutoExpire: ")
45 #define LOC_ERR QString("AutoExpire Error: ")
46 
47 extern AutoExpire *expirer;
48 
52 #define SPACE_TOO_BIG_KB (3*1024*1024)
53 
56 {
57  RunProlog();
58  m_parent->RunExpirer();
59  RunEpilog();
60 }
61 
71 AutoExpire::AutoExpire(QMap<int, EncoderLink *> *tvList) :
72  m_encoderList(tvList),
73  m_expireThread(new ExpireThread(this)),
74  m_expireThreadRun(true)
75 {
78 }
79 
84 {
85  {
86  QMutexLocker locker(&m_instanceLock);
87  m_expireThreadRun = false;
88  m_instanceCond.wakeAll();
89  }
90 
91  {
92  QMutexLocker locker(&m_updateLock);
93  m_updateQueue.clear();
94  }
95 
96  if (m_expireThread)
97  {
100  delete m_expireThread;
101  m_expireThread = nullptr;
102  }
103 }
104 
110 uint64_t AutoExpire::GetDesiredSpace(int fsID) const
111 {
112  QMutexLocker locker(&m_instanceLock);
113  if (m_desiredSpace.contains(fsID))
114  return m_desiredSpace[fsID];
115  return 0;
116 }
117 
122 {
123  LOG(VB_FILE, LOG_INFO, LOC + "CalcParams()");
124 
125  QList<FileSystemInfo> fsInfos;
126 
127  m_instanceLock.lock();
128  if (m_mainServer)
129  {
130  // The scheduler relies on something forcing the mainserver
131  // fsinfos cache to get updated periodically. Currently, that
132  // is done here. Don't remove or change this invocation
133  // without handling that issue too. It is done this way
134  // because the scheduler thread can't afford to be blocked by
135  // an unresponsive, remote filesystem and the autoexpirer
136  // thread can.
137  m_mainServer->GetFilesystemInfos(fsInfos, false);
138  }
139  m_instanceLock.unlock();
140 
141  if (fsInfos.empty())
142  {
143  LOG(VB_GENERAL, LOG_ERR, LOC + "Filesystem Info cache is empty, unable "
144  "to calculate necessary parameters.");
145  return;
146  }
147 
148  uint64_t maxKBperMin = 0;
149  uint64_t extraKB = static_cast<uint64_t>
150  (gCoreContext->GetNumSetting("AutoExpireExtraSpace", 0))
151  << 20;
152 
153  QMap<int, uint64_t> fsMap;
154  QMap<int, vector<int> > fsEncoderMap;
155 
156  // We use this copying on purpose. The used_encoders map ensures
157  // that every encoder writes only to one fs.
158  // Copying the data minimizes the time the lock is held.
159  m_instanceLock.lock();
160  QMap<int, int>::const_iterator ueit = m_usedEncoders.cbegin();
161  while (ueit != m_usedEncoders.cend())
162  {
163  fsEncoderMap[*ueit].push_back(ueit.key());
164  ++ueit;
165  }
166  m_instanceLock.unlock();
167 
168  QList<FileSystemInfo>::iterator fsit;
169  for (fsit = fsInfos.begin(); fsit != fsInfos.end(); ++fsit)
170  {
171  if (fsMap.contains(fsit->getFSysID()))
172  continue;
173 
174  fsMap[fsit->getFSysID()] = 0;
175  uint64_t thisKBperMin = 0;
176 
177  // append unknown recordings to all fsIDs
178  for (auto unknownfs : qAsConst(fsEncoderMap[-1]))
179  fsEncoderMap[fsit->getFSysID()].push_back(unknownfs);
180 
181  if (fsEncoderMap.contains(fsit->getFSysID()))
182  {
183  LOG(VB_FILE, LOG_INFO,
184  QString("fsID #%1: Total: %2 GB Used: %3 GB Free: %4 GB")
185  .arg(fsit->getFSysID())
186  .arg(fsit->getTotalSpace() / 1024.0 / 1024.0, 7, 'f', 1)
187  .arg(fsit->getUsedSpace() / 1024.0 / 1024.0, 7, 'f', 1)
188  .arg(fsit->getFreeSpace() / 1024.0 / 1024.0, 7, 'f', 1));
189 
190  for (auto cardid : qAsConst(fsEncoderMap[fsit->getFSysID()]))
191  {
192  EncoderLink *enc = *(m_encoderList->constFind(cardid));
193 
194  if (!enc->IsConnected() || !enc->IsBusy())
195  {
196  // remove encoder since it can't write to any file system
197  LOG(VB_FILE, LOG_INFO, LOC +
198  QString("Cardid %1: is not recording, removing it "
199  "from used list.").arg(cardid));
200  m_instanceLock.lock();
201  m_usedEncoders.remove(cardid);
202  m_instanceLock.unlock();
203  continue;
204  }
205 
206  uint64_t maxBitrate = enc->GetMaxBitrate();
207  if (maxBitrate==0)
208  maxBitrate = 19500000LL;
209  thisKBperMin += (maxBitrate*((uint64_t)15))>>11;
210  LOG(VB_FILE, LOG_INFO, QString(" Cardid %1: max bitrate "
211  "%2 Kb/sec, fsID %3 max is now %4 KB/min")
212  .arg(enc->GetInputID())
213  .arg(enc->GetMaxBitrate() >> 10)
214  .arg(fsit->getFSysID())
215  .arg(thisKBperMin));
216  }
217  }
218  fsMap[fsit->getFSysID()] = thisKBperMin;
219 
220  if (thisKBperMin > maxKBperMin)
221  {
222  LOG(VB_FILE, LOG_INFO,
223  QString(" Max of %1 KB/min for fsID %2 is higher "
224  "than the existing Max of %3 so we'll use this Max instead")
225  .arg(thisKBperMin).arg(fsit->getFSysID()).arg(maxKBperMin));
226  maxKBperMin = thisKBperMin;
227  }
228  }
229 
230  // Determine frequency to run autoexpire so it doesn't have to free
231  // too much space
232  uint expireFreq = 15;
233  if (maxKBperMin > 0)
234  {
235  expireFreq = SPACE_TOO_BIG_KB / (maxKBperMin + maxKBperMin/3);
236  expireFreq = std::max(3U, std::min(expireFreq, 15U));
237  }
238 
239  double expireMinGB = ((maxKBperMin + maxKBperMin/3)
240  * expireFreq + extraKB) >> 20;
241  LOG(VB_GENERAL, LOG_NOTICE, LOC +
242  QString("CalcParams(): Max required Free Space: %1 GB w/freq: %2 min")
243  .arg(expireMinGB, 0, 'f', 1).arg(expireFreq));
244 
245  // lock class and save these parameters.
246  m_instanceLock.lock();
247  m_desiredFreq = expireFreq;
248  // write per file system needed space back, use safety of 33%
249  QMap<int, uint64_t>::iterator it = fsMap.begin();
250  while (it != fsMap.end())
251  {
252  m_desiredSpace[it.key()] = (*it + *it/3) * expireFreq + extraKB;
253  ++it;
254  }
255  m_instanceLock.unlock();
256 }
257 
267 {
268  QElapsedTimer timer;
269  QDateTime curTime;
270  QDateTime next_expire = MythDate::current().addSecs(60);
271 
272  QMutexLocker locker(&m_instanceLock);
273 
274  // wait a little for main server to come up and things to settle down
275  Sleep(20s);
276 
277  timer.start();
278 
279  while (m_expireThreadRun)
280  {
281  TVRec::s_inputsLock.lockForRead();
282 
283  curTime = MythDate::current();
284  // recalculate auto expire parameters
285  if (curTime >= next_expire)
286  {
287  m_updateLock.lock();
288  while (!m_updateQueue.empty())
289  {
290  UpdateEntry ue = m_updateQueue.dequeue();
291  if (ue.m_encoder > 0)
293  }
294  m_updateLock.unlock();
295 
296  locker.unlock();
297  CalcParams();
298  locker.relock();
299  if (!m_expireThreadRun)
300  break;
301  }
302  timer.restart();
303 
305 
306  // Expire Short LiveTV files for this backend every 2 minutes
307  if ((curTime.time().minute() % 2) == 0)
309 
310  // Expire normal recordings depending on frequency calculated
311  if (curTime >= next_expire)
312  {
313  LOG(VB_FILE, LOG_INFO, LOC + "Running now!");
314  next_expire =
315  MythDate::current().addSecs(m_desiredFreq * 60);
316 
318 
319  int maxAge = gCoreContext->GetNumSetting("DeletedMaxAge", 0);
320  if (maxAge > 0)
322  else if (maxAge == 0)
324 
326 
328  }
329 
330  TVRec::s_inputsLock.unlock();
331 
332  Sleep(60s - std::chrono::milliseconds(timer.elapsed()));
333  }
334 }
335 
342 void AutoExpire::Sleep(std::chrono::milliseconds sleepTime)
343 {
344  if (sleepTime <= 0ms)
345  return;
346 
347  QDateTime little_tm = MythDate::current().addMSecs(sleepTime.count());
348  std::chrono::milliseconds timeleft = sleepTime;
349  while (m_expireThreadRun && (timeleft > 0ms))
350  {
351  m_instanceCond.wait(&m_instanceLock, timeleft.count());
352  timeleft = MythDate::secsInFuture(little_tm);
353  }
354 }
355 
360 {
361  pginfolist_t expireList;
362 
363  LOG(VB_FILE, LOG_INFO, LOC + QString("ExpireLiveTV(%1)").arg(type));
364  FillDBOrdered(expireList, type);
365  SendDeleteMessages(expireList);
366  ClearExpireList(expireList);
367 }
368 
373 {
374  pginfolist_t expireList;
375 
376  LOG(VB_FILE, LOG_INFO, LOC + QString("ExpireOldDeleted()"));
377  FillDBOrdered(expireList, emOldDeletedPrograms);
378  SendDeleteMessages(expireList);
379  ClearExpireList(expireList);
380 }
381 
386 {
387  pginfolist_t expireList;
388 
389  LOG(VB_FILE, LOG_INFO, LOC + QString("ExpireQuickDeleted()"));
391  SendDeleteMessages(expireList);
392  ClearExpireList(expireList);
393 }
394 
400 {
401  pginfolist_t expireList;
402  pginfolist_t deleteList;
403  QList<FileSystemInfo> fsInfos;
404  QList<FileSystemInfo>::iterator fsit;
405 
406  LOG(VB_FILE, LOG_INFO, LOC + "ExpireRecordings()");
407 
408  if (m_mainServer)
409  m_mainServer->GetFilesystemInfos(fsInfos, true);
410 
411  if (fsInfos.empty())
412  {
413  LOG(VB_GENERAL, LOG_ERR, LOC + "Filesystem Info cache is empty, unable "
414  "to determine what Recordings to expire");
415 
416  return;
417  }
418 
419  FillExpireList(expireList);
420 
421  QMap <int, bool> truncateMap;
422  MSqlQuery query(MSqlQuery::InitCon());
423  query.prepare("SELECT DISTINCT rechost, recdir "
424  "FROM inuseprograms "
425  "WHERE recusage = 'truncatingdelete' "
426  "AND lastupdatetime > DATE_ADD(NOW(), INTERVAL -2 MINUTE);");
427 
428  if (!query.exec())
429  {
430  MythDB::DBError(LOC + "ExpireRecordings", query);
431  }
432  else
433  {
434  while (query.next())
435  {
436  QString rechost = query.value(0).toString();
437  QString recdir = query.value(1).toString();
438 
439  LOG(VB_FILE, LOG_INFO, LOC +
440  QString("%1:%2 has an in-progress truncating delete.")
441  .arg(rechost, recdir));
442 
443  for (fsit = fsInfos.begin(); fsit != fsInfos.end(); ++fsit)
444  {
445  if ((fsit->getHostname() == rechost) &&
446  (fsit->getPath() == recdir))
447  {
448  truncateMap[fsit->getFSysID()] = true;
449  break;
450  }
451  }
452  }
453  }
454 
455  QMap <int, bool> fsMap;
456  for (fsit = fsInfos.begin(); fsit != fsInfos.end(); ++fsit)
457  {
458  if (fsMap.contains(fsit->getFSysID()))
459  continue;
460 
461  fsMap[fsit->getFSysID()] = true;
462 
463  LOG(VB_FILE, LOG_INFO,
464  QString("fsID #%1: Total: %2 GB Used: %3 GB Free: %4 GB")
465  .arg(fsit->getFSysID())
466  .arg(fsit->getTotalSpace() / 1024.0 / 1024.0, 7, 'f', 1)
467  .arg(fsit->getUsedSpace() / 1024.0 / 1024.0, 7, 'f', 1)
468  .arg(fsit->getFreeSpace() / 1024.0 / 1024.0, 7, 'f', 1));
469 
470  if ((fsit->getTotalSpace() == -1) || (fsit->getUsedSpace() == -1))
471  {
472  LOG(VB_FILE, LOG_ERR, LOC +
473  QString("fsID #%1 has invalid info, AutoExpire cannot run for "
474  "this filesystem. Continuing on to next...")
475  .arg(fsit->getFSysID()));
476  LOG(VB_FILE, LOG_INFO, QString("Directories on filesystem ID %1:")
477  .arg(fsit->getFSysID()));
478  QList<FileSystemInfo>::iterator fsit2;
479  for (fsit2 = fsInfos.begin(); fsit2 != fsInfos.end(); ++fsit2)
480  {
481  if (fsit2->getFSysID() == fsit->getFSysID())
482  {
483  LOG(VB_FILE, LOG_INFO, QString(" %1:%2")
484  .arg(fsit2->getHostname(), fsit2->getPath()));
485  }
486  }
487 
488  continue;
489  }
490 
491  if (truncateMap.contains(fsit->getFSysID()))
492  {
493  LOG(VB_FILE, LOG_INFO,
494  QString(" fsid %1 has a truncating delete in progress, "
495  "AutoExpire cannot run for this filesystem until the "
496  "delete has finished. Continuing on to next...")
497  .arg(fsit->getFSysID()));
498  continue;
499  }
500 
501  if (std::max((int64_t)0LL, fsit->getFreeSpace()) <
502  m_desiredSpace[fsit->getFSysID()])
503  {
504  LOG(VB_FILE, LOG_INFO,
505  QString(" Not Enough Free Space! We want %1 MB")
506  .arg(m_desiredSpace[fsit->getFSysID()] / 1024));
507 
508  QMap<QString, int> dirList;
509  QList<FileSystemInfo>::iterator fsit2;
510 
511  LOG(VB_FILE, LOG_INFO,
512  QString(" Directories on filesystem ID %1:")
513  .arg(fsit->getFSysID()));
514 
515  for (fsit2 = fsInfos.begin(); fsit2 != fsInfos.end(); ++fsit2)
516  {
517  if (fsit2->getFSysID() == fsit->getFSysID())
518  {
519  LOG(VB_FILE, LOG_INFO, QString(" %1:%2")
520  .arg(fsit2->getHostname(), fsit2->getPath()));
521  dirList[fsit2->getHostname() + ":" + fsit2->getPath()] = 1;
522  }
523  }
524 
525  LOG(VB_FILE, LOG_INFO,
526  " Searching for files expirable in these directories");
527  QString myHostName = gCoreContext->GetHostName();
528  auto it = expireList.begin();
529  while ((it != expireList.end()) &&
530  (std::max((int64_t)0LL, fsit->getFreeSpace()) <
531  m_desiredSpace[fsit->getFSysID()]))
532  {
533  ProgramInfo *p = *it;
534  ++it;
535 
536  LOG(VB_FILE, LOG_INFO, QString(" Checking %1 => %2")
537  .arg(p->toString(ProgramInfo::kRecordingKey),
538  p->GetTitle()));
539 
540  if (!p->IsLocal())
541  {
542  bool foundFile = false;
543  auto eit = m_encoderList->constBegin();
544  while (eit != m_encoderList->constEnd())
545  {
546  EncoderLink *el = *eit;
547  eit++;
548 
549  if ((p->GetHostname() == el->GetHostName()) ||
550  ((p->GetHostname() == myHostName) &&
551  (el->IsLocal())))
552  {
553  if (el->IsConnected())
554  foundFile = el->CheckFile(p);
555 
556  eit = m_encoderList->constEnd();
557  }
558  }
559 
560  if (!foundFile && (p->GetHostname() != myHostName))
561  {
562  // Wasn't found so check locally
563  QString file = GetPlaybackURL(p);
564 
565  if (file.startsWith("/"))
566  {
567  p->SetPathname(file);
568  p->SetHostname(myHostName);
569  foundFile = true;
570  }
571  }
572 
573  if (!foundFile)
574  {
575  LOG(VB_FILE, LOG_ERR, LOC +
576  QString(" ERROR: Can't find file for %1")
577  .arg(p->toString(ProgramInfo::kRecordingKey)));
578  continue;
579  }
580  }
581 
582  QFileInfo vidFile(p->GetPathname());
583  if (dirList.contains(p->GetHostname() + ':' + vidFile.path()))
584  {
585  fsit->setUsedSpace(fsit->getUsedSpace()
586  - (p->GetFilesize() / 1024));
587  deleteList.push_back(p);
588 
589  LOG(VB_FILE, LOG_INFO,
590  QString(" FOUND file expirable. "
591  "%1 is located at %2 which is on fsID #%3. "
592  "Adding to deleteList. After deleting we "
593  "should have %4 MB free on this filesystem.")
594  .arg(p->toString(ProgramInfo::kRecordingKey),
595  p->GetPathname()).arg(fsit->getFSysID())
596  .arg(fsit->getFreeSpace() / 1024));
597  }
598  }
599  }
600  }
601 
602  SendDeleteMessages(deleteList);
603 
604  ClearExpireList(deleteList, false);
605  ClearExpireList(expireList);
606 }
607 
612 {
613  QString msg;
614 
615  if (deleteList.empty())
616  {
617  LOG(VB_FILE, LOG_INFO, LOC + "SendDeleteMessages. Nothing to expire.");
618  return;
619  }
620 
621  LOG(VB_FILE, LOG_INFO, LOC +
622  "SendDeleteMessages, cycling through deleteList.");
623  auto it = deleteList.begin();
624  while (it != deleteList.end())
625  {
626  msg = QString("%1Expiring %2 MB for %3 => %4")
627  .arg(VERBOSE_LEVEL_CHECK(VB_FILE, LOG_ANY) ? " " : "",
628  QString::number((*it)->GetFilesize() >> 20),
629  (*it)->toString(ProgramInfo::kRecordingKey),
630  (*it)->toString(ProgramInfo::kTitleSubtitle));
631 
632  LOG(VB_GENERAL, LOG_NOTICE, msg);
633 
634  // send auto expire message to backend's event thread.
635  MythEvent me(QString("AUTO_EXPIRE %1 %2").arg((*it)->GetChanID())
636  .arg((*it)->GetRecordingStartTime(MythDate::ISODate)));
637  gCoreContext->dispatch(me);
638 
639  ++it; // move on to next program
640  }
641 }
642 
649 {
650  QMap<QString, int> maxEpisodes;
651  QMap<QString, int>::Iterator maxIter;
652  QMap<QString, int> episodeParts;
653  QString episodeKey;
654 
655  MSqlQuery query(MSqlQuery::InitCon());
656  query.prepare("SELECT recordid, maxepisodes, title "
657  "FROM record WHERE maxepisodes > 0 "
658  "ORDER BY recordid ASC, maxepisodes DESC");
659 
660  if (query.exec() && query.isActive() && query.size() > 0)
661  {
662  LOG(VB_FILE, LOG_INFO, LOC +
663  QString("Found %1 record profiles using max episode expiration")
664  .arg(query.size()));
665  while (query.next())
666  {
667  LOG(VB_FILE, LOG_INFO, QString(" %1 (%2 for rec id %3)")
668  .arg(query.value(2).toString())
669  .arg(query.value(1).toInt())
670  .arg(query.value(0).toInt()));
671  maxEpisodes[query.value(0).toString()] = query.value(1).toInt();
672  }
673  }
674 
675  LOG(VB_FILE, LOG_INFO, LOC +
676  "Checking episode count for each recording profile using max episodes");
677  for (maxIter = maxEpisodes.begin(); maxIter != maxEpisodes.end(); ++maxIter)
678  {
679  query.prepare("SELECT chanid, starttime, title, progstart, progend, "
680  "duplicate "
681  "FROM recorded "
682  "WHERE recordid = :RECID AND preserve = 0 "
683  "AND recgroup NOT IN ('LiveTV', 'Deleted') "
684  "ORDER BY starttime DESC;");
685  query.bindValue(":RECID", maxIter.key());
686 
687  if (!query.exec() || !query.isActive())
688  {
689  MythDB::DBError("AutoExpire query failed!", query);
690  continue;
691  }
692 
693  LOG(VB_FILE, LOG_INFO, QString(" Recordid %1 has %2 recordings.")
694  .arg(maxIter.key())
695  .arg(query.size()));
696  if (query.size() > 0)
697  {
698  int found = 1;
699  while (query.next())
700  {
701  uint chanid = query.value(0).toUInt();
702  QDateTime startts = MythDate::as_utc(query.value(1).toDateTime());
703  QString title = query.value(2).toString();
704  QDateTime progstart = MythDate::as_utc(query.value(3).toDateTime());
705  QDateTime progend = MythDate::as_utc(query.value(4).toDateTime());
706  int duplicate = query.value(5).toInt();
707 
708  episodeKey = QString("%1_%2_%3")
709  .arg(QString::number(chanid),
710  progstart.toString(Qt::ISODate),
711  progend.toString(Qt::ISODate));
712 
713  if ((!IsInDontExpireSet(chanid, startts)) &&
714  (!episodeParts.contains(episodeKey)) &&
715  (found > *maxIter))
716  {
717  QString msg =
718  QString("%1Deleting %2 at %3 => %4. "
719  "Too many episodes, we only want to keep %5.")
720  .arg(VERBOSE_LEVEL_CHECK(VB_FILE, LOG_ANY) ? " " : "",
721  QString::number(chanid),
722  startts.toString(Qt::ISODate),
723  title,
724  QString::number(*maxIter));
725 
726  LOG(VB_GENERAL, LOG_NOTICE, msg);
727 
728  // allow re-record if auto expired
729  RecordingInfo recInfo(chanid, startts);
730  if (gCoreContext->GetBoolSetting("RerecordWatched", false) ||
731  !recInfo.IsWatched())
732  {
733  recInfo.ForgetHistory();
734  }
735  msg = QString("DELETE_RECORDING %1 %2")
736  .arg(chanid)
737  .arg(startts.toString(Qt::ISODate));
738 
739  MythEvent me(msg);
740  gCoreContext->dispatch(me);
741  }
742  else
743  {
744  // keep track of shows we haven't expired so we can
745  // make sure we don't expire another part of the same
746  // episode.
747  if (episodeParts.contains(episodeKey))
748  {
749  episodeParts[episodeKey] = episodeParts[episodeKey] + 1;
750  }
751  else
752  {
753  episodeParts[episodeKey] = 1;
754  if( duplicate )
755  found++;
756  }
757  }
758  }
759  }
760  }
761 }
762 
768 {
769  int expMethod = gCoreContext->GetNumSetting("AutoExpireMethod", 1);
770 
771  ClearExpireList(expireList);
772 
774 
775  switch(expMethod)
776  {
777  case emOldestFirst:
780  FillDBOrdered(expireList, expMethod);
781  break;
782  // default falls through so list is empty so no AutoExpire
783  }
784 }
785 
789 void AutoExpire::PrintExpireList(const QString& expHost)
790 {
791  pginfolist_t expireList;
792 
793  FillExpireList(expireList);
794 
795  QString msg = "MythTV AutoExpire List ";
796  if (expHost != "ALL")
797  msg += QString("for '%1' ").arg(expHost);
798  msg += "(programs listed in order of expiration)";
799  std::cout << msg.toLocal8Bit().constData() << std::endl;
800 
801  for (auto *first : expireList)
802  {
803  if (expHost != "ALL" && first->GetHostname() != expHost)
804  continue;
805 
806  QString title = first->toString(ProgramInfo::kTitleSubtitle);
807  title = title.leftJustified(39, ' ', true);
808 
809  QString outstr = QString("%1 %2 MB %3 [%4]")
810  .arg(title,
811  QString::number(first->GetFilesize() >> 20)
812  .rightJustified(5, ' ', true),
813  first->GetRecordingStartTime(MythDate::ISODate)
814  .leftJustified(24, ' ', true),
815  QString::number(first->GetRecordingPriority())
816  .rightJustified(3, ' ', true));
817  QByteArray out = outstr.toLocal8Bit();
818 
819  std::cout << out.constData() << std::endl;
820  }
821 
822  ClearExpireList(expireList);
823 }
824 
828 void AutoExpire::GetAllExpiring(QStringList &strList)
829 {
830  QMutexLocker lockit(&m_instanceLock);
831  pginfolist_t expireList;
832 
834 
838  FillDBOrdered(expireList, gCoreContext->GetNumSetting("AutoExpireMethod",
839  emOldestFirst));
840 
841  strList << QString::number(expireList.size());
842 
843  for (auto & info : expireList)
844  info->ToStringList(strList);
845 
846  ClearExpireList(expireList);
847 }
848 
853 {
854  QMutexLocker lockit(&m_instanceLock);
855  pginfolist_t expireList;
856 
858 
862  FillDBOrdered(expireList, gCoreContext->GetNumSetting("AutoExpireMethod",
863  emOldestFirst));
864 
865  for (auto & info : expireList)
866  list.push_back( new ProgramInfo( *info ));
867 
868  ClearExpireList(expireList);
869 }
870 
874 void AutoExpire::ClearExpireList(pginfolist_t &expireList, bool deleteProg)
875 {
876  ProgramInfo *pginfo = nullptr;
877  while (!expireList.empty())
878  {
879  if (deleteProg)
880  pginfo = expireList.back();
881 
882  expireList.pop_back();
883 
884  if (deleteProg)
885  delete pginfo;
886  }
887 }
888 
893 void AutoExpire::FillDBOrdered(pginfolist_t &expireList, int expMethod)
894 {
895  QString where;
896  QString orderby;
897  QString msg;
898  int maxAge = 0;
899 
900  switch (expMethod)
901  {
902  default:
903  case emOldestFirst:
904  msg = "Adding programs expirable in Oldest First order";
905  where = "autoexpire > 0";
906  if (gCoreContext->GetBoolSetting("AutoExpireWatchedPriority", false))
907  orderby = "recorded.watched DESC, ";
908  orderby += "starttime ASC";
909  break;
911  msg = "Adding programs expirable in Lowest Priority First order";
912  where = "autoexpire > 0";
913  if (gCoreContext->GetBoolSetting("AutoExpireWatchedPriority", false))
914  orderby = "recorded.watched DESC, ";
915  orderby += "recorded.recpriority ASC, starttime ASC";
916  break;
918  msg = "Adding programs expirable in Weighted Time Priority order";
919  where = "autoexpire > 0";
920  if (gCoreContext->GetBoolSetting("AutoExpireWatchedPriority", false))
921  orderby = "recorded.watched DESC, ";
922  orderby += QString("DATE_ADD(starttime, INTERVAL '%1' * "
923  "recorded.recpriority DAY) ASC")
924  .arg(gCoreContext->GetNumSetting("AutoExpireDayPriority", 3));
925  break;
927  msg = "Adding Short LiveTV programs in starttime order";
928  where = "recgroup = 'LiveTV' "
929  "AND endtime < DATE_ADD(starttime, INTERVAL '30' SECOND) "
930  "AND endtime <= DATE_ADD(NOW(), INTERVAL '-5' MINUTE) ";
931  orderby = "starttime ASC";
932  break;
934  msg = "Adding LiveTV programs in starttime order";
935  where = QString("recgroup = 'LiveTV' "
936  "AND endtime <= DATE_ADD(NOW(), INTERVAL '-%1' DAY) ")
937  .arg(gCoreContext->GetNumSetting("AutoExpireLiveTVMaxAge", 1));
938  orderby = "starttime ASC";
939  break;
941  if ((maxAge = gCoreContext->GetNumSetting("DeletedMaxAge", 0)) <= 0)
942  return;
943  msg = QString("Adding programs deleted more than %1 days ago")
944  .arg(maxAge);
945  where = QString("recgroup = 'Deleted' "
946  "AND lastmodified <= DATE_ADD(NOW(), INTERVAL '-%1' DAY) ")
947  .arg(maxAge);
948  orderby = "starttime ASC";
949  break;
951  if (gCoreContext->GetNumSetting("DeletedMaxAge", 0) != 0)
952  return;
953  msg = QString("Adding programs deleted more than 5 minutes ago");
954  where = QString("recgroup = 'Deleted' "
955  "AND lastmodified <= DATE_ADD(NOW(), INTERVAL '-5' MINUTE) ");
956  orderby = "lastmodified ASC";
957  break;
959  msg = "Adding deleted programs in FIFO order";
960  where = "recgroup = 'Deleted'";
961  orderby = "lastmodified ASC";
962  break;
963  }
964 
965  LOG(VB_FILE, LOG_INFO, LOC + "FillDBOrdered: " + msg);
966 
967  MSqlQuery query(MSqlQuery::InitCon());
968  QString querystr = QString(
969  "SELECT recorded.chanid, starttime "
970  "FROM recorded "
971  "LEFT JOIN channel ON recorded.chanid = channel.chanid "
972  "WHERE %1 AND deletepending = 0 "
973  "ORDER BY autoexpire DESC, %2").arg(where, orderby);
974 
975  query.prepare(querystr);
976 
977  if (!query.exec())
978  return;
979 
980  while (query.next())
981  {
982  uint chanid = query.value(0).toUInt();
983  QDateTime recstartts = MythDate::as_utc(query.value(1).toDateTime());
984 
985  if (IsInDontExpireSet(chanid, recstartts))
986  {
987  LOG(VB_FILE, LOG_INFO, LOC +
988  QString(" Skipping %1 at %2 because it is in Don't Expire "
989  "List")
990  .arg(chanid).arg(recstartts.toString(Qt::ISODate)));
991  }
992  else if (IsInExpireList(expireList, chanid, recstartts))
993  {
994  LOG(VB_FILE, LOG_INFO, LOC +
995  QString(" Skipping %1 at %2 because it is already in Expire "
996  "List")
997  .arg(chanid).arg(recstartts.toString(Qt::ISODate)));
998  }
999  else
1000  {
1001  auto *pginfo = new ProgramInfo(chanid, recstartts);
1002  if (pginfo->GetChanID())
1003  {
1004  LOG(VB_FILE, LOG_INFO, LOC + QString(" Adding %1 at %2")
1005  .arg(chanid).arg(recstartts.toString(Qt::ISODate)));
1006  expireList.push_back(pginfo);
1007  }
1008  else
1009  {
1010  LOG(VB_FILE, LOG_INFO, LOC +
1011  QString(" Skipping %1 at %2 "
1012  "because it could not be loaded from the DB")
1013  .arg(chanid).arg(recstartts.toString(Qt::ISODate)));
1014  delete pginfo;
1015  }
1016  }
1017  }
1018 }
1019 
1031 void AutoExpire::Update(int encoder, int fsID, bool immediately)
1032 {
1033  if (!expirer)
1034  return;
1035 
1036  if (encoder > 0)
1037  {
1038  QString msg = QString("Cardid %1: is starting a recording on")
1039  .arg(encoder);
1040  if (fsID == -1)
1041  msg.append(" an unknown fsID soon.");
1042  else
1043  msg.append(QString(" fsID %2 soon.").arg(fsID));
1044  LOG(VB_FILE, LOG_INFO, LOC + msg);
1045  }
1046 
1047  if (immediately)
1048  {
1049  if (encoder > 0)
1050  {
1051  expirer->m_instanceLock.lock();
1052  expirer->m_usedEncoders[encoder] = fsID;
1053  expirer->m_instanceLock.unlock();
1054  }
1055  expirer->CalcParams();
1056  expirer->m_instanceCond.wakeAll();
1057  }
1058  else
1059  {
1060  expirer->m_updateLock.lock();
1061  expirer->m_updateQueue.append(UpdateEntry(encoder, fsID));
1062  expirer->m_updateLock.unlock();
1063  }
1064 }
1065 
1067 {
1068  m_dontExpireSet.clear();
1069 
1070  MSqlQuery query(MSqlQuery::InitCon());
1071  query.prepare(
1072  "SELECT chanid, starttime, lastupdatetime, recusage, hostname "
1073  "FROM inuseprograms");
1074 
1075  if (!query.exec() || !query.next())
1076  return;
1077 
1078  LOG(VB_FILE, LOG_INFO, LOC + "Adding Programs to 'Do Not Expire' List");
1079  QDateTime curTime = MythDate::current();
1080 
1081  do
1082  {
1083  uint chanid = query.value(0).toUInt();
1084  QDateTime recstartts = MythDate::as_utc(query.value(1).toDateTime());
1085  QDateTime lastupdate = MythDate::as_utc(query.value(2).toDateTime());
1086 
1087  if (lastupdate.secsTo(curTime) < 2 * 60 * 60)
1088  {
1089  QString key = QString("%1_%2")
1090  .arg(chanid).arg(recstartts.toString(Qt::ISODate));
1091  m_dontExpireSet.insert(key);
1092  LOG(VB_FILE, LOG_INFO, QString(" %1 at %2 in use by %3 on %4")
1093  .arg(QString::number(chanid),
1094  recstartts.toString(Qt::ISODate),
1095  query.value(3).toString(),
1096  query.value(4).toString()));
1097  }
1098  }
1099  while (query.next());
1100 }
1101 
1103  uint chanid, const QDateTime &recstartts) const
1104 {
1105  QString key = QString("%1_%2")
1106  .arg(chanid).arg(recstartts.toString(Qt::ISODate));
1107 
1108  return (m_dontExpireSet.find(key) != m_dontExpireSet.end());
1109 }
1110 
1112  const pginfolist_t &expireList, uint chanid, const QDateTime &recstartts)
1113 {
1114  return std::any_of(expireList.cbegin(), expireList.cend(),
1115  [chanid,&recstartts](auto *info)
1116  { return ((info->GetChanID() == chanid) &&
1117  (info->GetRecordingStartTime() == recstartts)); } );
1118 }
1119 
1120 /* vim: set expandtab tabstop=4 shiftwidth=4: */
filesysteminfo.h
AutoExpire::m_instanceCond
QWaitCondition m_instanceCond
Definition: autoexpire.h:120
MSqlQuery::isActive
bool isActive(void) const
Definition: mythdbcon.h:213
MSqlQuery::next
bool next(void)
Wrap QSqlQuery::next() so we can display the query results.
Definition: mythdbcon.cpp:806
MSqlQuery
QSqlQuery wrapper that fetches a DB connection from the connection pool.
Definition: mythdbcon.h:125
MSqlQuery::size
int size(void) const
Definition: mythdbcon.h:212
MThread::start
void start(QThread::Priority p=QThread::InheritPriority)
Tell MThread to start running the thread in the near future.
Definition: mthread.cpp:286
AutoExpire::PrintExpireList
void PrintExpireList(const QString &expHost="ALL")
Prints a summary of the files that can be deleted.
Definition: autoexpire.cpp:789
mythdb.h
MainServer::GetFilesystemInfos
void GetFilesystemInfos(QList< FileSystemInfo > &fsInfos, bool useCache=true)
Definition: mainserver.cpp:5370
MythDate::as_utc
QDateTime as_utc(const QDateTime &old_dt)
Returns copy of QDateTime with TimeSpec set to UTC.
Definition: mythdate.cpp:23
AutoExpire::IsInExpireList
static bool IsInExpireList(const pginfolist_t &expireList, uint chanid, const QDateTime &recstartts)
Definition: autoexpire.cpp:1111
AutoExpire::Update
static void Update(int encoder, int fsID, bool immediately)
This is used to update the global AutoExpire instance "expirer".
Definition: autoexpire.cpp:1031
MThread::wait
bool wait(std::chrono::milliseconds time=std::chrono::milliseconds::max())
Wait for the MThread to exit, with a maximum timeout.
Definition: mthread.cpp:303
ExpireThread::m_parent
QPointer< AutoExpire > m_parent
Definition: autoexpire.h:47
RecordingInfo
Holds information on a TV Program one might wish to record.
Definition: recordinginfo.h:35
AutoExpire::ClearExpireList
static void ClearExpireList(pginfolist_t &expireList, bool deleteProg=true)
Clears expireList, freeing any ProgramInfo's if necessary.
Definition: autoexpire.cpp:874
RecordingInfo::ForgetHistory
void ForgetHistory(void)
Forget the recording of a program so it will be recorded again.
Definition: recordinginfo.cpp:1412
tvList
QMap< int, EncoderLink * > tvList
Definition: backendcontext.cpp:7
MythEvent
This class is used as a container for messages.
Definition: mythevent.h:16
ProgramInfo::kTitleSubtitle
@ kTitleSubtitle
Definition: programinfo.h:510
MSqlQuery::value
QVariant value(int i) const
Definition: mythdbcon.h:202
AutoExpire
Used to expire recordings to make space for new recordings.
Definition: autoexpire.h:60
MSqlQuery::exec
bool exec(void)
Wrap QSqlQuery::exec() so we can display SQL.
Definition: mythdbcon.cpp:607
AutoExpire::ExpireOldDeleted
void ExpireOldDeleted(void)
This expires deleted programs older than DeletedMaxAge.
Definition: autoexpire.cpp:372
emLowestPriorityFirst
@ emLowestPriorityFirst
Definition: autoexpire.h:29
LOG
#define LOG(_MASK_, _LEVEL_, _QSTRING_)
Definition: mythlogging.h:23
MThread::RunProlog
void RunProlog(void)
Sets up a thread, call this if you reimplement run().
Definition: mthread.cpp:196
AutoExpire::ExpireEpisodesOverMax
void ExpireEpisodesOverMax(void)
This deletes programs exceeding the maximum number of episodes of that program desired....
Definition: autoexpire.cpp:648
AutoExpire::GetAllExpiring
void GetAllExpiring(QStringList &strList)
Gets the full list of programs that can expire in expiration order.
Definition: autoexpire.cpp:828
ExpireThread::run
void run(void) override
This calls AutoExpire::RunExpirer() from within a new thread.
Definition: autoexpire.cpp:55
AutoExpire::m_desiredSpace
QMap< int, int64_t > m_desiredSpace
Definition: autoexpire.h:116
build_compdb.file
file
Definition: build_compdb.py:55
AutoExpire::m_updateLock
QMutex m_updateLock
Definition: autoexpire.h:125
remoteutil.h
AutoExpire::m_desiredFreq
uint m_desiredFreq
Definition: autoexpire.h:113
MythDate::current
QDateTime current(bool stripped)
Returns current Date and Time in UTC.
Definition: mythdate.cpp:10
true
VERBOSE_PREAMBLE Most true
Definition: verbosedefs.h:91
emOldDeletedPrograms
@ emOldDeletedPrograms
Definition: autoexpire.h:33
AutoExpire::m_instanceLock
QMutex m_instanceLock
Definition: autoexpire.h:119
ProgramInfo::IsWatched
bool IsWatched(void) const
Definition: programinfo.h:483
MythObservable::addListener
void addListener(QObject *listener)
Add a listener to the observable.
Definition: mythobservable.cpp:38
MythDate::secsInFuture
std::chrono::seconds secsInFuture(const QDateTime &future)
Definition: mythdate.cpp:204
fileserverutil.h
mythdate.h
autoexpire.h
expirer
AutoExpire * expirer
Definition: backendcontext.cpp:8
programinfo.h
mythlogging.h
AutoExpire::m_expireThreadRun
bool m_expireThreadRun
Definition: autoexpire.h:114
UpdateEntry::m_encoder
int m_encoder
Definition: autoexpire.h:54
AutoExpire::Sleep
void Sleep(std::chrono::milliseconds sleepTime)
Sleeps for sleepTime milliseconds; unless the expire thread is told to quit.
Definition: autoexpire.cpp:342
hardwareprofile.config.p
p
Definition: config.py:33
AutoExpire::m_usedEncoders
QMap< int, int > m_usedEncoders
Definition: autoexpire.h:117
MSqlQuery::InitCon
static MSqlQueryInfo InitCon(ConnectionReuse _reuse=kNormalConnection)
Only use this in combination with MSqlQuery constructor.
Definition: mythdbcon.cpp:539
compat.h
AutoExpire::ExpireRecordings
void ExpireRecordings(void)
This expires normal recordings.
Definition: autoexpire.cpp:399
MythDB::DBError
static void DBError(const QString &where, const MSqlQuery &query)
Definition: mythdb.cpp:226
AutoExpire::IsInDontExpireSet
bool IsInDontExpireSet(uint chanid, const QDateTime &recstartts) const
Definition: autoexpire.cpp:1102
GetPlaybackURL
QString GetPlaybackURL(ProgramInfo *pginfo, bool storePath)
Definition: fileserverutil.cpp:45
pginfolist_t
std::vector< ProgramInfo * > pginfolist_t
Definition: autoexpire.h:24
emNormalLiveTVPrograms
@ emNormalLiveTVPrograms
Definition: autoexpire.h:32
MThread::RunEpilog
void RunEpilog(void)
Cleans up a thread's resources, call this if you reimplement run().
Definition: mthread.cpp:209
ExpireThread
Definition: autoexpire.h:40
storagegroup.h
TVRec::s_inputsLock
static QReadWriteLock s_inputsLock
Definition: tv_rec.h:433
LOC
#define LOC
Definition: autoexpire.cpp:44
AutoExpire::SendDeleteMessages
static void SendDeleteMessages(pginfolist_t &deleteList)
This sends delete message to main event thread.
Definition: autoexpire.cpp:611
uint
unsigned int uint
Definition: compat.h:144
AutoExpire::AutoExpire
AutoExpire()=default
gCoreContext
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
Definition: mythcorecontext.cpp:60
AutoExpire::FillExpireList
void FillExpireList(pginfolist_t &expireList)
Uses the "AutoExpireMethod" setting in the database to fill the list of files that are deletable.
Definition: autoexpire.cpp:767
AutoExpire::m_dontExpireSet
QSet< QString > m_dontExpireSet
Definition: autoexpire.h:111
emNormalDeletedPrograms
@ emNormalDeletedPrograms
Definition: autoexpire.h:34
MythCoreContext::GetNumSetting
int GetNumSetting(const QString &key, int defaultval=0)
Definition: mythcorecontext.cpp:936
UpdateEntry
Definition: autoexpire.h:50
AutoExpire::RunExpirer
void RunExpirer(void)
This contains the main loop for the auto expire process.
Definition: autoexpire.cpp:266
MythCoreContext::GetBoolSetting
bool GetBoolSetting(const QString &key, bool defaultval=false)
Definition: mythcorecontext.cpp:930
AutoExpire::CalcParams
void CalcParams(void)
Definition: autoexpire.cpp:121
VERBOSE_LEVEL_CHECK
#define VERBOSE_LEVEL_CHECK(_MASK_, _LEVEL_)
Definition: mythlogging.h:14
ProgramInfo
Holds information on recordings and videos.
Definition: programinfo.h:67
AutoExpire::FillDBOrdered
void FillDBOrdered(pginfolist_t &expireList, int expMethod)
Creates a list of programs to delete using the database to order list.
Definition: autoexpire.cpp:893
AutoExpire::m_expireThread
ExpireThread * m_expireThread
Definition: autoexpire.h:112
mythcorecontext.h
MSqlQuery::bindValue
void bindValue(const QString &placeholder, const QVariant &val)
Add a single binding.
Definition: mythdbcon.cpp:882
MythDate::ISODate
@ ISODate
Default UTC.
Definition: mythdate.h:17
emOldestFirst
@ emOldestFirst
Definition: autoexpire.h:28
AutoExpire::~AutoExpire
~AutoExpire() override
AutoExpire destructor stops auto delete thread if it is running.
Definition: autoexpire.cpp:83
emShortLiveTVPrograms
@ emShortLiveTVPrograms
Definition: autoexpire.h:31
tv_rec.h
AutoExpire::m_mainServer
MainServer * m_mainServer
Definition: autoexpire.h:122
emQuickDeletedPrograms
@ emQuickDeletedPrograms
Definition: autoexpire.h:35
remoteencoder.h
mainserver.h
AutoExpire::ExpireQuickDeleted
void ExpireQuickDeleted(void)
This expires deleted programs within a few minutes.
Definition: autoexpire.cpp:385
SPACE_TOO_BIG_KB
#define SPACE_TOO_BIG_KB
If calculated desired space for 10 min freq is > SPACE_TOO_BIG_KB then we use 5 min expire frequency.
Definition: autoexpire.cpp:52
MythCoreContext::GetHostName
QString GetHostName(void)
Definition: mythcorecontext.cpp:862
AutoExpire::ExpireLiveTV
void ExpireLiveTV(int type)
This expires LiveTV programs.
Definition: autoexpire.cpp:359
AutoExpire::GetDesiredSpace
uint64_t GetDesiredSpace(int fsID) const
Used by the scheduler to select the next recording dir.
Definition: autoexpire.cpp:110
AutoExpire::m_encoderList
QMap< int, EncoderLink * > * m_encoderList
Definition: autoexpire.h:88
AutoExpire::m_updateQueue
QQueue< UpdateEntry > m_updateQueue
Definition: autoexpire.h:126
MythCoreContext::dispatch
void dispatch(const MythEvent &event)
Definition: mythcorecontext.cpp:1749
MythObservable::removeListener
void removeListener(QObject *listener)
Remove a listener to the observable.
Definition: mythobservable.cpp:55
AutoExpire::UpdateDontExpireSet
void UpdateDontExpireSet(void)
Definition: autoexpire.cpp:1066
emWeightedTimePriority
@ emWeightedTimePriority
Definition: autoexpire.h:30
ProgramInfo::kRecordingKey
@ kRecordingKey
Definition: programinfo.h:511
MSqlQuery::prepare
bool prepare(const QString &query)
QSqlQuery::prepare() is not thread safe in Qt <= 3.3.2.
Definition: mythdbcon.cpp:831
UpdateEntry::m_fsID
int m_fsID
Definition: autoexpire.h:57