MythTV  master
autoexpire.cpp
Go to the documentation of this file.
1 // System headers
2 #include <sys/stat.h>
3 #ifdef __linux__
4 # include <sys/vfs.h>
5 #else // if !__linux__
6 # include <sys/param.h>
7 # ifndef _WIN32
8 # include <sys/mount.h>
9 # endif // _WIN32
10 #endif // !__linux__
11 
12 // POSIX headers
13 #include <unistd.h>
14 
15 // C++ headers
16 #include <algorithm>
17 #include <cstdlib>
18 #include <iostream>
19 
20 // Qt headers
21 #include <QDateTime>
22 #include <QFileInfo>
23 #include <QList>
24 
25 // MythTV headers
26 #include "libmythbase/compat.h"
29 #include "libmythbase/mythdate.h"
30 #include "libmythbase/mythdb.h"
33 #include "libmythbase/remoteutil.h"
37 #include "libmythtv/tv_rec.h"
38 
39 // MythBackend
40 #include "autoexpire.h"
41 #include "backendcontext.h"
42 #include "encoderlink.h"
43 #include "mainserver.h"
44 
45 #define LOC QString("AutoExpire: ")
46 #define LOC_ERR QString("AutoExpire Error: ")
47 
51 static constexpr uint64_t kSpaceTooBigKB { 3ULL * 1024 * 1024 };
52 
53 // Consider recordings within the last two hours to be too recent to
54 // add to the autoexpire list.
55 static constexpr int64_t kRecentInterval { 2LL * 60 * 60 };
56 
59 {
60  RunProlog();
61  m_parent->RunExpirer();
62  RunEpilog();
63 }
64 
74 AutoExpire::AutoExpire(QMap<int, EncoderLink *> *tvList) :
75  m_encoderList(tvList),
76  m_expireThread(new ExpireThread(this)),
77  m_expireThreadRun(true)
78 {
81 }
82 
87 {
88  {
89  QMutexLocker locker(&m_instanceLock);
90  m_expireThreadRun = false;
91  m_instanceCond.wakeAll();
92  }
93 
94  {
95  QMutexLocker locker(&m_updateLock);
96  m_updateQueue.clear();
97  }
98 
99  if (m_expireThread)
100  {
102  m_expireThread->wait();
103  delete m_expireThread;
104  m_expireThread = nullptr;
105  }
106 }
107 
113 uint64_t AutoExpire::GetDesiredSpace(int fsID) const
114 {
115  QMutexLocker locker(&m_instanceLock);
116  if (m_desiredSpace.contains(fsID))
117  return m_desiredSpace[fsID];
118  return 0;
119 }
120 
125 {
126  LOG(VB_FILE, LOG_INFO, LOC + "CalcParams()");
127 
128  QList<FileSystemInfo> fsInfos;
129 
130  m_instanceLock.lock();
131  if (m_mainServer)
132  {
133  // The scheduler relies on something forcing the mainserver
134  // fsinfos cache to get updated periodically. Currently, that
135  // is done here. Don't remove or change this invocation
136  // without handling that issue too. It is done this way
137  // because the scheduler thread can't afford to be blocked by
138  // an unresponsive, remote filesystem and the autoexpirer
139  // thread can.
140  m_mainServer->GetFilesystemInfos(fsInfos, false);
141  }
142  m_instanceLock.unlock();
143 
144  if (fsInfos.empty())
145  {
146  LOG(VB_GENERAL, LOG_ERR, LOC + "Filesystem Info cache is empty, unable "
147  "to calculate necessary parameters.");
148  return;
149  }
150 
151  uint64_t maxKBperMin = 0;
152  uint64_t extraKB = static_cast<uint64_t>
153  (gCoreContext->GetNumSetting("AutoExpireExtraSpace", 0))
154  << 20;
155 
156  QMap<int, uint64_t> fsMap;
157  QMap<int, std::vector<int> > fsEncoderMap;
158 
159  // We use this copying on purpose. The used_encoders map ensures
160  // that every encoder writes only to one fs.
161  // Copying the data minimizes the time the lock is held.
162  m_instanceLock.lock();
163  QMap<int, int>::const_iterator ueit = m_usedEncoders.cbegin();
164  while (ueit != m_usedEncoders.cend())
165  {
166  fsEncoderMap[*ueit].push_back(ueit.key());
167  ++ueit;
168  }
169  m_instanceLock.unlock();
170 
171  QList<FileSystemInfo>::iterator fsit;
172  for (fsit = fsInfos.begin(); fsit != fsInfos.end(); ++fsit)
173  {
174  if (fsMap.contains(fsit->getFSysID()))
175  continue;
176 
177  fsMap[fsit->getFSysID()] = 0;
178  uint64_t thisKBperMin = 0;
179 
180  // append unknown recordings to all fsIDs
181  for (auto unknownfs : qAsConst(fsEncoderMap[-1]))
182  fsEncoderMap[fsit->getFSysID()].push_back(unknownfs);
183 
184  if (fsEncoderMap.contains(fsit->getFSysID()))
185  {
186  LOG(VB_FILE, LOG_INFO,
187  QString("fsID #%1: Total: %2 GB Used: %3 GB Free: %4 GB")
188  .arg(fsit->getFSysID())
189  .arg(fsit->getTotalSpace() / 1024.0 / 1024.0, 7, 'f', 1)
190  .arg(fsit->getUsedSpace() / 1024.0 / 1024.0, 7, 'f', 1)
191  .arg(fsit->getFreeSpace() / 1024.0 / 1024.0, 7, 'f', 1));
192 
193  for (auto cardid : qAsConst(fsEncoderMap[fsit->getFSysID()]))
194  {
195  EncoderLink *enc = *(m_encoderList->constFind(cardid));
196 
197  if (!enc->IsConnected() || !enc->IsBusy())
198  {
199  // remove encoder since it can't write to any file system
200  LOG(VB_FILE, LOG_INFO, LOC +
201  QString("Cardid %1: is not recording, removing it "
202  "from used list.").arg(cardid));
203  m_instanceLock.lock();
204  m_usedEncoders.remove(cardid);
205  m_instanceLock.unlock();
206  continue;
207  }
208 
209  uint64_t maxBitrate = enc->GetMaxBitrate();
210  if (maxBitrate==0)
211  maxBitrate = 19500000LL;
212  thisKBperMin += (maxBitrate*((uint64_t)15))>>11;
213  LOG(VB_FILE, LOG_INFO, QString(" Cardid %1: max bitrate "
214  "%2 Kb/sec, fsID %3 max is now %4 KB/min")
215  .arg(enc->GetInputID())
216  .arg(enc->GetMaxBitrate() >> 10)
217  .arg(fsit->getFSysID())
218  .arg(thisKBperMin));
219  }
220  }
221  fsMap[fsit->getFSysID()] = thisKBperMin;
222 
223  if (thisKBperMin > maxKBperMin)
224  {
225  LOG(VB_FILE, LOG_INFO,
226  QString(" Max of %1 KB/min for fsID %2 is higher "
227  "than the existing Max of %3 so we'll use this Max instead")
228  .arg(thisKBperMin).arg(fsit->getFSysID()).arg(maxKBperMin));
229  maxKBperMin = thisKBperMin;
230  }
231  }
232 
233  // Determine frequency to run autoexpire so it doesn't have to free
234  // too much space
235  uint expireFreq = 15;
236  if (maxKBperMin > 0)
237  {
238  expireFreq = kSpaceTooBigKB / (maxKBperMin + maxKBperMin/3);
239  expireFreq = std::max(3U, std::min(expireFreq, 15U));
240  }
241 
242  double expireMinGB = ((maxKBperMin + maxKBperMin/3)
243  * expireFreq + extraKB) >> 20;
244  LOG(VB_GENERAL, LOG_NOTICE, LOC +
245  QString("CalcParams(): Max required Free Space: %1 GB w/freq: %2 min")
246  .arg(expireMinGB, 0, 'f', 1).arg(expireFreq));
247 
248  // lock class and save these parameters.
249  m_instanceLock.lock();
250  m_desiredFreq = expireFreq;
251  // write per file system needed space back, use safety of 33%
252  QMap<int, uint64_t>::iterator it = fsMap.begin();
253  while (it != fsMap.end())
254  {
255  m_desiredSpace[it.key()] = (*it + *it/3) * expireFreq + extraKB;
256  ++it;
257  }
258  m_instanceLock.unlock();
259 }
260 
270 {
271  QElapsedTimer timer;
272  QDateTime curTime;
273  QDateTime next_expire = MythDate::current().addSecs(60);
274 
275  QMutexLocker locker(&m_instanceLock);
276 
277  // wait a little for main server to come up and things to settle down
278  Sleep(20s);
279 
280  timer.start();
281 
282  while (m_expireThreadRun)
283  {
284  TVRec::s_inputsLock.lockForRead();
285 
286  curTime = MythDate::current();
287  // recalculate auto expire parameters
288  if (curTime >= next_expire)
289  {
290  m_updateLock.lock();
291  while (!m_updateQueue.empty())
292  {
293  UpdateEntry ue = m_updateQueue.dequeue();
294  if (ue.m_encoder > 0)
296  }
297  m_updateLock.unlock();
298 
299  locker.unlock();
300  CalcParams();
301  locker.relock();
302  if (!m_expireThreadRun)
303  break;
304  }
305  timer.restart();
306 
308 
309  // Expire Short LiveTV files for this backend every 2 minutes
310  if ((curTime.time().minute() % 2) == 0)
312 
313  // Expire normal recordings depending on frequency calculated
314  if (curTime >= next_expire)
315  {
316  LOG(VB_FILE, LOG_INFO, LOC + "Running now!");
317  next_expire =
318  MythDate::current().addSecs(m_desiredFreq * 60LL);
319 
321 
322  int maxAge = gCoreContext->GetNumSetting("DeletedMaxAge", 0);
323  if (maxAge > 0)
325  else if (maxAge == 0)
327 
329 
331  }
332 
333  TVRec::s_inputsLock.unlock();
334 
335  Sleep(60s - std::chrono::milliseconds(timer.elapsed()));
336  }
337 }
338 
345 void AutoExpire::Sleep(std::chrono::milliseconds sleepTime)
346 {
347  if (sleepTime <= 0ms)
348  return;
349 
350  QDateTime little_tm = MythDate::current().addMSecs(sleepTime.count());
351  std::chrono::milliseconds timeleft = sleepTime;
352  while (m_expireThreadRun && (timeleft > 0ms))
353  {
354  m_instanceCond.wait(&m_instanceLock, timeleft.count());
355  timeleft = MythDate::secsInFuture(little_tm);
356  }
357 }
358 
363 {
364  pginfolist_t expireList;
365 
366  LOG(VB_FILE, LOG_INFO, LOC + QString("ExpireLiveTV(%1)").arg(type));
367  FillDBOrdered(expireList, type);
368  SendDeleteMessages(expireList);
369  ClearExpireList(expireList);
370 }
371 
376 {
377  pginfolist_t expireList;
378 
379  LOG(VB_FILE, LOG_INFO, LOC + QString("ExpireOldDeleted()"));
380  FillDBOrdered(expireList, emOldDeletedPrograms);
381  SendDeleteMessages(expireList);
382  ClearExpireList(expireList);
383 }
384 
389 {
390  pginfolist_t expireList;
391 
392  LOG(VB_FILE, LOG_INFO, LOC + QString("ExpireQuickDeleted()"));
394  SendDeleteMessages(expireList);
395  ClearExpireList(expireList);
396 }
397 
403 {
404  pginfolist_t expireList;
405  pginfolist_t deleteList;
406  QList<FileSystemInfo> fsInfos;
407  QList<FileSystemInfo>::iterator fsit;
408 
409  LOG(VB_FILE, LOG_INFO, LOC + "ExpireRecordings()");
410 
411  if (m_mainServer)
412  m_mainServer->GetFilesystemInfos(fsInfos, true);
413 
414  if (fsInfos.empty())
415  {
416  LOG(VB_GENERAL, LOG_ERR, LOC + "Filesystem Info cache is empty, unable "
417  "to determine what Recordings to expire");
418 
419  return;
420  }
421 
422  FillExpireList(expireList);
423 
424  QMap <int, bool> truncateMap;
425  MSqlQuery query(MSqlQuery::InitCon());
426  query.prepare("SELECT DISTINCT rechost, recdir "
427  "FROM inuseprograms "
428  "WHERE recusage = 'truncatingdelete' "
429  "AND lastupdatetime > DATE_ADD(NOW(), INTERVAL -2 MINUTE);");
430 
431  if (!query.exec())
432  {
433  MythDB::DBError(LOC + "ExpireRecordings", query);
434  }
435  else
436  {
437  while (query.next())
438  {
439  QString rechost = query.value(0).toString();
440  QString recdir = query.value(1).toString();
441 
442  LOG(VB_FILE, LOG_INFO, LOC +
443  QString("%1:%2 has an in-progress truncating delete.")
444  .arg(rechost, recdir));
445 
446  for (fsit = fsInfos.begin(); fsit != fsInfos.end(); ++fsit)
447  {
448  if ((fsit->getHostname() == rechost) &&
449  (fsit->getPath() == recdir))
450  {
451  truncateMap[fsit->getFSysID()] = true;
452  break;
453  }
454  }
455  }
456  }
457 
458  QMap <int, bool> fsMap;
459  for (fsit = fsInfos.begin(); fsit != fsInfos.end(); ++fsit)
460  {
461  if (fsMap.contains(fsit->getFSysID()))
462  continue;
463 
464  fsMap[fsit->getFSysID()] = true;
465 
466  LOG(VB_FILE, LOG_INFO,
467  QString("fsID #%1: Total: %2 GB Used: %3 GB Free: %4 GB")
468  .arg(fsit->getFSysID())
469  .arg(fsit->getTotalSpace() / 1024.0 / 1024.0, 7, 'f', 1)
470  .arg(fsit->getUsedSpace() / 1024.0 / 1024.0, 7, 'f', 1)
471  .arg(fsit->getFreeSpace() / 1024.0 / 1024.0, 7, 'f', 1));
472 
473  if ((fsit->getTotalSpace() == -1) || (fsit->getUsedSpace() == -1))
474  {
475  LOG(VB_FILE, LOG_ERR, LOC +
476  QString("fsID #%1 has invalid info, AutoExpire cannot run for "
477  "this filesystem. Continuing on to next...")
478  .arg(fsit->getFSysID()));
479  LOG(VB_FILE, LOG_INFO, QString("Directories on filesystem ID %1:")
480  .arg(fsit->getFSysID()));
481  QList<FileSystemInfo>::iterator fsit2;
482  for (fsit2 = fsInfos.begin(); fsit2 != fsInfos.end(); ++fsit2)
483  {
484  if (fsit2->getFSysID() == fsit->getFSysID())
485  {
486  LOG(VB_FILE, LOG_INFO, QString(" %1:%2")
487  .arg(fsit2->getHostname(), fsit2->getPath()));
488  }
489  }
490 
491  continue;
492  }
493 
494  if (truncateMap.contains(fsit->getFSysID()))
495  {
496  LOG(VB_FILE, LOG_INFO,
497  QString(" fsid %1 has a truncating delete in progress, "
498  "AutoExpire cannot run for this filesystem until the "
499  "delete has finished. Continuing on to next...")
500  .arg(fsit->getFSysID()));
501  continue;
502  }
503 
504  if (std::max((int64_t)0LL, fsit->getFreeSpace()) <
505  m_desiredSpace[fsit->getFSysID()])
506  {
507  LOG(VB_FILE, LOG_INFO,
508  QString(" Not Enough Free Space! We want %1 MB")
509  .arg(m_desiredSpace[fsit->getFSysID()] / 1024));
510 
511  QMap<QString, int> dirList;
512  QList<FileSystemInfo>::iterator fsit2;
513 
514  LOG(VB_FILE, LOG_INFO,
515  QString(" Directories on filesystem ID %1:")
516  .arg(fsit->getFSysID()));
517 
518  for (fsit2 = fsInfos.begin(); fsit2 != fsInfos.end(); ++fsit2)
519  {
520  if (fsit2->getFSysID() == fsit->getFSysID())
521  {
522  LOG(VB_FILE, LOG_INFO, QString(" %1:%2")
523  .arg(fsit2->getHostname(), fsit2->getPath()));
524  dirList[fsit2->getHostname() + ":" + fsit2->getPath()] = 1;
525  }
526  }
527 
528  LOG(VB_FILE, LOG_INFO,
529  " Searching for files expirable in these directories");
530  QString myHostName = gCoreContext->GetHostName();
531  auto it = expireList.begin();
532  while ((it != expireList.end()) &&
533  (std::max((int64_t)0LL, fsit->getFreeSpace()) <
534  m_desiredSpace[fsit->getFSysID()]))
535  {
536  ProgramInfo *p = *it;
537  ++it;
538 
539  LOG(VB_FILE, LOG_INFO, QString(" Checking %1 => %2")
540  .arg(p->toString(ProgramInfo::kRecordingKey),
541  p->GetTitle()));
542 
543  if (!p->IsLocal())
544  {
545  bool foundFile = false;
546  auto eit = m_encoderList->constBegin();
547  while (eit != m_encoderList->constEnd())
548  {
549  EncoderLink *el = *eit;
550  eit++;
551 
552  if ((p->GetHostname() == el->GetHostName()) ||
553  ((p->GetHostname() == myHostName) &&
554  (el->IsLocal())))
555  {
556  if (el->IsConnected())
557  foundFile = el->CheckFile(p);
558 
559  eit = m_encoderList->constEnd();
560  }
561  }
562 
563  if (!foundFile && (p->GetHostname() != myHostName))
564  {
565  // Wasn't found so check locally
566  QString file = GetPlaybackURL(p);
567 
568  if (file.startsWith("/"))
569  {
570  p->SetPathname(file);
571  p->SetHostname(myHostName);
572  foundFile = true;
573  }
574  }
575 
576  if (!foundFile)
577  {
578  LOG(VB_FILE, LOG_ERR, LOC +
579  QString(" ERROR: Can't find file for %1")
580  .arg(p->toString(ProgramInfo::kRecordingKey)));
581  continue;
582  }
583  }
584 
585  QFileInfo vidFile(p->GetPathname());
586  if (dirList.contains(p->GetHostname() + ':' + vidFile.path()))
587  {
588  fsit->setUsedSpace(fsit->getUsedSpace()
589  - (p->GetFilesize() / 1024));
590  deleteList.push_back(p);
591 
592  LOG(VB_FILE, LOG_INFO,
593  QString(" FOUND file expirable. "
594  "%1 is located at %2 which is on fsID #%3. "
595  "Adding to deleteList. After deleting we "
596  "should have %4 MB free on this filesystem.")
597  .arg(p->toString(ProgramInfo::kRecordingKey),
598  p->GetPathname()).arg(fsit->getFSysID())
599  .arg(fsit->getFreeSpace() / 1024));
600  }
601  }
602  }
603  }
604 
605  SendDeleteMessages(deleteList);
606 
607  ClearExpireList(deleteList, false);
608  ClearExpireList(expireList);
609 }
610 
615 {
616  QString msg;
617 
618  if (deleteList.empty())
619  {
620  LOG(VB_FILE, LOG_INFO, LOC + "SendDeleteMessages. Nothing to expire.");
621  return;
622  }
623 
624  LOG(VB_FILE, LOG_INFO, LOC +
625  "SendDeleteMessages, cycling through deleteList.");
626  auto it = deleteList.begin();
627  while (it != deleteList.end())
628  {
629  msg = QString("%1Expiring %2 MB for %3 => %4")
630  .arg(VERBOSE_LEVEL_CHECK(VB_FILE, LOG_ANY) ? " " : "",
631  QString::number((*it)->GetFilesize() >> 20),
632  (*it)->toString(ProgramInfo::kRecordingKey),
633  (*it)->toString(ProgramInfo::kTitleSubtitle));
634 
635  LOG(VB_GENERAL, LOG_NOTICE, msg);
636 
637  // send auto expire message to backend's event thread.
638  MythEvent me(QString("AUTO_EXPIRE %1 %2").arg((*it)->GetChanID())
639  .arg((*it)->GetRecordingStartTime(MythDate::ISODate)));
640  gCoreContext->dispatch(me);
641 
642  ++it; // move on to next program
643  }
644 }
645 
652 {
653  QMap<QString, int> maxEpisodes;
654  QMap<QString, int>::Iterator maxIter;
655  QMap<QString, int> episodeParts;
656  QString episodeKey;
657 
658  MSqlQuery query(MSqlQuery::InitCon());
659  query.prepare("SELECT recordid, maxepisodes, title "
660  "FROM record WHERE maxepisodes > 0 "
661  "ORDER BY recordid ASC, maxepisodes DESC");
662 
663  if (query.exec() && query.isActive() && query.size() > 0)
664  {
665  LOG(VB_FILE, LOG_INFO, LOC +
666  QString("Found %1 record profiles using max episode expiration")
667  .arg(query.size()));
668  while (query.next())
669  {
670  LOG(VB_FILE, LOG_INFO, QString(" %1 (%2 for rec id %3)")
671  .arg(query.value(2).toString())
672  .arg(query.value(1).toInt())
673  .arg(query.value(0).toInt()));
674  maxEpisodes[query.value(0).toString()] = query.value(1).toInt();
675  }
676  }
677 
678  LOG(VB_FILE, LOG_INFO, LOC +
679  "Checking episode count for each recording profile using max episodes");
680  for (maxIter = maxEpisodes.begin(); maxIter != maxEpisodes.end(); ++maxIter)
681  {
682  query.prepare("SELECT chanid, starttime, title, progstart, progend, "
683  "duplicate "
684  "FROM recorded "
685  "WHERE recordid = :RECID AND preserve = 0 "
686  "AND recgroup NOT IN ('LiveTV', 'Deleted') "
687  "ORDER BY starttime DESC;");
688  query.bindValue(":RECID", maxIter.key());
689 
690  if (!query.exec() || !query.isActive())
691  {
692  MythDB::DBError("AutoExpire query failed!", query);
693  continue;
694  }
695 
696  LOG(VB_FILE, LOG_INFO, QString(" Recordid %1 has %2 recordings.")
697  .arg(maxIter.key())
698  .arg(query.size()));
699  if (query.size() > 0)
700  {
701  int found = 1;
702  while (query.next())
703  {
704  uint chanid = query.value(0).toUInt();
705  QDateTime startts = MythDate::as_utc(query.value(1).toDateTime());
706  QString title = query.value(2).toString();
707  QDateTime progstart = MythDate::as_utc(query.value(3).toDateTime());
708  QDateTime progend = MythDate::as_utc(query.value(4).toDateTime());
709  int duplicate = query.value(5).toInt();
710 
711  episodeKey = QString("%1_%2_%3")
712  .arg(QString::number(chanid),
713  progstart.toString(Qt::ISODate),
714  progend.toString(Qt::ISODate));
715 
716  if ((!IsInDontExpireSet(chanid, startts)) &&
717  (!episodeParts.contains(episodeKey)) &&
718  (found > *maxIter))
719  {
720  QString msg =
721  QString("%1Deleting %2 at %3 => %4. "
722  "Too many episodes, we only want to keep %5.")
723  .arg(VERBOSE_LEVEL_CHECK(VB_FILE, LOG_ANY) ? " " : "",
724  QString::number(chanid),
725  startts.toString(Qt::ISODate),
726  title,
727  QString::number(*maxIter));
728 
729  LOG(VB_GENERAL, LOG_NOTICE, msg);
730 
731  // allow re-record if auto expired
732  RecordingInfo recInfo(chanid, startts);
733  if (gCoreContext->GetBoolSetting("RerecordWatched", false) ||
734  !recInfo.IsWatched())
735  {
736  recInfo.ForgetHistory();
737  }
738  msg = QString("DELETE_RECORDING %1 %2")
739  .arg(chanid)
740  .arg(startts.toString(Qt::ISODate));
741 
742  MythEvent me(msg);
743  gCoreContext->dispatch(me);
744  }
745  else
746  {
747  // keep track of shows we haven't expired so we can
748  // make sure we don't expire another part of the same
749  // episode.
750  if (episodeParts.contains(episodeKey))
751  {
752  episodeParts[episodeKey] = episodeParts[episodeKey] + 1;
753  }
754  else
755  {
756  episodeParts[episodeKey] = 1;
757  if( duplicate )
758  found++;
759  }
760  }
761  }
762  }
763  }
764 }
765 
771 {
772  int expMethod = gCoreContext->GetNumSetting("AutoExpireMethod", 1);
773 
774  ClearExpireList(expireList);
775 
777 
778  switch(expMethod)
779  {
780  case emOldestFirst:
783  FillDBOrdered(expireList, expMethod);
784  break;
785  // default falls through so list is empty so no AutoExpire
786  }
787 }
788 
792 void AutoExpire::PrintExpireList(const QString& expHost)
793 {
794  pginfolist_t expireList;
795 
796  FillExpireList(expireList);
797 
798  QString msg = "MythTV AutoExpire List ";
799  if (expHost != "ALL")
800  msg += QString("for '%1' ").arg(expHost);
801  msg += "(programs listed in order of expiration)";
802  std::cout << msg.toLocal8Bit().constData() << std::endl;
803 
804  for (auto *first : expireList)
805  {
806  if (expHost != "ALL" && first->GetHostname() != expHost)
807  continue;
808 
809  QString title = first->toString(ProgramInfo::kTitleSubtitle);
810  title = title.leftJustified(39, ' ', true);
811 
812  QString outstr = QString("%1 %2 MB %3 [%4]")
813  .arg(title,
814  QString::number(first->GetFilesize() >> 20)
815  .rightJustified(5, ' ', true),
816  first->GetRecordingStartTime(MythDate::ISODate)
817  .leftJustified(24, ' ', true),
818  QString::number(first->GetRecordingPriority())
819  .rightJustified(3, ' ', true));
820  QByteArray out = outstr.toLocal8Bit();
821 
822  std::cout << out.constData() << std::endl;
823  }
824 
825  ClearExpireList(expireList);
826 }
827 
831 void AutoExpire::GetAllExpiring(QStringList &strList)
832 {
833  QMutexLocker lockit(&m_instanceLock);
834  pginfolist_t expireList;
835 
837 
841  FillDBOrdered(expireList, gCoreContext->GetNumSetting("AutoExpireMethod",
842  emOldestFirst));
843 
844  strList << QString::number(expireList.size());
845 
846  for (auto & info : expireList)
847  info->ToStringList(strList);
848 
849  ClearExpireList(expireList);
850 }
851 
856 {
857  QMutexLocker lockit(&m_instanceLock);
858  pginfolist_t expireList;
859 
861 
865  FillDBOrdered(expireList, gCoreContext->GetNumSetting("AutoExpireMethod",
866  emOldestFirst));
867 
868  for (auto & info : expireList)
869  list.push_back( new ProgramInfo( *info ));
870 
871  ClearExpireList(expireList);
872 }
873 
877 void AutoExpire::ClearExpireList(pginfolist_t &expireList, bool deleteProg)
878 {
879  ProgramInfo *pginfo = nullptr;
880  while (!expireList.empty())
881  {
882  if (deleteProg)
883  pginfo = expireList.back();
884 
885  expireList.pop_back();
886 
887  if (deleteProg)
888  delete pginfo;
889  }
890 }
891 
896 void AutoExpire::FillDBOrdered(pginfolist_t &expireList, int expMethod)
897 {
898  QString where;
899  QString orderby;
900  QString msg;
901  int maxAge = 0;
902 
903  switch (expMethod)
904  {
905  default:
906  case emOldestFirst:
907  msg = "Adding programs expirable in Oldest First order";
908  where = "autoexpire > 0";
909  if (gCoreContext->GetBoolSetting("AutoExpireWatchedPriority", false))
910  orderby = "recorded.watched DESC, ";
911  orderby += "starttime ASC";
912  break;
914  msg = "Adding programs expirable in Lowest Priority First order";
915  where = "autoexpire > 0";
916  if (gCoreContext->GetBoolSetting("AutoExpireWatchedPriority", false))
917  orderby = "recorded.watched DESC, ";
918  orderby += "recorded.recpriority ASC, starttime ASC";
919  break;
921  msg = "Adding programs expirable in Weighted Time Priority order";
922  where = "autoexpire > 0";
923  if (gCoreContext->GetBoolSetting("AutoExpireWatchedPriority", false))
924  orderby = "recorded.watched DESC, ";
925  orderby += QString("DATE_ADD(starttime, INTERVAL '%1' * "
926  "recorded.recpriority DAY) ASC")
927  .arg(gCoreContext->GetNumSetting("AutoExpireDayPriority", 3));
928  break;
930  msg = "Adding Short LiveTV programs in starttime order";
931  where = "recgroup = 'LiveTV' "
932  "AND endtime < DATE_ADD(starttime, INTERVAL '30' SECOND) "
933  "AND endtime <= DATE_ADD(NOW(), INTERVAL '-5' MINUTE) ";
934  orderby = "starttime ASC";
935  break;
937  msg = "Adding LiveTV programs in starttime order";
938  where = QString("recgroup = 'LiveTV' "
939  "AND endtime <= DATE_ADD(NOW(), INTERVAL '-%1' DAY) ")
940  .arg(gCoreContext->GetNumSetting("AutoExpireLiveTVMaxAge", 1));
941  orderby = "starttime ASC";
942  break;
944  if ((maxAge = gCoreContext->GetNumSetting("DeletedMaxAge", 0)) <= 0)
945  return;
946  msg = QString("Adding programs deleted more than %1 days ago")
947  .arg(maxAge);
948  where = QString("recgroup = 'Deleted' "
949  "AND lastmodified <= DATE_ADD(NOW(), INTERVAL '-%1' DAY) ")
950  .arg(maxAge);
951  orderby = "starttime ASC";
952  break;
954  if (gCoreContext->GetNumSetting("DeletedMaxAge", 0) != 0)
955  return;
956  msg = QString("Adding programs deleted more than 5 minutes ago");
957  where = QString("recgroup = 'Deleted' "
958  "AND lastmodified <= DATE_ADD(NOW(), INTERVAL '-5' MINUTE) ");
959  orderby = "lastmodified ASC";
960  break;
962  msg = "Adding deleted programs in FIFO order";
963  where = "recgroup = 'Deleted'";
964  orderby = "lastmodified ASC";
965  break;
966  }
967 
968  LOG(VB_FILE, LOG_INFO, LOC + "FillDBOrdered: " + msg);
969 
970  MSqlQuery query(MSqlQuery::InitCon());
971  QString querystr = QString(
972  "SELECT recorded.chanid, starttime "
973  "FROM recorded "
974  "LEFT JOIN channel ON recorded.chanid = channel.chanid "
975  "WHERE %1 AND deletepending = 0 "
976  "ORDER BY autoexpire DESC, %2").arg(where, orderby);
977 
978  query.prepare(querystr);
979 
980  if (!query.exec())
981  return;
982 
983  while (query.next())
984  {
985  uint chanid = query.value(0).toUInt();
986  QDateTime recstartts = MythDate::as_utc(query.value(1).toDateTime());
987 
988  if (IsInDontExpireSet(chanid, recstartts))
989  {
990  LOG(VB_FILE, LOG_INFO, LOC +
991  QString(" Skipping %1 at %2 because it is in Don't Expire "
992  "List")
993  .arg(chanid).arg(recstartts.toString(Qt::ISODate)));
994  }
995  else if (IsInExpireList(expireList, chanid, recstartts))
996  {
997  LOG(VB_FILE, LOG_INFO, LOC +
998  QString(" Skipping %1 at %2 because it is already in Expire "
999  "List")
1000  .arg(chanid).arg(recstartts.toString(Qt::ISODate)));
1001  }
1002  else
1003  {
1004  auto *pginfo = new ProgramInfo(chanid, recstartts);
1005  if (pginfo->GetChanID())
1006  {
1007  LOG(VB_FILE, LOG_INFO, LOC + QString(" Adding %1 at %2")
1008  .arg(chanid).arg(recstartts.toString(Qt::ISODate)));
1009  expireList.push_back(pginfo);
1010  }
1011  else
1012  {
1013  LOG(VB_FILE, LOG_INFO, LOC +
1014  QString(" Skipping %1 at %2 "
1015  "because it could not be loaded from the DB")
1016  .arg(chanid).arg(recstartts.toString(Qt::ISODate)));
1017  delete pginfo;
1018  }
1019  }
1020  }
1021 }
1022 
1034 void AutoExpire::Update(int encoder, int fsID, bool immediately)
1035 {
1036  if (!gExpirer)
1037  return;
1038 
1039  if (encoder > 0)
1040  {
1041  QString msg = QString("Cardid %1: is starting a recording on")
1042  .arg(encoder);
1043  if (fsID == -1)
1044  msg.append(" an unknown fsID soon.");
1045  else
1046  msg.append(QString(" fsID %2 soon.").arg(fsID));
1047  LOG(VB_FILE, LOG_INFO, LOC + msg);
1048  }
1049 
1050  if (immediately)
1051  {
1052  if (encoder > 0)
1053  {
1054  gExpirer->m_instanceLock.lock();
1055  gExpirer->m_usedEncoders[encoder] = fsID;
1056  gExpirer->m_instanceLock.unlock();
1057  }
1058  gExpirer->CalcParams();
1059  gExpirer->m_instanceCond.wakeAll();
1060  }
1061  else
1062  {
1063  gExpirer->m_updateLock.lock();
1064  gExpirer->m_updateQueue.append(UpdateEntry(encoder, fsID));
1065  gExpirer->m_updateLock.unlock();
1066  }
1067 }
1068 
1070 {
1071  m_dontExpireSet.clear();
1072 
1073  MSqlQuery query(MSqlQuery::InitCon());
1074  query.prepare(
1075  "SELECT chanid, starttime, lastupdatetime, recusage, hostname "
1076  "FROM inuseprograms");
1077 
1078  if (!query.exec() || !query.next())
1079  return;
1080 
1081  LOG(VB_FILE, LOG_INFO, LOC + "Adding Programs to 'Do Not Expire' List");
1082  QDateTime curTime = MythDate::current();
1083 
1084  do
1085  {
1086  uint chanid = query.value(0).toUInt();
1087  QDateTime recstartts = MythDate::as_utc(query.value(1).toDateTime());
1088  QDateTime lastupdate = MythDate::as_utc(query.value(2).toDateTime());
1089 
1090  if (lastupdate.secsTo(curTime) < kRecentInterval)
1091  {
1092  QString key = QString("%1_%2")
1093  .arg(chanid).arg(recstartts.toString(Qt::ISODate));
1094  m_dontExpireSet.insert(key);
1095  LOG(VB_FILE, LOG_INFO, QString(" %1 at %2 in use by %3 on %4")
1096  .arg(QString::number(chanid),
1097  recstartts.toString(Qt::ISODate),
1098  query.value(3).toString(),
1099  query.value(4).toString()));
1100  }
1101  }
1102  while (query.next());
1103 }
1104 
1106  uint chanid, const QDateTime &recstartts) const
1107 {
1108  QString key = QString("%1_%2")
1109  .arg(chanid).arg(recstartts.toString(Qt::ISODate));
1110 
1111  return (m_dontExpireSet.find(key) != m_dontExpireSet.end());
1112 }
1113 
1115  const pginfolist_t &expireList, uint chanid, const QDateTime &recstartts)
1116 {
1117  return std::any_of(expireList.cbegin(), expireList.cend(),
1118  [chanid,&recstartts](auto *info)
1119  { return ((info->GetChanID() == chanid) &&
1120  (info->GetRecordingStartTime() == recstartts)); } );
1121 }
1122 
1123 /* vim: set expandtab tabstop=4 shiftwidth=4: */
filesysteminfo.h
AutoExpire::m_instanceCond
QWaitCondition m_instanceCond
Definition: autoexpire.h:120
MSqlQuery::isActive
bool isActive(void) const
Definition: mythdbcon.h:216
MSqlQuery::next
bool next(void)
Wrap QSqlQuery::next() so we can display the query results.
Definition: mythdbcon.cpp:807
MSqlQuery
QSqlQuery wrapper that fetches a DB connection from the connection pool.
Definition: mythdbcon.h:128
MSqlQuery::size
int size(void) const
Definition: mythdbcon.h:215
MThread::start
void start(QThread::Priority p=QThread::InheritPriority)
Tell MThread to start running the thread in the near future.
Definition: mthread.cpp:283
AutoExpire::PrintExpireList
void PrintExpireList(const QString &expHost="ALL")
Prints a summary of the files that can be deleted.
Definition: autoexpire.cpp:792
backendcontext.h
mythdb.h
MainServer::GetFilesystemInfos
void GetFilesystemInfos(QList< FileSystemInfo > &fsInfos, bool useCache=true)
Definition: mainserver.cpp:5376
MythDate::as_utc
QDateTime as_utc(const QDateTime &old_dt)
Returns copy of QDateTime with TimeSpec set to UTC.
Definition: mythdate.cpp:27
AutoExpire::IsInExpireList
static bool IsInExpireList(const pginfolist_t &expireList, uint chanid, const QDateTime &recstartts)
Definition: autoexpire.cpp:1114
AutoExpire::Update
static void Update(int encoder, int fsID, bool immediately)
This is used to update the global AutoExpire instance "expirer".
Definition: autoexpire.cpp:1034
MThread::wait
bool wait(std::chrono::milliseconds time=std::chrono::milliseconds::max())
Wait for the MThread to exit, with a maximum timeout.
Definition: mthread.cpp:300
ExpireThread::m_parent
QPointer< AutoExpire > m_parent
Definition: autoexpire.h:47
RecordingInfo
Holds information on a TV Program one might wish to record.
Definition: recordinginfo.h:35
gExpirer
AutoExpire * gExpirer
Definition: backendcontext.cpp:8
AutoExpire::ClearExpireList
static void ClearExpireList(pginfolist_t &expireList, bool deleteProg=true)
Clears expireList, freeing any ProgramInfo's if necessary.
Definition: autoexpire.cpp:877
RecordingInfo::ForgetHistory
void ForgetHistory(void)
Forget the recording of a program so it will be recorded again.
Definition: recordinginfo.cpp:1461
MythEvent
This class is used as a container for messages.
Definition: mythevent.h:16
ProgramInfo::kTitleSubtitle
@ kTitleSubtitle
Definition: programinfo.h:509
VERBOSE_LEVEL_CHECK
static bool VERBOSE_LEVEL_CHECK(uint64_t mask, LogLevel_t level)
Definition: mythlogging.h:29
MSqlQuery::value
QVariant value(int i) const
Definition: mythdbcon.h:205
MSqlQuery::exec
bool exec(void)
Wrap QSqlQuery::exec() so we can display SQL.
Definition: mythdbcon.cpp:608
AutoExpire::ExpireOldDeleted
void ExpireOldDeleted(void)
This expires deleted programs older than DeletedMaxAge.
Definition: autoexpire.cpp:375
emLowestPriorityFirst
@ emLowestPriorityFirst
Definition: autoexpire.h:29
LOG
#define LOG(_MASK_, _LEVEL_, _QSTRING_)
Definition: mythlogging.h:39
MThread::RunProlog
void RunProlog(void)
Sets up a thread, call this if you reimplement run().
Definition: mthread.cpp:196
AutoExpire::ExpireEpisodesOverMax
void ExpireEpisodesOverMax(void)
This deletes programs exceeding the maximum number of episodes of that program desired....
Definition: autoexpire.cpp:651
AutoExpire::GetAllExpiring
void GetAllExpiring(QStringList &strList)
Gets the full list of programs that can expire in expiration order.
Definition: autoexpire.cpp:831
ExpireThread::run
void run(void) override
This calls AutoExpire::RunExpirer() from within a new thread.
Definition: autoexpire.cpp:58
AutoExpire::m_desiredSpace
QMap< int, int64_t > m_desiredSpace
Definition: autoexpire.h:116
build_compdb.file
file
Definition: build_compdb.py:55
AutoExpire::m_updateLock
QMutex m_updateLock
Definition: autoexpire.h:125
remoteutil.h
AutoExpire::m_desiredFreq
uint m_desiredFreq
Definition: autoexpire.h:113
MythDate::current
QDateTime current(bool stripped)
Returns current Date and Time in UTC.
Definition: mythdate.cpp:14
true
VERBOSE_PREAMBLE Most true
Definition: verbosedefs.h:95
emOldDeletedPrograms
@ emOldDeletedPrograms
Definition: autoexpire.h:33
AutoExpire::m_instanceLock
QMutex m_instanceLock
Definition: autoexpire.h:119
ProgramInfo::IsWatched
bool IsWatched(void) const
Definition: programinfo.h:482
kSpaceTooBigKB
static constexpr uint64_t kSpaceTooBigKB
If calculated desired space for 10 min freq is > kSpaceTooBigKB then we use 5 min expire frequency.
Definition: autoexpire.cpp:51
MythObservable::addListener
void addListener(QObject *listener)
Add a listener to the observable.
Definition: mythobservable.cpp:38
kRecentInterval
static constexpr int64_t kRecentInterval
Definition: autoexpire.cpp:55
MythDate::secsInFuture
std::chrono::seconds secsInFuture(const QDateTime &future)
Definition: mythdate.cpp:208
fileserverutil.h
mythdate.h
autoexpire.h
programinfo.h
mythlogging.h
AutoExpire::m_expireThreadRun
bool m_expireThreadRun
Definition: autoexpire.h:114
UpdateEntry::m_encoder
int m_encoder
Definition: autoexpire.h:54
AutoExpire::Sleep
void Sleep(std::chrono::milliseconds sleepTime)
Sleeps for sleepTime milliseconds; unless the expire thread is told to quit.
Definition: autoexpire.cpp:345
hardwareprofile.config.p
p
Definition: config.py:33
AutoExpire::m_usedEncoders
QMap< int, int > m_usedEncoders
Definition: autoexpire.h:117
MSqlQuery::InitCon
static MSqlQueryInfo InitCon(ConnectionReuse _reuse=kNormalConnection)
Only use this in combination with MSqlQuery constructor.
Definition: mythdbcon.cpp:540
compat.h
AutoExpire::ExpireRecordings
void ExpireRecordings(void)
This expires normal recordings.
Definition: autoexpire.cpp:402
MythDB::DBError
static void DBError(const QString &where, const MSqlQuery &query)
Definition: mythdb.cpp:227
AutoExpire::IsInDontExpireSet
bool IsInDontExpireSet(uint chanid, const QDateTime &recstartts) const
Definition: autoexpire.cpp:1105
GetPlaybackURL
QString GetPlaybackURL(ProgramInfo *pginfo, bool storePath)
Definition: fileserverutil.cpp:46
pginfolist_t
std::vector< ProgramInfo * > pginfolist_t
Definition: autoexpire.h:24
emNormalLiveTVPrograms
@ emNormalLiveTVPrograms
Definition: autoexpire.h:32
MThread::RunEpilog
void RunEpilog(void)
Cleans up a thread's resources, call this if you reimplement run().
Definition: mthread.cpp:209
ExpireThread
Definition: autoexpire.h:40
storagegroup.h
TVRec::s_inputsLock
static QReadWriteLock s_inputsLock
Definition: tv_rec.h:432
LOC
#define LOC
Definition: autoexpire.cpp:45
AutoExpire::SendDeleteMessages
static void SendDeleteMessages(pginfolist_t &deleteList)
This sends delete message to main event thread.
Definition: autoexpire.cpp:614
uint
unsigned int uint
Definition: compat.h:79
AutoExpire::AutoExpire
AutoExpire()=default
gCoreContext
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
Definition: mythcorecontext.cpp:54
AutoExpire::FillExpireList
void FillExpireList(pginfolist_t &expireList)
Uses the "AutoExpireMethod" setting in the database to fill the list of files that are deletable.
Definition: autoexpire.cpp:770
AutoExpire::m_dontExpireSet
QSet< QString > m_dontExpireSet
Definition: autoexpire.h:111
emNormalDeletedPrograms
@ emNormalDeletedPrograms
Definition: autoexpire.h:34
MythCoreContext::GetNumSetting
int GetNumSetting(const QString &key, int defaultval=0)
Definition: mythcorecontext.cpp:910
UpdateEntry
Definition: autoexpire.h:50
AutoExpire::RunExpirer
void RunExpirer(void)
This contains the main loop for the auto expire process.
Definition: autoexpire.cpp:269
MythCoreContext::GetBoolSetting
bool GetBoolSetting(const QString &key, bool defaultval=false)
Definition: mythcorecontext.cpp:904
AutoExpire::CalcParams
void CalcParams(void)
Definition: autoexpire.cpp:124
ProgramInfo
Holds information on recordings and videos.
Definition: programinfo.h:67
AutoExpire::FillDBOrdered
void FillDBOrdered(pginfolist_t &expireList, int expMethod)
Creates a list of programs to delete using the database to order list.
Definition: autoexpire.cpp:896
AutoExpire::m_expireThread
ExpireThread * m_expireThread
Definition: autoexpire.h:112
mythcorecontext.h
MSqlQuery::bindValue
void bindValue(const QString &placeholder, const QVariant &val)
Add a single binding.
Definition: mythdbcon.cpp:883
MythDate::ISODate
@ ISODate
Default UTC.
Definition: mythdate.h:17
emOldestFirst
@ emOldestFirst
Definition: autoexpire.h:28
AutoExpire::~AutoExpire
~AutoExpire() override
AutoExpire destructor stops auto delete thread if it is running.
Definition: autoexpire.cpp:86
emShortLiveTVPrograms
@ emShortLiveTVPrograms
Definition: autoexpire.h:31
tv_rec.h
AutoExpire::m_mainServer
MainServer * m_mainServer
Definition: autoexpire.h:122
emQuickDeletedPrograms
@ emQuickDeletedPrograms
Definition: autoexpire.h:35
remoteencoder.h
mainserver.h
AutoExpire::ExpireQuickDeleted
void ExpireQuickDeleted(void)
This expires deleted programs within a few minutes.
Definition: autoexpire.cpp:388
MythCoreContext::GetHostName
QString GetHostName(void)
Definition: mythcorecontext.cpp:836
AutoExpire::ExpireLiveTV
void ExpireLiveTV(int type)
This expires LiveTV programs.
Definition: autoexpire.cpp:362
AutoExpire::GetDesiredSpace
uint64_t GetDesiredSpace(int fsID) const
Used by the scheduler to select the next recording dir.
Definition: autoexpire.cpp:113
AutoExpire::m_encoderList
QMap< int, EncoderLink * > * m_encoderList
Definition: autoexpire.h:88
AutoExpire::m_updateQueue
QQueue< UpdateEntry > m_updateQueue
Definition: autoexpire.h:126
MythCoreContext::dispatch
void dispatch(const MythEvent &event)
Definition: mythcorecontext.cpp:1723
MythObservable::removeListener
void removeListener(QObject *listener)
Remove a listener to the observable.
Definition: mythobservable.cpp:55
AutoExpire::UpdateDontExpireSet
void UpdateDontExpireSet(void)
Definition: autoexpire.cpp:1069
emWeightedTimePriority
@ emWeightedTimePriority
Definition: autoexpire.h:30
ProgramInfo::kRecordingKey
@ kRecordingKey
Definition: programinfo.h:510
MSqlQuery::prepare
bool prepare(const QString &query)
QSqlQuery::prepare() is not thread safe in Qt <= 3.3.2.
Definition: mythdbcon.cpp:832
UpdateEntry::m_fsID
int m_fsID
Definition: autoexpire.h:57