6 # include <sys/param.h>
8 # include <sys/mount.h>
45 #define LOC QString("AutoExpire: ")
46 #define LOC_ERR QString("AutoExpire Error: ")
51 #define SPACE_TOO_BIG_KB (3*1024*1024)
71 m_encoderList(tvList),
73 m_expireThreadRun(
true)
122 LOG(VB_FILE, LOG_INFO,
LOC +
"CalcParams()");
124 QList<FileSystemInfo> fsInfos;
142 LOG(VB_GENERAL, LOG_ERR,
LOC +
"Filesystem Info cache is empty, unable "
143 "to calculate necessary parameters.");
147 uint64_t maxKBperMin = 0;
148 uint64_t extraKB =
static_cast<uint64_t
>
152 QMap<int, uint64_t> fsMap;
153 QMap<int, std::vector<int> > fsEncoderMap;
162 fsEncoderMap[*ueit].push_back(ueit.key());
167 QList<FileSystemInfo>::iterator fsit;
168 for (fsit = fsInfos.begin(); fsit != fsInfos.end(); ++fsit)
170 if (fsMap.contains(fsit->getFSysID()))
173 fsMap[fsit->getFSysID()] = 0;
174 uint64_t thisKBperMin = 0;
177 for (
auto unknownfs : qAsConst(fsEncoderMap[-1]))
178 fsEncoderMap[fsit->getFSysID()].push_back(unknownfs);
180 if (fsEncoderMap.contains(fsit->getFSysID()))
182 LOG(VB_FILE, LOG_INFO,
183 QString(
"fsID #%1: Total: %2 GB Used: %3 GB Free: %4 GB")
184 .arg(fsit->getFSysID())
185 .arg(fsit->getTotalSpace() / 1024.0 / 1024.0, 7,
'f', 1)
186 .arg(fsit->getUsedSpace() / 1024.0 / 1024.0, 7,
'f', 1)
187 .arg(fsit->getFreeSpace() / 1024.0 / 1024.0, 7,
'f', 1));
189 for (
auto cardid : qAsConst(fsEncoderMap[fsit->getFSysID()]))
196 LOG(VB_FILE, LOG_INFO,
LOC +
197 QString(
"Cardid %1: is not recording, removing it "
198 "from used list.").arg(cardid));
207 maxBitrate = 19500000LL;
208 thisKBperMin += (maxBitrate*((uint64_t)15))>>11;
209 LOG(VB_FILE, LOG_INFO, QString(
" Cardid %1: max bitrate "
210 "%2 Kb/sec, fsID %3 max is now %4 KB/min")
213 .arg(fsit->getFSysID())
217 fsMap[fsit->getFSysID()] = thisKBperMin;
219 if (thisKBperMin > maxKBperMin)
221 LOG(VB_FILE, LOG_INFO,
222 QString(
" Max of %1 KB/min for fsID %2 is higher "
223 "than the existing Max of %3 so we'll use this Max instead")
224 .arg(thisKBperMin).arg(fsit->getFSysID()).arg(maxKBperMin));
225 maxKBperMin = thisKBperMin;
231 uint expireFreq = 15;
235 expireFreq = std::max(3U, std::min(expireFreq, 15U));
238 double expireMinGB = ((maxKBperMin + maxKBperMin/3)
239 * expireFreq + extraKB) >> 20;
240 LOG(VB_GENERAL, LOG_NOTICE,
LOC +
241 QString(
"CalcParams(): Max required Free Space: %1 GB w/freq: %2 min")
242 .arg(expireMinGB, 0,
'f', 1).arg(expireFreq));
248 QMap<int, uint64_t>::iterator it = fsMap.begin();
249 while (it != fsMap.end())
284 if (curTime >= next_expire)
306 if ((curTime.time().minute() % 2) == 0)
310 if (curTime >= next_expire)
312 LOG(VB_FILE, LOG_INFO,
LOC +
"Running now!");
321 else if (maxAge == 0)
331 Sleep(60s - std::chrono::milliseconds(timer.elapsed()));
343 if (sleepTime <= 0ms)
347 std::chrono::milliseconds timeleft = sleepTime;
362 LOG(VB_FILE, LOG_INFO,
LOC + QString(
"ExpireLiveTV(%1)").arg(
type));
375 LOG(VB_FILE, LOG_INFO,
LOC + QString(
"ExpireOldDeleted()"));
388 LOG(VB_FILE, LOG_INFO,
LOC + QString(
"ExpireQuickDeleted()"));
402 QList<FileSystemInfo> fsInfos;
403 QList<FileSystemInfo>::iterator fsit;
405 LOG(VB_FILE, LOG_INFO,
LOC +
"ExpireRecordings()");
412 LOG(VB_GENERAL, LOG_ERR,
LOC +
"Filesystem Info cache is empty, unable "
413 "to determine what Recordings to expire");
420 QMap <int, bool> truncateMap;
422 query.
prepare(
"SELECT DISTINCT rechost, recdir "
423 "FROM inuseprograms "
424 "WHERE recusage = 'truncatingdelete' "
425 "AND lastupdatetime > DATE_ADD(NOW(), INTERVAL -2 MINUTE);");
435 QString rechost = query.
value(0).toString();
436 QString recdir = query.
value(1).toString();
438 LOG(VB_FILE, LOG_INFO,
LOC +
439 QString(
"%1:%2 has an in-progress truncating delete.")
440 .arg(rechost, recdir));
442 for (fsit = fsInfos.begin(); fsit != fsInfos.end(); ++fsit)
444 if ((fsit->getHostname() == rechost) &&
445 (fsit->getPath() == recdir))
447 truncateMap[fsit->getFSysID()] =
true;
454 QMap <int, bool> fsMap;
455 for (fsit = fsInfos.begin(); fsit != fsInfos.end(); ++fsit)
457 if (fsMap.contains(fsit->getFSysID()))
460 fsMap[fsit->getFSysID()] =
true;
462 LOG(VB_FILE, LOG_INFO,
463 QString(
"fsID #%1: Total: %2 GB Used: %3 GB Free: %4 GB")
464 .arg(fsit->getFSysID())
465 .arg(fsit->getTotalSpace() / 1024.0 / 1024.0, 7,
'f', 1)
466 .arg(fsit->getUsedSpace() / 1024.0 / 1024.0, 7,
'f', 1)
467 .arg(fsit->getFreeSpace() / 1024.0 / 1024.0, 7,
'f', 1));
469 if ((fsit->getTotalSpace() == -1) || (fsit->getUsedSpace() == -1))
471 LOG(VB_FILE, LOG_ERR,
LOC +
472 QString(
"fsID #%1 has invalid info, AutoExpire cannot run for "
473 "this filesystem. Continuing on to next...")
474 .arg(fsit->getFSysID()));
475 LOG(VB_FILE, LOG_INFO, QString(
"Directories on filesystem ID %1:")
476 .arg(fsit->getFSysID()));
477 QList<FileSystemInfo>::iterator fsit2;
478 for (fsit2 = fsInfos.begin(); fsit2 != fsInfos.end(); ++fsit2)
480 if (fsit2->getFSysID() == fsit->getFSysID())
482 LOG(VB_FILE, LOG_INFO, QString(
" %1:%2")
483 .arg(fsit2->getHostname(), fsit2->getPath()));
490 if (truncateMap.contains(fsit->getFSysID()))
492 LOG(VB_FILE, LOG_INFO,
493 QString(
" fsid %1 has a truncating delete in progress, "
494 "AutoExpire cannot run for this filesystem until the "
495 "delete has finished. Continuing on to next...")
496 .arg(fsit->getFSysID()));
500 if (std::max((int64_t)0LL, fsit->getFreeSpace()) <
503 LOG(VB_FILE, LOG_INFO,
504 QString(
" Not Enough Free Space! We want %1 MB")
507 QMap<QString, int> dirList;
508 QList<FileSystemInfo>::iterator fsit2;
510 LOG(VB_FILE, LOG_INFO,
511 QString(
" Directories on filesystem ID %1:")
512 .arg(fsit->getFSysID()));
514 for (fsit2 = fsInfos.begin(); fsit2 != fsInfos.end(); ++fsit2)
516 if (fsit2->getFSysID() == fsit->getFSysID())
518 LOG(VB_FILE, LOG_INFO, QString(
" %1:%2")
519 .arg(fsit2->getHostname(), fsit2->getPath()));
520 dirList[fsit2->getHostname() +
":" + fsit2->getPath()] = 1;
524 LOG(VB_FILE, LOG_INFO,
525 " Searching for files expirable in these directories");
527 auto it = expireList.begin();
528 while ((it != expireList.end()) &&
529 (std::max((int64_t)0LL, fsit->getFreeSpace()) <
535 LOG(VB_FILE, LOG_INFO, QString(
" Checking %1 => %2")
541 bool foundFile =
false;
549 ((
p->GetHostname() == myHostName) &&
559 if (!foundFile && (
p->GetHostname() != myHostName))
564 if (
file.startsWith(
"/"))
566 p->SetPathname(
file);
567 p->SetHostname(myHostName);
574 LOG(VB_FILE, LOG_ERR,
LOC +
575 QString(
" ERROR: Can't find file for %1")
581 QFileInfo vidFile(
p->GetPathname());
582 if (dirList.contains(
p->GetHostname() +
':' + vidFile.path()))
584 fsit->setUsedSpace(fsit->getUsedSpace()
585 - (
p->GetFilesize() / 1024));
586 deleteList.push_back(
p);
588 LOG(VB_FILE, LOG_INFO,
589 QString(
" FOUND file expirable. "
590 "%1 is located at %2 which is on fsID #%3. "
591 "Adding to deleteList. After deleting we "
592 "should have %4 MB free on this filesystem.")
594 p->GetPathname()).arg(fsit->getFSysID())
595 .arg(fsit->getFreeSpace() / 1024));
614 if (deleteList.empty())
616 LOG(VB_FILE, LOG_INFO,
LOC +
"SendDeleteMessages. Nothing to expire.");
620 LOG(VB_FILE, LOG_INFO,
LOC +
621 "SendDeleteMessages, cycling through deleteList.");
622 auto it = deleteList.begin();
623 while (it != deleteList.end())
625 msg = QString(
"%1Expiring %2 MB for %3 => %4")
627 QString::number((*it)->GetFilesize() >> 20),
631 LOG(VB_GENERAL, LOG_NOTICE, msg);
634 MythEvent me(QString(
"AUTO_EXPIRE %1 %2").arg((*it)->GetChanID())
649 QMap<QString, int> maxEpisodes;
650 QMap<QString, int>::Iterator maxIter;
651 QMap<QString, int> episodeParts;
655 query.
prepare(
"SELECT recordid, maxepisodes, title "
656 "FROM record WHERE maxepisodes > 0 "
657 "ORDER BY recordid ASC, maxepisodes DESC");
661 LOG(VB_FILE, LOG_INFO,
LOC +
662 QString(
"Found %1 record profiles using max episode expiration")
666 LOG(VB_FILE, LOG_INFO, QString(
" %1 (%2 for rec id %3)")
667 .arg(query.
value(2).toString())
668 .arg(query.
value(1).toInt())
669 .arg(query.
value(0).toInt()));
670 maxEpisodes[query.
value(0).toString()] = query.
value(1).toInt();
674 LOG(VB_FILE, LOG_INFO,
LOC +
675 "Checking episode count for each recording profile using max episodes");
676 for (maxIter = maxEpisodes.begin(); maxIter != maxEpisodes.end(); ++maxIter)
678 query.
prepare(
"SELECT chanid, starttime, title, progstart, progend, "
681 "WHERE recordid = :RECID AND preserve = 0 "
682 "AND recgroup NOT IN ('LiveTV', 'Deleted') "
683 "ORDER BY starttime DESC;");
684 query.
bindValue(
":RECID", maxIter.key());
692 LOG(VB_FILE, LOG_INFO, QString(
" Recordid %1 has %2 recordings.")
695 if (query.
size() > 0)
702 QString title = query.
value(2).toString();
705 int duplicate = query.
value(5).toInt();
707 episodeKey = QString(
"%1_%2_%3")
708 .arg(QString::number(chanid),
713 (!episodeParts.contains(episodeKey)) &&
717 QString(
"%1Deleting %2 at %3 => %4. "
718 "Too many episodes, we only want to keep %5.")
720 QString::number(chanid),
723 QString::number(*maxIter));
725 LOG(VB_GENERAL, LOG_NOTICE, msg);
734 msg = QString(
"DELETE_RECORDING %1 %2")
746 if (episodeParts.contains(episodeKey))
748 episodeParts[episodeKey] = episodeParts[episodeKey] + 1;
752 episodeParts[episodeKey] = 1;
794 QString msg =
"MythTV AutoExpire List ";
795 if (expHost !=
"ALL")
796 msg += QString(
"for '%1' ").arg(expHost);
797 msg +=
"(programs listed in order of expiration)";
798 std::cout << msg.toLocal8Bit().constData() << std::endl;
800 for (
auto *first : expireList)
802 if (expHost !=
"ALL" && first->GetHostname() != expHost)
806 title = title.leftJustified(39,
' ',
true);
808 QString outstr = QString(
"%1 %2 MB %3 [%4]")
810 QString::number(first->GetFilesize() >> 20)
811 .rightJustified(5,
' ',
true),
813 .leftJustified(24,
' ',
true),
814 QString::number(first->GetRecordingPriority())
815 .rightJustified(3,
' ',
true));
816 QByteArray out = outstr.toLocal8Bit();
818 std::cout << out.constData() << std::endl;
840 strList << QString::number(expireList.size());
842 for (
auto & info : expireList)
843 info->ToStringList(strList);
864 for (
auto & info : expireList)
876 while (!expireList.empty())
879 pginfo = expireList.back();
881 expireList.pop_back();
903 msg =
"Adding programs expirable in Oldest First order";
904 where =
"autoexpire > 0";
906 orderby =
"recorded.watched DESC, ";
907 orderby +=
"starttime ASC";
910 msg =
"Adding programs expirable in Lowest Priority First order";
911 where =
"autoexpire > 0";
913 orderby =
"recorded.watched DESC, ";
914 orderby +=
"recorded.recpriority ASC, starttime ASC";
917 msg =
"Adding programs expirable in Weighted Time Priority order";
918 where =
"autoexpire > 0";
920 orderby =
"recorded.watched DESC, ";
921 orderby += QString(
"DATE_ADD(starttime, INTERVAL '%1' * "
922 "recorded.recpriority DAY) ASC")
926 msg =
"Adding Short LiveTV programs in starttime order";
927 where =
"recgroup = 'LiveTV' "
928 "AND endtime < DATE_ADD(starttime, INTERVAL '30' SECOND) "
929 "AND endtime <= DATE_ADD(NOW(), INTERVAL '-5' MINUTE) ";
930 orderby =
"starttime ASC";
933 msg =
"Adding LiveTV programs in starttime order";
934 where = QString(
"recgroup = 'LiveTV' "
935 "AND endtime <= DATE_ADD(NOW(), INTERVAL '-%1' DAY) ")
937 orderby =
"starttime ASC";
942 msg = QString(
"Adding programs deleted more than %1 days ago")
944 where = QString(
"recgroup = 'Deleted' "
945 "AND lastmodified <= DATE_ADD(NOW(), INTERVAL '-%1' DAY) ")
947 orderby =
"starttime ASC";
952 msg = QString(
"Adding programs deleted more than 5 minutes ago");
953 where = QString(
"recgroup = 'Deleted' "
954 "AND lastmodified <= DATE_ADD(NOW(), INTERVAL '-5' MINUTE) ");
955 orderby =
"lastmodified ASC";
958 msg =
"Adding deleted programs in FIFO order";
959 where =
"recgroup = 'Deleted'";
960 orderby =
"lastmodified ASC";
964 LOG(VB_FILE, LOG_INFO,
LOC +
"FillDBOrdered: " + msg);
967 QString querystr = QString(
968 "SELECT recorded.chanid, starttime "
970 "LEFT JOIN channel ON recorded.chanid = channel.chanid "
971 "WHERE %1 AND deletepending = 0 "
972 "ORDER BY autoexpire DESC, %2").arg(where, orderby);
986 LOG(VB_FILE, LOG_INFO,
LOC +
987 QString(
" Skipping %1 at %2 because it is in Don't Expire "
989 .arg(chanid).arg(recstartts.toString(
Qt::ISODate)));
993 LOG(VB_FILE, LOG_INFO,
LOC +
994 QString(
" Skipping %1 at %2 because it is already in Expire "
996 .arg(chanid).arg(recstartts.toString(
Qt::ISODate)));
1000 auto *pginfo =
new ProgramInfo(chanid, recstartts);
1001 if (pginfo->GetChanID())
1003 LOG(VB_FILE, LOG_INFO,
LOC + QString(
" Adding %1 at %2")
1004 .arg(chanid).arg(recstartts.toString(
Qt::ISODate)));
1005 expireList.push_back(pginfo);
1009 LOG(VB_FILE, LOG_INFO,
LOC +
1010 QString(
" Skipping %1 at %2 "
1011 "because it could not be loaded from the DB")
1012 .arg(chanid).arg(recstartts.toString(
Qt::ISODate)));
1037 QString msg = QString(
"Cardid %1: is starting a recording on")
1040 msg.append(
" an unknown fsID soon.");
1042 msg.append(QString(
" fsID %2 soon.").arg(fsID));
1043 LOG(VB_FILE, LOG_INFO,
LOC + msg);
1071 "SELECT chanid, starttime, lastupdatetime, recusage, hostname "
1072 "FROM inuseprograms");
1074 if (!query.
exec() || !query.
next())
1077 LOG(VB_FILE, LOG_INFO,
LOC +
"Adding Programs to 'Do Not Expire' List");
1086 if (lastupdate.secsTo(curTime) < 2 * 60 * 60)
1088 QString key = QString(
"%1_%2")
1089 .arg(chanid).arg(recstartts.toString(
Qt::ISODate));
1091 LOG(VB_FILE, LOG_INFO, QString(
" %1 at %2 in use by %3 on %4")
1092 .arg(QString::number(chanid),
1094 query.
value(3).toString(),
1095 query.
value(4).toString()));
1098 while (query.
next());
1102 uint chanid,
const QDateTime &recstartts)
const
1104 QString key = QString(
"%1_%2")
1105 .arg(chanid).arg(recstartts.toString(
Qt::ISODate));
1113 return std::any_of(expireList.cbegin(), expireList.cend(),
1114 [chanid,&recstartts](
auto *info)
1115 { return ((info->GetChanID() == chanid) &&
1116 (info->GetRecordingStartTime() == recstartts)); } );