13 #include <QTextStream> 36 #define LOC QString("FillData: ") 37 #define LOC_WARN QString("FillData, Warning: ") 38 #define LOC_ERR QString("FillData, Error: ") 74 if (kRefreshClear == day)
77 m_refresh_day.clear();
79 else if (kRefreshAll == day)
85 m_refresh_day[(
uint)day] = set;
93 QMap<QString, QList<ProgInfo> > proglist;
95 m_xmltv_parser.lateInit();
96 if (!m_xmltv_parser.parseFile(
filename, &chanlist, &proglist))
99 m_chan_data.handleChannels(
id, &chanlist);
100 if (proglist.count() == 0)
102 LOG(VB_GENERAL, LOG_INFO,
"No programs found in data.");
116 const QString templatename =
"/tmp/mythXXXXXX";
118 if (templatename == tempfilename)
120 m_fatalErrors.push_back(
"Failed to create temporary file.");
124 QString
filename = QString(tempfilename);
129 query1.
prepare(
"SELECT configpath FROM videosource" 130 " WHERE sourceid = :ID AND configpath IS NOT NULL");
139 configfile = query1.
value(0).toString();
141 configfile = QString(
"%1/%2.xmltv").arg(
GetConfDir())
144 LOG(VB_GENERAL, LOG_INFO,
145 QString(
"XMLTV config file is: %1").arg(configfile));
147 QString command = QString(
"nice %1 --config-file '%2' --output %3")
148 .arg(xmltv_grabber).arg(configfile).arg(
filename);
157 command += QString(
" --days 1 --offset %1").arg(offset);
161 command +=
" --quiet";
165 if (!m_graboptions.isEmpty())
167 command += m_graboptions;
168 LOG(VB_XMLTV, LOG_INFO,
169 QString(
"Using graboptions: %1").arg(m_graboptions));
172 QString status = QObject::tr(
"currently running.");
177 LOG(VB_XMLTV, LOG_INFO, QString(
"Grabber Command: %1").arg(command));
179 LOG(VB_XMLTV, LOG_INFO,
180 "----------------- Start of XMLTV output -----------------");
185 LOG(VB_XMLTV, LOG_INFO,
186 "------------------ End of XMLTV output ------------------");
190 status = QObject::tr(
"Successful.");
196 m_interrupted =
true;
197 status = QObject::tr(
"FAILED: XMLTV grabber ran but was interrupted.");
201 status = QObject::tr(
"FAILED: XMLTV grabber returned error code %1.")
202 .arg(systemcall_status);
203 LOG(VB_GENERAL, LOG_ERR,
LOC +
204 QString(
"XMLTV grabber returned error code %1")
205 .arg(systemcall_status));
211 succeeded &= GrabDataFromFile(source.
id,
filename);
226 SourceList::iterator it;
228 QString status, querystr;
230 QDateTime GuideDataBefore, GuideDataAfter;
232 int externally_handled = 0;
233 int total_sources = sourcelist.size();
234 int source_channels = 0;
236 QString sidStr = QString(
"Updating source #%1 (%2) with grabber %3");
238 m_need_post_grab_proc =
false;
241 for (it = sourcelist.begin(); it != sourcelist.end(); ++it)
243 if (!m_fatalErrors.empty())
246 QString xmltv_grabber = (*it).xmltvgrabber;
248 if (xmltv_grabber ==
"datadirect" ||
249 xmltv_grabber ==
"schedulesdirect1")
251 LOG(VB_GENERAL, LOG_ERR,
252 QString(
"Source %1 is configured to use the DataDirect guide" 253 "service from Schedules Direct. That service is no " 254 "longer supported by MythTV. Update to use one of " 255 "the XMLTV grabbers that use the JSON-based guide " 256 "service from Schedules Direct.")
261 query.
prepare(
"SELECT MAX(endtime) " 263 "LEFT JOIN channel c ON p.chanid=c.chanid " 264 "WHERE c.deleted IS NULL AND c.sourceid= :SRCID " 265 " AND manualid = 0 AND c.xmltvid != '';");
275 m_channel_update_run =
false;
278 if (xmltv_grabber ==
"eitonly")
280 LOG(VB_GENERAL, LOG_INFO,
281 QString(
"Source %1 configured to use only the " 282 "broadcasted guide data. Skipping.") .arg((*it).id));
284 externally_handled++;
289 if (xmltv_grabber.trimmed().isEmpty() ||
290 xmltv_grabber ==
"/bin/true" ||
291 xmltv_grabber ==
"none")
293 LOG(VB_GENERAL, LOG_INFO,
294 QString(
"Source %1 configured with no grabber. Nothing to do.")
297 externally_handled++;
303 LOG(VB_GENERAL, LOG_INFO, sidStr.arg((*it).id)
305 .arg(xmltv_grabber));
308 "SELECT COUNT(chanid) FROM channel " 309 "WHERE deleted IS NULL AND " 310 " sourceid = :SRCID AND xmltvid != ''");
315 source_channels = query.
value(0).toInt();
316 if (source_channels > 0)
318 LOG(VB_GENERAL, LOG_INFO,
319 QString(
"Found %1 channels for source %2 which use grabber")
320 .arg(source_channels).arg((*it).id));
324 LOG(VB_GENERAL, LOG_INFO,
325 QString(
"No channels are configured to use grabber."));
331 LOG(VB_GENERAL, LOG_INFO,
332 QString(
"Can't get a channel count for source id %1")
336 bool hasprefmethod =
false;
342 QStringList(
"--capabilities"),
344 grabber_capabilities_proc.
Run(25);
346 LOG(VB_GENERAL, LOG_ERR,
347 QString(
"%1 --capabilities failed or we timed out waiting." 348 " You may need to upgrade your xmltv grabber")
349 .arg(xmltv_grabber));
352 QByteArray result = grabber_capabilities_proc.
ReadAll();
353 QTextStream ostream(result);
354 QString capabilities;
355 while (!ostream.atEnd())
358 = ostream.readLine().simplified();
360 if (capability.isEmpty())
363 capabilities += capability +
' ';
365 if (capability ==
"baseline")
366 (*it).xmltvgrabber_baseline =
true;
368 if (capability ==
"manualconfig")
369 (*it).xmltvgrabber_manualconfig =
true;
371 if (capability ==
"cache")
372 (*it).xmltvgrabber_cache =
true;
374 if (capability ==
"preferredmethod")
375 hasprefmethod =
true;
377 LOG(VB_GENERAL, LOG_INFO,
378 QString(
"Grabber has capabilities: %1") .arg(capabilities));
386 QStringList(
"--preferredmethod"),
388 grabber_method_proc.
Run(15);
390 LOG(VB_GENERAL, LOG_ERR,
391 QString(
"%1 --preferredmethod failed or we timed out " 392 "waiting. You may need to upgrade your xmltv " 393 "grabber").arg(xmltv_grabber));
396 QTextStream ostream(grabber_method_proc.
ReadAll());
397 (*it).xmltvgrabber_prefmethod =
398 ostream.readLine().simplified();
400 LOG(VB_GENERAL, LOG_INFO, QString(
"Grabber prefers method: %1")
401 .arg((*it).xmltvgrabber_prefmethod));
405 m_need_post_grab_proc |=
true;
407 if ((*it).xmltvgrabber_prefmethod ==
"allatonce" && !m_no_allatonce)
409 if (!GrabData(*it, 0))
412 else if ((*it).xmltvgrabber_baseline)
421 grabdays = (m_maxDays > 0) ? m_maxDays : grabdays;
422 grabdays = (m_only_update_channels) ? 1 : grabdays;
424 vector<bool> refresh_request;
425 refresh_request.resize(grabdays, m_refresh_all);
430 for (
int i = 0; i < grabdays; i++)
431 refresh_request[i] = m_refresh_day[i];
433 for (
int i = 0; i < grabdays; i++)
435 if (!m_fatalErrors.empty())
444 i += (newDate.daysTo(qCurrentDate));
447 qCurrentDate = newDate;
450 QString prevDate(qCurrentDate.addDays(i-1).toString());
451 QString currDate(qCurrentDate.addDays(i).toString());
453 LOG(VB_GENERAL, LOG_INFO,
"");
454 LOG(VB_GENERAL, LOG_INFO,
"Checking day @ " +
455 QString(
"offset %1, date: %2").arg(i).arg(currDate));
457 bool download_needed =
false;
459 if (refresh_request[i])
463 LOG(VB_GENERAL, LOG_INFO,
464 "Data Refresh always needed for tomorrow");
468 LOG(VB_GENERAL, LOG_INFO,
469 "Data Refresh needed because of user request");
471 download_needed =
true;
477 querystr =
"SELECT c.chanid, COUNT(p.starttime) " 479 "LEFT JOIN program p ON c.chanid = p.chanid " 481 "DATE_ADD(DATE_ADD(CURRENT_DATE(), " 482 "INTERVAL '%1' DAY), INTERVAL '20' HOUR) " 483 " AND starttime < DATE_ADD(CURRENT_DATE(), " 484 "INTERVAL '%2' DAY) " 485 "WHERE c.deleted IS NULL AND c.sourceid = %3 AND c.xmltvid != '' " 486 "GROUP BY c.chanid;";
488 if (query.
exec(querystr.arg(i-1).arg(i).arg((*it).id)) &&
491 int prevChanCount = 0;
492 int currentChanCount = 0;
493 int previousDayCount = 0;
494 int currentDayCount = 0;
496 LOG(VB_CHANNEL, LOG_INFO,
497 QString(
"Checking program counts for day %1")
502 if (query.
value(1).toInt() > 0)
504 previousDayCount += query.
value(1).toInt();
506 LOG(VB_CHANNEL, LOG_INFO,
507 QString(
" chanid %1 -> %2 programs")
508 .arg(query.
value(0).toString())
509 .arg(query.
value(1).toInt()));
512 if (query.
exec(querystr.arg(i).arg(i+1).arg((*it).id))
515 LOG(VB_CHANNEL, LOG_INFO,
516 QString(
"Checking program counts for day %1")
520 if (query.
value(1).toInt() > 0)
522 currentDayCount += query.
value(1).toInt();
524 LOG(VB_CHANNEL, LOG_INFO,
525 QString(
" chanid %1 -> %2 programs")
526 .arg(query.
value(0).toString())
527 .arg(query.
value(1).toInt()));
532 LOG(VB_GENERAL, LOG_INFO,
533 QString(
"Data Refresh because we are unable to " 534 "query the data for day %1 to " 535 "determine if we have enough").arg(i));
536 download_needed =
true;
539 if (currentChanCount < (prevChanCount * 0.90))
541 LOG(VB_GENERAL, LOG_INFO,
542 QString(
"Data refresh needed because only %1 " 543 "out of %2 channels have at least one " 544 "program listed for day @ offset %3 " 545 "from 8PM - midnight. Previous day " 546 "had %4 channels with data in that " 548 .arg(currentChanCount).arg(source_channels)
549 .arg(i).arg(prevChanCount));
550 download_needed =
true;
552 else if (currentDayCount == 0)
554 LOG(VB_GENERAL, LOG_INFO,
555 QString(
"Data refresh needed because no data " 556 "exists for day @ offset %1 from 8PM - " 557 "midnight.").arg(i));
558 download_needed =
true;
560 else if (previousDayCount == 0)
562 LOG(VB_GENERAL, LOG_INFO,
563 QString(
"Data refresh needed because no data " 564 "exists for day @ offset %1 from 8PM - " 565 "midnight. Unable to calculate how " 566 "much we should have for the current " 567 "day so a refresh is being forced.")
569 download_needed =
true;
571 else if (currentDayCount < (currentChanCount * 3))
573 LOG(VB_GENERAL, LOG_INFO,
574 QString(
"Data Refresh needed because offset " 575 "day %1 has less than 3 programs " 576 "per channel for the 8PM - midnight " 577 "time window for channels that " 578 "normally have data. " 579 "We want at least %2 programs, but " 581 .arg(i).arg(currentChanCount * 3)
582 .arg(currentDayCount));
583 download_needed =
true;
585 else if (currentDayCount < (previousDayCount / 2))
587 LOG(VB_GENERAL, LOG_INFO,
588 QString(
"Data Refresh needed because offset " 589 "day %1 has less than half the number " 590 "of programs as the previous day for " 591 "the 8PM - midnight time window. " 592 "We want at least %2 programs, but " 593 "only found %3").arg(i)
594 .arg(previousDayCount / 2)
595 .arg(currentDayCount));
596 download_needed =
true;
601 LOG(VB_GENERAL, LOG_INFO,
602 QString(
"Data Refresh needed because we are unable " 603 "to query the data for day @ offset %1 to " 604 "determine how much we should have for " 605 "offset day %2.").arg(i-1).arg(i));
606 download_needed =
true;
612 LOG(VB_GENERAL, LOG_NOTICE,
613 QString(
"Refreshing data for ") + currDate);
614 if (!GrabData(*it, i))
617 if (!m_fatalErrors.empty() || m_interrupted)
625 LOG(VB_GENERAL, LOG_INFO,
626 "Grabber is no longer returning program data, " 633 LOG(VB_GENERAL, LOG_NOTICE,
634 QString(
"Data is already present for ") + currDate +
638 if (!m_fatalErrors.empty())
643 LOG(VB_GENERAL, LOG_ERR,
644 QString(
"Grabbing XMLTV data using ") + xmltv_grabber +
645 " is not supported. You may need to upgrade to" 646 " the latest version of XMLTV.");
654 query.
prepare(
"SELECT MAX(endtime) FROM program p " 655 "LEFT JOIN channel c ON p.chanid=c.chanid " 656 "WHERE c.deleted IS NULL AND c.sourceid= :SRCID " 657 "AND manualid = 0 AND c.xmltvid != '';");
666 if (GuideDataAfter == GuideDataBefore)
672 if (!m_fatalErrors.empty())
674 for (
int i = 0; i < m_fatalErrors.size(); i++)
676 LOG(VB_GENERAL, LOG_CRIT,
LOC +
"Encountered Fatal Error: " +
682 if (m_only_update_channels && !m_need_post_grab_proc)
688 (total_sources != externally_handled))
689 status = QObject::tr(
690 "mythfilldatabase ran, but did not insert " 691 "any new data into the Guide for %1 of %2 sources. " 692 "This can indicate a potential grabber failure.")
696 status = QObject::tr(
"Successful.");
701 return (failures == 0);
bool next(void)
Wrap QSqlQuery::next() so we can display the query results.
vector< Source > SourceList
void Run(time_t timeout=0)
Runs a command inside the /bin/sh shell. Returns immediately.
void bindValue(const QString &placeholder, const QVariant &val)
Add a single binding.
bool GrabDataFromFile(int id, QString &filename)
#define GENERIC_EXIT_OK
Exited with no error.
QString xmltvgrabber_prefmethod
bool Run(SourceList &sourcelist)
Goes through the sourcelist and updates its channels with program info grabbed with the associated gr...
QSqlQuery wrapper that fetches a DB connection from the connection pool.
bool updateLastRunEnd(void)
#define GENERIC_EXIT_KILLED
Process killed or stopped.
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
static void HandlePrograms(uint sourceid, QMap< QString, QList< ProgInfo > > &proglist)
Called from mythfilldatabase to bulk insert data into the program database.
QVariant value(int i) const
#define VERBOSE_LEVEL_CHECK(_MASK_, _LEVEL_)
QDateTime current(bool stripped)
Returns current Date and Time in UTC.
bool isActive(void) const
static MSqlQueryInfo InitCon(ConnectionReuse=kNormalConnection)
Only use this in combination with MSqlQuery constructor.
uint myth_system(const QString &command, uint flags, uint timeout)
uint Wait(time_t timeout=0)
run process through shell
QString createTempFile(QString name_template, bool dir)
bool prepare(const QString &query)
QSqlQuery::prepare() is not thread safe in Qt <= 3.3.2.
#define LOG(_MASK_, _LEVEL_, _STRING_)
bool updateLastRunStatus(QString &status)
bool GrabData(const Source &source, int offset)
bool exec(void)
Wrap QSqlQuery::exec() so we can display SQL.
bool SaveSettingOnHost(const QString &key, const QString &newValue, const QString &host)
bool updateNextScheduledRun()
static void DBError(const QString &where, const MSqlQuery &query)
bool isNull(int field) const
QDateTime fromString(const QString &dtstr)
Converts kFilename && kISODate formats to QDateTime.
vector< ChannelInfo > ChannelInfoList
static bool is_grabber_external(const QString &grabber)
bool updateLastRunStart(void)
void SetRefresh(int day, bool set)