MythTV  master
iptvchannelfetcher.cpp
Go to the documentation of this file.
1 // -*- Mode: c++ -*-
2 
3 // Std C headers
4 #include <cmath>
5 #include <unistd.h>
6 
7 // Qt headers
8 #include <QFile>
9 #include <QTextStream>
10 
11 // MythTV headers
12 #include "mythcontext.h"
13 #include "cardutil.h"
14 #include "channelutil.h"
15 #include "iptvchannelfetcher.h"
16 #include "scanmonitor.h"
17 #include "mythlogging.h"
18 #include "mythdownloadmanager.h"
19 
20 #define LOC QString("IPTVChanFetch: ")
21 
22 static bool parse_chan_info(const QString &rawdata,
23  IPTVChannelInfo &info,
24  QString &channum,
25  int &nextChanNum,
26  uint &lineNum);
27 
28 static bool parse_extinf(const QString &line,
29  QString &channum,
30  QString &name,
31  int &nextChanNum);
32 
34  uint cardid, const QString &inputname, uint sourceid,
35  bool is_mpts, ScanMonitor *monitor) :
36  m_scan_monitor(monitor),
37  m_cardid(cardid), m_inputname(inputname),
38  m_sourceid(sourceid), m_is_mpts(is_mpts),
39  m_thread(new MThread("IPTVChannelFetcher", this))
40 {
41  LOG(VB_CHANNEL, LOG_INFO, LOC + QString("Has ScanMonitor %1")
42  .arg(monitor?"true":"false"));
43 }
44 
46 {
47  Stop();
48  delete m_thread;
49  m_thread = nullptr;
50 }
51 
56 {
57  m_lock.lock();
58 
59  while (m_thread_running)
60  {
61  m_stop_now = true;
62  m_lock.unlock();
63  m_thread->wait(5);
64  m_lock.lock();
65  }
66 
67  m_lock.unlock();
68 
69  m_thread->wait();
70 }
71 
73 {
74  while (!m_thread->isFinished())
75  m_thread->wait(500);
76 
77  LOG(VB_CHANNEL, LOG_INFO, LOC + QString("Found %1 channels")
78  .arg(m_channels.size()));
79  return m_channels;
80 }
81 
84 {
85  Stop();
86  m_stop_now = false;
87  m_thread->start();
88 }
89 
91 {
92  m_lock.lock();
93  m_thread_running = true;
94  m_lock.unlock();
95 
96  // Step 1/4 : Get info from DB
97  QString url = CardUtil::GetVideoDevice(m_cardid);
98 
99  if (m_stop_now || url.isEmpty())
100  {
101  LOG(VB_CHANNEL, LOG_INFO, LOC + "Playlist URL was empty");
102  QMutexLocker locker(&m_lock);
103  m_thread_running = false;
104  m_stop_now = true;
105  return;
106  }
107 
108  LOG(VB_CHANNEL, LOG_INFO, LOC + QString("Playlist URL: %1").arg(url));
109 
110  // Step 2/4 : Download
111  if (m_scan_monitor)
112  {
114  m_scan_monitor->ScanAppendTextToLog(tr("Downloading Playlist"));
115  }
116 
117  QString playlist = DownloadPlaylist(url);
118 
119  if (m_stop_now || playlist.isEmpty())
120  {
121  if (playlist.isNull() && m_scan_monitor)
122  {
124  QCoreApplication::translate("(Common)", "Error"));
126  m_scan_monitor->ScanErrored(tr("Downloading Playlist Failed"));
127  }
128  QMutexLocker locker(&m_lock);
129  m_thread_running = false;
130  m_stop_now = true;
131  return;
132  }
133 
134  // Step 3/4 : Process
135  if (m_scan_monitor)
136  {
138  m_scan_monitor->ScanAppendTextToLog(tr("Processing Playlist"));
139  }
140 
141  m_channels.clear();
142  m_channels = ParsePlaylist(playlist, this);
143 
144  // Step 4/4 : Finish up
145  if (m_scan_monitor)
146  m_scan_monitor->ScanAppendTextToLog(tr("Adding Channels"));
148 
149  LOG(VB_CHANNEL, LOG_INFO, LOC + QString("Found %1 channels")
150  .arg(m_channels.size()));
151 
152  if (!m_is_mpts)
153  {
154  fbox_chan_map_t::const_iterator it = m_channels.begin();
155  for (uint i = 1; it != m_channels.end(); ++it, ++i)
156  {
157  const QString& channum = it.key();
158  QString name = (*it).m_name;
159  QString xmltvid = (*it).m_xmltvid.isEmpty() ? "" : (*it).m_xmltvid;
160  uint programnumber = (*it).m_programNumber;
161  //: %1 is the channel number, %2 is the channel name
162  QString msg = tr("Channel #%1 : %2").arg(channum).arg(name);
163 
164  LOG(VB_CHANNEL, LOG_INFO, QString("Handling channel %1 %2")
165  .arg(channum).arg(name));
166 
167  int chanid = ChannelUtil::GetChanID(m_sourceid, channum);
168  if (chanid <= 0)
169  {
170  if (m_scan_monitor)
171  {
173  tr("Adding %1").arg(msg));
174  }
175  chanid = ChannelUtil::CreateChanID(m_sourceid, channum);
177  channum, programnumber, 0, 0,
178  false, false, false, QString(),
179  QString(), "Default", xmltvid);
180  ChannelUtil::CreateIPTVTuningData(chanid, (*it).m_tuning);
181  }
182  else
183  {
184  if (m_scan_monitor)
185  {
187  tr("Updating %1").arg(msg));
188  }
190  channum, programnumber, 0, 0,
191  false, false, false, QString(),
192  QString(), "Default", xmltvid);
193  ChannelUtil::UpdateIPTVTuningData(chanid, (*it).m_tuning);
194  }
195 
197  }
198 
199  if (m_scan_monitor)
200  {
201  m_scan_monitor->ScanAppendTextToLog(tr("Done"));
205  }
206  }
207 
208  QMutexLocker locker(&m_lock);
209  m_thread_running = false;
210  m_stop_now = true;
211 }
212 
214 {
215  uint minval = 35, range = 70 - minval;
216  uint pct = minval + (uint) truncf((((float)val) / m_chan_cnt) * range);
217  if (m_scan_monitor)
219 }
220 
222 {
223  uint minval = 70, range = 100 - minval;
224  uint pct = minval + (uint) truncf((((float)val) / m_chan_cnt) * range);
225  if (m_scan_monitor)
227 }
228 
229 void IPTVChannelFetcher::SetMessage(const QString &status)
230 {
231  if (m_scan_monitor)
233 }
234 
235 // This function is always called from a thread context.
236 QString IPTVChannelFetcher::DownloadPlaylist(const QString &url)
237 {
238  if (url.startsWith("file", Qt::CaseInsensitive))
239  {
240  QString ret = "";
241  QUrl qurl(url);
242  QFile file(qurl.toLocalFile());
243  if (!file.open(QIODevice::ReadOnly))
244  {
245  LOG(VB_GENERAL, LOG_ERR, LOC + QString("Opening '%1'")
246  .arg(qurl.toLocalFile()) + ENO);
247  return ret;
248  }
249 
250  QTextStream stream(&file);
251  while (!stream.atEnd())
252  ret += stream.readLine() + "\n";
253 
254  file.close();
255  return ret;
256  }
257 
258  // Use MythDownloadManager for http URLs
259  QByteArray data;
260  QString tmp;
261 
262  if (!GetMythDownloadManager()->download(url, &data))
263  {
264  LOG(VB_GENERAL, LOG_INFO, LOC +
265  QString("DownloadPlaylist failed to "
266  "download from %1").arg(url));
267  }
268  else
269  tmp = QString(data);
270 
271  return tmp.isNull() ? tmp : QString::fromUtf8(tmp.toLatin1().constData());
272 }
273 
274 static uint estimate_number_of_channels(const QString &rawdata)
275 {
276  uint result = 0;
277  uint numLine = 1;
278  while (true)
279  {
280  QString url = rawdata.section("\n", numLine, numLine, QString::SectionSkipEmpty);
281  if (url.isEmpty())
282  return result;
283 
284  ++numLine;
285  if (!url.startsWith("#")) // ignore comments
286  ++result;
287  }
288 }
289 
291  const QString &reallyrawdata, IPTVChannelFetcher *fetcher)
292 {
293  fbox_chan_map_t chanmap;
294  int nextChanNum = 1;
295 
296  QString rawdata = reallyrawdata;
297  rawdata.replace("\r\n", "\n");
298 
299  // Verify header is ok
300  QString header = rawdata.section("\n", 0, 0);
301  if (!header.startsWith("#EXTM3U"))
302  {
303  LOG(VB_GENERAL, LOG_ERR, LOC +
304  QString("Invalid channel list header (%1)").arg(header));
305 
306  if (fetcher)
307  {
308  fetcher->SetMessage(
309  tr("ERROR: M3U channel list is malformed"));
310  }
311 
312  return chanmap;
313  }
314 
315  // estimate number of channels
316  if (fetcher)
317  {
318  uint num_channels = estimate_number_of_channels(rawdata);
319  fetcher->SetTotalNumChannels(num_channels);
320 
321  LOG(VB_CHANNEL, LOG_INFO, LOC +
322  QString("Estimating there are %1 channels in playlist")
323  .arg(num_channels));
324  }
325 
326  // get the next available channel number for the source (only used if we can't find one in the playlist)
327  if (fetcher)
328  {
329  MSqlQuery query(MSqlQuery::InitCon());
330  QString sql = "select MAX(CONVERT(channum, UNSIGNED INTEGER)) from channel where sourceid = :SOURCEID;";
331 
332  query.prepare(sql);
333  query.bindValue(":SOURCEID", fetcher->m_sourceid);
334 
335  if (!query.exec())
336  {
337  MythDB::DBError("Get next max channel number", query);
338  }
339  else
340  {
341  if (query.first())
342  {
343  nextChanNum = query.value(0).toInt() + 1;
344  LOG(VB_GENERAL, LOG_INFO, LOC + QString("Next available channel number from DB is: %1").arg(nextChanNum));
345  }
346  else
347  {
348  nextChanNum = 1;
349  LOG(VB_GENERAL, LOG_INFO, LOC + QString("No channels found for this source, using default channel number: %1").arg(nextChanNum));
350  }
351  }
352  }
353 
354  // Parse each channel
355  uint lineNum = 1;
356  for (uint i = 1; true; ++i)
357  {
358  IPTVChannelInfo info;
359  QString channum;
360 
361  if (!parse_chan_info(rawdata, info, channum, nextChanNum, lineNum))
362  break;
363 
364  QString msg = tr("Encountered malformed channel");
365  if (!channum.isEmpty())
366  {
367  chanmap[channum] = info;
368 
369  msg = QString("Parsing Channel #%1 : %2 : %3")
370  .arg(channum).arg(info.m_name)
371  .arg(info.m_tuning.GetDataURL().toString());
372  LOG(VB_CHANNEL, LOG_INFO, LOC + msg);
373 
374  msg.clear(); // don't tell fetcher
375  }
376 
377  if (fetcher)
378  {
379  if (!msg.isEmpty())
380  fetcher->SetMessage(msg);
381  fetcher->SetNumChannelsParsed(i);
382  }
383  }
384 
385  return chanmap;
386 }
387 
388 static bool parse_chan_info(const QString &rawdata,
389  IPTVChannelInfo &info,
390  QString &channum,
391  int &nextChanNum,
392  uint &lineNum)
393 {
394  // #EXTINF:0,2 - France 2 <-- duration,channum - channame
395  // #EXTMYTHTV:xmltvid=C2.telepoche.com <-- optional line (myth specific)
396  // #EXTMYTHTV:bitrate=BITRATE <-- optional line (myth specific)
397  // #EXTMYTHTV:fectype=FECTYPE <-- optional line (myth specific)
398  // The FECTYPE can be rfc2733, rfc5109, or smpte2022
399  // #EXTMYTHTV:fecurl0=URL <-- optional line (myth specific)
400  // #EXTMYTHTV:fecurl1=URL <-- optional line (myth specific)
401  // #EXTMYTHTV:fecbitrate0=BITRATE <-- optional line (myth specific)
402  // #EXTMYTHTV:fecbitrate1=BITRATE <-- optional line (myth specific)
403  // #EXTVLCOPT:program=program_number <-- optional line (used by MythTV and VLC)
404  // #... <-- ignored comments
405  // rtsp://maiptv.iptv.fr/iptvtv/201 <-- url
406 
407  QString name;
408  QMap<QString,QString> values;
409 
410  while (true)
411  {
412  QString line = rawdata.section("\n", lineNum, lineNum, QString::SectionSkipEmpty);
413  if (line.isEmpty())
414  return false;
415 
416  ++lineNum;
417  if (line.startsWith("#"))
418  {
419  if (line.startsWith("#EXTINF:"))
420  {
421  parse_extinf(line.mid(line.indexOf(':')+1), channum, name, nextChanNum);
422  }
423  else if (line.startsWith("#EXTMYTHTV:"))
424  {
425  QString data = line.mid(line.indexOf(':')+1);
426  QString key = data.left(data.indexOf('='));
427  if (!key.isEmpty())
428  values[key] = data.mid(data.indexOf('=')+1);
429  }
430  else if (line.startsWith("#EXTVLCOPT:program="))
431  {
432  values["programnumber"] = line.mid(line.indexOf('=')+1);
433  }
434  continue;
435  }
436 
437  if (name.isEmpty())
438  return false;
439 
440  QMap<QString,QString>::const_iterator it = values.begin();
441  for (; it != values.end(); ++it)
442  {
443  LOG(VB_GENERAL, LOG_INFO, LOC +
444  QString("parse_chan_info [%1]='%2'")
445  .arg(it.key()).arg(*it));
446  }
447  info = IPTVChannelInfo(
448  name, values["xmltvid"],
449  line, values["bitrate"].toUInt(),
450  values["fectype"],
451  values["fecurl0"], values["fecbitrate0"].toUInt(),
452  values["fecurl1"], values["fecbitrate1"].toUInt(),
453  values["programnumber"].toUInt());
454  return true;
455  }
456 }
457 
458 static bool parse_extinf(const QString &line,
459  QString &channum,
460  QString &name,
461  int &nextChanNum)
462 {
463  // Parse extension portion, Freebox or SAT>IP format
464  QRegExp chanNumName1("^-?\\d+,(\\d+)(?:\\.\\s|\\s-\\s)(.*)$");
465  int pos = chanNumName1.indexIn(line);
466  if (pos != -1)
467  {
468  channum = chanNumName1.cap(1);
469  name = chanNumName1.cap(2);
470  return true;
471  }
472 
473  // Parse extension portion, A1 TV format
474  QRegExp chanNumName2("^-?\\d+\\s+[^,]*tvg-num=\"(\\d+)\"[^,]*,(.*)$");
475  pos = chanNumName2.indexIn(line);
476  if (pos != -1)
477  {
478  channum = chanNumName2.cap(1);
479  name = chanNumName2.cap(2);
480  return true;
481  }
482 
483  // Parse extension portion, Moviestar TV number then name
484  QRegExp chanNumName3("^-?\\d+,\\[(\\d+)\\]\\s+(.*)$");
485  pos = chanNumName3.indexIn(line);
486  if (pos != -1)
487  {
488  channum = chanNumName3.cap(1);
489  name = chanNumName3.cap(2);
490  return true;
491  }
492 
493  // Parse extension portion, Moviestar TV name then number
494  QRegExp chanNumName4("^-?\\d+,(.*)\\s+\\[(\\d+)\\]$");
495  pos = chanNumName4.indexIn(line);
496  if (pos != -1)
497  {
498  channum = chanNumName4.cap(2);
499  name = chanNumName4.cap(1);
500  return true;
501  }
502 
503  // Parse extension portion, russion iptv plugin style
504  QRegExp chanNumName5("^(-?\\d+)\\s+[^,]*,\\s*(.*)$");
505  pos = chanNumName5.indexIn(line);
506  if (pos != -1)
507  {
508  channum = chanNumName5.cap(1).simplified();
509  name = chanNumName5.cap(2).simplified();
510  bool ok;
511  int channel_number = channum.toInt (&ok);
512  if (ok && (channel_number > 0))
513  {
514  return true;
515  }
516  }
517 
518  // Parse extension portion, https://github.com/iptv-org/iptv/blob/master/channels/ style
519  // EG. #EXTINF:-1 tvg-id="" tvg-name="" tvg-logo="https://i.imgur.com/VejnhiB.png" group-title="News",BBC News
520  QRegExp chanNumName6("(^-?\\d+)\\s+[^,]*[^,]*,(.*)$");
521  pos = chanNumName6.indexIn(line);
522  if (pos != -1)
523  {
524  channum = chanNumName6.cap(1).simplified();
525  name = chanNumName6.cap(2).simplified();
526 
527  bool ok;
528  int channel_number = channum.toInt(&ok);
529  if (ok && channel_number > 0)
530  {
531  if (channel_number >= nextChanNum)
532  nextChanNum = channel_number + 1;
533  return true;
534  }
535  else
536  {
537  // no valid channel number found use the default next one
538  LOG(VB_GENERAL, LOG_ERR, QString("No channel number found, using next available: %1 for channel: %2").arg(nextChanNum).arg(name));
539  channum = QString::number(nextChanNum);
540  nextChanNum++;
541  return true;
542  }
543  }
544 
545  // not one of the formats we support
546  QString msg = LOC + QString("Invalid header in channel list line \n\t\t\tEXTINF:%1").arg(line);
547  LOG(VB_GENERAL, LOG_ERR, msg);
548  return false;
549 }
550 
551 /* vim: set expandtab tabstop=4 shiftwidth=4: */
void start(QThread::Priority=QThread::InheritPriority)
Tell MThread to start running the thread in the near future.
Definition: mthread.cpp:294
This is a wrapper around QThread that does several additional things.
Definition: mthread.h:46
void bindValue(const QString &placeholder, const QVariant &val)
Add a single binding.
Definition: mythdbcon.cpp:863
static bool parse_extinf(const QString &line, QString &channum, QString &name, int &nextChanNum)
static bool UpdateIPTVTuningData(uint channel_id, const IPTVTuningData &tuning)
IPTVChannelFetcher(uint cardid, const QString &inputname, uint sourceid, bool is_mpts, ScanMonitor *monitor=nullptr)
static bool CreateChannel(uint db_mplexid, uint db_sourceid, uint new_channel_id, const QString &callsign, const QString &service_name, const QString &chan_num, uint service_id, uint atsc_major_channel, uint atsc_minor_channel, bool use_on_air_guide, bool hidden, bool hidden_in_guide, const QString &freqid, const QString &icon=QString(), QString format="Default", const QString &xmltvid=QString(), const QString &default_authority=QString(), uint service_type=0)
QSqlQuery wrapper that fetches a DB connection from the connection pool.
Definition: mythdbcon.h:125
bool wait(unsigned long time=ULONG_MAX)
Wait for the MThread to exit, with a maximum timeout.
Definition: mthread.cpp:311
void ScanPercentComplete(int pct)
unsigned int uint
Definition: compat.h:140
QMap< QString, IPTVChannelInfo > fbox_chan_map_t
static guint32 * tmp
Definition: goom_core.c:35
static fbox_chan_map_t ParsePlaylist(const QString &rawdata, IPTVChannelFetcher *fetcher=nullptr)
QVariant value(int i) const
Definition: mythdbcon.h:198
void ScanErrored(const QString &error)
static int GetChanID(int db_mplexid, int service_transport_id, int major_channel, int minor_channel, int program_number)
static QString DownloadPlaylist(const QString &url)
MythDownloadManager * GetMythDownloadManager(void)
Gets the pointer to the MythDownloadManager singleton.
void SetTotalNumChannels(uint val)
static uint estimate_number_of_channels(const QString &rawdata)
void ScanAppendTextToLog(const QString &status)
static MSqlQueryInfo InitCon(ConnectionReuse=kNormalConnection)
Only use this in combination with MSqlQuery constructor.
Definition: mythdbcon.cpp:535
static bool CreateIPTVTuningData(uint channel_id, const IPTVTuningData &tuning)
Definition: channelutil.h:142
const char * name
Definition: ParseText.cpp:328
#define ENO
This can be appended to the LOG args with "+".
Definition: mythlogging.h:99
static int CreateChanID(uint sourceid, const QString &chan_num)
Creates a unique channel ID for database use.
void Scan(void)
Scans the given frequency list.
fbox_chan_map_t GetChannels(void)
bool first(void)
Wrap QSqlQuery::first() so we can display the query results.
Definition: mythdbcon.cpp:792
static QString GetVideoDevice(uint inputid)
Definition: cardutil.h:273
bool prepare(const QString &query)
QSqlQuery::prepare() is not thread safe in Qt <= 3.3.2.
Definition: mythdbcon.cpp:807
#define LOG(_MASK_, _LEVEL_, _STRING_)
Definition: mythlogging.h:41
IPTVTuningData m_tuning
static bool parse_chan_info(const QString &rawdata, IPTVChannelInfo &info, QString &channum, int &nextChanNum, uint &lineNum)
ScanMonitor * m_scan_monitor
void ScanComplete(void)
Definition: scanmonitor.cpp:98
bool isFinished(void) const
Definition: mthread.cpp:269
void run(void) override
fbox_chan_map_t m_channels
bool exec(void)
Wrap QSqlQuery::exec() so we can display SQL.
Definition: mythdbcon.cpp:603
static void DBError(const QString &where, const MSqlQuery &query)
Definition: mythdb.cpp:179
#define LOC
void Stop(void)
Stops the scanning thread running.
QUrl GetDataURL(void) const
void SetMessage(const QString &status)
static bool UpdateChannel(uint db_mplexid, uint source_id, uint channel_id, const QString &callsign, const QString &service_name, const QString &chan_num, uint service_id, uint atsc_major_channel, uint atsc_minor_channel, bool use_on_air_guide, bool hidden, bool hidden_in_guide, const QString &freqid=QString(), const QString &icon=QString(), QString format=QString(), const QString &xmltvid=QString(), const QString &default_authority=QString(), uint service_type=0)