MythTV  master
iptvchannelfetcher.cpp
Go to the documentation of this file.
1 // -*- Mode: c++ -*-
2 
3 // Std C headers
4 #include <cmath>
5 #include <unistd.h>
6 #include <utility>
7 
8 // Qt headers
9 #include <QFile>
10 #include <QTextStream>
11 
12 // MythTV headers
13 #include "mythcontext.h"
14 #include "cardutil.h"
15 #include "channelutil.h"
16 #include "iptvchannelfetcher.h"
17 #include "scanmonitor.h"
18 #include "mythlogging.h"
19 #include "mythdownloadmanager.h"
20 
21 #define LOC QString("IPTVChanFetch: ")
22 
23 static bool parse_chan_info(const QString &rawdata,
24  IPTVChannelInfo &info,
25  QString &channum,
26  int &nextChanNum,
27  uint &lineNum);
28 
29 static bool parse_extinf(const QString &line,
30  QString &channum,
31  QString &name,
32  int &nextChanNum);
33 
35  uint cardid, QString inputname, uint sourceid,
36  bool is_mpts, ScanMonitor *monitor) :
37  m_scanMonitor(monitor),
38  m_cardId(cardid), m_inputName(std::move(inputname)),
39  m_sourceId(sourceid), m_isMpts(is_mpts),
40  m_thread(new MThread("IPTVChannelFetcher", this))
41 {
42  LOG(VB_CHANNEL, LOG_INFO, LOC + QString("Has ScanMonitor %1")
43  .arg(monitor?"true":"false"));
44 }
45 
47 {
48  Stop();
49  delete m_thread;
50  m_thread = nullptr;
51 }
52 
57 {
58  m_lock.lock();
59 
60  while (m_threadRunning)
61  {
62  m_stopNow = true;
63  m_lock.unlock();
64  m_thread->wait(5);
65  m_lock.lock();
66  }
67 
68  m_lock.unlock();
69 
70  m_thread->wait();
71 }
72 
74 {
75  while (!m_thread->isFinished())
76  m_thread->wait(500);
77 
78  LOG(VB_CHANNEL, LOG_INFO, LOC + QString("Found %1 channels")
79  .arg(m_channels.size()));
80  return m_channels;
81 }
82 
85 {
86  Stop();
87  m_stopNow = false;
88  m_thread->start();
89 }
90 
92 {
93  m_lock.lock();
94  m_threadRunning = true;
95  m_lock.unlock();
96 
97  // Step 1/4 : Get info from DB
98  QString url = CardUtil::GetVideoDevice(m_cardId);
99 
100  if (m_stopNow || url.isEmpty())
101  {
102  LOG(VB_CHANNEL, LOG_INFO, LOC + "Playlist URL was empty");
103  QMutexLocker locker(&m_lock);
104  m_threadRunning = false;
105  m_stopNow = true;
106  return;
107  }
108 
109  LOG(VB_CHANNEL, LOG_INFO, LOC + QString("Playlist URL: %1").arg(url));
110 
111  // Step 2/4 : Download
112  if (m_scanMonitor)
113  {
115  m_scanMonitor->ScanAppendTextToLog(tr("Downloading Playlist"));
116  }
117 
118  QString playlist = DownloadPlaylist(url);
119 
120  if (m_stopNow || playlist.isEmpty())
121  {
122  if (playlist.isNull() && m_scanMonitor)
123  {
125  QCoreApplication::translate("(Common)", "Error"));
127  m_scanMonitor->ScanErrored(tr("Downloading Playlist Failed"));
128  }
129  QMutexLocker locker(&m_lock);
130  m_threadRunning = false;
131  m_stopNow = true;
132  return;
133  }
134 
135  // Step 3/4 : Process
136  if (m_scanMonitor)
137  {
139  m_scanMonitor->ScanAppendTextToLog(tr("Processing Playlist"));
140  }
141 
142  m_channels.clear();
143  m_channels = ParsePlaylist(playlist, this);
144 
145  // Step 4/4 : Finish up
146  if (m_scanMonitor)
147  m_scanMonitor->ScanAppendTextToLog(tr("Adding Channels"));
149 
150  LOG(VB_CHANNEL, LOG_INFO, LOC + QString("Found %1 channels")
151  .arg(m_channels.size()));
152 
153  if (!m_isMpts)
154  {
155  fbox_chan_map_t::const_iterator it = m_channels.begin();
156  for (uint i = 1; it != m_channels.end(); ++it, ++i)
157  {
158  const QString& channum = it.key();
159  QString name = (*it).m_name;
160  QString xmltvid = (*it).m_xmltvid.isEmpty() ? "" : (*it).m_xmltvid;
161  uint programnumber = (*it).m_programNumber;
162  //: %1 is the channel number, %2 is the channel name
163  QString msg = tr("Channel #%1 : %2").arg(channum).arg(name);
164 
165  LOG(VB_CHANNEL, LOG_INFO, QString("Handling channel %1 %2")
166  .arg(channum).arg(name));
167 
168  int chanid = ChannelUtil::GetChanID(m_sourceId, channum);
169  if (chanid <= 0)
170  {
171  if (m_scanMonitor)
172  {
173  m_scanMonitor->ScanAppendTextToLog(tr("Adding %1").arg(msg));
174  }
175  chanid = ChannelUtil::CreateChanID(m_sourceId, channum);
176  ChannelUtil::CreateChannel(0, m_sourceId, chanid, name, name,
177  channum, programnumber, 0, 0,
178  false, kChannelVisible, QString(),
179  QString(), "Default", xmltvid);
180  ChannelUtil::CreateIPTVTuningData(chanid, (*it).m_tuning);
181  }
182  else
183  {
184  if (m_scanMonitor)
185  {
187  tr("Updating %1").arg(msg));
188  }
189  ChannelUtil::UpdateChannel(0, m_sourceId, chanid, name, name,
190  channum, programnumber, 0, 0,
191  false, kChannelVisible, QString(),
192  QString(), "Default", xmltvid);
193  ChannelUtil::UpdateIPTVTuningData(chanid, (*it).m_tuning);
194  }
195 
197  }
198 
199  if (m_scanMonitor)
200  {
201  m_scanMonitor->ScanAppendTextToLog(tr("Done"));
205  }
206  }
207 
208  QMutexLocker locker(&m_lock);
209  m_threadRunning = false;
210  m_stopNow = true;
211 }
212 
214 {
215  uint minval = 35;
216  uint range = 70 - minval;
217  uint pct = minval + (uint) truncf((((float)val) / m_chanCnt) * range);
218  if (m_scanMonitor)
220 }
221 
223 {
224  uint minval = 70;
225  uint range = 100 - minval;
226  uint pct = minval + (uint) truncf((((float)val) / m_chanCnt) * range);
227  if (m_scanMonitor)
229 }
230 
231 void IPTVChannelFetcher::SetMessage(const QString &status)
232 {
233  if (m_scanMonitor)
235 }
236 
237 // This function is always called from a thread context.
238 QString IPTVChannelFetcher::DownloadPlaylist(const QString &url)
239 {
240  if (url.startsWith("file", Qt::CaseInsensitive))
241  {
242  QString ret = "";
243  QUrl qurl(url);
244  QFile file(qurl.toLocalFile());
245  if (!file.open(QIODevice::ReadOnly))
246  {
247  LOG(VB_GENERAL, LOG_ERR, LOC + QString("Opening '%1'")
248  .arg(qurl.toLocalFile()) + ENO);
249  return ret;
250  }
251 
252  QTextStream stream(&file);
253  while (!stream.atEnd())
254  ret += stream.readLine() + "\n";
255 
256  file.close();
257  return ret;
258  }
259 
260  // Use MythDownloadManager for http URLs
261  QByteArray data;
262  QString tmp;
263 
264  if (!GetMythDownloadManager()->download(url, &data))
265  {
266  LOG(VB_GENERAL, LOG_INFO, LOC +
267  QString("DownloadPlaylist failed to "
268  "download from %1").arg(url));
269  }
270  else
271  tmp = QString(data);
272 
273  return tmp.isNull() ? tmp : QString::fromUtf8(tmp.toLatin1().constData());
274 }
275 
276 static uint estimate_number_of_channels(const QString &rawdata)
277 {
278  uint result = 0;
279  uint numLine = 1;
280  while (true)
281  {
282  QString url = rawdata.section("\n", numLine, numLine, QString::SectionSkipEmpty);
283  if (url.isEmpty())
284  return result;
285 
286  ++numLine;
287  if (!url.startsWith("#")) // ignore comments
288  ++result;
289  }
290 }
291 
293  const QString &reallyrawdata, IPTVChannelFetcher *fetcher)
294 {
295  fbox_chan_map_t chanmap;
296  int nextChanNum = 1;
297 
298  QString rawdata = reallyrawdata;
299  rawdata.replace("\r\n", "\n");
300 
301  // Verify header is ok
302  QString header = rawdata.section("\n", 0, 0);
303  if (!header.startsWith("#EXTM3U"))
304  {
305  LOG(VB_GENERAL, LOG_ERR, LOC +
306  QString("Invalid channel list header (%1)").arg(header));
307 
308  if (fetcher)
309  {
310  fetcher->SetMessage(
311  tr("ERROR: M3U channel list is malformed"));
312  }
313 
314  return chanmap;
315  }
316 
317  // estimate number of channels
318  if (fetcher)
319  {
320  uint num_channels = estimate_number_of_channels(rawdata);
321  fetcher->SetTotalNumChannels(num_channels);
322 
323  LOG(VB_CHANNEL, LOG_INFO, LOC +
324  QString("Estimating there are %1 channels in playlist")
325  .arg(num_channels));
326  }
327 
328  // get the next available channel number for the source (only used if we can't find one in the playlist)
329  if (fetcher)
330  {
331  MSqlQuery query(MSqlQuery::InitCon());
332  QString sql = "select MAX(CONVERT(channum, UNSIGNED INTEGER)) from channel where sourceid = :SOURCEID;";
333 
334  query.prepare(sql);
335  query.bindValue(":SOURCEID", fetcher->m_sourceId);
336 
337  if (!query.exec())
338  {
339  MythDB::DBError("Get next max channel number", query);
340  }
341  else
342  {
343  if (query.first())
344  {
345  nextChanNum = query.value(0).toInt() + 1;
346  LOG(VB_GENERAL, LOG_INFO, LOC + QString("Next available channel number from DB is: %1").arg(nextChanNum));
347  }
348  else
349  {
350  nextChanNum = 1;
351  LOG(VB_GENERAL, LOG_INFO, LOC + QString("No channels found for this source, using default channel number: %1").arg(nextChanNum));
352  }
353  }
354  }
355 
356  // Parse each channel
357  uint lineNum = 1;
358  for (uint i = 1; true; ++i)
359  {
360  IPTVChannelInfo info;
361  QString channum;
362 
363  if (!parse_chan_info(rawdata, info, channum, nextChanNum, lineNum))
364  break;
365 
366  QString msg = tr("Encountered malformed channel");
367  if (!channum.isEmpty())
368  {
369  chanmap[channum] = info;
370 
371  msg = QString("Parsing Channel #%1 : %2 : %3")
372  .arg(channum).arg(info.m_name)
373  .arg(info.m_tuning.GetDataURL().toString());
374  LOG(VB_CHANNEL, LOG_INFO, LOC + msg);
375 
376  msg.clear(); // don't tell fetcher
377  }
378 
379  if (fetcher)
380  {
381  if (!msg.isEmpty())
382  fetcher->SetMessage(msg);
383  fetcher->SetNumChannelsParsed(i);
384  }
385  }
386 
387  return chanmap;
388 }
389 
390 static bool parse_chan_info(const QString &rawdata,
391  IPTVChannelInfo &info,
392  QString &channum,
393  int &nextChanNum,
394  uint &lineNum)
395 {
396  // #EXTINF:0,2 - France 2 <-- duration,channum - channame
397  // #EXTMYTHTV:xmltvid=C2.telepoche.com <-- optional line (myth specific)
398  // #EXTMYTHTV:bitrate=BITRATE <-- optional line (myth specific)
399  // #EXTMYTHTV:fectype=FECTYPE <-- optional line (myth specific)
400  // The FECTYPE can be rfc2733, rfc5109, or smpte2022
401  // #EXTMYTHTV:fecurl0=URL <-- optional line (myth specific)
402  // #EXTMYTHTV:fecurl1=URL <-- optional line (myth specific)
403  // #EXTMYTHTV:fecbitrate0=BITRATE <-- optional line (myth specific)
404  // #EXTMYTHTV:fecbitrate1=BITRATE <-- optional line (myth specific)
405  // #EXTVLCOPT:program=program_number <-- optional line (used by MythTV and VLC)
406  // #... <-- ignored comments
407  // rtsp://maiptv.iptv.fr/iptvtv/201 <-- url
408 
409  QString name;
410  QMap<QString,QString> values;
411 
412  while (true)
413  {
414  QString line = rawdata.section("\n", lineNum, lineNum, QString::SectionSkipEmpty);
415  if (line.isEmpty())
416  return false;
417 
418  ++lineNum;
419  if (line.startsWith("#"))
420  {
421  if (line.startsWith("#EXTINF:"))
422  {
423  parse_extinf(line.mid(line.indexOf(':')+1), channum, name, nextChanNum);
424  }
425  else if (line.startsWith("#EXTMYTHTV:"))
426  {
427  QString data = line.mid(line.indexOf(':')+1);
428  QString key = data.left(data.indexOf('='));
429  if (!key.isEmpty())
430  values[key] = data.mid(data.indexOf('=')+1);
431  }
432  else if (line.startsWith("#EXTVLCOPT:program="))
433  {
434  values["programnumber"] = line.mid(line.indexOf('=')+1);
435  }
436  continue;
437  }
438 
439  if (name.isEmpty())
440  return false;
441 
442  QMap<QString,QString>::const_iterator it = values.begin();
443  for (; it != values.end(); ++it)
444  {
445  LOG(VB_GENERAL, LOG_INFO, LOC +
446  QString("parse_chan_info [%1]='%2'")
447  .arg(it.key()).arg(*it));
448  }
449  info = IPTVChannelInfo(
450  name, values["xmltvid"],
451  line, values["bitrate"].toUInt(),
452  values["fectype"],
453  values["fecurl0"], values["fecbitrate0"].toUInt(),
454  values["fecurl1"], values["fecbitrate1"].toUInt(),
455  values["programnumber"].toUInt());
456  return true;
457  }
458 }
459 
460 static bool parse_extinf(const QString &line,
461  QString &channum,
462  QString &name,
463  int &nextChanNum)
464 {
465  // Parse extension portion, Freebox or SAT>IP format
466  QRegExp chanNumName1("^-?\\d+,(\\d+)(?:\\.\\s|\\s-\\s)(.*)$");
467  int pos = chanNumName1.indexIn(line);
468  if (pos != -1)
469  {
470  channum = chanNumName1.cap(1);
471  name = chanNumName1.cap(2);
472  return true;
473  }
474 
475  // Parse extension portion, A1 TV format
476  QRegExp chanNumName2("^-?\\d+\\s+[^,]*tvg-num=\"(\\d+)\"[^,]*,(.*)$");
477  pos = chanNumName2.indexIn(line);
478  if (pos != -1)
479  {
480  channum = chanNumName2.cap(1);
481  name = chanNumName2.cap(2);
482  return true;
483  }
484 
485  // Parse extension portion, Moviestar TV number then name
486  QRegExp chanNumName3("^-?\\d+,\\[(\\d+)\\]\\s+(.*)$");
487  pos = chanNumName3.indexIn(line);
488  if (pos != -1)
489  {
490  channum = chanNumName3.cap(1);
491  name = chanNumName3.cap(2);
492  return true;
493  }
494 
495  // Parse extension portion, Moviestar TV name then number
496  QRegExp chanNumName4("^-?\\d+,(.*)\\s+\\[(\\d+)\\]$");
497  pos = chanNumName4.indexIn(line);
498  if (pos != -1)
499  {
500  channum = chanNumName4.cap(2);
501  name = chanNumName4.cap(1);
502  return true;
503  }
504 
505  // Parse extension portion, russion iptv plugin style
506  QRegExp chanNumName5("^(-?\\d+)\\s+[^,]*,\\s*(.*)$");
507  pos = chanNumName5.indexIn(line);
508  if (pos != -1)
509  {
510  channum = chanNumName5.cap(1).simplified();
511  name = chanNumName5.cap(2).simplified();
512  bool ok = false;
513  int channel_number = channum.toInt (&ok);
514  if (ok && (channel_number > 0))
515  {
516  return true;
517  }
518  }
519 
520  // Parse extension portion, https://github.com/iptv-org/iptv/blob/master/channels/ style
521  // EG. #EXTINF:-1 tvg-id="" tvg-name="" tvg-logo="https://i.imgur.com/VejnhiB.png" group-title="News",BBC News
522  QRegExp chanNumName6("(^-?\\d+)\\s+[^,]*[^,]*,(.*)$");
523  pos = chanNumName6.indexIn(line);
524  if (pos != -1)
525  {
526  channum = chanNumName6.cap(1).simplified();
527  name = chanNumName6.cap(2).simplified();
528 
529  bool ok = false;
530  int channel_number = channum.toInt(&ok);
531  if (ok && channel_number > 0)
532  {
533  if (channel_number >= nextChanNum)
534  nextChanNum = channel_number + 1;
535  return true;
536  }
537 
538  // no valid channel number found use the default next one
539  LOG(VB_GENERAL, LOG_ERR, QString("No channel number found, using next available: %1 for channel: %2").arg(nextChanNum).arg(name));
540  channum = QString::number(nextChanNum);
541  nextChanNum++;
542  return true;
543  }
544 
545  // not one of the formats we support
546  QString msg = LOC + QString("Invalid header in channel list line \n\t\t\tEXTINF:%1").arg(line);
547  LOG(VB_GENERAL, LOG_ERR, msg);
548  return false;
549 }
550 
551 /* vim: set expandtab tabstop=4 shiftwidth=4: */
void SetNumChannelsParsed(uint val)
This is a wrapper around QThread that does several additional things.
Definition: mthread.h:46
void bindValue(const QString &placeholder, const QVariant &val)
Add a single binding.
Definition: mythdbcon.cpp:864
static bool parse_extinf(const QString &line, QString &channum, QString &name, int &nextChanNum)
ScanMonitor * m_scanMonitor
static bool UpdateIPTVTuningData(uint channel_id, const IPTVTuningData &tuning)
QSqlQuery wrapper that fetches a DB connection from the connection pool.
Definition: mythdbcon.h:125
bool wait(unsigned long time=ULONG_MAX)
Wait for the MThread to exit, with a maximum timeout.
Definition: mthread.cpp:311
void ScanPercentComplete(int pct)
static guint32 * tmp
Definition: goom_core.c:35
void SetNumChannelsInserted(uint val)
static fbox_chan_map_t ParsePlaylist(const QString &rawdata, IPTVChannelFetcher *fetcher=nullptr)
QVariant value(int i) const
Definition: mythdbcon.h:198
void ScanErrored(const QString &error)
static bool CreateChannel(uint db_mplexid, uint db_sourceid, uint new_channel_id, const QString &callsign, const QString &service_name, const QString &chan_num, uint service_id, uint atsc_major_channel, uint atsc_minor_channel, bool use_on_air_guide, ChannelVisibleType visible, const QString &freqid, const QString &icon=QString(), QString format="Default", const QString &xmltvid=QString(), const QString &default_authority=QString(), uint service_type=0)
static int GetChanID(int db_mplexid, int service_transport_id, int major_channel, int minor_channel, int program_number)
static QString DownloadPlaylist(const QString &url)
static bool UpdateChannel(uint db_mplexid, uint source_id, uint channel_id, const QString &callsign, const QString &service_name, const QString &chan_num, uint service_id, uint atsc_major_channel, uint atsc_minor_channel, bool use_on_air_guide, ChannelVisibleType visible, const QString &freqid=QString(), const QString &icon=QString(), QString format=QString(), const QString &xmltvid=QString(), const QString &default_authority=QString(), uint service_type=0)
MythDownloadManager * GetMythDownloadManager(void)
Gets the pointer to the MythDownloadManager singleton.
void SetTotalNumChannels(uint val)
static uint estimate_number_of_channels(const QString &rawdata)
void ScanAppendTextToLog(const QString &status)
unsigned int uint
Definition: compat.h:140
static MSqlQueryInfo InitCon(ConnectionReuse _reuse=kNormalConnection)
Only use this in combination with MSqlQuery constructor.
Definition: mythdbcon.cpp:535
static bool CreateIPTVTuningData(uint channel_id, const IPTVTuningData &tuning)
Definition: channelutil.h:144
#define ENO
This can be appended to the LOG args with "+".
Definition: mythlogging.h:99
static int CreateChanID(uint sourceid, const QString &chan_num)
Creates a unique channel ID for database use.
void Scan(void)
Scans the given frequency list.
fbox_chan_map_t GetChannels(void)
bool first(void)
Wrap QSqlQuery::first() so we can display the query results.
Definition: mythdbcon.cpp:793
static QString GetVideoDevice(uint inputid)
Definition: cardutil.h:273
bool prepare(const QString &query)
QSqlQuery::prepare() is not thread safe in Qt <= 3.3.2.
Definition: mythdbcon.cpp:808
#define LOG(_MASK_, _LEVEL_, _STRING_)
Definition: mythlogging.h:41
IPTVTuningData m_tuning
static bool parse_chan_info(const QString &rawdata, IPTVChannelInfo &info, QString &channum, int &nextChanNum, uint &lineNum)
void start(QThread::Priority p=QThread::InheritPriority)
Tell MThread to start running the thread in the near future.
Definition: mthread.cpp:294
void ScanComplete(void)
Definition: scanmonitor.cpp:98
IPTVChannelFetcher(uint cardid, QString inputname, uint sourceid, bool is_mpts, ScanMonitor *monitor=nullptr)
bool isFinished(void) const
Definition: mthread.cpp:269
void run(void) override
fbox_chan_map_t m_channels
bool exec(void)
Wrap QSqlQuery::exec() so we can display SQL.
Definition: mythdbcon.cpp:603
static void DBError(const QString &where, const MSqlQuery &query)
Definition: mythdb.cpp:179
#define LOC
void Stop(void)
Stops the scanning thread running.
QUrl GetDataURL(void) const
void SetMessage(const QString &status)
QMap< QString, IPTVChannelInfo > fbox_chan_map_t