MythTV  master
iptvchannelfetcher.cpp
Go to the documentation of this file.
1 // -*- Mode: c++ -*-
2 
3 // Std C headers
4 #include <cmath>
5 #include <unistd.h>
6 
7 // Qt headers
8 #include <QFile>
9 #include <QTextStream>
10 
11 // MythTV headers
12 #include "mythcontext.h"
13 #include "cardutil.h"
14 #include "channelutil.h"
15 #include "iptvchannelfetcher.h"
16 #include "scanmonitor.h"
17 #include "mythlogging.h"
18 #include "mythdownloadmanager.h"
19 
20 #define LOC QString("IPTVChanFetch: ")
21 
22 static bool parse_chan_info(const QString &rawdata,
23  IPTVChannelInfo &info,
24  QString &channum,
25  uint &lineNum);
26 
27 static bool parse_extinf(const QString &line,
28  QString &channum,
29  QString &name);
30 
32  uint cardid, const QString &inputname, uint sourceid,
33  bool is_mpts, ScanMonitor *monitor) :
34  m_scan_monitor(monitor),
35  m_cardid(cardid), m_inputname(inputname),
36  m_sourceid(sourceid), m_is_mpts(is_mpts),
37  m_thread(new MThread("IPTVChannelFetcher", this))
38 {
39  LOG(VB_CHANNEL, LOG_INFO, LOC + QString("Has ScanMonitor %1")
40  .arg(monitor?"true":"false"));
41 }
42 
44 {
45  Stop();
46  delete m_thread;
47  m_thread = nullptr;
48 }
49 
54 {
55  m_lock.lock();
56 
57  while (m_thread_running)
58  {
59  m_stop_now = true;
60  m_lock.unlock();
61  m_thread->wait(5);
62  m_lock.lock();
63  }
64 
65  m_lock.unlock();
66 
67  m_thread->wait();
68 }
69 
71 {
72  while (!m_thread->isFinished())
73  m_thread->wait(500);
74 
75  LOG(VB_CHANNEL, LOG_INFO, LOC + QString("Found %1 channels")
76  .arg(m_channels.size()));
77  return m_channels;
78 }
79 
82 {
83  Stop();
84  m_stop_now = false;
85  m_thread->start();
86 }
87 
89 {
90  m_lock.lock();
91  m_thread_running = true;
92  m_lock.unlock();
93 
94  // Step 1/4 : Get info from DB
95  QString url = CardUtil::GetVideoDevice(m_cardid);
96 
97  if (m_stop_now || url.isEmpty())
98  {
99  LOG(VB_CHANNEL, LOG_INFO, LOC + "Playlist URL was empty");
100  QMutexLocker locker(&m_lock);
101  m_thread_running = false;
102  m_stop_now = true;
103  return;
104  }
105 
106  LOG(VB_CHANNEL, LOG_INFO, LOC + QString("Playlist URL: %1").arg(url));
107 
108  // Step 2/4 : Download
109  if (m_scan_monitor)
110  {
112  m_scan_monitor->ScanAppendTextToLog(tr("Downloading Playlist"));
113  }
114 
115  QString playlist = DownloadPlaylist(url);
116 
117  if (m_stop_now || playlist.isEmpty())
118  {
119  if (playlist.isNull() && m_scan_monitor)
120  {
122  QCoreApplication::translate("(Common)", "Error"));
124  m_scan_monitor->ScanErrored(tr("Downloading Playlist Failed"));
125  }
126  QMutexLocker locker(&m_lock);
127  m_thread_running = false;
128  m_stop_now = true;
129  return;
130  }
131 
132  // Step 3/4 : Process
133  if (m_scan_monitor)
134  {
136  m_scan_monitor->ScanAppendTextToLog(tr("Processing Playlist"));
137  }
138 
139  m_channels.clear();
140  m_channels = ParsePlaylist(playlist, this);
141 
142  // Step 4/4 : Finish up
143  if (m_scan_monitor)
144  m_scan_monitor->ScanAppendTextToLog(tr("Adding Channels"));
146 
147  LOG(VB_CHANNEL, LOG_INFO, LOC + QString("Found %1 channels")
148  .arg(m_channels.size()));
149 
150  if (!m_is_mpts)
151  {
152  fbox_chan_map_t::const_iterator it = m_channels.begin();
153  for (uint i = 1; it != m_channels.end(); ++it, ++i)
154  {
155  const QString& channum = it.key();
156  QString name = (*it).m_name;
157  QString xmltvid = (*it).m_xmltvid.isEmpty() ? "" : (*it).m_xmltvid;
158  uint programnumber = (*it).m_programNumber;
159  //: %1 is the channel number, %2 is the channel name
160  QString msg = tr("Channel #%1 : %2").arg(channum).arg(name);
161 
162  LOG(VB_CHANNEL, LOG_INFO, QString("Handling channel %1 %2")
163  .arg(channum).arg(name));
164 
165  int chanid = ChannelUtil::GetChanID(m_sourceid, channum);
166  if (chanid <= 0)
167  {
168  if (m_scan_monitor)
169  {
171  tr("Adding %1").arg(msg));
172  }
173  chanid = ChannelUtil::CreateChanID(m_sourceid, channum);
175  channum, programnumber, 0, 0,
176  false, false, false, QString(),
177  QString(), "Default", xmltvid);
178  ChannelUtil::CreateIPTVTuningData(chanid, (*it).m_tuning);
179  }
180  else
181  {
182  if (m_scan_monitor)
183  {
185  tr("Updating %1").arg(msg));
186  }
188  channum, programnumber, 0, 0,
189  false, false, false, QString(),
190  QString(), "Default", xmltvid);
191  ChannelUtil::UpdateIPTVTuningData(chanid, (*it).m_tuning);
192  }
193 
195  }
196 
197  if (m_scan_monitor)
198  {
199  m_scan_monitor->ScanAppendTextToLog(tr("Done"));
203  }
204  }
205 
206  QMutexLocker locker(&m_lock);
207  m_thread_running = false;
208  m_stop_now = true;
209 }
210 
212 {
213  uint minval = 35, range = 70 - minval;
214  uint pct = minval + (uint) truncf((((float)val) / m_chan_cnt) * range);
215  if (m_scan_monitor)
217 }
218 
220 {
221  uint minval = 70, range = 100 - minval;
222  uint pct = minval + (uint) truncf((((float)val) / m_chan_cnt) * range);
223  if (m_scan_monitor)
225 }
226 
227 void IPTVChannelFetcher::SetMessage(const QString &status)
228 {
229  if (m_scan_monitor)
231 }
232 
233 // This function is always called from a thread context.
234 QString IPTVChannelFetcher::DownloadPlaylist(const QString &url)
235 {
236  if (url.startsWith("file", Qt::CaseInsensitive))
237  {
238  QString ret = "";
239  QUrl qurl(url);
240  QFile file(qurl.toLocalFile());
241  if (!file.open(QIODevice::ReadOnly))
242  {
243  LOG(VB_GENERAL, LOG_ERR, LOC + QString("Opening '%1'")
244  .arg(qurl.toLocalFile()) + ENO);
245  return ret;
246  }
247 
248  QTextStream stream(&file);
249  while (!stream.atEnd())
250  ret += stream.readLine() + "\n";
251 
252  file.close();
253  return ret;
254  }
255 
256  // Use MythDownloadManager for http URLs
257  QByteArray data;
258  QString tmp;
259 
260  if (!GetMythDownloadManager()->download(url, &data))
261  {
262  LOG(VB_GENERAL, LOG_INFO, LOC +
263  QString("DownloadPlaylist failed to "
264  "download from %1").arg(url));
265  }
266  else
267  tmp = QString(data);
268 
269  return tmp.isNull() ? tmp : QString::fromUtf8(tmp.toLatin1().constData());
270 }
271 
272 static uint estimate_number_of_channels(const QString &rawdata)
273 {
274  uint result = 0;
275  uint numLine = 1;
276  while (true)
277  {
278  QString url = rawdata.section("\n", numLine, numLine, QString::SectionSkipEmpty);
279  if (url.isEmpty())
280  return result;
281 
282  ++numLine;
283  if (!url.startsWith("#")) // ignore comments
284  ++result;
285  }
286 }
287 
289  const QString &reallyrawdata, IPTVChannelFetcher *fetcher)
290 {
291  fbox_chan_map_t chanmap;
292 
293  QString rawdata = reallyrawdata;
294  rawdata.replace("\r\n", "\n");
295 
296  // Verify header is ok
297  QString header = rawdata.section("\n", 0, 0);
298  if (header != "#EXTM3U")
299  {
300  LOG(VB_GENERAL, LOG_ERR, LOC +
301  QString("Invalid channel list header (%1)").arg(header));
302 
303  if (fetcher)
304  {
305  fetcher->SetMessage(
306  tr("ERROR: M3U channel list is malformed"));
307  }
308 
309  return chanmap;
310  }
311 
312  // estimate number of channels
313  if (fetcher)
314  {
315  uint num_channels = estimate_number_of_channels(rawdata);
316  fetcher->SetTotalNumChannels(num_channels);
317 
318  LOG(VB_CHANNEL, LOG_INFO, LOC +
319  QString("Estimating there are %1 channels in playlist")
320  .arg(num_channels));
321  }
322 
323  // Parse each channel
324  uint lineNum = 1;
325  for (uint i = 1; true; ++i)
326  {
327  IPTVChannelInfo info;
328  QString channum;
329 
330  if (!parse_chan_info(rawdata, info, channum, lineNum))
331  break;
332 
333  QString msg = tr("Encountered malformed channel");
334  if (!channum.isEmpty())
335  {
336  chanmap[channum] = info;
337 
338  msg = QString("Parsing Channel #%1 : %2 : %3")
339  .arg(channum).arg(info.m_name)
340  .arg(info.m_tuning.GetDataURL().toString());
341  LOG(VB_CHANNEL, LOG_INFO, LOC + msg);
342 
343  msg.clear(); // don't tell fetcher
344  }
345 
346  if (fetcher)
347  {
348  if (!msg.isEmpty())
349  fetcher->SetMessage(msg);
350  fetcher->SetNumChannelsParsed(i);
351  }
352  }
353 
354  return chanmap;
355 }
356 
357 static bool parse_chan_info(const QString &rawdata,
358  IPTVChannelInfo &info,
359  QString &channum,
360  uint &lineNum)
361 {
362  // #EXTINF:0,2 - France 2 <-- duration,channum - channame
363  // #EXTMYTHTV:xmltvid=C2.telepoche.com <-- optional line (myth specific)
364  // #EXTMYTHTV:bitrate=BITRATE <-- optional line (myth specific)
365  // #EXTMYTHTV:fectype=FECTYPE <-- optional line (myth specific)
366  // The FECTYPE can be rfc2733, rfc5109, or smpte2022
367  // #EXTMYTHTV:fecurl0=URL <-- optional line (myth specific)
368  // #EXTMYTHTV:fecurl1=URL <-- optional line (myth specific)
369  // #EXTMYTHTV:fecbitrate0=BITRATE <-- optional line (myth specific)
370  // #EXTMYTHTV:fecbitrate1=BITRATE <-- optional line (myth specific)
371  // #EXTVLCOPT:program=program_number <-- optional line (used by MythTV and VLC)
372  // #... <-- ignored comments
373  // rtsp://maiptv.iptv.fr/iptvtv/201 <-- url
374 
375  QString name;
376  QMap<QString,QString> values;
377 
378  while (true)
379  {
380  QString line = rawdata.section("\n", lineNum, lineNum, QString::SectionSkipEmpty);
381  if (line.isEmpty())
382  return false;
383 
384  ++lineNum;
385  if (line.startsWith("#"))
386  {
387  if (line.startsWith("#EXTINF:"))
388  {
389  parse_extinf(line.mid(line.indexOf(':')+1), channum, name);
390  }
391  else if (line.startsWith("#EXTMYTHTV:"))
392  {
393  QString data = line.mid(line.indexOf(':')+1);
394  QString key = data.left(data.indexOf('='));
395  if (!key.isEmpty())
396  values[key] = data.mid(data.indexOf('=')+1);
397  }
398  else if (line.startsWith("#EXTVLCOPT:program="))
399  {
400  values["programnumber"] = line.mid(line.indexOf('=')+1);
401  }
402  continue;
403  }
404 
405  if (name.isEmpty())
406  return false;
407 
408  QMap<QString,QString>::const_iterator it = values.begin();
409  for (; it != values.end(); ++it)
410  {
411  LOG(VB_GENERAL, LOG_INFO, LOC +
412  QString("parse_chan_info [%1]='%2'")
413  .arg(it.key()).arg(*it));
414  }
415  info = IPTVChannelInfo(
416  name, values["xmltvid"],
417  line, values["bitrate"].toUInt(),
418  values["fectype"],
419  values["fecurl0"], values["fecbitrate0"].toUInt(),
420  values["fecurl1"], values["fecbitrate1"].toUInt(),
421  values["programnumber"].toUInt());
422  return true;
423  }
424 }
425 
426 static bool parse_extinf(const QString &line,
427  QString &channum,
428  QString &name)
429 {
430  // Parse extension portion, Freebox or SAT>IP format
431  QRegExp chanNumName1("^-?\\d+,(\\d+)(?:\\.\\s|\\s-\\s)(.*)$");
432  int pos = chanNumName1.indexIn(line);
433  if (pos != -1)
434  {
435  channum = chanNumName1.cap(1);
436  name = chanNumName1.cap(2);
437  return true;
438  }
439 
440  // Parse extension portion, A1 TV format
441  QRegExp chanNumName2("^-?\\d+\\s+[^,]*tvg-num=\"(\\d+)\"[^,]*,(.*)$");
442  pos = chanNumName2.indexIn(line);
443  if (pos != -1)
444  {
445  channum = chanNumName2.cap(1);
446  name = chanNumName2.cap(2);
447  return true;
448  }
449 
450  // Parse extension portion, Moviestar TV number then name
451  QRegExp chanNumName3("^-?\\d+,\\[(\\d+)\\]\\s+(.*)$");
452  pos = chanNumName3.indexIn(line);
453  if (pos != -1)
454  {
455  channum = chanNumName3.cap(1);
456  name = chanNumName3.cap(2);
457  return true;
458  }
459 
460  // Parse extension portion, Moviestar TV name then number
461  QRegExp chanNumName4("^-?\\d+,(.*)\\s+\\[(\\d+)\\]$");
462  pos = chanNumName4.indexIn(line);
463  if (pos != -1)
464  {
465  channum = chanNumName4.cap(2);
466  name = chanNumName4.cap(1);
467  return true;
468  }
469 
470  // Parse extension portion, russion iptv plugin style
471  QRegExp chanNumName5("^(-?\\d+)\\s+[^,]*,\\s*(.*)$");
472  pos = chanNumName5.indexIn(line);
473  if (pos != -1)
474  {
475  channum = chanNumName5.cap(1).simplified();
476  name = chanNumName5.cap(2).simplified();
477  bool ok;
478  int channel_number = channum.toInt (&ok);
479  if (ok && (channel_number > 0))
480  {
481  return true;
482  }
483  }
484 
485  // line is supposed to contain the "0,2 - France 2" part
486  QString msg = LOC +
487  QString("Invalid header in channel list line \n\t\t\tEXTINF:%1")
488  .arg(line);
489  LOG(VB_GENERAL, LOG_ERR, msg);
490  return false;
491 }
492 
493 /* vim: set expandtab tabstop=4 shiftwidth=4: */
void start(QThread::Priority=QThread::InheritPriority)
Tell MThread to start running the thread in the near future.
Definition: mthread.cpp:294
static bool CreateChannel(uint db_mplexid, uint db_sourceid, uint new_channel_id, const QString &callsign, const QString &service_name, const QString &chan_num, uint service_id, uint atsc_major_channel, uint atsc_minor_channel, bool use_on_air_guide, bool hidden, bool hidden_in_guide, const QString &freqid, QString icon=QString(), QString format="Default", QString xmltvid=QString(), QString default_authority=QString())
This is a wrapper around QThread that does several additional things.
Definition: mthread.h:46
static bool parse_extinf(const QString &line, QString &channum, QString &name)
static bool UpdateIPTVTuningData(uint channel_id, const IPTVTuningData &tuning)
IPTVChannelFetcher(uint cardid, const QString &inputname, uint sourceid, bool is_mpts, ScanMonitor *monitor=nullptr)
static bool parse_chan_info(const QString &rawdata, IPTVChannelInfo &info, QString &channum, uint &lineNum)
bool wait(unsigned long time=ULONG_MAX)
Wait for the MThread to exit, with a maximum timeout.
Definition: mthread.cpp:311
void ScanPercentComplete(int pct)
unsigned int uint
Definition: compat.h:140
QMap< QString, IPTVChannelInfo > fbox_chan_map_t
static guint32 * tmp
Definition: goom_core.c:35
static fbox_chan_map_t ParsePlaylist(const QString &rawdata, IPTVChannelFetcher *fetcher=nullptr)
void ScanErrored(const QString &error)
static int GetChanID(int db_mplexid, int service_transport_id, int major_channel, int minor_channel, int program_number)
static QString DownloadPlaylist(const QString &url)
MythDownloadManager * GetMythDownloadManager(void)
Gets the pointer to the MythDownloadManager singleton.
void SetTotalNumChannels(uint val)
static uint estimate_number_of_channels(const QString &rawdata)
void ScanAppendTextToLog(const QString &status)
static bool CreateIPTVTuningData(uint channel_id, const IPTVTuningData &tuning)
Definition: channelutil.h:140
const char * name
Definition: ParseText.cpp:328
#define ENO
This can be appended to the LOG args with "+".
Definition: mythlogging.h:99
static int CreateChanID(uint sourceid, const QString &chan_num)
Creates a unique channel ID for database use.
void Scan(void)
Scans the given frequency list.
static bool UpdateChannel(uint db_mplexid, uint source_id, uint channel_id, const QString &callsign, const QString &service_name, const QString &chan_num, uint service_id, uint atsc_major_channel, uint atsc_minor_channel, bool use_on_air_guide, bool hidden, bool hidden_in_guide, const QString &freqid=QString(), const QString &icon=QString(), QString format=QString(), const QString &xmltvid=QString(), const QString &default_authority=QString())
fbox_chan_map_t GetChannels(void)
static QString GetVideoDevice(uint inputid)
Definition: cardutil.h:273
#define LOG(_MASK_, _LEVEL_, _STRING_)
Definition: mythlogging.h:41
IPTVTuningData m_tuning
ScanMonitor * m_scan_monitor
void ScanComplete(void)
Definition: scanmonitor.cpp:98
bool isFinished(void) const
Definition: mthread.cpp:269
void run(void) override
fbox_chan_map_t m_channels
#define LOC
void Stop(void)
Stops the scanning thread running.
QUrl GetDataURL(void) const
void SetMessage(const QString &status)