MythTV  master
TemplateFinder.cpp
Go to the documentation of this file.
1 // ANSI C headers
2 #include <cmath>
3 #include <cstdlib>
4 
5 // Qt headers
6 #include <QFile>
7 #include <QFileInfo>
8 #include <QTextStream>
9 #include <utility>
10 
11 // MythTV headers
12 #include "mythplayer.h"
13 #include "mythcorecontext.h" /* gContext */
14 #include "mythframe.h" /* VideoFrame */
15 #include "mythdate.h"
16 #include "mythsystemlegacy.h"
17 #include "exitcodes.h"
18 
19 // Commercial Flagging headers
20 #include "CommDetector2.h"
21 #include "pgm.h"
22 #include "PGMConverter.h"
23 #include "BorderDetector.h"
24 #include "EdgeDetector.h"
25 #include "TemplateFinder.h"
26 
27 extern "C" {
28  #include "libavutil/imgutils.h"
29  }
30 
31 using namespace commDetector2;
32 
33 namespace {
34 
35 //returns true on success, false otherwise
36 bool writeJPG(const QString& prefix, const AVFrame *img, int imgheight)
37 {
38  const int imgwidth = img->linesize[0];
39  QFileInfo jpgfi(prefix + ".jpg");
40  if (!jpgfi.exists())
41  {
42  QFile pgmfile(prefix + ".pgm");
43  if (!pgmfile.exists())
44  {
45  QByteArray pfname = pgmfile.fileName().toLocal8Bit();
46  if (pgm_write(img->data[0], imgwidth, imgheight,
47  pfname.constData()))
48  {
49  return false;
50  }
51  }
52 
53  QString cmd = QString("convert -quality 50 -resize 192x144 %1 %2")
54  .arg(pgmfile.fileName(), jpgfi.filePath());
55  if (myth_system(cmd) != GENERIC_EXIT_OK)
56  return false;
57 
58  if (!pgmfile.remove())
59  {
60  LOG(VB_COMMFLAG, LOG_ERR,
61  QString("TemplateFinder.writeJPG error removing %1 (%2)")
62  .arg(pgmfile.fileName(), strerror(errno)));
63  return false;
64  }
65  }
66  return true;
67 }
68 
69 int
70 pgm_scorepixels(unsigned int *scores, int width, int row, int col,
71  const AVFrame *src, int srcheight)
72 {
73  /* Every time a pixel is an edge, give it a point. */
74  const int srcwidth = src->linesize[0];
75 
76  for (int rr = 0; rr < srcheight; rr++)
77  {
78  for (int cc = 0; cc < srcwidth; cc++)
79  {
80  if (src->data[0][rr * srcwidth + cc])
81  scores[(row + rr) * width + col + cc]++;
82  }
83  }
84 
85  return 0;
86 }
87 
88 int
89 sort_ascending(const void *aa, const void *bb)
90 {
91  return *(unsigned int*)aa - *(unsigned int*)bb;
92 }
93 
94 float
95 bounding_score(const AVFrame *img, int row, int col, int width, int height)
96 {
97  /* Return a value between [0..1] */
98  const int imgwidth = img->linesize[0];
99 
100  uint score = 0;
101  int rr2 = row + height;
102  int cc2 = col + width;
103  for (int rr = row; rr < rr2; rr++)
104  {
105  for (int cc = col; cc < cc2; cc++)
106  {
107  if (img->data[0][rr * imgwidth + cc])
108  score++;
109  }
110  }
111  return (float)score / (width * height);
112 }
113 
114 bool
115 rowisempty(const AVFrame *img, int row, int col, int width)
116 {
117  const int imgwidth = img->linesize[0];
118  for (int cc = col; cc < col + width; cc++)
119  if (img->data[0][row * imgwidth + cc])
120  return false;
121  return true;
122 }
123 
124 bool
125 colisempty(const AVFrame *img, int col, int row, int height)
126 {
127  const int imgwidth = img->linesize[0];
128  for (int rr = row; rr < row + height; rr++)
129  if (img->data[0][rr * imgwidth + col])
130  return false;
131  return true;
132 }
133 
134 int
135 bounding_box(const AVFrame *img, int imgheight,
136  int minrow, int mincol, int maxrow1, int maxcol1,
137  int *prow, int *pcol, int *pwidth, int *pheight)
138 {
139  const int imgwidth = img->linesize[0];
140  /*
141  * TUNABLE:
142  *
143  * Maximum logo size, expressed as a percentage of the content area
144  * (adjusting for letterboxing and pillarboxing).
145  */
146  static constexpr int kMaxWidthPct = 20;
147  static constexpr int kMaxHeightPct = 20;
148 
149  /*
150  * TUNABLE:
151  *
152  * Safety margin to avoid cutting too much of the logo.
153  * Higher values cut more, but avoid noise as part of the template..
154  * Lower values cut less, but can include noise as part of the template.
155  */
156  const int VERTSLOP = std::max(4, imgheight * 1 / 15);
157  const int HORIZSLOP = std::max(4, imgwidth * 1 / 20);
158 
159  int maxwidth = (maxcol1 - mincol) * kMaxWidthPct / 100;
160  int maxheight = (maxrow1 - minrow) * kMaxHeightPct / 100;
161 
162  int row = minrow;
163  int col = mincol;
164  int width = maxcol1 - mincol;
165  int height = maxrow1 - minrow;
166  int newrow = 0;
167  int newcol = 0;
168  int newright = 0;
169  int newbottom = 0;
170 
171  for (;;)
172  {
173  float newscore = NAN;
174  bool improved = false;
175 
176  LOG(VB_COMMFLAG, LOG_INFO, QString("bounding_box %1x%2@(%3,%4)")
177  .arg(width).arg(height).arg(col).arg(row));
178 
179  /* Chop top. */
180  float score = bounding_score(img, row, col, width, height);
181  newrow = row;
182  for (int ii = 1; ii < height; ii++)
183  {
184  if ((newscore = bounding_score(img, row + ii, col,
185  width, height - ii)) < score)
186  break;
187  score = newscore;
188  newrow = row + ii;
189  improved = true;
190  }
191 
192  /* Chop left. */
193  score = bounding_score(img, row, col, width, height);
194  newcol = col;
195  for (int ii = 1; ii < width; ii++)
196  {
197  if ((newscore = bounding_score(img, row, col + ii,
198  width - ii, height)) < score)
199  break;
200  score = newscore;
201  newcol = col + ii;
202  improved = true;
203  }
204 
205  /* Chop bottom. */
206  score = bounding_score(img, row, col, width, height);
207  newbottom = row + height;
208  for (int ii = 1; ii < height; ii++)
209  {
210  if ((newscore = bounding_score(img, row, col,
211  width, height - ii)) < score)
212  break;
213  score = newscore;
214  newbottom = row + height - ii;
215  improved = true;
216  }
217 
218  /* Chop right. */
219  score = bounding_score(img, row, col, width, height);
220  newright = col + width;
221  for (int ii = 1; ii < width; ii++)
222  {
223  if ((newscore = bounding_score(img, row, col,
224  width - ii, height)) < score)
225  break;
226  score = newscore;
227  newright = col + width - ii;
228  improved = true;
229  }
230 
231  if (!improved)
232  break;
233 
234  row = newrow;
235  col = newcol;
236  width = newright - newcol;
237  height = newbottom - newrow;
238 
239  /*
240  * Noise edge pixels in the frequency template can sometimes stretch
241  * the template area to be larger than it should be.
242  *
243  * However, noise needs to be distinguished from a uniform distribution
244  * of noise pixels (e.g., no real statically-located template). So if
245  * the template area is too "large", then some quadrant must have a
246  * clear majority of the edge pixels; otherwise we declare failure (no
247  * template found).
248  *
249  * Intuitively, we should simply repeat until a single bounding box is
250  * converged upon. However, this requires a more sophisticated
251  * bounding_score function that I don't feel like figuring out.
252  * Indefinitely repeating with the present bounding_score function will
253  * tend to chop off too much. Instead, simply do some sanity checks on
254  * the candidate template's size, and prune the template area and
255  * repeat if it is too "large".
256  */
257 
258  if (width > maxwidth)
259  {
260  /* Too wide; test left and right portions. */
261  int chop = width / 3;
262  int chopwidth = width - chop;
263 
264  float left = bounding_score(img, row, col, chopwidth, height);
265  float right = bounding_score(img, row, col + chop, chopwidth, height);
266  LOG(VB_COMMFLAG, LOG_INFO,
267  QString("bounding_box too wide (%1 > %2); left=%3, right=%4")
268  .arg(width).arg(maxwidth)
269  .arg(left, 0, 'f', 3).arg(right, 0, 'f', 3));
270  float minscore = std::min(left, right);
271  float maxscore = std::max(left, right);
272  if (maxscore < 3 * minscore / 2)
273  {
274  /*
275  * Edge pixel distribution too uniform; give up.
276  *
277  * XXX: also fails for horizontally-centered templates ...
278  */
279  LOG(VB_COMMFLAG, LOG_ERR, "bounding_box giving up (edge "
280  "pixels distributed too uniformly)");
281  return -1;
282  }
283 
284  if (left < right)
285  col += chop;
286  width -= chop;
287  continue;
288  }
289 
290  if (height > maxheight)
291  {
292  /* Too tall; test upper and lower portions. */
293  int chop = height / 3;
294  int chopheight = height - chop;
295 
296  float upper = bounding_score(img, row, col, width, chopheight);
297  float lower = bounding_score(img, row + chop, col, width, chopheight);
298  LOG(VB_COMMFLAG, LOG_INFO,
299  QString("bounding_box too tall (%1 > %2); upper=%3, lower=%4")
300  .arg(height).arg(maxheight)
301  .arg(upper, 0, 'f', 3).arg(lower, 0, 'f', 3));
302  float minscore = std::min(upper, lower);
303  float maxscore = std::max(upper, lower);
304  if (maxscore < 3 * minscore / 2)
305  {
306  /*
307  * Edge pixel distribution too uniform; give up.
308  *
309  * XXX: also fails for vertically-centered templates ...
310  */
311  LOG(VB_COMMFLAG, LOG_ERR, "bounding_box giving up (edge "
312  "pixel distribution too uniform)");
313  return -1;
314  }
315 
316  if (upper < lower)
317  row += chop;
318  height -= chop;
319  continue;
320  }
321 
322  break;
323  }
324 
325  /*
326  * The above "chop" algorithm often cuts off the outside edges of the
327  * logos because the outside edges don't contribute enough to the score. So
328  * compensate by now expanding the bounding box (up to a *SLOP pixels in
329  * each direction) to include all edge pixels.
330  */
331 
332  LOG(VB_COMMFLAG, LOG_INFO,
333  QString("bounding_box %1x%2@(%3,%4); horizslop=%5,vertslop=%6")
334  .arg(width).arg(height).arg(col).arg(row)
335  .arg(HORIZSLOP).arg(VERTSLOP));
336 
337  /* Expand upwards. */
338  newrow = row - 1;
339  for (;;)
340  {
341  if (newrow <= minrow)
342  {
343  newrow = minrow;
344  break;
345  }
346  if (row - newrow >= VERTSLOP)
347  {
348  newrow = row - VERTSLOP;
349  break;
350  }
351  if (rowisempty(img, newrow, col, width))
352  {
353  newrow++;
354  break;
355  }
356  newrow--;
357  }
358  newrow = std::max(minrow, newrow - 1); /* Empty row on top. */
359 
360  /* Expand leftwards. */
361  newcol = col - 1;
362  for (;;)
363  {
364  if (newcol <= mincol)
365  {
366  newcol = mincol;
367  break;
368  }
369  if (col - newcol >= HORIZSLOP)
370  {
371  newcol = col - HORIZSLOP;
372  break;
373  }
374  if (colisempty(img, newcol, row, height))
375  {
376  newcol++;
377  break;
378  }
379  newcol--;
380  }
381  newcol = std::max(mincol, newcol - 1); /* Empty column to left. */
382 
383  /* Expand rightwards. */
384  newright = col + width;
385  for (;;)
386  {
387  if (newright >= maxcol1)
388  {
389  newright = maxcol1;
390  break;
391  }
392  if (newright - (col + width) >= HORIZSLOP)
393  {
394  newright = col + width + HORIZSLOP;
395  break;
396  }
397  if (colisempty(img, newright, row, height))
398  break;
399  newright++;
400  }
401  newright = std::min(maxcol1, newright + 1); /* Empty column to right. */
402 
403  /* Expand downwards. */
404  newbottom = row + height;
405  for (;;)
406  {
407  if (newbottom >= maxrow1)
408  {
409  newbottom = maxrow1;
410  break;
411  }
412  if (newbottom - (row + height) >= VERTSLOP)
413  {
414  newbottom = row + height + VERTSLOP;
415  break;
416  }
417  if (rowisempty(img, newbottom, col, width))
418  break;
419  newbottom++;
420  }
421  newbottom = std::min(maxrow1, newbottom + 1); /* Empty row on bottom. */
422 
423  row = newrow;
424  col = newcol;
425  width = newright - newcol;
426  height = newbottom - newrow;
427 
428  LOG(VB_COMMFLAG, LOG_INFO, QString("bounding_box %1x%2@(%3,%4)")
429  .arg(width).arg(height).arg(col).arg(row));
430 
431  *prow = row;
432  *pcol = col;
433  *pwidth = width;
434  *pheight = height;
435  return 0;
436 }
437 
438 bool
439 template_alloc(const unsigned int *scores, int width, int height,
440  int minrow, int mincol, int maxrow1, int maxcol1, AVFrame *tmpl,
441  int *ptmplrow, int *ptmplcol, int *ptmplwidth, int *ptmplheight,
442  bool debug_edgecounts, const QString& debugdir)
443 {
444  /*
445  * TUNABLE:
446  *
447  * Higher values select for "stronger" pixels to be in the template, but
448  * weak pixels might be missed.
449  *
450  * Lower values allow more pixels to be included as part of the template,
451  * but strong non-template pixels might be included.
452  */
453  static constexpr float kMinScorePctile = 0.998;
454 
455  const int nn = width * height;
456  int ii = 0;
457  int first = 0;
458  int last = 0;
459  unsigned int threshscore = 0;
460  AVFrame thresh;
461 
462  if (av_image_alloc(thresh.data, thresh.linesize,
463  width, height, AV_PIX_FMT_GRAY8, IMAGE_ALIGN))
464  {
465  LOG(VB_COMMFLAG, LOG_ERR,
466  QString("template_alloc av_image_alloc thresh (%1x%2) failed")
467  .arg(width).arg(height));
468  return false;
469  }
470 
471  uint *sortedscores = new unsigned int[nn];
472  memcpy(sortedscores, scores, nn * sizeof(*sortedscores));
473  qsort(sortedscores, nn, sizeof(*sortedscores), sort_ascending);
474 
475  if (sortedscores[0] == sortedscores[nn - 1])
476  {
477  /* All pixels in the template area look the same; no template. */
478  LOG(VB_COMMFLAG, LOG_ERR,
479  QString("template_alloc: %1x%2 pixels all identical!")
480  .arg(width).arg(height));
481  goto free_thresh;
482  }
483 
484  /* Threshold the edge frequences. */
485 
486  ii = (int)roundf(nn * kMinScorePctile);
487  threshscore = sortedscores[ii];
488  for (first = ii; first > 0 && sortedscores[first] == threshscore; first--)
489  ;
490  if (sortedscores[first] != threshscore)
491  first++;
492  for (last = ii; last < nn - 1 && sortedscores[last] == threshscore; last++)
493  ;
494  if (sortedscores[last] != threshscore)
495  last--;
496 
497  LOG(VB_COMMFLAG, LOG_INFO, QString("template_alloc wanted %1, got %2-%3")
498  .arg(kMinScorePctile, 0, 'f', 6)
499  .arg((float)first / nn, 0, 'f', 6)
500  .arg((float)last / nn, 0, 'f', 6));
501 
502  for (ii = 0; ii < nn; ii++)
503  thresh.data[0][ii] = scores[ii] >= threshscore ? UCHAR_MAX : 0;
504 
505  if (debug_edgecounts)
506  {
507  /* Scores, rescaled to [0..UCHAR_MAX]. */
508  AVFrame scored;
509  if (av_image_alloc(scored.data, scored.linesize,
510  width, height, AV_PIX_FMT_GRAY8, IMAGE_ALIGN))
511  {
512  LOG(VB_COMMFLAG, LOG_ERR,
513  QString("template_alloc av_image_alloc scored (%1x%2) failed")
514  .arg(width).arg(height));
515  goto free_thresh;
516  }
517  unsigned int maxscore = sortedscores[nn - 1];
518  for (ii = 0; ii < nn; ii++)
519  scored.data[0][ii] = scores[ii] * UCHAR_MAX / maxscore;
520  bool success = writeJPG(debugdir + "/TemplateFinder-scores", &scored,
521  height);
522  av_freep(&scored.data[0]);
523  if (!success)
524  goto free_thresh;
525 
526  /* Thresholded scores. */
527  if (!writeJPG(debugdir + "/TemplateFinder-edgecounts", &thresh, height))
528  goto free_thresh;
529  }
530 
531  /* Crop to a minimal bounding box. */
532 
533  if (bounding_box(&thresh, height, minrow, mincol, maxrow1, maxcol1,
534  ptmplrow, ptmplcol, ptmplwidth, ptmplheight))
535  goto free_thresh;
536 
537  if ((uint)(*ptmplwidth * *ptmplheight) > USHRT_MAX)
538  {
539  /* Max value of data type of TemplateMatcher::edgematch */
540  LOG(VB_COMMFLAG, LOG_ERR,
541  QString("template_alloc bounding_box too big (%1x%2)")
542  .arg(*ptmplwidth).arg(*ptmplheight));
543  goto free_thresh;
544  }
545 
546  if (av_image_alloc(tmpl->data, tmpl->linesize,
547  *ptmplwidth, *ptmplheight, AV_PIX_FMT_GRAY8, IMAGE_ALIGN))
548  {
549  LOG(VB_COMMFLAG, LOG_ERR,
550  QString("template_alloc av_image_alloc tmpl (%1x%2) failed")
551  .arg(*ptmplwidth).arg(*ptmplheight));
552  goto free_thresh;
553  }
554 
555  if (pgm_crop(tmpl, &thresh, height, *ptmplrow, *ptmplcol,
556  *ptmplwidth, *ptmplheight))
557  goto free_thresh;
558 
559  delete []sortedscores;
560  av_freep(&thresh.data[0]);
561 
562  return true;
563 
564 free_thresh:
565  delete []sortedscores;
566  av_freep(&thresh.data[0]);
567  return false;
568 }
569 
570 bool
571 analyzeFrameDebug(long long frameno, const AVFrame *pgm, int pgmheight,
572  const AVFrame *cropped, const AVFrame *edges, int cropheight,
573  int croprow, int cropcol, bool debug_frames, const QString& debugdir)
574 {
575  static constexpr int kDelta = 24;
576  static int s_lastrow;
577  static int s_lastcol;
578  static int s_lastwidth;
579  static int s_lastheight;
580  const int cropwidth = cropped->linesize[0];
581 
582  int rowsame = abs(s_lastrow - croprow) <= kDelta ? 1 : 0;
583  int colsame = abs(s_lastcol - cropcol) <= kDelta ? 1 : 0;
584  int widthsame = abs(s_lastwidth - cropwidth) <= kDelta ? 1 : 0;
585  int heightsame = abs(s_lastheight - cropheight) <= kDelta ? 1 : 0;
586 
587  if (frameno > 0 && rowsame + colsame + widthsame + heightsame >= 3)
588  return true;
589 
590  LOG(VB_COMMFLAG, LOG_INFO,
591  QString("TemplateFinder Frame %1: %2x%3@(%4,%5)")
592  .arg(frameno, 5)
593  .arg(cropwidth).arg(cropheight)
594  .arg(cropcol).arg(croprow));
595 
596  s_lastrow = croprow;
597  s_lastcol = cropcol;
598  s_lastwidth = cropwidth;
599  s_lastheight = cropheight;
600 
601  if (debug_frames)
602  {
603  QString base = QString("%1/TemplateFinder-%2")
604  .arg(debugdir).arg(frameno, 5, 10, QChar('0'));
605 
606  /* PGM greyscale image of frame. */
607  if (!writeJPG(base, pgm, pgmheight))
608  return false;
609 
610  /* Cropped template area of frame. */
611  if (!writeJPG(base + "-cropped", cropped, cropheight))
612  return false;
613 
614  /* Edges of cropped template area of frame. */
615  if (!writeJPG(base + "-edges", edges, cropheight))
616  return false;
617  }
618 
619  return true;
620 }
621 
622 /* NOLINTNEXTLINE(readability-non-const-parameter) */
623 bool readTemplate(const QString& datafile, int *prow, int *pcol, int *pwidth, int *pheight,
624  const QString& tmplfile, AVFrame *tmpl, bool *pvalid)
625 {
626  QFile dfile(datafile);
627  QFileInfo dfileinfo(dfile);
628 
629  if (!dfile.open(QIODevice::ReadOnly))
630  return false;
631 
632  if (!dfileinfo.size())
633  {
634  /* Dummy file: no template. */
635  *pvalid = false;
636  return true;
637  }
638 
639  QTextStream stream(&dfile);
640  stream >> *prow >> *pcol >> *pwidth >> *pheight;
641  dfile.close();
642 
643  if (av_image_alloc(tmpl->data, tmpl->linesize,
644  *pwidth, *pheight, AV_PIX_FMT_GRAY8, IMAGE_ALIGN))
645  {
646  LOG(VB_COMMFLAG, LOG_ERR,
647  QString("readTemplate av_image_alloc %1 (%2x%3) failed")
648  .arg(tmplfile).arg(*pwidth).arg(*pheight));
649  return false;
650  }
651 
652  QByteArray tmfile = tmplfile.toLatin1();
653  if (pgm_read(tmpl->data[0], *pwidth, *pheight, tmfile.constData()))
654  {
655  av_freep(&tmpl->data[0]);
656  return false;
657  }
658 
659  *pvalid = true;
660  return true;
661 }
662 
663 void
664 writeDummyTemplate(const QString& datafile)
665 {
666  /* Leave a 0-byte file. */
667  QFile dfile(datafile);
668 
669  if (!dfile.open(QIODevice::WriteOnly | QIODevice::Truncate) &&
670  dfile.exists())
671  (void)dfile.remove();
672 }
673 
674 bool
675 writeTemplate(const QString& tmplfile, const AVFrame *tmpl, const QString& datafile,
676  int row, int col, int width, int height)
677 {
678  QFile tfile(tmplfile);
679 
680  QByteArray tmfile = tmplfile.toLatin1();
681  if (pgm_write(tmpl->data[0], width, height, tmfile.constData()))
682  return false;
683 
684  QFile dfile(datafile);
685  if (!dfile.open(QIODevice::WriteOnly))
686  return false;
687 
688  QTextStream stream(&dfile);
689  stream << row << " " << col << "\n" << width << " " << height << "\n";
690  dfile.close();
691  return true;
692 }
693 
694 }; /* namespace */
695 
696 TemplateFinder::TemplateFinder(std::shared_ptr<PGMConverter> pgmc,
697  std::shared_ptr<BorderDetector> bd,
698  std::shared_ptr<EdgeDetector> ed,
699  MythPlayer *player, std::chrono::seconds proglen,
700  const QString& debugdir)
701  : m_pgmConverter(std::move(pgmc)),
702  m_borderDetector(std::move(bd)),
703  m_edgeDetector(std::move(ed)),
704  m_sampleTime(std::min(proglen / 2, 20 * 60s)),
705  m_debugDir(debugdir),
706  m_debugData(debugdir + "/TemplateFinder.txt"),
707  m_debugTmpl(debugdir + "/TemplateFinder.pgm")
708 {
709  /*
710  * TUNABLE:
711  *
712  * The number of frames desired for sampling to build the template.
713  *
714  * Higher values should yield a more accurate template, but requires more
715  * time.
716  */
717  unsigned int samplesNeeded = 300;
718 
719  /*
720  * TUNABLE:
721  *
722  * The leading amount of time (in seconds) to sample frames for building up
723  * the possible template, and the interval between frames for analysis.
724  * This affects how soon flagging can start after a recording has begun
725  * (a.k.a. "real-time flagging").
726  *
727  * Sample half of the program length or 20 minutes, whichever is less.
728  */
729  // m_sampleTime
730 
731  const float fps = player->GetFrameRate();
732 
733  m_frameInterval = (int)roundf(m_sampleTime.count() * fps / samplesNeeded);
734  m_endFrame = 0 + (long long)m_frameInterval * samplesNeeded - 1;
735 
736  LOG(VB_COMMFLAG, LOG_INFO,
737  QString("TemplateFinder: sampleTime=%1s, samplesNeeded=%2, endFrame=%3")
738  .arg(m_sampleTime.count()).arg(samplesNeeded).arg(m_endFrame));
739 
740  /*
741  * debugLevel:
742  * 0: no extra debugging
743  * 1: cache computations into debugdir [O(1) files]
744  * 2: extra verbosity [O(nframes)]
745  * 3: dump frames into debugdir [O(nframes) files]
746  */
747  m_debugLevel = gCoreContext->GetNumSetting("TemplateFinderDebugLevel", 0);
748 
749  if (m_debugLevel >= 1)
750  {
752  QString("TemplateFinder debugLevel %1").arg(m_debugLevel));
753 
754  m_debugTemplate = true;
755  m_debugEdgeCounts = true;
756 
757  if (m_debugLevel >= 3)
758  m_debugFrames = true;
759  }
760 }
761 
763 {
764  delete []m_scores;
765  av_freep(&m_tmpl.data[0]);
766  av_freep(&m_cropped.data[0]);
767 }
768 
770 TemplateFinder::MythPlayerInited(MythPlayer *player, long long nframes)
771 {
772  /*
773  * Only detect edges in portions of the frame where we expect to find
774  * a template. This serves two purposes:
775  *
776  * - Speed: reduce search space.
777  * - Correctness (insofar as the assumption of template location is
778  * correct): don't "pollute" the set of candidate template edges with
779  * the "content" edges in the non-template portions of the frame.
780  */
781  QString tmpldims;
782  QString playerdims;
783 
784  (void)nframes; /* gcc */
785  QSize buf_dim = player->GetVideoBufferSize();
786  m_width = buf_dim.width();
787  m_height = buf_dim.height();
788  playerdims = QString("%1x%2").arg(m_width).arg(m_height);
789 
790  if (m_debugTemplate)
791  {
792  if ((m_tmplDone = readTemplate(m_debugData, &m_tmplRow, &m_tmplCol,
794  &m_tmplValid)))
795  {
796  tmpldims = m_tmplValid ? QString("%1x%2@(%3,%4)")
797  .arg(m_tmplWidth).arg(m_tmplHeight).arg(m_tmplCol).arg(m_tmplRow) :
798  "no template";
799 
800  LOG(VB_COMMFLAG, LOG_INFO,
801  QString("TemplateFinder::MythPlayerInited read %1: %2")
802  .arg(m_debugTmpl, tmpldims));
803  }
804  }
805 
806  if (m_pgmConverter->MythPlayerInited(player))
807  goto free_tmpl;
808 
809  if (m_borderDetector->MythPlayerInited(player))
810  goto free_tmpl;
811 
812  if (m_tmplDone)
813  {
814  if (m_tmplValid)
815  {
816  LOG(VB_COMMFLAG, LOG_INFO,
817  QString("TemplateFinder::MythPlayerInited %1 of %2 (%3)")
818  .arg(tmpldims, playerdims, m_debugTmpl));
819  }
820  return ANALYZE_FINISHED;
821  }
822 
823  LOG(VB_COMMFLAG, LOG_INFO,
824  QString("TemplateFinder::MythPlayerInited framesize %1")
825  .arg(playerdims));
826  m_scores = new unsigned int[m_width * m_height];
827 
828  return ANALYZE_OK;
829 
830 free_tmpl:
831  av_freep(&m_tmpl.data[0]);
832  return ANALYZE_FATAL;
833 }
834 
835 int
836 TemplateFinder::resetBuffers(int newwidth, int newheight)
837 {
838  if (m_cwidth == newwidth && m_cheight == newheight)
839  return 0;
840 
841  av_freep(&m_cropped.data[0]);
842 
843  if (av_image_alloc(m_cropped.data, m_cropped.linesize,
844  newwidth, newheight, AV_PIX_FMT_GRAY8, IMAGE_ALIGN))
845  {
846  LOG(VB_COMMFLAG, LOG_ERR,
847  QString("TemplateFinder::resetBuffers "
848  "av_image_alloc cropped (%1x%2) failed")
849  .arg(newwidth).arg(newheight));
850  return -1;
851  }
852 
853  m_cwidth = newwidth;
854  m_cheight = newheight;
855  return 0;
856 }
857 
859 TemplateFinder::analyzeFrame(const MythVideoFrame *frame, long long frameno,
860  long long *pNextFrame)
861 {
862  /*
863  * TUNABLE:
864  *
865  * When looking for edges in frames, select some percentile of
866  * squared-gradient magnitudes (intensities) as candidate edges. (This
867  * number conventionally should not go any lower than the 95th percentile;
868  * see edge_mark.)
869  *
870  * Higher values result in fewer edges; faint logos might not be picked up.
871  * Lower values result in more edges; non-logo edges might be picked up.
872  *
873  * The TemplateFinder accumulates all its state in the "scores" array to
874  * be processed later by TemplateFinder::finished.
875  */
876  const int FRAMESGMPCTILE = 90;
877 
878  /*
879  * TUNABLE:
880  *
881  * Exclude some portion of the center of the frame from edge analysis.
882  * Elminate false edge-detection logo positives from talking-host types of
883  * shows where the high-contrast host and clothes (e.g., tie against white
884  * shirt against dark jacket) dominates the edges.
885  *
886  * This has a nice side-effect of reducing the area to be examined (speed
887  * optimization).
888  */
889  static constexpr float kExcludeWidth = 0.5;
890  static constexpr float kExcludeHeight = 0.5;
891 
892  int pgmwidth= 0;
893  int pgmheight = 0;
894  int croprow= 0;
895  int cropcol = 0;
896  int cropwidth = 0;
897  int cropheight = 0;
898 
899  if (frameno < m_nextFrame)
900  {
901  *pNextFrame = m_nextFrame;
902  return ANALYZE_OK;
903  }
904 
905  m_nextFrame = frameno + m_frameInterval;
906  *pNextFrame = std::min(m_endFrame, m_nextFrame);
907 
908  const AVFrame *pgm = m_pgmConverter->getImage(frame, frameno, &pgmwidth, &pgmheight);
909  if (pgm == nullptr)
910  goto error;
911 
912  if (!m_borderDetector->getDimensions(pgm, pgmheight, frameno,
913  &croprow, &cropcol, &cropwidth, &cropheight))
914  {
915  /* Not a blank frame. */
916 
917  auto start = nowAsDuration<std::chrono::microseconds>();
918 
919  if (croprow < m_minContentRow)
920  m_minContentRow = croprow;
921  if (cropcol < m_minContentCol)
922  m_minContentCol = cropcol;
923  if (cropcol + cropwidth > m_maxContentCol1)
924  m_maxContentCol1 = cropcol + cropwidth;
925  if (croprow + cropheight > m_maxContentRow1)
926  m_maxContentRow1 = croprow + cropheight;
927 
928  if (resetBuffers(cropwidth, cropheight))
929  goto error;
930 
931  if (pgm_crop(&m_cropped, pgm, pgmheight, croprow, cropcol,
932  cropwidth, cropheight))
933  goto error;
934 
935  /*
936  * Translate the excluded area of the screen into "cropped"
937  * coordinates.
938  */
939  int excludewidth = (int)(pgmwidth * kExcludeWidth);
940  int excludeheight = (int)(pgmheight * kExcludeHeight);
941  int excluderow = (pgmheight - excludeheight) / 2 - croprow;
942  int excludecol = (pgmwidth - excludewidth) / 2 - cropcol;
943  (void)m_edgeDetector->setExcludeArea(excluderow, excludecol,
944  excludewidth, excludeheight);
945 
946  const AVFrame *edges =
947  m_edgeDetector->detectEdges(&m_cropped, cropheight, FRAMESGMPCTILE);
948  if (edges == nullptr)
949  goto error;
950 
951  if (pgm_scorepixels(m_scores, pgmwidth, croprow, cropcol,
952  edges, cropheight))
953  goto error;
954 
955  if (m_debugLevel >= 2)
956  {
957  if (!analyzeFrameDebug(frameno, pgm, pgmheight, &m_cropped, edges,
958  cropheight, croprow, cropcol, m_debugFrames, m_debugDir))
959  goto error;
960  }
961 
962  auto end = nowAsDuration<std::chrono::microseconds>();
963  m_analyzeTime += (end - start);
964  }
965 
966  if (m_nextFrame > m_endFrame)
967  return ANALYZE_FINISHED;
968 
969  return ANALYZE_OK;
970 
971 error:
972  LOG(VB_COMMFLAG, LOG_ERR,
973  QString("TemplateFinder::analyzeFrame error at frame %1")
974  .arg(frameno));
975 
976  if (m_nextFrame > m_endFrame)
977  return ANALYZE_FINISHED;
978 
979  return ANALYZE_ERROR;
980 }
981 
982 int
983 TemplateFinder::finished(long long nframes, bool final)
984 {
985  (void)nframes; /* gcc */
986  if (!m_tmplDone)
987  {
988  if (!template_alloc(m_scores, m_width, m_height,
993  {
994  if (final)
995  writeDummyTemplate(m_debugData);
996  }
997  else
998  {
999  if (final && m_debugTemplate)
1000  {
1001  if (!(m_tmplValid = writeTemplate(m_debugTmpl, &m_tmpl, m_debugData,
1003  goto free_tmpl;
1004 
1005  LOG(VB_COMMFLAG, LOG_INFO,
1006  QString("TemplateFinder::finished wrote %1"
1007  " and %2 [%3x%4@(%5,%6)]")
1008  .arg(m_debugTmpl, m_debugData)
1009  .arg(m_tmplWidth).arg(m_tmplHeight)
1010  .arg(m_tmplCol).arg(m_tmplRow));
1011  }
1012  }
1013 
1014  if (final)
1015  m_tmplDone = true;
1016  }
1017 
1018  m_borderDetector->setLogoState(this);
1019 
1020  return 0;
1021 
1022 free_tmpl:
1023  av_freep(&m_tmpl.data[0]);
1024  return -1;
1025 }
1026 
1027 int
1029 {
1030  if (m_pgmConverter->reportTime())
1031  return -1;
1032 
1033  if (m_borderDetector->reportTime())
1034  return -1;
1035 
1036  LOG(VB_COMMFLAG, LOG_INFO, QString("TF Time: analyze=%1s")
1037  .arg(strftimeval(m_analyzeTime)));
1038  return 0;
1039 }
1040 
1041 const struct AVFrame *
1042 TemplateFinder::getTemplate(int *prow, int *pcol, int *pwidth, int *pheight)
1043  const
1044 {
1045  if (m_tmplValid)
1046  {
1047  *prow = m_tmplRow;
1048  *pcol = m_tmplCol;
1049  *pwidth = m_tmplWidth;
1050  *pheight = m_tmplHeight;
1051  return &m_tmpl;
1052  }
1053  return nullptr;
1054 }
1055 
1056 /* vim: set expandtab tabstop=4 shiftwidth=4: */
TemplateFinder::m_tmplValid
bool m_tmplValid
Definition: TemplateFinder.h:102
GENERIC_EXIT_OK
#define GENERIC_EXIT_OK
Exited with no error.
Definition: exitcodes.h:10
TemplateFinder::m_debugTemplate
bool m_debugTemplate
Definition: TemplateFinder.h:99
TemplateFinder::m_debugData
QString m_debugData
Definition: TemplateFinder.h:97
error
static void error(const char *str,...)
Definition: vbi.cpp:35
TemplateFinder::m_width
int m_width
Definition: TemplateFinder.h:75
TemplateFinder::m_analyzeTime
std::chrono::microseconds m_analyzeTime
Definition: TemplateFinder.h:104
FrameAnalyzer::ANALYZE_OK
@ ANALYZE_OK
Definition: FrameAnalyzer.h:37
CommDetector2.h
pgm_read
int pgm_read(unsigned char *buf, int width, int height, const char *filename)
Definition: pgm.cpp:32
TemplateFinder::m_tmplWidth
int m_tmplWidth
Definition: TemplateFinder.h:87
TemplateFinder::m_minContentCol
int m_minContentCol
Definition: TemplateFinder.h:80
cc
Definition: cc.h:9
TemplateFinder::m_edgeDetector
std::shared_ptr< EdgeDetector > m_edgeDetector
Definition: TemplateFinder.h:68
TemplateFinder::m_maxContentCol1
int m_maxContentCol1
Definition: TemplateFinder.h:82
MythPlayer::GetFrameRate
float GetFrameRate(void) const
Definition: mythplayer.h:136
LOG
#define LOG(_MASK_, _LEVEL_, _QSTRING_)
Definition: mythlogging.h:23
mythplayer.h
mythframe.h
TemplateFinder::m_scores
unsigned int * m_scores
Definition: TemplateFinder.h:77
MythPlayer
Definition: mythplayer.h:86
TemplateFinder::m_maxContentRow1
int m_maxContentRow1
Definition: TemplateFinder.h:81
pgm_crop
int pgm_crop(AVFrame *dst, const AVFrame *src, int srcheight, int srcrow, int srccol, int cropwidth, int cropheight)
Definition: pgm.cpp:165
TemplateFinder::m_tmplDone
bool m_tmplDone
Definition: TemplateFinder.h:103
myth_system
uint myth_system(const QString &command, uint flags, std::chrono::seconds timeout)
Definition: mythsystemlegacy.cpp:505
commDetector2::createDebugDirectory
void createDebugDirectory(const QString &dirname, const QString &comment)
Definition: CommDetector2.cpp:227
hardwareprofile.distros.mythtv_data.data_mythtv.prefix
string prefix
Definition: data_mythtv.py:40
TemplateFinder::m_borderDetector
std::shared_ptr< BorderDetector > m_borderDetector
Definition: TemplateFinder.h:67
TemplateFinder::MythPlayerInited
enum analyzeFrameResult MythPlayerInited(MythPlayer *player, long long nframes) override
Definition: TemplateFinder.cpp:770
TemplateFinder::m_debugDir
QString m_debugDir
Definition: TemplateFinder.h:96
mythsystemlegacy.h
TemplateFinder::m_tmplRow
int m_tmplRow
Definition: TemplateFinder.h:85
AVFrame
struct AVFrame AVFrame
Definition: BorderDetector.h:15
pgm_write
int pgm_write(const unsigned char *buf, int width, int height, const char *filename)
Definition: pgm.cpp:80
mythdate.h
edgeDetector::sort_ascending
static int sort_ascending(const void *aa, const void *bb)
Definition: EdgeDetector.cpp:68
pgm.h
TemplateFinder::m_sampleTime
std::chrono::seconds m_sampleTime
Definition: TemplateFinder.h:70
PGMConverter.h
MythPlayer::GetVideoBufferSize
QSize GetVideoBufferSize(void) const
Definition: mythplayer.h:133
TemplateFinder::m_debugLevel
int m_debugLevel
Definition: TemplateFinder.h:95
TemplateFinder::m_tmpl
AVFrame m_tmpl
Definition: TemplateFinder.h:84
TemplateFinder::reportTime
int reportTime(void) const override
Definition: TemplateFinder.cpp:1028
hardwareprofile.smolt.long
long
Definition: smolt.py:76
FrameAnalyzer::analyzeFrameResult
analyzeFrameResult
Definition: FrameAnalyzer.h:36
TemplateFinder::TemplateFinder
TemplateFinder(std::shared_ptr< PGMConverter > pgmc, std::shared_ptr< BorderDetector > bd, std::shared_ptr< EdgeDetector > ed, MythPlayer *player, std::chrono::seconds proglen, const QString &debugdir)
Definition: TemplateFinder.cpp:696
FrameAnalyzer::ANALYZE_ERROR
@ ANALYZE_ERROR
Definition: FrameAnalyzer.h:38
TemplateFinder::m_cheight
int m_cheight
Definition: TemplateFinder.h:92
uint
unsigned int uint
Definition: compat.h:144
gCoreContext
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
Definition: mythcorecontext.cpp:60
commDetector2::strftimeval
QString strftimeval(std::chrono::microseconds usecs)
Definition: CommDetector2.cpp:266
EdgeDetector.h
TemplateFinder::finished
int finished(long long nframes, bool final) override
Definition: TemplateFinder.cpp:983
MythCoreContext::GetNumSetting
int GetNumSetting(const QString &key, int defaultval=0)
Definition: mythcorecontext.cpp:936
TemplateFinder::resetBuffers
int resetBuffers(int newwidth, int newheight)
Definition: TemplateFinder.cpp:836
TemplateFinder::m_debugTmpl
QString m_debugTmpl
Definition: TemplateFinder.h:98
BorderDetector.h
mythcorecontext.h
TemplateFinder::m_cropped
AVFrame m_cropped
Definition: TemplateFinder.h:90
std
Definition: mythchrono.h:23
FrameAnalyzer::ANALYZE_FATAL
@ ANALYZE_FATAL
Definition: FrameAnalyzer.h:40
commDetector2
Definition: CommDetector2.cpp:191
TemplateFinder::m_tmplHeight
int m_tmplHeight
Definition: TemplateFinder.h:88
TemplateFinder::m_debugEdgeCounts
bool m_debugEdgeCounts
Definition: TemplateFinder.h:100
FrameAnalyzer::ANALYZE_FINISHED
@ ANALYZE_FINISHED
Definition: FrameAnalyzer.h:39
TemplateFinder::m_debugFrames
bool m_debugFrames
Definition: TemplateFinder.h:101
TemplateFinder::m_minContentRow
int m_minContentRow
Definition: TemplateFinder.h:79
TemplateFinder::analyzeFrame
enum analyzeFrameResult analyzeFrame(const MythVideoFrame *frame, long long frameno, long long *pNextFrame) override
Definition: TemplateFinder.cpp:859
TemplateFinder.h
TemplateFinder::m_height
int m_height
Definition: TemplateFinder.h:76
TemplateFinder::m_frameInterval
int m_frameInterval
Definition: TemplateFinder.h:71
MythVideoFrame
Definition: mythframe.h:88
exitcodes.h
TemplateFinder::m_cwidth
int m_cwidth
Definition: TemplateFinder.h:91
TemplateFinder::m_nextFrame
long long m_nextFrame
Definition: TemplateFinder.h:73
TemplateFinder::getTemplate
const struct AVFrame * getTemplate(int *prow, int *pcol, int *pwidth, int *pheight) const
Definition: TemplateFinder.cpp:1042
TemplateFinder::~TemplateFinder
~TemplateFinder(void) override
Definition: TemplateFinder.cpp:762
TemplateFinder::m_endFrame
long long m_endFrame
Definition: TemplateFinder.h:72
TemplateFinder::m_tmplCol
int m_tmplCol
Definition: TemplateFinder.h:86
TemplateFinder::m_pgmConverter
std::shared_ptr< PGMConverter > m_pgmConverter
Definition: TemplateFinder.h:66