MythTV  master
TemplateFinder.cpp
Go to the documentation of this file.
1 // POSIX headers
2 #include <sys/time.h> /* gettimeofday */
3 
4 // ANSI C headers
5 #include <cmath>
6 #include <cstdlib>
7 
8 // Qt headers
9 #include <QFile>
10 #include <QFileInfo>
11 #include <QTextStream>
12 #include <utility>
13 
14 // MythTV headers
15 #include "mythplayer.h"
16 #include "mythcorecontext.h" /* gContext */
17 #include "mythframe.h" /* VideoFrame */
18 #include "mythdate.h"
19 #include "mythsystemlegacy.h"
20 #include "exitcodes.h"
21 
22 // Commercial Flagging headers
23 #include "CommDetector2.h"
24 #include "pgm.h"
25 #include "PGMConverter.h"
26 #include "BorderDetector.h"
27 #include "EdgeDetector.h"
28 #include "TemplateFinder.h"
29 
30 extern "C" {
31  #include "libavutil/imgutils.h"
32  }
33 
34 using namespace commDetector2;
35 
36 namespace {
37 
38 //returns true on success, false otherwise
39 bool writeJPG(const QString& prefix, const AVFrame *img, int imgheight)
40 {
41  const int imgwidth = img->linesize[0];
42  QFileInfo jpgfi(prefix + ".jpg");
43  if (!jpgfi.exists())
44  {
45  QFile pgmfile(prefix + ".pgm");
46  if (!pgmfile.exists())
47  {
48  QByteArray pfname = pgmfile.fileName().toLocal8Bit();
49  if (pgm_write(img->data[0], imgwidth, imgheight,
50  pfname.constData()))
51  {
52  return false;
53  }
54  }
55 
56  QString cmd = QString("convert -quality 50 -resize 192x144 %1 %2")
57  .arg(pgmfile.fileName()).arg(jpgfi.filePath());
58  if (myth_system(cmd) != GENERIC_EXIT_OK)
59  return false;
60 
61  if (!pgmfile.remove())
62  {
63  LOG(VB_COMMFLAG, LOG_ERR,
64  QString("TemplateFinder.writeJPG error removing %1 (%2)")
65  .arg(pgmfile.fileName()).arg(strerror(errno)));
66  return false;
67  }
68  }
69  return true;
70 }
71 
72 int
73 pgm_scorepixels(unsigned int *scores, int width, int row, int col,
74  const AVFrame *src, int srcheight)
75 {
76  /* Every time a pixel is an edge, give it a point. */
77  const int srcwidth = src->linesize[0];
78 
79  for (int rr = 0; rr < srcheight; rr++)
80  {
81  for (int cc = 0; cc < srcwidth; cc++)
82  {
83  if (src->data[0][rr * srcwidth + cc])
84  scores[(row + rr) * width + col + cc]++;
85  }
86  }
87 
88  return 0;
89 }
90 
91 int
92 sort_ascending(const void *aa, const void *bb)
93 {
94  return *(unsigned int*)aa - *(unsigned int*)bb;
95 }
96 
97 float
98 bounding_score(const AVFrame *img, int row, int col, int width, int height)
99 {
100  /* Return a value between [0..1] */
101  const int imgwidth = img->linesize[0];
102 
103  uint score = 0;
104  int rr2 = row + height;
105  int cc2 = col + width;
106  for (int rr = row; rr < rr2; rr++)
107  {
108  for (int cc = col; cc < cc2; cc++)
109  {
110  if (img->data[0][rr * imgwidth + cc])
111  score++;
112  }
113  }
114  return (float)score / (width * height);
115 }
116 
117 bool
118 rowisempty(const AVFrame *img, int row, int col, int width)
119 {
120  const int imgwidth = img->linesize[0];
121  for (int cc = col; cc < col + width; cc++)
122  if (img->data[0][row * imgwidth + cc])
123  return false;
124  return true;
125 }
126 
127 bool
128 colisempty(const AVFrame *img, int col, int row, int height)
129 {
130  const int imgwidth = img->linesize[0];
131  for (int rr = row; rr < row + height; rr++)
132  if (img->data[0][rr * imgwidth + col])
133  return false;
134  return true;
135 }
136 
137 int
138 bounding_box(const AVFrame *img, int imgheight,
139  int minrow, int mincol, int maxrow1, int maxcol1,
140  int *prow, int *pcol, int *pwidth, int *pheight)
141 {
142  const int imgwidth = img->linesize[0];
143  /*
144  * TUNABLE:
145  *
146  * Maximum logo size, expressed as a percentage of the content area
147  * (adjusting for letterboxing and pillarboxing).
148  */
149  static constexpr int kMaxWidthPct = 20;
150  static constexpr int kMaxHeightPct = 20;
151 
152  /*
153  * TUNABLE:
154  *
155  * Safety margin to avoid cutting too much of the logo.
156  * Higher values cut more, but avoid noise as part of the template..
157  * Lower values cut less, but can include noise as part of the template.
158  */
159  const int VERTSLOP = max(4, imgheight * 1 / 15);
160  const int HORIZSLOP = max(4, imgwidth * 1 / 20);
161 
162  int maxwidth = (maxcol1 - mincol) * kMaxWidthPct / 100;
163  int maxheight = (maxrow1 - minrow) * kMaxHeightPct / 100;
164 
165  int row = minrow;
166  int col = mincol;
167  int width = maxcol1 - mincol;
168  int height = maxrow1 - minrow;
169  int newrow = 0;
170  int newcol = 0;
171  int newright = 0;
172  int newbottom = 0;
173 
174  for (;;)
175  {
176  float newscore = NAN;
177  bool improved = false;
178 
179  LOG(VB_COMMFLAG, LOG_INFO, QString("bounding_box %1x%2@(%3,%4)")
180  .arg(width).arg(height).arg(col).arg(row));
181 
182  /* Chop top. */
183  float score = bounding_score(img, row, col, width, height);
184  newrow = row;
185  for (int ii = 1; ii < height; ii++)
186  {
187  if ((newscore = bounding_score(img, row + ii, col,
188  width, height - ii)) < score)
189  break;
190  score = newscore;
191  newrow = row + ii;
192  improved = true;
193  }
194 
195  /* Chop left. */
196  score = bounding_score(img, row, col, width, height);
197  newcol = col;
198  for (int ii = 1; ii < width; ii++)
199  {
200  if ((newscore = bounding_score(img, row, col + ii,
201  width - ii, height)) < score)
202  break;
203  score = newscore;
204  newcol = col + ii;
205  improved = true;
206  }
207 
208  /* Chop bottom. */
209  score = bounding_score(img, row, col, width, height);
210  newbottom = row + height;
211  for (int ii = 1; ii < height; ii++)
212  {
213  if ((newscore = bounding_score(img, row, col,
214  width, height - ii)) < score)
215  break;
216  score = newscore;
217  newbottom = row + height - ii;
218  improved = true;
219  }
220 
221  /* Chop right. */
222  score = bounding_score(img, row, col, width, height);
223  newright = col + width;
224  for (int ii = 1; ii < width; ii++)
225  {
226  if ((newscore = bounding_score(img, row, col,
227  width - ii, height)) < score)
228  break;
229  score = newscore;
230  newright = col + width - ii;
231  improved = true;
232  }
233 
234  if (!improved)
235  break;
236 
237  row = newrow;
238  col = newcol;
239  width = newright - newcol;
240  height = newbottom - newrow;
241 
242  /*
243  * Noise edge pixels in the frequency template can sometimes stretch
244  * the template area to be larger than it should be.
245  *
246  * However, noise needs to be distinguished from a uniform distribution
247  * of noise pixels (e.g., no real statically-located template). So if
248  * the template area is too "large", then some quadrant must have a
249  * clear majority of the edge pixels; otherwise we declare failure (no
250  * template found).
251  *
252  * Intuitively, we should simply repeat until a single bounding box is
253  * converged upon. However, this requires a more sophisticated
254  * bounding_score function that I don't feel like figuring out.
255  * Indefinitely repeating with the present bounding_score function will
256  * tend to chop off too much. Instead, simply do some sanity checks on
257  * the candidate template's size, and prune the template area and
258  * repeat if it is too "large".
259  */
260 
261  if (width > maxwidth)
262  {
263  /* Too wide; test left and right portions. */
264  int chop = width / 3;
265  int chopwidth = width - chop;
266 
267  float left = bounding_score(img, row, col, chopwidth, height);
268  float right = bounding_score(img, row, col + chop, chopwidth, height);
269  LOG(VB_COMMFLAG, LOG_INFO,
270  QString("bounding_box too wide (%1 > %2); left=%3, right=%4")
271  .arg(width).arg(maxwidth)
272  .arg(left, 0, 'f', 3).arg(right, 0, 'f', 3));
273  float minscore = min(left, right);
274  float maxscore = max(left, right);
275  if (maxscore < 3 * minscore / 2)
276  {
277  /*
278  * Edge pixel distribution too uniform; give up.
279  *
280  * XXX: also fails for horizontally-centered templates ...
281  */
282  LOG(VB_COMMFLAG, LOG_ERR, "bounding_box giving up (edge "
283  "pixels distributed too uniformly)");
284  return -1;
285  }
286 
287  if (left < right)
288  col += chop;
289  width -= chop;
290  continue;
291  }
292 
293  if (height > maxheight)
294  {
295  /* Too tall; test upper and lower portions. */
296  int chop = height / 3;
297  int chopheight = height - chop;
298 
299  float upper = bounding_score(img, row, col, width, chopheight);
300  float lower = bounding_score(img, row + chop, col, width, chopheight);
301  LOG(VB_COMMFLAG, LOG_INFO,
302  QString("bounding_box too tall (%1 > %2); upper=%3, lower=%4")
303  .arg(height).arg(maxheight)
304  .arg(upper, 0, 'f', 3).arg(lower, 0, 'f', 3));
305  float minscore = min(upper, lower);
306  float maxscore = max(upper, lower);
307  if (maxscore < 3 * minscore / 2)
308  {
309  /*
310  * Edge pixel distribution too uniform; give up.
311  *
312  * XXX: also fails for vertically-centered templates ...
313  */
314  LOG(VB_COMMFLAG, LOG_ERR, "bounding_box giving up (edge "
315  "pixel distribution too uniform)");
316  return -1;
317  }
318 
319  if (upper < lower)
320  row += chop;
321  height -= chop;
322  continue;
323  }
324 
325  break;
326  }
327 
328  /*
329  * The above "chop" algorithm often cuts off the outside edges of the
330  * logos because the outside edges don't contribute enough to the score. So
331  * compensate by now expanding the bounding box (up to a *SLOP pixels in
332  * each direction) to include all edge pixels.
333  */
334 
335  LOG(VB_COMMFLAG, LOG_INFO,
336  QString("bounding_box %1x%2@(%3,%4); horizslop=%5,vertslop=%6")
337  .arg(width).arg(height).arg(col).arg(row)
338  .arg(HORIZSLOP).arg(VERTSLOP));
339 
340  /* Expand upwards. */
341  newrow = row - 1;
342  for (;;)
343  {
344  if (newrow <= minrow)
345  {
346  newrow = minrow;
347  break;
348  }
349  if (row - newrow >= VERTSLOP)
350  {
351  newrow = row - VERTSLOP;
352  break;
353  }
354  if (rowisempty(img, newrow, col, width))
355  {
356  newrow++;
357  break;
358  }
359  newrow--;
360  }
361  newrow = max(minrow, newrow - 1); /* Empty row on top. */
362 
363  /* Expand leftwards. */
364  newcol = col - 1;
365  for (;;)
366  {
367  if (newcol <= mincol)
368  {
369  newcol = mincol;
370  break;
371  }
372  if (col - newcol >= HORIZSLOP)
373  {
374  newcol = col - HORIZSLOP;
375  break;
376  }
377  if (colisempty(img, newcol, row, height))
378  {
379  newcol++;
380  break;
381  }
382  newcol--;
383  }
384  newcol = max(mincol, newcol - 1); /* Empty column to left. */
385 
386  /* Expand rightwards. */
387  newright = col + width;
388  for (;;)
389  {
390  if (newright >= maxcol1)
391  {
392  newright = maxcol1;
393  break;
394  }
395  if (newright - (col + width) >= HORIZSLOP)
396  {
397  newright = col + width + HORIZSLOP;
398  break;
399  }
400  if (colisempty(img, newright, row, height))
401  break;
402  newright++;
403  }
404  newright = min(maxcol1, newright + 1); /* Empty column to right. */
405 
406  /* Expand downwards. */
407  newbottom = row + height;
408  for (;;)
409  {
410  if (newbottom >= maxrow1)
411  {
412  newbottom = maxrow1;
413  break;
414  }
415  if (newbottom - (row + height) >= VERTSLOP)
416  {
417  newbottom = row + height + VERTSLOP;
418  break;
419  }
420  if (rowisempty(img, newbottom, col, width))
421  break;
422  newbottom++;
423  }
424  newbottom = min(maxrow1, newbottom + 1); /* Empty row on bottom. */
425 
426  row = newrow;
427  col = newcol;
428  width = newright - newcol;
429  height = newbottom - newrow;
430 
431  LOG(VB_COMMFLAG, LOG_INFO, QString("bounding_box %1x%2@(%3,%4)")
432  .arg(width).arg(height).arg(col).arg(row));
433 
434  *prow = row;
435  *pcol = col;
436  *pwidth = width;
437  *pheight = height;
438  return 0;
439 }
440 
441 bool
442 template_alloc(const unsigned int *scores, int width, int height,
443  int minrow, int mincol, int maxrow1, int maxcol1, AVFrame *tmpl,
444  int *ptmplrow, int *ptmplcol, int *ptmplwidth, int *ptmplheight,
445  bool debug_edgecounts, const QString& debugdir)
446 {
447  /*
448  * TUNABLE:
449  *
450  * Higher values select for "stronger" pixels to be in the template, but
451  * weak pixels might be missed.
452  *
453  * Lower values allow more pixels to be included as part of the template,
454  * but strong non-template pixels might be included.
455  */
456  static constexpr float kMinScorePctile = 0.998;
457 
458  const int nn = width * height;
459  int ii = 0;
460  int first = 0;
461  int last = 0;
462  unsigned int threshscore = 0;
463  AVFrame thresh;
464 
465  if (av_image_alloc(thresh.data, thresh.linesize,
466  width, height, AV_PIX_FMT_GRAY8, IMAGE_ALIGN))
467  {
468  LOG(VB_COMMFLAG, LOG_ERR,
469  QString("template_alloc av_image_alloc thresh (%1x%2) failed")
470  .arg(width).arg(height));
471  return false;
472  }
473 
474  uint *sortedscores = new unsigned int[nn];
475  memcpy(sortedscores, scores, nn * sizeof(*sortedscores));
476  qsort(sortedscores, nn, sizeof(*sortedscores), sort_ascending);
477 
478  if (sortedscores[0] == sortedscores[nn - 1])
479  {
480  /* All pixels in the template area look the same; no template. */
481  LOG(VB_COMMFLAG, LOG_ERR,
482  QString("template_alloc: %1x%2 pixels all identical!")
483  .arg(width).arg(height));
484  goto free_thresh;
485  }
486 
487  /* Threshold the edge frequences. */
488 
489  ii = (int)roundf(nn * kMinScorePctile);
490  threshscore = sortedscores[ii];
491  for (first = ii; first > 0 && sortedscores[first] == threshscore; first--)
492  ;
493  if (sortedscores[first] != threshscore)
494  first++;
495  for (last = ii; last < nn - 1 && sortedscores[last] == threshscore; last++)
496  ;
497  if (sortedscores[last] != threshscore)
498  last--;
499 
500  LOG(VB_COMMFLAG, LOG_INFO, QString("template_alloc wanted %1, got %2-%3")
501  .arg(kMinScorePctile, 0, 'f', 6)
502  .arg((float)first / nn, 0, 'f', 6)
503  .arg((float)last / nn, 0, 'f', 6));
504 
505  for (ii = 0; ii < nn; ii++)
506  thresh.data[0][ii] = scores[ii] >= threshscore ? UCHAR_MAX : 0;
507 
508  if (debug_edgecounts)
509  {
510  /* Scores, rescaled to [0..UCHAR_MAX]. */
511  AVFrame scored;
512  if (av_image_alloc(scored.data, scored.linesize,
513  width, height, AV_PIX_FMT_GRAY8, IMAGE_ALIGN))
514  {
515  LOG(VB_COMMFLAG, LOG_ERR,
516  QString("template_alloc av_image_alloc scored (%1x%2) failed")
517  .arg(width).arg(height));
518  goto free_thresh;
519  }
520  unsigned int maxscore = sortedscores[nn - 1];
521  for (ii = 0; ii < nn; ii++)
522  scored.data[0][ii] = scores[ii] * UCHAR_MAX / maxscore;
523  bool success = writeJPG(debugdir + "/TemplateFinder-scores", &scored,
524  height);
525  av_freep(&scored.data[0]);
526  if (!success)
527  goto free_thresh;
528 
529  /* Thresholded scores. */
530  if (!writeJPG(debugdir + "/TemplateFinder-edgecounts", &thresh, height))
531  goto free_thresh;
532  }
533 
534  /* Crop to a minimal bounding box. */
535 
536  if (bounding_box(&thresh, height, minrow, mincol, maxrow1, maxcol1,
537  ptmplrow, ptmplcol, ptmplwidth, ptmplheight))
538  goto free_thresh;
539 
540  if ((uint)(*ptmplwidth * *ptmplheight) > USHRT_MAX)
541  {
542  /* Max value of data type of TemplateMatcher::edgematch */
543  LOG(VB_COMMFLAG, LOG_ERR,
544  QString("template_alloc bounding_box too big (%1x%2)")
545  .arg(*ptmplwidth).arg(*ptmplheight));
546  goto free_thresh;
547  }
548 
549  if (av_image_alloc(tmpl->data, tmpl->linesize,
550  *ptmplwidth, *ptmplheight, AV_PIX_FMT_GRAY8, IMAGE_ALIGN))
551  {
552  LOG(VB_COMMFLAG, LOG_ERR,
553  QString("template_alloc av_image_alloc tmpl (%1x%2) failed")
554  .arg(*ptmplwidth).arg(*ptmplheight));
555  goto free_thresh;
556  }
557 
558  if (pgm_crop(tmpl, &thresh, height, *ptmplrow, *ptmplcol,
559  *ptmplwidth, *ptmplheight))
560  goto free_thresh;
561 
562  delete []sortedscores;
563  av_freep(&thresh.data[0]);
564 
565  return true;
566 
567 free_thresh:
568  delete []sortedscores;
569  av_freep(&thresh.data[0]);
570  return false;
571 }
572 
573 bool
574 analyzeFrameDebug(long long frameno, const AVFrame *pgm, int pgmheight,
575  const AVFrame *cropped, const AVFrame *edges, int cropheight,
576  int croprow, int cropcol, bool debug_frames, const QString& debugdir)
577 {
578  static constexpr int kDelta = 24;
579  static int s_lastrow;
580  static int s_lastcol;
581  static int s_lastwidth;
582  static int s_lastheight;
583  const int cropwidth = cropped->linesize[0];
584 
585  int rowsame = abs(s_lastrow - croprow) <= kDelta ? 1 : 0;
586  int colsame = abs(s_lastcol - cropcol) <= kDelta ? 1 : 0;
587  int widthsame = abs(s_lastwidth - cropwidth) <= kDelta ? 1 : 0;
588  int heightsame = abs(s_lastheight - cropheight) <= kDelta ? 1 : 0;
589 
590  if (frameno > 0 && rowsame + colsame + widthsame + heightsame >= 3)
591  return true;
592 
593  LOG(VB_COMMFLAG, LOG_INFO,
594  QString("TemplateFinder Frame %1: %2x%3@(%4,%5)")
595  .arg(frameno, 5)
596  .arg(cropwidth).arg(cropheight)
597  .arg(cropcol).arg(croprow));
598 
599  s_lastrow = croprow;
600  s_lastcol = cropcol;
601  s_lastwidth = cropwidth;
602  s_lastheight = cropheight;
603 
604  if (debug_frames)
605  {
606  QString base = QString("%1/TemplateFinder-%2")
607  .arg(debugdir).arg(frameno, 5, 10, QChar('0'));
608 
609  /* PGM greyscale image of frame. */
610  if (!writeJPG(base, pgm, pgmheight))
611  return false;
612 
613  /* Cropped template area of frame. */
614  if (!writeJPG(base + "-cropped", cropped, cropheight))
615  return false;
616 
617  /* Edges of cropped template area of frame. */
618  if (!writeJPG(base + "-edges", edges, cropheight))
619  return false;
620  }
621 
622  return true;
623 }
624 
625 /* NOLINTNEXTLINE(readability-non-const-parameter) */
626 bool readTemplate(const QString& datafile, int *prow, int *pcol, int *pwidth, int *pheight,
627  const QString& tmplfile, AVFrame *tmpl, bool *pvalid)
628 {
629  QFile dfile(datafile);
630  QFileInfo dfileinfo(dfile);
631 
632  if (!dfile.open(QIODevice::ReadOnly))
633  return false;
634 
635  if (!dfileinfo.size())
636  {
637  /* Dummy file: no template. */
638  *pvalid = false;
639  return true;
640  }
641 
642  QTextStream stream(&dfile);
643  stream >> *prow >> *pcol >> *pwidth >> *pheight;
644  dfile.close();
645 
646  if (av_image_alloc(tmpl->data, tmpl->linesize,
647  *pwidth, *pheight, AV_PIX_FMT_GRAY8, IMAGE_ALIGN))
648  {
649  LOG(VB_COMMFLAG, LOG_ERR,
650  QString("readTemplate av_image_alloc %1 (%2x%3) failed")
651  .arg(tmplfile).arg(*pwidth).arg(*pheight));
652  return false;
653  }
654 
655  QByteArray tmfile = tmplfile.toLatin1();
656  if (pgm_read(tmpl->data[0], *pwidth, *pheight, tmfile.constData()))
657  {
658  av_freep(&tmpl->data[0]);
659  return false;
660  }
661 
662  *pvalid = true;
663  return true;
664 }
665 
666 void
667 writeDummyTemplate(const QString& datafile)
668 {
669  /* Leave a 0-byte file. */
670  QFile dfile(datafile);
671 
672  if (!dfile.open(QIODevice::WriteOnly | QIODevice::Truncate) &&
673  dfile.exists())
674  (void)dfile.remove();
675 }
676 
677 bool
678 writeTemplate(const QString& tmplfile, const AVFrame *tmpl, const QString& datafile,
679  int row, int col, int width, int height)
680 {
681  QFile tfile(tmplfile);
682 
683  QByteArray tmfile = tmplfile.toLatin1();
684  if (pgm_write(tmpl->data[0], width, height, tmfile.constData()))
685  return false;
686 
687  QFile dfile(datafile);
688  if (!dfile.open(QIODevice::WriteOnly))
689  return false;
690 
691  QTextStream stream(&dfile);
692  stream << row << " " << col << "\n" << width << " " << height << "\n";
693  dfile.close();
694  return true;
695 }
696 
697 }; /* namespace */
698 
699 TemplateFinder::TemplateFinder(std::shared_ptr<PGMConverter> pgmc,
700  std::shared_ptr<BorderDetector> bd,
701  std::shared_ptr<EdgeDetector> ed,
702  MythPlayer *player, int proglen,
703  const QString& debugdir)
704  : m_pgmConverter(std::move(pgmc))
705  , m_borderDetector(std::move(bd))
706  , m_edgeDetector(std::move(ed))
707  , m_debugDir(debugdir)
708  , m_debugData(debugdir + "/TemplateFinder.txt")
709  , m_debugTmpl(debugdir + "/TemplateFinder.pgm")
710 {
711  /*
712  * TUNABLE:
713  *
714  * The number of frames desired for sampling to build the template.
715  *
716  * Higher values should yield a more accurate template, but requires more
717  * time.
718  */
719  unsigned int samplesNeeded = 300;
720 
721  /*
722  * TUNABLE:
723  *
724  * The leading amount of time (in seconds) to sample frames for building up
725  * the possible template, and the interval between frames for analysis.
726  * This affects how soon flagging can start after a recording has begun
727  * (a.k.a. "real-time flagging").
728  *
729  * Sample half of the program length or 20 minutes, whichever is less.
730  */
731  m_sampleTime = min(proglen / 2, 20 * 60);
732 
733  const float fps = player->GetFrameRate();
734 
735  m_frameInterval = (int)roundf(m_sampleTime * fps / samplesNeeded);
736  m_endFrame = 0 + (long long)m_frameInterval * samplesNeeded - 1;
737 
738  LOG(VB_COMMFLAG, LOG_INFO,
739  QString("TemplateFinder: sampleTime=%1s, samplesNeeded=%2, endFrame=%3")
740  .arg(m_sampleTime).arg(samplesNeeded).arg(m_endFrame));
741 
742  /*
743  * debugLevel:
744  * 0: no extra debugging
745  * 1: cache computations into debugdir [O(1) files]
746  * 2: extra verbosity [O(nframes)]
747  * 3: dump frames into debugdir [O(nframes) files]
748  */
749  m_debugLevel = gCoreContext->GetNumSetting("TemplateFinderDebugLevel", 0);
750 
751  if (m_debugLevel >= 1)
752  {
754  QString("TemplateFinder debugLevel %1").arg(m_debugLevel));
755 
756  m_debugTemplate = true;
757  m_debugEdgeCounts = true;
758 
759  if (m_debugLevel >= 3)
760  m_debugFrames = true;
761  }
762 }
763 
765 {
766  delete []m_scores;
767  av_freep(&m_tmpl.data[0]);
768  av_freep(&m_cropped.data[0]);
769 }
770 
772 TemplateFinder::MythPlayerInited(MythPlayer *player, long long nframes)
773 {
774  /*
775  * Only detect edges in portions of the frame where we expect to find
776  * a template. This serves two purposes:
777  *
778  * - Speed: reduce search space.
779  * - Correctness (insofar as the assumption of template location is
780  * correct): don't "pollute" the set of candidate template edges with
781  * the "content" edges in the non-template portions of the frame.
782  */
783  QString tmpldims;
784  QString playerdims;
785 
786  (void)nframes; /* gcc */
787  QSize buf_dim = player->GetVideoBufferSize();
788  m_width = buf_dim.width();
789  m_height = buf_dim.height();
790  playerdims = QString("%1x%2").arg(m_width).arg(m_height);
791 
792  if (m_debugTemplate)
793  {
794  if ((m_tmplDone = readTemplate(m_debugData, &m_tmplRow, &m_tmplCol,
796  &m_tmplValid)))
797  {
798  tmpldims = m_tmplValid ? QString("%1x%2@(%3,%4)")
799  .arg(m_tmplWidth).arg(m_tmplHeight).arg(m_tmplCol).arg(m_tmplRow) :
800  "no template";
801 
802  LOG(VB_COMMFLAG, LOG_INFO,
803  QString("TemplateFinder::MythPlayerInited read %1: %2")
804  .arg(m_debugTmpl)
805  .arg(tmpldims));
806  }
807  }
808 
809  if (m_pgmConverter->MythPlayerInited(player))
810  goto free_tmpl;
811 
812  if (m_borderDetector->MythPlayerInited(player))
813  goto free_tmpl;
814 
815  if (m_tmplDone)
816  {
817  if (m_tmplValid)
818  {
819  LOG(VB_COMMFLAG, LOG_INFO,
820  QString("TemplateFinder::MythPlayerInited %1 of %2 (%3)")
821  .arg(tmpldims).arg(playerdims).arg(m_debugTmpl));
822  }
823  return ANALYZE_FINISHED;
824  }
825 
826  LOG(VB_COMMFLAG, LOG_INFO,
827  QString("TemplateFinder::MythPlayerInited framesize %1")
828  .arg(playerdims));
829  m_scores = new unsigned int[m_width * m_height];
830 
831  return ANALYZE_OK;
832 
833 free_tmpl:
834  av_freep(&m_tmpl.data[0]);
835  return ANALYZE_FATAL;
836 }
837 
838 int
839 TemplateFinder::resetBuffers(int newwidth, int newheight)
840 {
841  if (m_cwidth == newwidth && m_cheight == newheight)
842  return 0;
843 
844  av_freep(&m_cropped.data[0]);
845 
846  if (av_image_alloc(m_cropped.data, m_cropped.linesize,
847  newwidth, newheight, AV_PIX_FMT_GRAY8, IMAGE_ALIGN))
848  {
849  LOG(VB_COMMFLAG, LOG_ERR,
850  QString("TemplateFinder::resetBuffers "
851  "av_image_alloc cropped (%1x%2) failed")
852  .arg(newwidth).arg(newheight));
853  return -1;
854  }
855 
856  m_cwidth = newwidth;
857  m_cheight = newheight;
858  return 0;
859 }
860 
862 TemplateFinder::analyzeFrame(const VideoFrame *frame, long long frameno,
863  long long *pNextFrame)
864 {
865  /*
866  * TUNABLE:
867  *
868  * When looking for edges in frames, select some percentile of
869  * squared-gradient magnitudes (intensities) as candidate edges. (This
870  * number conventionally should not go any lower than the 95th percentile;
871  * see edge_mark.)
872  *
873  * Higher values result in fewer edges; faint logos might not be picked up.
874  * Lower values result in more edges; non-logo edges might be picked up.
875  *
876  * The TemplateFinder accumulates all its state in the "scores" array to
877  * be processed later by TemplateFinder::finished.
878  */
879  const int FRAMESGMPCTILE = 90;
880 
881  /*
882  * TUNABLE:
883  *
884  * Exclude some portion of the center of the frame from edge analysis.
885  * Elminate false edge-detection logo positives from talking-host types of
886  * shows where the high-contrast host and clothes (e.g., tie against white
887  * shirt against dark jacket) dominates the edges.
888  *
889  * This has a nice side-effect of reducing the area to be examined (speed
890  * optimization).
891  */
892  static constexpr float kExcludeWidth = 0.5;
893  static constexpr float kExcludeHeight = 0.5;
894 
895  int pgmwidth= 0;
896  int pgmheight = 0;
897  int croprow= 0;
898  int cropcol = 0;
899  int cropwidth = 0;
900  int cropheight = 0;
901  struct timeval start {};
902  struct timeval end {};
903  struct timeval elapsed {};
904 
905  if (frameno < m_nextFrame)
906  {
907  *pNextFrame = m_nextFrame;
908  return ANALYZE_OK;
909  }
910 
911  m_nextFrame = frameno + m_frameInterval;
912  *pNextFrame = min(m_endFrame, m_nextFrame);
913 
914  const AVFrame *pgm = m_pgmConverter->getImage(frame, frameno, &pgmwidth, &pgmheight);
915  if (pgm == nullptr)
916  goto error;
917 
918  if (!m_borderDetector->getDimensions(pgm, pgmheight, frameno,
919  &croprow, &cropcol, &cropwidth, &cropheight))
920  {
921  /* Not a blank frame. */
922 
923  (void)gettimeofday(&start, nullptr);
924 
925  if (croprow < m_minContentRow)
926  m_minContentRow = croprow;
927  if (cropcol < m_minContentCol)
928  m_minContentCol = cropcol;
929  if (cropcol + cropwidth > m_maxContentCol1)
930  m_maxContentCol1 = cropcol + cropwidth;
931  if (croprow + cropheight > m_maxContentRow1)
932  m_maxContentRow1 = croprow + cropheight;
933 
934  if (resetBuffers(cropwidth, cropheight))
935  goto error;
936 
937  if (pgm_crop(&m_cropped, pgm, pgmheight, croprow, cropcol,
938  cropwidth, cropheight))
939  goto error;
940 
941  /*
942  * Translate the excluded area of the screen into "cropped"
943  * coordinates.
944  */
945  int excludewidth = (int)(pgmwidth * kExcludeWidth);
946  int excludeheight = (int)(pgmheight * kExcludeHeight);
947  int excluderow = (pgmheight - excludeheight) / 2 - croprow;
948  int excludecol = (pgmwidth - excludewidth) / 2 - cropcol;
949  (void)m_edgeDetector->setExcludeArea(excluderow, excludecol,
950  excludewidth, excludeheight);
951 
952  const AVFrame *edges =
953  m_edgeDetector->detectEdges(&m_cropped, cropheight, FRAMESGMPCTILE);
954  if (edges == nullptr)
955  goto error;
956 
957  if (pgm_scorepixels(m_scores, pgmwidth, croprow, cropcol,
958  edges, cropheight))
959  goto error;
960 
961  if (m_debugLevel >= 2)
962  {
963  if (!analyzeFrameDebug(frameno, pgm, pgmheight, &m_cropped, edges,
964  cropheight, croprow, cropcol, m_debugFrames, m_debugDir))
965  goto error;
966  }
967 
968  (void)gettimeofday(&end, nullptr);
969  timersub(&end, &start, &elapsed);
970  timeradd(&m_analyzeTime, &elapsed, &m_analyzeTime);
971  }
972 
973  if (m_nextFrame > m_endFrame)
974  return ANALYZE_FINISHED;
975 
976  return ANALYZE_OK;
977 
978 error:
979  LOG(VB_COMMFLAG, LOG_ERR,
980  QString("TemplateFinder::analyzeFrame error at frame %1")
981  .arg(frameno));
982 
983  if (m_nextFrame > m_endFrame)
984  return ANALYZE_FINISHED;
985 
986  return ANALYZE_ERROR;
987 }
988 
989 int
990 TemplateFinder::finished(long long nframes, bool final)
991 {
992  (void)nframes; /* gcc */
993  if (!m_tmplDone)
994  {
995  if (!template_alloc(m_scores, m_width, m_height,
1000  {
1001  if (final)
1002  writeDummyTemplate(m_debugData);
1003  }
1004  else
1005  {
1006  if (final && m_debugTemplate)
1007  {
1008  if (!(m_tmplValid = writeTemplate(m_debugTmpl, &m_tmpl, m_debugData,
1010  goto free_tmpl;
1011 
1012  LOG(VB_COMMFLAG, LOG_INFO,
1013  QString("TemplateFinder::finished wrote %1"
1014  " and %2 [%3x%4@(%5,%6)]")
1015  .arg(m_debugTmpl).arg(m_debugData)
1016  .arg(m_tmplWidth).arg(m_tmplHeight)
1017  .arg(m_tmplCol).arg(m_tmplRow));
1018  }
1019  }
1020 
1021  if (final)
1022  m_tmplDone = true;
1023  }
1024 
1025  m_borderDetector->setLogoState(this);
1026 
1027  return 0;
1028 
1029 free_tmpl:
1030  av_freep(&m_tmpl.data[0]);
1031  return -1;
1032 }
1033 
1034 int
1036 {
1037  if (m_pgmConverter->reportTime())
1038  return -1;
1039 
1040  if (m_borderDetector->reportTime())
1041  return -1;
1042 
1043  LOG(VB_COMMFLAG, LOG_INFO, QString("TF Time: analyze=%1s")
1044  .arg(strftimeval(&m_analyzeTime)));
1045  return 0;
1046 }
1047 
1048 const struct AVFrame *
1049 TemplateFinder::getTemplate(int *prow, int *pcol, int *pwidth, int *pheight)
1050  const
1051 {
1052  if (m_tmplValid)
1053  {
1054  *prow = m_tmplRow;
1055  *pcol = m_tmplCol;
1056  *pwidth = m_tmplWidth;
1057  *pheight = m_tmplHeight;
1058  return &m_tmpl;
1059  }
1060  return nullptr;
1061 }
1062 
1063 /* vim: set expandtab tabstop=4 shiftwidth=4: */
Definition: cc.h:9
static void error(const char *str,...)
Definition: vbi.cpp:42
#define GENERIC_EXIT_OK
Exited with no error.
Definition: exitcodes.h:10
enum analyzeFrameResult analyzeFrame(const VideoFrame *frame, long long frameno, long long *pNextFrame) override
long long m_endFrame
#define timeradd(a, b, result)
Definition: compat.h:300
struct AVFrame AVFrame
std::shared_ptr< PGMConverter > m_pgmConverter
static int sort_ascending(const void *aa, const void *bb)
std::shared_ptr< EdgeDetector > m_edgeDetector
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
int finished(long long nframes, bool final) override
~TemplateFinder(void) override
int pgm_read(unsigned char *buf, int width, int height, const char *filename)
Definition: pgm.cpp:32
std::shared_ptr< BorderDetector > m_borderDetector
if(query.exec() &&query.next())
int reportTime(void) const override
const struct AVFrame * getTemplate(int *prow, int *pcol, int *pwidth, int *pheight) const
float GetFrameRate(void) const
Definition: mythplayer.h:219
unsigned int * m_scores
unsigned int uint
Definition: compat.h:140
enum analyzeFrameResult MythPlayerInited(MythPlayer *player, long long nframes) override
uint myth_system(const QString &command, uint flags, uint timeout)
int resetBuffers(int newwidth, int newheight)
int GetNumSetting(const QString &key, int defaultval=0)
void createDebugDirectory(const QString &dirname, const QString &comment)
QSize GetVideoBufferSize(void) const
Definition: mythplayer.h:216
unsigned int m_sampleTime
QString m_debugTmpl
int pgm_crop(AVFrame *dst, const AVFrame *src, int srcheight, int srcrow, int srccol, int cropwidth, int cropheight)
Definition: pgm.cpp:165
int pgm_write(const unsigned char *buf, int width, int height, const char *filename)
Definition: pgm.cpp:80
QString m_debugDir
QString strftimeval(const struct timeval *tv)
QString m_debugData
#define timersub(a, b, result)
Definition: compat.h:310
TemplateFinder(std::shared_ptr< PGMConverter > pgmc, std::shared_ptr< BorderDetector > bd, std::shared_ptr< EdgeDetector > ed, MythPlayer *player, int proglen, const QString &debugdir)
long long m_nextFrame
#define LOG(_MASK_, _LEVEL_, _QSTRING_)
Definition: mythlogging.h:23