MythTV  master
TemplateFinder.cpp
Go to the documentation of this file.
1 // POSIX headers
2 #include <sys/time.h> /* gettimeofday */
3 
4 // ANSI C headers
5 #include <cmath>
6 #include <cstdlib>
7 
8 // Qt headers
9 #include <QFile>
10 #include <QFileInfo>
11 #include <QTextStream>
12 
13 // MythTV headers
14 #include "mythplayer.h"
15 #include "mythcorecontext.h" /* gContext */
16 #include "mythframe.h" /* VideoFrame */
17 #include "mythdate.h"
18 #include "mythsystemlegacy.h"
19 #include "exitcodes.h"
20 
21 // Commercial Flagging headers
22 #include "CommDetector2.h"
23 #include "pgm.h"
24 #include "PGMConverter.h"
25 #include "BorderDetector.h"
26 #include "EdgeDetector.h"
27 #include "TemplateFinder.h"
28 
29 extern "C" {
30  #include "libavutil/imgutils.h"
31  }
32 
33 using namespace commDetector2;
34 
35 namespace {
36 
37 //returns true on success, false otherwise
38 bool writeJPG(const QString& prefix, const AVFrame *img, int imgheight)
39 {
40  const int imgwidth = img->linesize[0];
41  QFileInfo jpgfi(prefix + ".jpg");
42  if (!jpgfi.exists())
43  {
44  QFile pgmfile(prefix + ".pgm");
45  if (!pgmfile.exists())
46  {
47  QByteArray pfname = pgmfile.fileName().toLocal8Bit();
48  if (pgm_write(img->data[0], imgwidth, imgheight,
49  pfname.constData()))
50  {
51  return false;
52  }
53  }
54 
55  QString cmd = QString("convert -quality 50 -resize 192x144 %1 %2")
56  .arg(pgmfile.fileName()).arg(jpgfi.filePath());
57  if (myth_system(cmd) != GENERIC_EXIT_OK)
58  return false;
59 
60  if (!pgmfile.remove())
61  {
62  LOG(VB_COMMFLAG, LOG_ERR,
63  QString("TemplateFinder.writeJPG error removing %1 (%2)")
64  .arg(pgmfile.fileName()).arg(strerror(errno)));
65  return false;
66  }
67  }
68  return true;
69 }
70 
71 int
72 pgm_scorepixels(unsigned int *scores, int width, int row, int col,
73  const AVFrame *src, int srcheight)
74 {
75  /* Every time a pixel is an edge, give it a point. */
76  const int srcwidth = src->linesize[0];
77 
78  for (int rr = 0; rr < srcheight; rr++)
79  {
80  for (int cc = 0; cc < srcwidth; cc++)
81  {
82  if (src->data[0][rr * srcwidth + cc])
83  scores[(row + rr) * width + col + cc]++;
84  }
85  }
86 
87  return 0;
88 }
89 
90 int
91 sort_ascending(const void *aa, const void *bb)
92 {
93  return *(unsigned int*)aa - *(unsigned int*)bb;
94 }
95 
96 float
97 bounding_score(const AVFrame *img, int row, int col, int width, int height)
98 {
99  /* Return a value between [0..1] */
100  const int imgwidth = img->linesize[0];
101 
102  uint score = 0;
103  int rr2 = row + height;
104  int cc2 = col + width;
105  for (int rr = row; rr < rr2; rr++)
106  {
107  for (int cc = col; cc < cc2; cc++)
108  {
109  if (img->data[0][rr * imgwidth + cc])
110  score++;
111  }
112  }
113  return (float)score / (width * height);
114 }
115 
116 bool
117 rowisempty(const AVFrame *img, int row, int col, int width)
118 {
119  const int imgwidth = img->linesize[0];
120  for (int cc = col; cc < col + width; cc++)
121  if (img->data[0][row * imgwidth + cc])
122  return false;
123  return true;
124 }
125 
126 bool
127 colisempty(const AVFrame *img, int col, int row, int height)
128 {
129  const int imgwidth = img->linesize[0];
130  for (int rr = row; rr < row + height; rr++)
131  if (img->data[0][rr * imgwidth + col])
132  return false;
133  return true;
134 }
135 
136 int
137 bounding_box(const AVFrame *img, int imgheight,
138  int minrow, int mincol, int maxrow1, int maxcol1,
139  int *prow, int *pcol, int *pwidth, int *pheight)
140 {
141  const int imgwidth = img->linesize[0];
142  /*
143  * TUNABLE:
144  *
145  * Maximum logo size, expressed as a percentage of the content area
146  * (adjusting for letterboxing and pillarboxing).
147  */
148  static constexpr int kMaxWidthPct = 20;
149  static constexpr int kMaxHeightPct = 20;
150 
151  /*
152  * TUNABLE:
153  *
154  * Safety margin to avoid cutting too much of the logo.
155  * Higher values cut more, but avoid noise as part of the template..
156  * Lower values cut less, but can include noise as part of the template.
157  */
158  const int VERTSLOP = max(4, imgheight * 1 / 15);
159  const int HORIZSLOP = max(4, imgwidth * 1 / 20);
160 
161  int maxwidth = (maxcol1 - mincol) * kMaxWidthPct / 100;
162  int maxheight = (maxrow1 - minrow) * kMaxHeightPct / 100;
163 
164  int row = minrow;
165  int col = mincol;
166  int width = maxcol1 - mincol;
167  int height = maxrow1 - minrow;
168  int newrow = 0;
169  int newcol = 0;
170  int newright = 0;
171  int newbottom = 0;
172 
173  for (;;)
174  {
175  float newscore = NAN;
176  bool improved = false;
177 
178  LOG(VB_COMMFLAG, LOG_INFO, QString("bounding_box %1x%2@(%3,%4)")
179  .arg(width).arg(height).arg(col).arg(row));
180 
181  /* Chop top. */
182  float score = bounding_score(img, row, col, width, height);
183  newrow = row;
184  for (int ii = 1; ii < height; ii++)
185  {
186  if ((newscore = bounding_score(img, row + ii, col,
187  width, height - ii)) < score)
188  break;
189  score = newscore;
190  newrow = row + ii;
191  improved = true;
192  }
193 
194  /* Chop left. */
195  score = bounding_score(img, row, col, width, height);
196  newcol = col;
197  for (int ii = 1; ii < width; ii++)
198  {
199  if ((newscore = bounding_score(img, row, col + ii,
200  width - ii, height)) < score)
201  break;
202  score = newscore;
203  newcol = col + ii;
204  improved = true;
205  }
206 
207  /* Chop bottom. */
208  score = bounding_score(img, row, col, width, height);
209  newbottom = row + height;
210  for (int ii = 1; ii < height; ii++)
211  {
212  if ((newscore = bounding_score(img, row, col,
213  width, height - ii)) < score)
214  break;
215  score = newscore;
216  newbottom = row + height - ii;
217  improved = true;
218  }
219 
220  /* Chop right. */
221  score = bounding_score(img, row, col, width, height);
222  newright = col + width;
223  for (int ii = 1; ii < width; ii++)
224  {
225  if ((newscore = bounding_score(img, row, col,
226  width - ii, height)) < score)
227  break;
228  score = newscore;
229  newright = col + width - ii;
230  improved = true;
231  }
232 
233  if (!improved)
234  break;
235 
236  row = newrow;
237  col = newcol;
238  width = newright - newcol;
239  height = newbottom - newrow;
240 
241  /*
242  * Noise edge pixels in the frequency template can sometimes stretch
243  * the template area to be larger than it should be.
244  *
245  * However, noise needs to be distinguished from a uniform distribution
246  * of noise pixels (e.g., no real statically-located template). So if
247  * the template area is too "large", then some quadrant must have a
248  * clear majority of the edge pixels; otherwise we declare failure (no
249  * template found).
250  *
251  * Intuitively, we should simply repeat until a single bounding box is
252  * converged upon. However, this requires a more sophisticated
253  * bounding_score function that I don't feel like figuring out.
254  * Indefinitely repeating with the present bounding_score function will
255  * tend to chop off too much. Instead, simply do some sanity checks on
256  * the candidate template's size, and prune the template area and
257  * repeat if it is too "large".
258  */
259 
260  if (width > maxwidth)
261  {
262  /* Too wide; test left and right portions. */
263  int chop = width / 3;
264  int chopwidth = width - chop;
265 
266  float left = bounding_score(img, row, col, chopwidth, height);
267  float right = bounding_score(img, row, col + chop, chopwidth, height);
268  LOG(VB_COMMFLAG, LOG_INFO,
269  QString("bounding_box too wide (%1 > %2); left=%3, right=%4")
270  .arg(width).arg(maxwidth)
271  .arg(left, 0, 'f', 3).arg(right, 0, 'f', 3));
272  float minscore = min(left, right);
273  float maxscore = max(left, right);
274  if (maxscore < 3 * minscore / 2)
275  {
276  /*
277  * Edge pixel distribution too uniform; give up.
278  *
279  * XXX: also fails for horizontally-centered templates ...
280  */
281  LOG(VB_COMMFLAG, LOG_ERR, "bounding_box giving up (edge "
282  "pixels distributed too uniformly)");
283  return -1;
284  }
285 
286  if (left < right)
287  col += chop;
288  width -= chop;
289  continue;
290  }
291 
292  if (height > maxheight)
293  {
294  /* Too tall; test upper and lower portions. */
295  int chop = height / 3;
296  int chopheight = height - chop;
297 
298  float upper = bounding_score(img, row, col, width, chopheight);
299  float lower = bounding_score(img, row + chop, col, width, chopheight);
300  LOG(VB_COMMFLAG, LOG_INFO,
301  QString("bounding_box too tall (%1 > %2); upper=%3, lower=%4")
302  .arg(height).arg(maxheight)
303  .arg(upper, 0, 'f', 3).arg(lower, 0, 'f', 3));
304  float minscore = min(upper, lower);
305  float maxscore = max(upper, lower);
306  if (maxscore < 3 * minscore / 2)
307  {
308  /*
309  * Edge pixel distribution too uniform; give up.
310  *
311  * XXX: also fails for vertically-centered templates ...
312  */
313  LOG(VB_COMMFLAG, LOG_ERR, "bounding_box giving up (edge "
314  "pixel distribution too uniform)");
315  return -1;
316  }
317 
318  if (upper < lower)
319  row += chop;
320  height -= chop;
321  continue;
322  }
323 
324  break;
325  }
326 
327  /*
328  * The above "chop" algorithm often cuts off the outside edges of the
329  * logos because the outside edges don't contribute enough to the score. So
330  * compensate by now expanding the bounding box (up to a *SLOP pixels in
331  * each direction) to include all edge pixels.
332  */
333 
334  LOG(VB_COMMFLAG, LOG_INFO,
335  QString("bounding_box %1x%2@(%3,%4); horizslop=%5,vertslop=%6")
336  .arg(width).arg(height).arg(col).arg(row)
337  .arg(HORIZSLOP).arg(VERTSLOP));
338 
339  /* Expand upwards. */
340  newrow = row - 1;
341  for (;;)
342  {
343  if (newrow <= minrow)
344  {
345  newrow = minrow;
346  break;
347  }
348  if (row - newrow >= VERTSLOP)
349  {
350  newrow = row - VERTSLOP;
351  break;
352  }
353  if (rowisempty(img, newrow, col, width))
354  {
355  newrow++;
356  break;
357  }
358  newrow--;
359  }
360  newrow = max(minrow, newrow - 1); /* Empty row on top. */
361 
362  /* Expand leftwards. */
363  newcol = col - 1;
364  for (;;)
365  {
366  if (newcol <= mincol)
367  {
368  newcol = mincol;
369  break;
370  }
371  if (col - newcol >= HORIZSLOP)
372  {
373  newcol = col - HORIZSLOP;
374  break;
375  }
376  if (colisempty(img, newcol, row, height))
377  {
378  newcol++;
379  break;
380  }
381  newcol--;
382  }
383  newcol = max(mincol, newcol - 1); /* Empty column to left. */
384 
385  /* Expand rightwards. */
386  newright = col + width;
387  for (;;)
388  {
389  if (newright >= maxcol1)
390  {
391  newright = maxcol1;
392  break;
393  }
394  if (newright - (col + width) >= HORIZSLOP)
395  {
396  newright = col + width + HORIZSLOP;
397  break;
398  }
399  if (colisempty(img, newright, row, height))
400  break;
401  newright++;
402  }
403  newright = min(maxcol1, newright + 1); /* Empty column to right. */
404 
405  /* Expand downwards. */
406  newbottom = row + height;
407  for (;;)
408  {
409  if (newbottom >= maxrow1)
410  {
411  newbottom = maxrow1;
412  break;
413  }
414  if (newbottom - (row + height) >= VERTSLOP)
415  {
416  newbottom = row + height + VERTSLOP;
417  break;
418  }
419  if (rowisempty(img, newbottom, col, width))
420  break;
421  newbottom++;
422  }
423  newbottom = min(maxrow1, newbottom + 1); /* Empty row on bottom. */
424 
425  row = newrow;
426  col = newcol;
427  width = newright - newcol;
428  height = newbottom - newrow;
429 
430  LOG(VB_COMMFLAG, LOG_INFO, QString("bounding_box %1x%2@(%3,%4)")
431  .arg(width).arg(height).arg(col).arg(row));
432 
433  *prow = row;
434  *pcol = col;
435  *pwidth = width;
436  *pheight = height;
437  return 0;
438 }
439 
440 bool
441 template_alloc(const unsigned int *scores, int width, int height,
442  int minrow, int mincol, int maxrow1, int maxcol1, AVFrame *tmpl,
443  int *ptmplrow, int *ptmplcol, int *ptmplwidth, int *ptmplheight,
444  bool debug_edgecounts, const QString& debugdir)
445 {
446  /*
447  * TUNABLE:
448  *
449  * Higher values select for "stronger" pixels to be in the template, but
450  * weak pixels might be missed.
451  *
452  * Lower values allow more pixels to be included as part of the template,
453  * but strong non-template pixels might be included.
454  */
455  static constexpr float kMinScorePctile = 0.998;
456 
457  const int nn = width * height;
458  int ii = 0;
459  int first = 0;
460  int last = 0;
461  unsigned int threshscore = 0;
462  AVFrame thresh;
463 
464  if (av_image_alloc(thresh.data, thresh.linesize,
465  width, height, AV_PIX_FMT_GRAY8, IMAGE_ALIGN))
466  {
467  LOG(VB_COMMFLAG, LOG_ERR,
468  QString("template_alloc av_image_alloc thresh (%1x%2) failed")
469  .arg(width).arg(height));
470  return false;
471  }
472 
473  uint *sortedscores = new unsigned int[nn];
474  memcpy(sortedscores, scores, nn * sizeof(*sortedscores));
475  qsort(sortedscores, nn, sizeof(*sortedscores), sort_ascending);
476 
477  if (sortedscores[0] == sortedscores[nn - 1])
478  {
479  /* All pixels in the template area look the same; no template. */
480  LOG(VB_COMMFLAG, LOG_ERR,
481  QString("template_alloc: %1x%2 pixels all identical!")
482  .arg(width).arg(height));
483  goto free_thresh;
484  }
485 
486  /* Threshold the edge frequences. */
487 
488  ii = (int)roundf(nn * kMinScorePctile);
489  threshscore = sortedscores[ii];
490  for (first = ii; first > 0 && sortedscores[first] == threshscore; first--)
491  ;
492  if (sortedscores[first] != threshscore)
493  first++;
494  for (last = ii; last < nn - 1 && sortedscores[last] == threshscore; last++)
495  ;
496  if (sortedscores[last] != threshscore)
497  last--;
498 
499  LOG(VB_COMMFLAG, LOG_INFO, QString("template_alloc wanted %1, got %2-%3")
500  .arg(kMinScorePctile, 0, 'f', 6)
501  .arg((float)first / nn, 0, 'f', 6)
502  .arg((float)last / nn, 0, 'f', 6));
503 
504  for (ii = 0; ii < nn; ii++)
505  thresh.data[0][ii] = scores[ii] >= threshscore ? UCHAR_MAX : 0;
506 
507  if (debug_edgecounts)
508  {
509  /* Scores, rescaled to [0..UCHAR_MAX]. */
510  AVFrame scored;
511  if (av_image_alloc(scored.data, scored.linesize,
512  width, height, AV_PIX_FMT_GRAY8, IMAGE_ALIGN))
513  {
514  LOG(VB_COMMFLAG, LOG_ERR,
515  QString("template_alloc av_image_alloc scored (%1x%2) failed")
516  .arg(width).arg(height));
517  goto free_thresh;
518  }
519  unsigned int maxscore = sortedscores[nn - 1];
520  for (ii = 0; ii < nn; ii++)
521  scored.data[0][ii] = scores[ii] * UCHAR_MAX / maxscore;
522  bool success = writeJPG(debugdir + "/TemplateFinder-scores", &scored,
523  height);
524  av_freep(&scored.data[0]);
525  if (!success)
526  goto free_thresh;
527 
528  /* Thresholded scores. */
529  if (!writeJPG(debugdir + "/TemplateFinder-edgecounts", &thresh, height))
530  goto free_thresh;
531  }
532 
533  /* Crop to a minimal bounding box. */
534 
535  if (bounding_box(&thresh, height, minrow, mincol, maxrow1, maxcol1,
536  ptmplrow, ptmplcol, ptmplwidth, ptmplheight))
537  goto free_thresh;
538 
539  if ((uint)(*ptmplwidth * *ptmplheight) > USHRT_MAX)
540  {
541  /* Max value of data type of TemplateMatcher::edgematch */
542  LOG(VB_COMMFLAG, LOG_ERR,
543  QString("template_alloc bounding_box too big (%1x%2)")
544  .arg(*ptmplwidth).arg(*ptmplheight));
545  goto free_thresh;
546  }
547 
548  if (av_image_alloc(tmpl->data, tmpl->linesize,
549  *ptmplwidth, *ptmplheight, AV_PIX_FMT_GRAY8, IMAGE_ALIGN))
550  {
551  LOG(VB_COMMFLAG, LOG_ERR,
552  QString("template_alloc av_image_alloc tmpl (%1x%2) failed")
553  .arg(*ptmplwidth).arg(*ptmplheight));
554  goto free_thresh;
555  }
556 
557  if (pgm_crop(tmpl, &thresh, height, *ptmplrow, *ptmplcol,
558  *ptmplwidth, *ptmplheight))
559  goto free_thresh;
560 
561  delete []sortedscores;
562  av_freep(&thresh.data[0]);
563 
564  return true;
565 
566 free_thresh:
567  delete []sortedscores;
568  av_freep(&thresh.data[0]);
569  return false;
570 }
571 
572 bool
573 analyzeFrameDebug(long long frameno, const AVFrame *pgm, int pgmheight,
574  const AVFrame *cropped, const AVFrame *edges, int cropheight,
575  int croprow, int cropcol, bool debug_frames, const QString& debugdir)
576 {
577  static constexpr int kDelta = 24;
578  static int s_lastrow;
579  static int s_lastcol;
580  static int s_lastwidth;
581  static int s_lastheight;
582  const int cropwidth = cropped->linesize[0];
583 
584  int rowsame = abs(s_lastrow - croprow) <= kDelta ? 1 : 0;
585  int colsame = abs(s_lastcol - cropcol) <= kDelta ? 1 : 0;
586  int widthsame = abs(s_lastwidth - cropwidth) <= kDelta ? 1 : 0;
587  int heightsame = abs(s_lastheight - cropheight) <= kDelta ? 1 : 0;
588 
589  if (frameno > 0 && rowsame + colsame + widthsame + heightsame >= 3)
590  return true;
591 
592  LOG(VB_COMMFLAG, LOG_INFO,
593  QString("TemplateFinder Frame %1: %2x%3@(%4,%5)")
594  .arg(frameno, 5)
595  .arg(cropwidth).arg(cropheight)
596  .arg(cropcol).arg(croprow));
597 
598  s_lastrow = croprow;
599  s_lastcol = cropcol;
600  s_lastwidth = cropwidth;
601  s_lastheight = cropheight;
602 
603  if (debug_frames)
604  {
605  QString base = QString("%1/TemplateFinder-%2")
606  .arg(debugdir).arg(frameno, 5, 10, QChar('0'));
607 
608  /* PGM greyscale image of frame. */
609  if (!writeJPG(base, pgm, pgmheight))
610  return false;
611 
612  /* Cropped template area of frame. */
613  if (!writeJPG(base + "-cropped", cropped, cropheight))
614  return false;
615 
616  /* Edges of cropped template area of frame. */
617  if (!writeJPG(base + "-edges", edges, cropheight))
618  return false;
619  }
620 
621  return true;
622 }
623 
624 /* NOLINTNEXTLINE(readability-non-const-parameter) */
625 bool readTemplate(const QString& datafile, int *prow, int *pcol, int *pwidth, int *pheight,
626  const QString& tmplfile, AVFrame *tmpl, bool *pvalid)
627 {
628  QFile dfile(datafile);
629  QFileInfo dfileinfo(dfile);
630 
631  if (!dfile.open(QIODevice::ReadOnly))
632  return false;
633 
634  if (!dfileinfo.size())
635  {
636  /* Dummy file: no template. */
637  *pvalid = false;
638  return true;
639  }
640 
641  QTextStream stream(&dfile);
642  stream >> *prow >> *pcol >> *pwidth >> *pheight;
643  dfile.close();
644 
645  if (av_image_alloc(tmpl->data, tmpl->linesize,
646  *pwidth, *pheight, AV_PIX_FMT_GRAY8, IMAGE_ALIGN))
647  {
648  LOG(VB_COMMFLAG, LOG_ERR,
649  QString("readTemplate av_image_alloc %1 (%2x%3) failed")
650  .arg(tmplfile).arg(*pwidth).arg(*pheight));
651  return false;
652  }
653 
654  QByteArray tmfile = tmplfile.toLatin1();
655  if (pgm_read(tmpl->data[0], *pwidth, *pheight, tmfile.constData()))
656  {
657  av_freep(&tmpl->data[0]);
658  return false;
659  }
660 
661  *pvalid = true;
662  return true;
663 }
664 
665 void
666 writeDummyTemplate(const QString& datafile)
667 {
668  /* Leave a 0-byte file. */
669  QFile dfile(datafile);
670 
671  if (!dfile.open(QIODevice::WriteOnly | QIODevice::Truncate) &&
672  dfile.exists())
673  (void)dfile.remove();
674 }
675 
676 bool
677 writeTemplate(const QString& tmplfile, const AVFrame *tmpl, const QString& datafile,
678  int row, int col, int width, int height)
679 {
680  QFile tfile(tmplfile);
681 
682  QByteArray tmfile = tmplfile.toLatin1();
683  if (pgm_write(tmpl->data[0], width, height, tmfile.constData()))
684  return false;
685 
686  QFile dfile(datafile);
687  if (!dfile.open(QIODevice::WriteOnly))
688  return false;
689 
690  QTextStream stream(&dfile);
691  stream << row << " " << col << "\n" << width << " " << height << "\n";
692  dfile.close();
693  return true;
694 }
695 
696 }; /* namespace */
697 
699  EdgeDetector *ed, MythPlayer *player, int proglen,
700  const QString& debugdir)
701  : m_pgmConverter(pgmc)
702  , m_borderDetector(bd)
703  , m_edgeDetector(ed)
704  , m_debugDir(debugdir)
705  , m_debugData(debugdir + "/TemplateFinder.txt")
706  , m_debugTmpl(debugdir + "/TemplateFinder.pgm")
707 {
708  /*
709  * TUNABLE:
710  *
711  * The number of frames desired for sampling to build the template.
712  *
713  * Higher values should yield a more accurate template, but requires more
714  * time.
715  */
716  unsigned int samplesNeeded = 300;
717 
718  /*
719  * TUNABLE:
720  *
721  * The leading amount of time (in seconds) to sample frames for building up
722  * the possible template, and the interval between frames for analysis.
723  * This affects how soon flagging can start after a recording has begun
724  * (a.k.a. "real-time flagging").
725  *
726  * Sample half of the program length or 20 minutes, whichever is less.
727  */
728  m_sampleTime = min(proglen / 2, 20 * 60);
729 
730  const float fps = player->GetFrameRate();
731 
732  m_frameInterval = (int)roundf(m_sampleTime * fps / samplesNeeded);
733  m_endFrame = 0 + (long long)m_frameInterval * samplesNeeded - 1;
734 
735  LOG(VB_COMMFLAG, LOG_INFO,
736  QString("TemplateFinder: sampleTime=%1s, samplesNeeded=%2, endFrame=%3")
737  .arg(m_sampleTime).arg(samplesNeeded).arg(m_endFrame));
738 
739  /*
740  * debugLevel:
741  * 0: no extra debugging
742  * 1: cache computations into debugdir [O(1) files]
743  * 2: extra verbosity [O(nframes)]
744  * 3: dump frames into debugdir [O(nframes) files]
745  */
746  m_debugLevel = gCoreContext->GetNumSetting("TemplateFinderDebugLevel", 0);
747 
748  if (m_debugLevel >= 1)
749  {
751  QString("TemplateFinder debugLevel %1").arg(m_debugLevel));
752 
753  m_debugTemplate = true;
754  m_debugEdgeCounts = true;
755 
756  if (m_debugLevel >= 3)
757  m_debugFrames = true;
758  }
759 }
760 
762 {
763  delete []m_scores;
764  av_freep(&m_tmpl.data[0]);
765  av_freep(&m_cropped.data[0]);
766 }
767 
769 TemplateFinder::MythPlayerInited(MythPlayer *player, long long nframes)
770 {
771  /*
772  * Only detect edges in portions of the frame where we expect to find
773  * a template. This serves two purposes:
774  *
775  * - Speed: reduce search space.
776  * - Correctness (insofar as the assumption of template location is
777  * correct): don't "pollute" the set of candidate template edges with
778  * the "content" edges in the non-template portions of the frame.
779  */
780  QString tmpldims;
781  QString playerdims;
782 
783  (void)nframes; /* gcc */
784  QSize buf_dim = player->GetVideoBufferSize();
785  m_width = buf_dim.width();
786  m_height = buf_dim.height();
787  playerdims = QString("%1x%2").arg(m_width).arg(m_height);
788 
789  if (m_debugTemplate)
790  {
791  if ((m_tmplDone = readTemplate(m_debugData, &m_tmplRow, &m_tmplCol,
793  &m_tmplValid)))
794  {
795  tmpldims = m_tmplValid ? QString("%1x%2@(%3,%4)")
796  .arg(m_tmplWidth).arg(m_tmplHeight).arg(m_tmplCol).arg(m_tmplRow) :
797  "no template";
798 
799  LOG(VB_COMMFLAG, LOG_INFO,
800  QString("TemplateFinder::MythPlayerInited read %1: %2")
801  .arg(m_debugTmpl)
802  .arg(tmpldims));
803  }
804  }
805 
806  if (m_pgmConverter->MythPlayerInited(player))
807  goto free_tmpl;
808 
809  if (m_borderDetector->MythPlayerInited(player))
810  goto free_tmpl;
811 
812  if (m_tmplDone)
813  {
814  if (m_tmplValid)
815  {
816  LOG(VB_COMMFLAG, LOG_INFO,
817  QString("TemplateFinder::MythPlayerInited %1 of %2 (%3)")
818  .arg(tmpldims).arg(playerdims).arg(m_debugTmpl));
819  }
820  return ANALYZE_FINISHED;
821  }
822 
823  LOG(VB_COMMFLAG, LOG_INFO,
824  QString("TemplateFinder::MythPlayerInited framesize %1")
825  .arg(playerdims));
826  m_scores = new unsigned int[m_width * m_height];
827 
828  return ANALYZE_OK;
829 
830 free_tmpl:
831  av_freep(&m_tmpl.data[0]);
832  return ANALYZE_FATAL;
833 }
834 
835 int
836 TemplateFinder::resetBuffers(int newwidth, int newheight)
837 {
838  if (m_cwidth == newwidth && m_cheight == newheight)
839  return 0;
840 
841  av_freep(&m_cropped.data[0]);
842 
843  if (av_image_alloc(m_cropped.data, m_cropped.linesize,
844  newwidth, newheight, AV_PIX_FMT_GRAY8, IMAGE_ALIGN))
845  {
846  LOG(VB_COMMFLAG, LOG_ERR,
847  QString("TemplateFinder::resetBuffers "
848  "av_image_alloc cropped (%1x%2) failed")
849  .arg(newwidth).arg(newheight));
850  return -1;
851  }
852 
853  m_cwidth = newwidth;
854  m_cheight = newheight;
855  return 0;
856 }
857 
859 TemplateFinder::analyzeFrame(const VideoFrame *frame, long long frameno,
860  long long *pNextFrame)
861 {
862  /*
863  * TUNABLE:
864  *
865  * When looking for edges in frames, select some percentile of
866  * squared-gradient magnitudes (intensities) as candidate edges. (This
867  * number conventionally should not go any lower than the 95th percentile;
868  * see edge_mark.)
869  *
870  * Higher values result in fewer edges; faint logos might not be picked up.
871  * Lower values result in more edges; non-logo edges might be picked up.
872  *
873  * The TemplateFinder accumulates all its state in the "scores" array to
874  * be processed later by TemplateFinder::finished.
875  */
876  const int FRAMESGMPCTILE = 90;
877 
878  /*
879  * TUNABLE:
880  *
881  * Exclude some portion of the center of the frame from edge analysis.
882  * Elminate false edge-detection logo positives from talking-host types of
883  * shows where the high-contrast host and clothes (e.g., tie against white
884  * shirt against dark jacket) dominates the edges.
885  *
886  * This has a nice side-effect of reducing the area to be examined (speed
887  * optimization).
888  */
889  static constexpr float kExcludeWidth = 0.5;
890  static constexpr float kExcludeHeight = 0.5;
891 
892  int pgmwidth= 0;
893  int pgmheight = 0;
894  int croprow= 0;
895  int cropcol = 0;
896  int cropwidth = 0;
897  int cropheight = 0;
898  struct timeval start {};
899  struct timeval end {};
900  struct timeval elapsed {};
901 
902  if (frameno < m_nextFrame)
903  {
904  *pNextFrame = m_nextFrame;
905  return ANALYZE_OK;
906  }
907 
908  m_nextFrame = frameno + m_frameInterval;
909  *pNextFrame = min(m_endFrame, m_nextFrame);
910 
911  const AVFrame *pgm = m_pgmConverter->getImage(frame, frameno, &pgmwidth, &pgmheight);
912  if (pgm == nullptr)
913  goto error;
914 
915  if (!m_borderDetector->getDimensions(pgm, pgmheight, frameno,
916  &croprow, &cropcol, &cropwidth, &cropheight))
917  {
918  /* Not a blank frame. */
919 
920  (void)gettimeofday(&start, nullptr);
921 
922  if (croprow < m_minContentRow)
923  m_minContentRow = croprow;
924  if (cropcol < m_minContentCol)
925  m_minContentCol = cropcol;
926  if (cropcol + cropwidth > m_maxContentCol1)
927  m_maxContentCol1 = cropcol + cropwidth;
928  if (croprow + cropheight > m_maxContentRow1)
929  m_maxContentRow1 = croprow + cropheight;
930 
931  if (resetBuffers(cropwidth, cropheight))
932  goto error;
933 
934  if (pgm_crop(&m_cropped, pgm, pgmheight, croprow, cropcol,
935  cropwidth, cropheight))
936  goto error;
937 
938  /*
939  * Translate the excluded area of the screen into "cropped"
940  * coordinates.
941  */
942  int excludewidth = (int)(pgmwidth * kExcludeWidth);
943  int excludeheight = (int)(pgmheight * kExcludeHeight);
944  int excluderow = (pgmheight - excludeheight) / 2 - croprow;
945  int excludecol = (pgmwidth - excludewidth) / 2 - cropcol;
946  (void)m_edgeDetector->setExcludeArea(excluderow, excludecol,
947  excludewidth, excludeheight);
948 
949  const AVFrame *edges =
950  m_edgeDetector->detectEdges(&m_cropped, cropheight, FRAMESGMPCTILE);
951  if (edges == nullptr)
952  goto error;
953 
954  if (pgm_scorepixels(m_scores, pgmwidth, croprow, cropcol,
955  edges, cropheight))
956  goto error;
957 
958  if (m_debugLevel >= 2)
959  {
960  if (!analyzeFrameDebug(frameno, pgm, pgmheight, &m_cropped, edges,
961  cropheight, croprow, cropcol, m_debugFrames, m_debugDir))
962  goto error;
963  }
964 
965  (void)gettimeofday(&end, nullptr);
966  timersub(&end, &start, &elapsed);
967  timeradd(&m_analyzeTime, &elapsed, &m_analyzeTime);
968  }
969 
970  if (m_nextFrame > m_endFrame)
971  return ANALYZE_FINISHED;
972 
973  return ANALYZE_OK;
974 
975 error:
976  LOG(VB_COMMFLAG, LOG_ERR,
977  QString("TemplateFinder::analyzeFrame error at frame %1")
978  .arg(frameno));
979 
980  if (m_nextFrame > m_endFrame)
981  return ANALYZE_FINISHED;
982 
983  return ANALYZE_ERROR;
984 }
985 
986 int
987 TemplateFinder::finished(long long nframes, bool final)
988 {
989  (void)nframes; /* gcc */
990  if (!m_tmplDone)
991  {
992  if (!template_alloc(m_scores, m_width, m_height,
997  {
998  if (final)
999  writeDummyTemplate(m_debugData);
1000  }
1001  else
1002  {
1003  if (final && m_debugTemplate)
1004  {
1005  if (!(m_tmplValid = writeTemplate(m_debugTmpl, &m_tmpl, m_debugData,
1007  goto free_tmpl;
1008 
1009  LOG(VB_COMMFLAG, LOG_INFO,
1010  QString("TemplateFinder::finished wrote %1"
1011  " and %2 [%3x%4@(%5,%6)]")
1012  .arg(m_debugTmpl).arg(m_debugData)
1013  .arg(m_tmplWidth).arg(m_tmplHeight)
1014  .arg(m_tmplCol).arg(m_tmplRow));
1015  }
1016  }
1017 
1018  if (final)
1019  m_tmplDone = true;
1020  }
1021 
1023 
1024  return 0;
1025 
1026 free_tmpl:
1027  av_freep(&m_tmpl.data[0]);
1028  return -1;
1029 }
1030 
1031 int
1033 {
1034  if (m_pgmConverter->reportTime())
1035  return -1;
1036 
1038  return -1;
1039 
1040  LOG(VB_COMMFLAG, LOG_INFO, QString("TF Time: analyze=%1s")
1041  .arg(strftimeval(&m_analyzeTime)));
1042  return 0;
1043 }
1044 
1045 const struct AVFrame *
1046 TemplateFinder::getTemplate(int *prow, int *pcol, int *pwidth, int *pheight)
1047  const
1048 {
1049  if (m_tmplValid)
1050  {
1051  *prow = m_tmplRow;
1052  *pcol = m_tmplCol;
1053  *pwidth = m_tmplWidth;
1054  *pheight = m_tmplHeight;
1055  return &m_tmpl;
1056  }
1057  return nullptr;
1058 }
1059 
1060 /* vim: set expandtab tabstop=4 shiftwidth=4: */
Definition: cc.h:13
#define GENERIC_EXIT_OK
Exited with no error.
Definition: exitcodes.h:10
enum analyzeFrameResult analyzeFrame(const VideoFrame *frame, long long frameno, long long *pNextFrame) override
static void error(const char *str,...)
Definition: vbi.c:42
virtual int setExcludeArea(int row, int col, int width, int height)
long long m_endFrame
#define timeradd(a, b, result)
Definition: compat.h:300
PGMConverter * m_pgmConverter
struct AVFrame AVFrame
static int sort_ascending(const void *aa, const void *bb)
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
int finished(long long nframes, bool final) override
~TemplateFinder(void) override
TemplateFinder(PGMConverter *pgmc, BorderDetector *bd, EdgeDetector *ed, MythPlayer *player, int proglen, const QString &debugdir)
int pgm_read(unsigned char *buf, int width, int height, const char *filename)
Definition: pgm.cpp:32
int MythPlayerInited(const MythPlayer *player)
int reportTime(void) const override
const struct AVFrame * getTemplate(int *prow, int *pcol, int *pwidth, int *pheight) const
float GetFrameRate(void) const
Definition: mythplayer.h:211
void setLogoState(TemplateFinder *finder)
unsigned int * m_scores
unsigned int uint
Definition: compat.h:140
enum analyzeFrameResult MythPlayerInited(MythPlayer *player, long long nframes) override
uint myth_system(const QString &command, uint flags, uint timeout)
int MythPlayerInited(const MythPlayer *player)
int resetBuffers(int newwidth, int newheight)
EdgeDetector * m_edgeDetector
BorderDetector * m_borderDetector
int GetNumSetting(const QString &key, int defaultval=0)
void createDebugDirectory(const QString &dirname, const QString &comment)
#define LOG(_MASK_, _LEVEL_, _STRING_)
Definition: mythlogging.h:41
QSize GetVideoBufferSize(void) const
Definition: mythplayer.h:208
unsigned int m_sampleTime
QString m_debugTmpl
int pgm_crop(AVFrame *dst, const AVFrame *src, int srcheight, int srcrow, int srccol, int cropwidth, int cropheight)
Definition: pgm.cpp:165
int pgm_write(const unsigned char *buf, int width, int height, const char *filename)
Definition: pgm.cpp:80
QString m_debugDir
QString strftimeval(const struct timeval *tv)
int reportTime(void)
QString m_debugData
#define timersub(a, b, result)
Definition: compat.h:310
int getDimensions(const AVFrame *pgm, int pgmheight, long long frameno, int *prow, int *pcol, int *pwidth, int *pheight)
long long m_nextFrame
virtual const AVFrame * detectEdges(const AVFrame *pgm, int pgmheight, int percentile)=0
const AVFrame * getImage(const VideoFrame *frame, long long frameno, int *pwidth, int *pheight)
int reportTime(void)