MythTV  master
TemplateFinder.cpp
Go to the documentation of this file.
1 // POSIX headers
2 #include <sys/time.h> /* gettimeofday */
3 
4 // ANSI C headers
5 #include <cmath>
6 #include <cstdlib>
7 
8 // Qt headers
9 #include <QFile>
10 #include <QFileInfo>
11 #include <QTextStream>
12 
13 // MythTV headers
14 #include "mythplayer.h"
15 #include "mythcorecontext.h" /* gContext */
16 #include "mythframe.h" /* VideoFrame */
17 #include "mythdate.h"
18 #include "mythsystemlegacy.h"
19 #include "exitcodes.h"
20 
21 // Commercial Flagging headers
22 #include "CommDetector2.h"
23 #include "pgm.h"
24 #include "PGMConverter.h"
25 #include "BorderDetector.h"
26 #include "EdgeDetector.h"
27 #include "TemplateFinder.h"
28 
29 extern "C" {
30  #include "libavutil/imgutils.h"
31  }
32 
33 using namespace commDetector2;
34 
35 namespace {
36 
37 //returns true on success, false otherwise
38 bool writeJPG(const QString& prefix, const AVFrame *img, int imgheight)
39 {
40  const int imgwidth = img->linesize[0];
41  QFileInfo jpgfi(prefix + ".jpg");
42  if (!jpgfi.exists())
43  {
44  QFile pgmfile(prefix + ".pgm");
45  if (!pgmfile.exists())
46  {
47  QByteArray pfname = pgmfile.fileName().toLocal8Bit();
48  if (pgm_write(img->data[0], imgwidth, imgheight,
49  pfname.constData()))
50  {
51  return false;
52  }
53  }
54 
55  QString cmd = QString("convert -quality 50 -resize 192x144 %1 %2")
56  .arg(pgmfile.fileName()).arg(jpgfi.filePath());
57  if (myth_system(cmd) != GENERIC_EXIT_OK)
58  return false;
59 
60  if (!pgmfile.remove())
61  {
62  LOG(VB_COMMFLAG, LOG_ERR,
63  QString("TemplateFinder.writeJPG error removing %1 (%2)")
64  .arg(pgmfile.fileName()).arg(strerror(errno)));
65  return false;
66  }
67  }
68  return true;
69 }
70 
71 int
72 pgm_scorepixels(unsigned int *scores, int width, int row, int col,
73  const AVFrame *src, int srcheight)
74 {
75  /* Every time a pixel is an edge, give it a point. */
76  const int srcwidth = src->linesize[0];
77 
78  for (int rr = 0; rr < srcheight; rr++)
79  {
80  for (int cc = 0; cc < srcwidth; cc++)
81  {
82  if (src->data[0][rr * srcwidth + cc])
83  scores[(row + rr) * width + col + cc]++;
84  }
85  }
86 
87  return 0;
88 }
89 
90 int
91 sort_ascending(const void *aa, const void *bb)
92 {
93  return *(unsigned int*)aa - *(unsigned int*)bb;
94 }
95 
96 float
97 bounding_score(const AVFrame *img, int row, int col, int width, int height)
98 {
99  /* Return a value between [0..1] */
100  const int imgwidth = img->linesize[0];
101 
102  uint score = 0;
103  int rr2 = row + height;
104  int cc2 = col + width;
105  for (int rr = row; rr < rr2; rr++)
106  {
107  for (int cc = col; cc < cc2; cc++)
108  {
109  if (img->data[0][rr * imgwidth + cc])
110  score++;
111  }
112  }
113  return (float)score / (width * height);
114 }
115 
116 bool
117 rowisempty(const AVFrame *img, int row, int col, int width)
118 {
119  const int imgwidth = img->linesize[0];
120  for (int cc = col; cc < col + width; cc++)
121  if (img->data[0][row * imgwidth + cc])
122  return false;
123  return true;
124 }
125 
126 bool
127 colisempty(const AVFrame *img, int col, int row, int height)
128 {
129  const int imgwidth = img->linesize[0];
130  for (int rr = row; rr < row + height; rr++)
131  if (img->data[0][rr * imgwidth + col])
132  return false;
133  return true;
134 }
135 
136 int
137 bounding_box(const AVFrame *img, int imgheight,
138  int minrow, int mincol, int maxrow1, int maxcol1,
139  int *prow, int *pcol, int *pwidth, int *pheight)
140 {
141  const int imgwidth = img->linesize[0];
142  /*
143  * TUNABLE:
144  *
145  * Maximum logo size, expressed as a percentage of the content area
146  * (adjusting for letterboxing and pillarboxing).
147  */
148  static constexpr int kMaxWidthPct = 20;
149  static constexpr int kMaxHeightPct = 20;
150 
151  /*
152  * TUNABLE:
153  *
154  * Safety margin to avoid cutting too much of the logo.
155  * Higher values cut more, but avoid noise as part of the template..
156  * Lower values cut less, but can include noise as part of the template.
157  */
158  const int VERTSLOP = max(4, imgheight * 1 / 15);
159  const int HORIZSLOP = max(4, imgwidth * 1 / 20);
160 
161  int maxwidth = (maxcol1 - mincol) * kMaxWidthPct / 100;
162  int maxheight = (maxrow1 - minrow) * kMaxHeightPct / 100;
163 
164  int row = minrow;
165  int col = mincol;
166  int width = maxcol1 - mincol;
167  int height = maxrow1 - minrow;
168  int newrow = 0, newcol = 0, newright = 0, newbottom = 0;
169 
170  for (;;)
171  {
172  float newscore = NAN;
173  bool improved = false;
174 
175  LOG(VB_COMMFLAG, LOG_INFO, QString("bounding_box %1x%2@(%3,%4)")
176  .arg(width).arg(height).arg(col).arg(row));
177 
178  /* Chop top. */
179  float score = bounding_score(img, row, col, width, height);
180  newrow = row;
181  for (int ii = 1; ii < height; ii++)
182  {
183  if ((newscore = bounding_score(img, row + ii, col,
184  width, height - ii)) < score)
185  break;
186  score = newscore;
187  newrow = row + ii;
188  improved = true;
189  }
190 
191  /* Chop left. */
192  score = bounding_score(img, row, col, width, height);
193  newcol = col;
194  for (int ii = 1; ii < width; ii++)
195  {
196  if ((newscore = bounding_score(img, row, col + ii,
197  width - ii, height)) < score)
198  break;
199  score = newscore;
200  newcol = col + ii;
201  improved = true;
202  }
203 
204  /* Chop bottom. */
205  score = bounding_score(img, row, col, width, height);
206  newbottom = row + height;
207  for (int ii = 1; ii < height; ii++)
208  {
209  if ((newscore = bounding_score(img, row, col,
210  width, height - ii)) < score)
211  break;
212  score = newscore;
213  newbottom = row + height - ii;
214  improved = true;
215  }
216 
217  /* Chop right. */
218  score = bounding_score(img, row, col, width, height);
219  newright = col + width;
220  for (int ii = 1; ii < width; ii++)
221  {
222  if ((newscore = bounding_score(img, row, col,
223  width - ii, height)) < score)
224  break;
225  score = newscore;
226  newright = col + width - ii;
227  improved = true;
228  }
229 
230  if (!improved)
231  break;
232 
233  row = newrow;
234  col = newcol;
235  width = newright - newcol;
236  height = newbottom - newrow;
237 
238  /*
239  * Noise edge pixels in the frequency template can sometimes stretch
240  * the template area to be larger than it should be.
241  *
242  * However, noise needs to be distinguished from a uniform distribution
243  * of noise pixels (e.g., no real statically-located template). So if
244  * the template area is too "large", then some quadrant must have a
245  * clear majority of the edge pixels; otherwise we declare failure (no
246  * template found).
247  *
248  * Intuitively, we should simply repeat until a single bounding box is
249  * converged upon. However, this requires a more sophisticated
250  * bounding_score function that I don't feel like figuring out.
251  * Indefinitely repeating with the present bounding_score function will
252  * tend to chop off too much. Instead, simply do some sanity checks on
253  * the candidate template's size, and prune the template area and
254  * repeat if it is too "large".
255  */
256 
257  if (width > maxwidth)
258  {
259  /* Too wide; test left and right portions. */
260  int chop = width / 3;
261  int chopwidth = width - chop;
262 
263  float left = bounding_score(img, row, col, chopwidth, height);
264  float right = bounding_score(img, row, col + chop, chopwidth, height);
265  LOG(VB_COMMFLAG, LOG_INFO,
266  QString("bounding_box too wide (%1 > %2); left=%3, right=%4")
267  .arg(width).arg(maxwidth)
268  .arg(left, 0, 'f', 3).arg(right, 0, 'f', 3));
269  float minscore = min(left, right);
270  float maxscore = max(left, right);
271  if (maxscore < 3 * minscore / 2)
272  {
273  /*
274  * Edge pixel distribution too uniform; give up.
275  *
276  * XXX: also fails for horizontally-centered templates ...
277  */
278  LOG(VB_COMMFLAG, LOG_ERR, "bounding_box giving up (edge "
279  "pixels distributed too uniformly)");
280  return -1;
281  }
282 
283  if (left < right)
284  col += chop;
285  width -= chop;
286  continue;
287  }
288 
289  if (height > maxheight)
290  {
291  /* Too tall; test upper and lower portions. */
292  int chop = height / 3;
293  int chopheight = height - chop;
294 
295  float upper = bounding_score(img, row, col, width, chopheight);
296  float lower = bounding_score(img, row + chop, col, width, chopheight);
297  LOG(VB_COMMFLAG, LOG_INFO,
298  QString("bounding_box too tall (%1 > %2); upper=%3, lower=%4")
299  .arg(height).arg(maxheight)
300  .arg(upper, 0, 'f', 3).arg(lower, 0, 'f', 3));
301  float minscore = min(upper, lower);
302  float maxscore = max(upper, lower);
303  if (maxscore < 3 * minscore / 2)
304  {
305  /*
306  * Edge pixel distribution too uniform; give up.
307  *
308  * XXX: also fails for vertically-centered templates ...
309  */
310  LOG(VB_COMMFLAG, LOG_ERR, "bounding_box giving up (edge "
311  "pixel distribution too uniform)");
312  return -1;
313  }
314 
315  if (upper < lower)
316  row += chop;
317  height -= chop;
318  continue;
319  }
320 
321  break;
322  }
323 
324  /*
325  * The above "chop" algorithm often cuts off the outside edges of the
326  * logos because the outside edges don't contribute enough to the score. So
327  * compensate by now expanding the bounding box (up to a *SLOP pixels in
328  * each direction) to include all edge pixels.
329  */
330 
331  LOG(VB_COMMFLAG, LOG_INFO,
332  QString("bounding_box %1x%2@(%3,%4); horizslop=%5,vertslop=%6")
333  .arg(width).arg(height).arg(col).arg(row)
334  .arg(HORIZSLOP).arg(VERTSLOP));
335 
336  /* Expand upwards. */
337  newrow = row - 1;
338  for (;;)
339  {
340  if (newrow <= minrow)
341  {
342  newrow = minrow;
343  break;
344  }
345  if (row - newrow >= VERTSLOP)
346  {
347  newrow = row - VERTSLOP;
348  break;
349  }
350  if (rowisempty(img, newrow, col, width))
351  {
352  newrow++;
353  break;
354  }
355  newrow--;
356  }
357  newrow = max(minrow, newrow - 1); /* Empty row on top. */
358 
359  /* Expand leftwards. */
360  newcol = col - 1;
361  for (;;)
362  {
363  if (newcol <= mincol)
364  {
365  newcol = mincol;
366  break;
367  }
368  if (col - newcol >= HORIZSLOP)
369  {
370  newcol = col - HORIZSLOP;
371  break;
372  }
373  if (colisempty(img, newcol, row, height))
374  {
375  newcol++;
376  break;
377  }
378  newcol--;
379  }
380  newcol = max(mincol, newcol - 1); /* Empty column to left. */
381 
382  /* Expand rightwards. */
383  newright = col + width;
384  for (;;)
385  {
386  if (newright >= maxcol1)
387  {
388  newright = maxcol1;
389  break;
390  }
391  if (newright - (col + width) >= HORIZSLOP)
392  {
393  newright = col + width + HORIZSLOP;
394  break;
395  }
396  if (colisempty(img, newright, row, height))
397  break;
398  newright++;
399  }
400  newright = min(maxcol1, newright + 1); /* Empty column to right. */
401 
402  /* Expand downwards. */
403  newbottom = row + height;
404  for (;;)
405  {
406  if (newbottom >= maxrow1)
407  {
408  newbottom = maxrow1;
409  break;
410  }
411  if (newbottom - (row + height) >= VERTSLOP)
412  {
413  newbottom = row + height + VERTSLOP;
414  break;
415  }
416  if (rowisempty(img, newbottom, col, width))
417  break;
418  newbottom++;
419  }
420  newbottom = min(maxrow1, newbottom + 1); /* Empty row on bottom. */
421 
422  row = newrow;
423  col = newcol;
424  width = newright - newcol;
425  height = newbottom - newrow;
426 
427  LOG(VB_COMMFLAG, LOG_INFO, QString("bounding_box %1x%2@(%3,%4)")
428  .arg(width).arg(height).arg(col).arg(row));
429 
430  *prow = row;
431  *pcol = col;
432  *pwidth = width;
433  *pheight = height;
434  return 0;
435 }
436 
437 bool
438 template_alloc(const unsigned int *scores, int width, int height,
439  int minrow, int mincol, int maxrow1, int maxcol1, AVFrame *tmpl,
440  int *ptmplrow, int *ptmplcol, int *ptmplwidth, int *ptmplheight,
441  bool debug_edgecounts, const QString& debugdir)
442 {
443  /*
444  * TUNABLE:
445  *
446  * Higher values select for "stronger" pixels to be in the template, but
447  * weak pixels might be missed.
448  *
449  * Lower values allow more pixels to be included as part of the template,
450  * but strong non-template pixels might be included.
451  */
452  static constexpr float kMinScorePctile = 0.998;
453 
454  const int nn = width * height;
455  int ii = 0, first = 0, last = 0;
456  unsigned int threshscore = 0;
457  AVFrame thresh;
458 
459  if (av_image_alloc(thresh.data, thresh.linesize,
460  width, height, AV_PIX_FMT_GRAY8, IMAGE_ALIGN))
461  {
462  LOG(VB_COMMFLAG, LOG_ERR,
463  QString("template_alloc av_image_alloc thresh (%1x%2) failed")
464  .arg(width).arg(height));
465  return false;
466  }
467 
468  uint *sortedscores = new unsigned int[nn];
469  memcpy(sortedscores, scores, nn * sizeof(*sortedscores));
470  qsort(sortedscores, nn, sizeof(*sortedscores), sort_ascending);
471 
472  if (sortedscores[0] == sortedscores[nn - 1])
473  {
474  /* All pixels in the template area look the same; no template. */
475  LOG(VB_COMMFLAG, LOG_ERR,
476  QString("template_alloc: %1x%2 pixels all identical!")
477  .arg(width).arg(height));
478  goto free_thresh;
479  }
480 
481  /* Threshold the edge frequences. */
482 
483  ii = (int)roundf(nn * kMinScorePctile);
484  threshscore = sortedscores[ii];
485  for (first = ii; first > 0 && sortedscores[first] == threshscore; first--)
486  ;
487  if (sortedscores[first] != threshscore)
488  first++;
489  for (last = ii; last < nn - 1 && sortedscores[last] == threshscore; last++)
490  ;
491  if (sortedscores[last] != threshscore)
492  last--;
493 
494  LOG(VB_COMMFLAG, LOG_INFO, QString("template_alloc wanted %1, got %2-%3")
495  .arg(kMinScorePctile, 0, 'f', 6)
496  .arg((float)first / nn, 0, 'f', 6)
497  .arg((float)last / nn, 0, 'f', 6));
498 
499  for (ii = 0; ii < nn; ii++)
500  thresh.data[0][ii] = scores[ii] >= threshscore ? UCHAR_MAX : 0;
501 
502  if (debug_edgecounts)
503  {
504  /* Scores, rescaled to [0..UCHAR_MAX]. */
505  AVFrame scored;
506  if (av_image_alloc(scored.data, scored.linesize,
507  width, height, AV_PIX_FMT_GRAY8, IMAGE_ALIGN))
508  {
509  LOG(VB_COMMFLAG, LOG_ERR,
510  QString("template_alloc av_image_alloc scored (%1x%2) failed")
511  .arg(width).arg(height));
512  goto free_thresh;
513  }
514  unsigned int maxscore = sortedscores[nn - 1];
515  for (ii = 0; ii < nn; ii++)
516  scored.data[0][ii] = scores[ii] * UCHAR_MAX / maxscore;
517  bool success = writeJPG(debugdir + "/TemplateFinder-scores", &scored,
518  height);
519  av_freep(&scored.data[0]);
520  if (!success)
521  goto free_thresh;
522 
523  /* Thresholded scores. */
524  if (!writeJPG(debugdir + "/TemplateFinder-edgecounts", &thresh, height))
525  goto free_thresh;
526  }
527 
528  /* Crop to a minimal bounding box. */
529 
530  if (bounding_box(&thresh, height, minrow, mincol, maxrow1, maxcol1,
531  ptmplrow, ptmplcol, ptmplwidth, ptmplheight))
532  goto free_thresh;
533 
534  if ((uint)(*ptmplwidth * *ptmplheight) > USHRT_MAX)
535  {
536  /* Max value of data type of TemplateMatcher::edgematch */
537  LOG(VB_COMMFLAG, LOG_ERR,
538  QString("template_alloc bounding_box too big (%1x%2)")
539  .arg(*ptmplwidth).arg(*ptmplheight));
540  goto free_thresh;
541  }
542 
543  if (av_image_alloc(tmpl->data, tmpl->linesize,
544  *ptmplwidth, *ptmplheight, AV_PIX_FMT_GRAY8, IMAGE_ALIGN))
545  {
546  LOG(VB_COMMFLAG, LOG_ERR,
547  QString("template_alloc av_image_alloc tmpl (%1x%2) failed")
548  .arg(*ptmplwidth).arg(*ptmplheight));
549  goto free_thresh;
550  }
551 
552  if (pgm_crop(tmpl, &thresh, height, *ptmplrow, *ptmplcol,
553  *ptmplwidth, *ptmplheight))
554  goto free_thresh;
555 
556  delete []sortedscores;
557  av_freep(&thresh.data[0]);
558 
559  return true;
560 
561 free_thresh:
562  delete []sortedscores;
563  av_freep(&thresh.data[0]);
564  return false;
565 }
566 
567 bool
568 analyzeFrameDebug(long long frameno, const AVFrame *pgm, int pgmheight,
569  const AVFrame *cropped, const AVFrame *edges, int cropheight,
570  int croprow, int cropcol, bool debug_frames, const QString& debugdir)
571 {
572  static constexpr int kDelta = 24;
573  static int s_lastrow, s_lastcol, s_lastwidth, s_lastheight;
574  const int cropwidth = cropped->linesize[0];
575 
576  int rowsame = abs(s_lastrow - croprow) <= kDelta ? 1 : 0;
577  int colsame = abs(s_lastcol - cropcol) <= kDelta ? 1 : 0;
578  int widthsame = abs(s_lastwidth - cropwidth) <= kDelta ? 1 : 0;
579  int heightsame = abs(s_lastheight - cropheight) <= kDelta ? 1 : 0;
580 
581  if (frameno > 0 && rowsame + colsame + widthsame + heightsame >= 3)
582  return true;
583 
584  LOG(VB_COMMFLAG, LOG_INFO,
585  QString("TemplateFinder Frame %1: %2x%3@(%4,%5)")
586  .arg(frameno, 5)
587  .arg(cropwidth).arg(cropheight)
588  .arg(cropcol).arg(croprow));
589 
590  s_lastrow = croprow;
591  s_lastcol = cropcol;
592  s_lastwidth = cropwidth;
593  s_lastheight = cropheight;
594 
595  if (debug_frames)
596  {
597  QString base = QString("%1/TemplateFinder-%2")
598  .arg(debugdir).arg(frameno, 5, 10, QChar('0'));
599 
600  /* PGM greyscale image of frame. */
601  if (!writeJPG(base, pgm, pgmheight))
602  return false;
603 
604  /* Cropped template area of frame. */
605  if (!writeJPG(base + "-cropped", cropped, cropheight))
606  return false;
607 
608  /* Edges of cropped template area of frame. */
609  if (!writeJPG(base + "-edges", edges, cropheight))
610  return false;
611  }
612 
613  return true;
614 }
615 
616 bool
617 readTemplate(const QString& datafile, int *prow, int *pcol, int *pwidth, int *pheight,
618  const QString& tmplfile, AVFrame *tmpl, bool *pvalid)
619 {
620  QFile dfile(datafile);
621  QFileInfo dfileinfo(dfile);
622 
623  if (!dfile.open(QIODevice::ReadOnly))
624  return false;
625 
626  if (!dfileinfo.size())
627  {
628  /* Dummy file: no template. */
629  *pvalid = false;
630  return true;
631  }
632 
633  QTextStream stream(&dfile);
634  stream >> *prow >> *pcol >> *pwidth >> *pheight;
635  dfile.close();
636 
637  if (av_image_alloc(tmpl->data, tmpl->linesize,
638  *pwidth, *pheight, AV_PIX_FMT_GRAY8, IMAGE_ALIGN))
639  {
640  LOG(VB_COMMFLAG, LOG_ERR,
641  QString("readTemplate av_image_alloc %1 (%2x%3) failed")
642  .arg(tmplfile).arg(*pwidth).arg(*pheight));
643  return false;
644  }
645 
646  QByteArray tmfile = tmplfile.toLatin1();
647  if (pgm_read(tmpl->data[0], *pwidth, *pheight, tmfile.constData()))
648  {
649  av_freep(&tmpl->data[0]);
650  return false;
651  }
652 
653  *pvalid = true;
654  return true;
655 }
656 
657 void
658 writeDummyTemplate(const QString& datafile)
659 {
660  /* Leave a 0-byte file. */
661  QFile dfile(datafile);
662 
663  if (!dfile.open(QIODevice::WriteOnly | QIODevice::Truncate) &&
664  dfile.exists())
665  (void)dfile.remove();
666 }
667 
668 bool
669 writeTemplate(const QString& tmplfile, const AVFrame *tmpl, const QString& datafile,
670  int row, int col, int width, int height)
671 {
672  QFile tfile(tmplfile);
673 
674  QByteArray tmfile = tmplfile.toLatin1();
675  if (pgm_write(tmpl->data[0], width, height, tmfile.constData()))
676  return false;
677 
678  QFile dfile(datafile);
679  if (!dfile.open(QIODevice::WriteOnly))
680  return false;
681 
682  QTextStream stream(&dfile);
683  stream << row << " " << col << "\n" << width << " " << height << "\n";
684  dfile.close();
685  return true;
686 }
687 
688 }; /* namespace */
689 
691  EdgeDetector *ed, MythPlayer *player, int proglen,
692  const QString& debugdir)
693  : m_pgmConverter(pgmc)
694  , m_borderDetector(bd)
695  , m_edgeDetector(ed)
696  , m_debugdir(debugdir)
697  , m_debugdata(debugdir + "/TemplateFinder.txt")
698  , m_debugtmpl(debugdir + "/TemplateFinder.pgm")
699 {
700  /*
701  * TUNABLE:
702  *
703  * The number of frames desired for sampling to build the template.
704  *
705  * Higher values should yield a more accurate template, but requires more
706  * time.
707  */
708  unsigned int samplesNeeded = 300;
709 
710  /*
711  * TUNABLE:
712  *
713  * The leading amount of time (in seconds) to sample frames for building up
714  * the possible template, and the interval between frames for analysis.
715  * This affects how soon flagging can start after a recording has begun
716  * (a.k.a. "real-time flagging").
717  *
718  * Sample half of the program length or 20 minutes, whichever is less.
719  */
720  m_sampleTime = min(proglen / 2, 20 * 60);
721 
722  const float fps = player->GetFrameRate();
723 
724  m_frameInterval = (int)roundf(m_sampleTime * fps / samplesNeeded);
725  m_endFrame = 0 + (long long)m_frameInterval * samplesNeeded - 1;
726 
727  LOG(VB_COMMFLAG, LOG_INFO,
728  QString("TemplateFinder: sampleTime=%1s, samplesNeeded=%2, endFrame=%3")
729  .arg(m_sampleTime).arg(samplesNeeded).arg(m_endFrame));
730 
731  /*
732  * debugLevel:
733  * 0: no extra debugging
734  * 1: cache computations into debugdir [O(1) files]
735  * 2: extra verbosity [O(nframes)]
736  * 3: dump frames into debugdir [O(nframes) files]
737  */
738  m_debugLevel = gCoreContext->GetNumSetting("TemplateFinderDebugLevel", 0);
739 
740  if (m_debugLevel >= 1)
741  {
743  QString("TemplateFinder debugLevel %1").arg(m_debugLevel));
744 
745  m_debug_template = true;
746  m_debug_edgecounts = true;
747 
748  if (m_debugLevel >= 3)
749  m_debug_frames = true;
750  }
751 }
752 
754 {
755  delete []scores;
756  av_freep(&m_tmpl.data[0]);
757  av_freep(&m_cropped.data[0]);
758 }
759 
761 TemplateFinder::MythPlayerInited(MythPlayer *player, long long nframes)
762 {
763  /*
764  * Only detect edges in portions of the frame where we expect to find
765  * a template. This serves two purposes:
766  *
767  * - Speed: reduce search space.
768  * - Correctness (insofar as the assumption of template location is
769  * correct): don't "pollute" the set of candidate template edges with
770  * the "content" edges in the non-template portions of the frame.
771  */
772  QString tmpldims, playerdims;
773 
774  (void)nframes; /* gcc */
775  QSize buf_dim = player->GetVideoBufferSize();
776  m_width = buf_dim.width();
777  m_height = buf_dim.height();
778  playerdims = QString("%1x%2").arg(m_width).arg(m_height);
779 
780  if (m_debug_template)
781  {
782  if ((m_tmpl_done = readTemplate(m_debugdata, &m_tmplrow, &m_tmplcol,
784  &m_tmpl_valid)))
785  {
786  tmpldims = m_tmpl_valid ? QString("%1x%2@(%3,%4)")
787  .arg(m_tmplwidth).arg(m_tmplheight).arg(m_tmplcol).arg(m_tmplrow) :
788  "no template";
789 
790  LOG(VB_COMMFLAG, LOG_INFO,
791  QString("TemplateFinder::MythPlayerInited read %1: %2")
792  .arg(m_debugtmpl)
793  .arg(tmpldims));
794  }
795  }
796 
797  if (m_pgmConverter->MythPlayerInited(player))
798  goto free_tmpl;
799 
800  if (m_borderDetector->MythPlayerInited(player))
801  goto free_tmpl;
802 
803  if (m_tmpl_done)
804  {
805  if (m_tmpl_valid)
806  {
807  LOG(VB_COMMFLAG, LOG_INFO,
808  QString("TemplateFinder::MythPlayerInited %1 of %2 (%3)")
809  .arg(tmpldims).arg(playerdims).arg(m_debugtmpl));
810  }
811  return ANALYZE_FINISHED;
812  }
813 
814  LOG(VB_COMMFLAG, LOG_INFO,
815  QString("TemplateFinder::MythPlayerInited framesize %1")
816  .arg(playerdims));
817  scores = new unsigned int[m_width * m_height];
818 
819  return ANALYZE_OK;
820 
821 free_tmpl:
822  av_freep(&m_tmpl.data[0]);
823  return ANALYZE_FATAL;
824 }
825 
826 int
827 TemplateFinder::resetBuffers(int newwidth, int newheight)
828 {
829  if (m_cwidth == newwidth && m_cheight == newheight)
830  return 0;
831 
832  av_freep(&m_cropped.data[0]);
833 
834  if (av_image_alloc(m_cropped.data, m_cropped.linesize,
835  newwidth, newheight, AV_PIX_FMT_GRAY8, IMAGE_ALIGN))
836  {
837  LOG(VB_COMMFLAG, LOG_ERR,
838  QString("TemplateFinder::resetBuffers "
839  "av_image_alloc cropped (%1x%2) failed")
840  .arg(newwidth).arg(newheight));
841  return -1;
842  }
843 
844  m_cwidth = newwidth;
845  m_cheight = newheight;
846  return 0;
847 }
848 
850 TemplateFinder::analyzeFrame(const VideoFrame *frame, long long frameno,
851  long long *pNextFrame)
852 {
853  /*
854  * TUNABLE:
855  *
856  * When looking for edges in frames, select some percentile of
857  * squared-gradient magnitudes (intensities) as candidate edges. (This
858  * number conventionally should not go any lower than the 95th percentile;
859  * see edge_mark.)
860  *
861  * Higher values result in fewer edges; faint logos might not be picked up.
862  * Lower values result in more edges; non-logo edges might be picked up.
863  *
864  * The TemplateFinder accumulates all its state in the "scores" array to
865  * be processed later by TemplateFinder::finished.
866  */
867  const int FRAMESGMPCTILE = 90;
868 
869  /*
870  * TUNABLE:
871  *
872  * Exclude some portion of the center of the frame from edge analysis.
873  * Elminate false edge-detection logo positives from talking-host types of
874  * shows where the high-contrast host and clothes (e.g., tie against white
875  * shirt against dark jacket) dominates the edges.
876  *
877  * This has a nice side-effect of reducing the area to be examined (speed
878  * optimization).
879  */
880  static constexpr float kExcludeWidth = 0.5;
881  static constexpr float kExcludeHeight = 0.5;
882 
883  int pgmwidth= 0, pgmheight = 0;
884  int croprow= 0, cropcol = 0, cropwidth = 0, cropheight = 0;
885  struct timeval start {}, end {}, elapsed {};
886 
887  if (frameno < m_nextFrame)
888  {
889  *pNextFrame = m_nextFrame;
890  return ANALYZE_OK;
891  }
892 
893  m_nextFrame = frameno + m_frameInterval;
894  *pNextFrame = min(m_endFrame, m_nextFrame);
895 
896  const AVFrame *pgm = m_pgmConverter->getImage(frame, frameno, &pgmwidth, &pgmheight);
897  if (pgm == nullptr)
898  goto error;
899 
900  if (!m_borderDetector->getDimensions(pgm, pgmheight, frameno,
901  &croprow, &cropcol, &cropwidth, &cropheight))
902  {
903  /* Not a blank frame. */
904 
905  (void)gettimeofday(&start, nullptr);
906 
907  if (croprow < mincontentrow)
908  mincontentrow = croprow;
909  if (cropcol < mincontentcol)
910  mincontentcol = cropcol;
911  if (cropcol + cropwidth > maxcontentcol1)
912  maxcontentcol1 = cropcol + cropwidth;
913  if (croprow + cropheight > maxcontentrow1)
914  maxcontentrow1 = croprow + cropheight;
915 
916  if (resetBuffers(cropwidth, cropheight))
917  goto error;
918 
919  if (pgm_crop(&m_cropped, pgm, pgmheight, croprow, cropcol,
920  cropwidth, cropheight))
921  goto error;
922 
923  /*
924  * Translate the excluded area of the screen into "cropped"
925  * coordinates.
926  */
927  int excludewidth = (int)(pgmwidth * kExcludeWidth);
928  int excludeheight = (int)(pgmheight * kExcludeHeight);
929  int excluderow = (pgmheight - excludeheight) / 2 - croprow;
930  int excludecol = (pgmwidth - excludewidth) / 2 - cropcol;
931  (void)m_edgeDetector->setExcludeArea(excluderow, excludecol,
932  excludewidth, excludeheight);
933 
934  const AVFrame *edges =
935  m_edgeDetector->detectEdges(&m_cropped, cropheight, FRAMESGMPCTILE);
936  if (edges == nullptr)
937  goto error;
938 
939  if (pgm_scorepixels(scores, pgmwidth, croprow, cropcol,
940  edges, cropheight))
941  goto error;
942 
943  if (m_debugLevel >= 2)
944  {
945  if (!analyzeFrameDebug(frameno, pgm, pgmheight, &m_cropped, edges,
946  cropheight, croprow, cropcol, m_debug_frames, m_debugdir))
947  goto error;
948  }
949 
950  (void)gettimeofday(&end, nullptr);
951  timersub(&end, &start, &elapsed);
952  timeradd(&m_analyze_time, &elapsed, &m_analyze_time);
953  }
954 
955  if (m_nextFrame > m_endFrame)
956  return ANALYZE_FINISHED;
957 
958  return ANALYZE_OK;
959 
960 error:
961  LOG(VB_COMMFLAG, LOG_ERR,
962  QString("TemplateFinder::analyzeFrame error at frame %1")
963  .arg(frameno));
964 
965  if (m_nextFrame > m_endFrame)
966  return ANALYZE_FINISHED;
967 
968  return ANALYZE_ERROR;
969 }
970 
971 int
972 TemplateFinder::finished(long long nframes, bool final)
973 {
974  (void)nframes; /* gcc */
975  if (!m_tmpl_done)
976  {
977  if (!template_alloc(scores, m_width, m_height,
982  {
983  if (final)
984  writeDummyTemplate(m_debugdata);
985  }
986  else
987  {
988  if (final && m_debug_template)
989  {
990  if (!(m_tmpl_valid = writeTemplate(m_debugtmpl, &m_tmpl, m_debugdata,
992  goto free_tmpl;
993 
994  LOG(VB_COMMFLAG, LOG_INFO,
995  QString("TemplateFinder::finished wrote %1"
996  " and %2 [%3x%4@(%5,%6)]")
997  .arg(m_debugtmpl).arg(m_debugdata)
998  .arg(m_tmplwidth).arg(m_tmplheight)
999  .arg(m_tmplcol).arg(m_tmplrow));
1000  }
1001  }
1002 
1003  if (final)
1004  m_tmpl_done = true;
1005  }
1006 
1008 
1009  return 0;
1010 
1011 free_tmpl:
1012  av_freep(&m_tmpl.data[0]);
1013  return -1;
1014 }
1015 
1016 int
1018 {
1019  if (m_pgmConverter->reportTime())
1020  return -1;
1021 
1023  return -1;
1024 
1025  LOG(VB_COMMFLAG, LOG_INFO, QString("TF Time: analyze=%1s")
1026  .arg(strftimeval(&m_analyze_time)));
1027  return 0;
1028 }
1029 
1030 const struct AVFrame *
1031 TemplateFinder::getTemplate(int *prow, int *pcol, int *pwidth, int *pheight)
1032  const
1033 {
1034  if (m_tmpl_valid)
1035  {
1036  *prow = m_tmplrow;
1037  *pcol = m_tmplcol;
1038  *pwidth = m_tmplwidth;
1039  *pheight = m_tmplheight;
1040  return &m_tmpl;
1041  }
1042  return nullptr;
1043 }
1044 
1045 /* vim: set expandtab tabstop=4 shiftwidth=4: */
Definition: cc.h:13
QString m_debugtmpl
#define GENERIC_EXIT_OK
Exited with no error.
Definition: exitcodes.h:10
enum analyzeFrameResult analyzeFrame(const VideoFrame *frame, long long frameno, long long *pNextFrame) override
static void error(const char *str,...)
Definition: vbi.c:42
virtual int setExcludeArea(int row, int col, int width, int height)
long long m_endFrame
#define timeradd(a, b, result)
Definition: compat.h:300
PGMConverter * m_pgmConverter
struct AVFrame AVFrame
static int sort_ascending(const void *aa, const void *bb)
QString m_debugdata
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
int finished(long long nframes, bool final) override
TemplateFinder(PGMConverter *pgmc, BorderDetector *bd, EdgeDetector *ed, MythPlayer *player, int proglen, const QString &debugdir)
int pgm_read(unsigned char *buf, int width, int height, const char *filename)
Definition: pgm.cpp:32
int MythPlayerInited(const MythPlayer *player)
int reportTime(void) const override
const struct AVFrame * getTemplate(int *prow, int *pcol, int *pwidth, int *pheight) const
float GetFrameRate(void) const
Definition: mythplayer.h:190
void setLogoState(TemplateFinder *finder)
unsigned int uint
Definition: compat.h:140
enum analyzeFrameResult MythPlayerInited(MythPlayer *player, long long nframes) override
uint myth_system(const QString &command, uint flags, uint timeout)
int MythPlayerInited(const MythPlayer *player)
int resetBuffers(int newwidth, int newheight)
EdgeDetector * m_edgeDetector
BorderDetector * m_borderDetector
int GetNumSetting(const QString &key, int defaultval=0)
void createDebugDirectory(const QString &dirname, const QString &comment)
#define LOG(_MASK_, _LEVEL_, _STRING_)
Definition: mythlogging.h:41
QSize GetVideoBufferSize(void) const
Definition: mythplayer.h:187
unsigned int m_sampleTime
int pgm_crop(AVFrame *dst, const AVFrame *src, int srcheight, int srcrow, int srccol, int cropwidth, int cropheight)
Definition: pgm.cpp:157
int pgm_write(const unsigned char *buf, int width, int height, const char *filename)
Definition: pgm.cpp:78
QString m_debugdir
QString strftimeval(const struct timeval *tv)
int reportTime(void)
#define timersub(a, b, result)
Definition: compat.h:310
int getDimensions(const AVFrame *pgm, int pgmheight, long long frameno, int *prow, int *pcol, int *pwidth, int *pheight)
long long m_nextFrame
virtual const AVFrame * detectEdges(const AVFrame *pgm, int pgmheight, int percentile)=0
unsigned int * scores
const AVFrame * getImage(const VideoFrame *frame, long long frameno, int *pwidth, int *pheight)
int reportTime(void)