MythTV  master
TemplateFinder.cpp
Go to the documentation of this file.
1 // POSIX headers
2 #include <sys/time.h> /* gettimeofday */
3 
4 // ANSI C headers
5 #include <cmath>
6 #include <cstdlib>
7 
8 // Qt headers
9 #include <QFile>
10 #include <QFileInfo>
11 #include <QTextStream>
12 
13 // MythTV headers
14 #include "mythplayer.h"
15 #include "mythcorecontext.h" /* gContext */
16 #include "mythframe.h" /* VideoFrame */
17 #include "mythdate.h"
18 #include "mythsystemlegacy.h"
19 #include "exitcodes.h"
20 
21 // Commercial Flagging headers
22 #include "CommDetector2.h"
23 #include "pgm.h"
24 #include "PGMConverter.h"
25 #include "BorderDetector.h"
26 #include "EdgeDetector.h"
27 #include "TemplateFinder.h"
28 
29 extern "C" {
30  #include "libavutil/imgutils.h"
31  }
32 
33 using namespace commDetector2;
34 
35 namespace {
36 
37 //returns true on success, false otherwise
38 bool writeJPG(const QString& prefix, const AVFrame *img, int imgheight)
39 {
40  const int imgwidth = img->linesize[0];
41  QFileInfo jpgfi(prefix + ".jpg");
42  if (!jpgfi.exists())
43  {
44  QFile pgmfile(prefix + ".pgm");
45  if (!pgmfile.exists())
46  {
47  QByteArray pfname = pgmfile.fileName().toLocal8Bit();
48  if (pgm_write(img->data[0], imgwidth, imgheight,
49  pfname.constData()))
50  {
51  return false;
52  }
53  }
54 
55  QString cmd = QString("convert -quality 50 -resize 192x144 %1 %2")
56  .arg(pgmfile.fileName()).arg(jpgfi.filePath());
57  if (myth_system(cmd) != GENERIC_EXIT_OK)
58  return false;
59 
60  if (!pgmfile.remove())
61  {
62  LOG(VB_COMMFLAG, LOG_ERR,
63  QString("TemplateFinder.writeJPG error removing %1 (%2)")
64  .arg(pgmfile.fileName()).arg(strerror(errno)));
65  return false;
66  }
67  }
68  return true;
69 }
70 
71 int
72 pgm_scorepixels(unsigned int *scores, int width, int row, int col,
73  const AVFrame *src, int srcheight)
74 {
75  /* Every time a pixel is an edge, give it a point. */
76  const int srcwidth = src->linesize[0];
77  int rr, cc;
78 
79  for (rr = 0; rr < srcheight; rr++)
80  {
81  for (cc = 0; cc < srcwidth; cc++)
82  {
83  if (src->data[0][rr * srcwidth + cc])
84  scores[(row + rr) * width + col + cc]++;
85  }
86  }
87 
88  return 0;
89 }
90 
91 int
92 sort_ascending(const void *aa, const void *bb)
93 {
94  return *(unsigned int*)aa - *(unsigned int*)bb;
95 }
96 
97 float
98 bounding_score(const AVFrame *img, int row, int col, int width, int height)
99 {
100  /* Return a value between [0..1] */
101  const int imgwidth = img->linesize[0];
102  unsigned int score;
103  int rr, cc, rr2, cc2;
104 
105  score = 0;
106  rr2 = row + height;
107  cc2 = col + width;
108  for (rr = row; rr < rr2; rr++)
109  {
110  for (cc = col; cc < cc2; cc++)
111  {
112  if (img->data[0][rr * imgwidth + cc])
113  score++;
114  }
115  }
116  return (float)score / (width * height);
117 }
118 
119 bool
120 rowisempty(const AVFrame *img, int row, int col, int width)
121 {
122  const int imgwidth = img->linesize[0];
123  for (int cc = col; cc < col + width; cc++)
124  if (img->data[0][row * imgwidth + cc])
125  return false;
126  return true;
127 }
128 
129 bool
130 colisempty(const AVFrame *img, int col, int row, int height)
131 {
132  const int imgwidth = img->linesize[0];
133  for (int rr = row; rr < row + height; rr++)
134  if (img->data[0][rr * imgwidth + col])
135  return false;
136  return true;
137 }
138 
139 int
140 bounding_box(const AVFrame *img, int imgheight,
141  int minrow, int mincol, int maxrow1, int maxcol1,
142  int *prow, int *pcol, int *pwidth, int *pheight)
143 {
144  const int imgwidth = img->linesize[0];
145  /*
146  * TUNABLE:
147  *
148  * Maximum logo size, expressed as a percentage of the content area
149  * (adjusting for letterboxing and pillarboxing).
150  */
151  static const int MAXWIDTHPCT = 20;
152  static const int MAXHEIGHTPCT = 20;
153 
154  /*
155  * TUNABLE:
156  *
157  * Safety margin to avoid cutting too much of the logo.
158  * Higher values cut more, but avoid noise as part of the template..
159  * Lower values cut less, but can include noise as part of the template.
160  */
161  const int VERTSLOP = max(4, imgheight * 1 / 15);
162  const int HORIZSLOP = max(4, imgwidth * 1 / 20);
163 
164  int maxwidth, maxheight;
165  int width, height, row, col;
166  int newrow, newcol, newright, newbottom;
167 
168  maxwidth = (maxcol1 - mincol) * MAXWIDTHPCT / 100;
169  maxheight = (maxrow1 - minrow) * MAXHEIGHTPCT / 100;
170 
171  row = minrow;
172  col = mincol;
173  width = maxcol1 - mincol;
174  height = maxrow1 - minrow;
175 
176  for (;;)
177  {
178  float score, newscore;
179  int ii;
180  bool improved = false;
181 
182  LOG(VB_COMMFLAG, LOG_INFO, QString("bounding_box %1x%2@(%3,%4)")
183  .arg(width).arg(height).arg(col).arg(row));
184 
185  /* Chop top. */
186  score = bounding_score(img, row, col, width, height);
187  newrow = row;
188  for (ii = 1; ii < height; ii++)
189  {
190  if ((newscore = bounding_score(img, row + ii, col,
191  width, height - ii)) < score)
192  break;
193  score = newscore;
194  newrow = row + ii;
195  improved = true;
196  }
197 
198  /* Chop left. */
199  score = bounding_score(img, row, col, width, height);
200  newcol = col;
201  for (ii = 1; ii < width; ii++)
202  {
203  if ((newscore = bounding_score(img, row, col + ii,
204  width - ii, height)) < score)
205  break;
206  score = newscore;
207  newcol = col + ii;
208  improved = true;
209  }
210 
211  /* Chop bottom. */
212  score = bounding_score(img, row, col, width, height);
213  newbottom = row + height;
214  for (ii = 1; ii < height; ii++)
215  {
216  if ((newscore = bounding_score(img, row, col,
217  width, height - ii)) < score)
218  break;
219  score = newscore;
220  newbottom = row + height - ii;
221  improved = true;
222  }
223 
224  /* Chop right. */
225  score = bounding_score(img, row, col, width, height);
226  newright = col + width;
227  for (ii = 1; ii < width; ii++)
228  {
229  if ((newscore = bounding_score(img, row, col,
230  width - ii, height)) < score)
231  break;
232  score = newscore;
233  newright = col + width - ii;
234  improved = true;
235  }
236 
237  if (!improved)
238  break;
239 
240  row = newrow;
241  col = newcol;
242  width = newright - newcol;
243  height = newbottom - newrow;
244 
245  /*
246  * Noise edge pixels in the frequency template can sometimes stretch
247  * the template area to be larger than it should be.
248  *
249  * However, noise needs to be distinguished from a uniform distribution
250  * of noise pixels (e.g., no real statically-located template). So if
251  * the template area is too "large", then some quadrant must have a
252  * clear majority of the edge pixels; otherwise we declare failure (no
253  * template found).
254  *
255  * Intuitively, we should simply repeat until a single bounding box is
256  * converged upon. However, this requires a more sophisticated
257  * bounding_score function that I don't feel like figuring out.
258  * Indefinitely repeating with the present bounding_score function will
259  * tend to chop off too much. Instead, simply do some sanity checks on
260  * the candidate template's size, and prune the template area and
261  * repeat if it is too "large".
262  */
263 
264  if (width > maxwidth)
265  {
266  /* Too wide; test left and right portions. */
267  int chop = width / 3;
268  int chopwidth = width - chop;
269  float left, right, minscore, maxscore;
270 
271  left = bounding_score(img, row, col, chopwidth, height);
272  right = bounding_score(img, row, col + chop, chopwidth, height);
273  LOG(VB_COMMFLAG, LOG_INFO,
274  QString("bounding_box too wide (%1 > %2); left=%3, right=%4")
275  .arg(width).arg(maxwidth)
276  .arg(left, 0, 'f', 3).arg(right, 0, 'f', 3));
277  minscore = min(left, right);
278  maxscore = max(left, right);
279  if (maxscore < 3 * minscore / 2)
280  {
281  /*
282  * Edge pixel distribution too uniform; give up.
283  *
284  * XXX: also fails for horizontally-centered templates ...
285  */
286  LOG(VB_COMMFLAG, LOG_ERR, "bounding_box giving up (edge "
287  "pixels distributed too uniformly)");
288  return -1;
289  }
290 
291  if (left < right)
292  col += chop;
293  width -= chop;
294  continue;
295  }
296 
297  if (height > maxheight)
298  {
299  /* Too tall; test upper and lower portions. */
300  int chop = height / 3;
301  int chopheight = height - chop;
302  float upper, lower, minscore, maxscore;
303 
304  upper = bounding_score(img, row, col, width, chopheight);
305  lower = bounding_score(img, row + chop, col, width, chopheight);
306  LOG(VB_COMMFLAG, LOG_INFO,
307  QString("bounding_box too tall (%1 > %2); upper=%3, lower=%4")
308  .arg(height).arg(maxheight)
309  .arg(upper, 0, 'f', 3).arg(lower, 0, 'f', 3));
310  minscore = min(upper, lower);
311  maxscore = max(upper, lower);
312  if (maxscore < 3 * minscore / 2)
313  {
314  /*
315  * Edge pixel distribution too uniform; give up.
316  *
317  * XXX: also fails for vertically-centered templates ...
318  */
319  LOG(VB_COMMFLAG, LOG_ERR, "bounding_box giving up (edge "
320  "pixel distribution too uniform)");
321  return -1;
322  }
323 
324  if (upper < lower)
325  row += chop;
326  height -= chop;
327  continue;
328  }
329 
330  break;
331  }
332 
333  /*
334  * The above "chop" algorithm often cuts off the outside edges of the
335  * logos because the outside edges don't contribute enough to the score. So
336  * compensate by now expanding the bounding box (up to a *SLOP pixels in
337  * each direction) to include all edge pixels.
338  */
339 
340  LOG(VB_COMMFLAG, LOG_INFO,
341  QString("bounding_box %1x%2@(%3,%4); horizslop=%5,vertslop=%6")
342  .arg(width).arg(height).arg(col).arg(row)
343  .arg(HORIZSLOP).arg(VERTSLOP));
344 
345  /* Expand upwards. */
346  newrow = row - 1;
347  for (;;)
348  {
349  if (newrow <= minrow)
350  {
351  newrow = minrow;
352  break;
353  }
354  if (row - newrow >= VERTSLOP)
355  {
356  newrow = row - VERTSLOP;
357  break;
358  }
359  if (rowisempty(img, newrow, col, width))
360  {
361  newrow++;
362  break;
363  }
364  newrow--;
365  }
366  newrow = max(minrow, newrow - 1); /* Empty row on top. */
367 
368  /* Expand leftwards. */
369  newcol = col - 1;
370  for (;;)
371  {
372  if (newcol <= mincol)
373  {
374  newcol = mincol;
375  break;
376  }
377  if (col - newcol >= HORIZSLOP)
378  {
379  newcol = col - HORIZSLOP;
380  break;
381  }
382  if (colisempty(img, newcol, row, height))
383  {
384  newcol++;
385  break;
386  }
387  newcol--;
388  }
389  newcol = max(mincol, newcol - 1); /* Empty column to left. */
390 
391  /* Expand rightwards. */
392  newright = col + width;
393  for (;;)
394  {
395  if (newright >= maxcol1)
396  {
397  newright = maxcol1;
398  break;
399  }
400  if (newright - (col + width) >= HORIZSLOP)
401  {
402  newright = col + width + HORIZSLOP;
403  break;
404  }
405  if (colisempty(img, newright, row, height))
406  break;
407  newright++;
408  }
409  newright = min(maxcol1, newright + 1); /* Empty column to right. */
410 
411  /* Expand downwards. */
412  newbottom = row + height;
413  for (;;)
414  {
415  if (newbottom >= maxrow1)
416  {
417  newbottom = maxrow1;
418  break;
419  }
420  if (newbottom - (row + height) >= VERTSLOP)
421  {
422  newbottom = row + height + VERTSLOP;
423  break;
424  }
425  if (rowisempty(img, newbottom, col, width))
426  break;
427  newbottom++;
428  }
429  newbottom = min(maxrow1, newbottom + 1); /* Empty row on bottom. */
430 
431  row = newrow;
432  col = newcol;
433  width = newright - newcol;
434  height = newbottom - newrow;
435 
436  LOG(VB_COMMFLAG, LOG_INFO, QString("bounding_box %1x%2@(%3,%4)")
437  .arg(width).arg(height).arg(col).arg(row));
438 
439  *prow = row;
440  *pcol = col;
441  *pwidth = width;
442  *pheight = height;
443  return 0;
444 }
445 
446 bool
447 template_alloc(const unsigned int *scores, int width, int height,
448  int minrow, int mincol, int maxrow1, int maxcol1, AVFrame *tmpl,
449  int *ptmplrow, int *ptmplcol, int *ptmplwidth, int *ptmplheight,
450  bool debug_edgecounts, const QString& debugdir)
451 {
452  /*
453  * TUNABLE:
454  *
455  * Higher values select for "stronger" pixels to be in the template, but
456  * weak pixels might be missed.
457  *
458  * Lower values allow more pixels to be included as part of the template,
459  * but strong non-template pixels might be included.
460  */
461  static const float MINSCOREPCTILE = 0.998;
462 
463  const int nn = width * height;
464  int ii, first, last;
465  unsigned int *sortedscores, threshscore;
466  AVFrame thresh;
467 
468  if (av_image_alloc(thresh.data, thresh.linesize,
469  width, height, AV_PIX_FMT_GRAY8, IMAGE_ALIGN))
470  {
471  LOG(VB_COMMFLAG, LOG_ERR,
472  QString("template_alloc av_image_alloc thresh (%1x%2) failed")
473  .arg(width).arg(height));
474  return false;
475  }
476 
477  sortedscores = new unsigned int[nn];
478  memcpy(sortedscores, scores, nn * sizeof(*sortedscores));
479  qsort(sortedscores, nn, sizeof(*sortedscores), sort_ascending);
480 
481  if (sortedscores[0] == sortedscores[nn - 1])
482  {
483  /* All pixels in the template area look the same; no template. */
484  LOG(VB_COMMFLAG, LOG_ERR,
485  QString("template_alloc: %1x%2 pixels all identical!")
486  .arg(width).arg(height));
487  goto free_thresh;
488  }
489 
490  /* Threshold the edge frequences. */
491 
492  ii = (int)roundf(nn * MINSCOREPCTILE);
493  threshscore = sortedscores[ii];
494  for (first = ii; first > 0 && sortedscores[first] == threshscore; first--)
495  ;
496  if (sortedscores[first] != threshscore)
497  first++;
498  for (last = ii; last < nn - 1 && sortedscores[last] == threshscore; last++)
499  ;
500  if (sortedscores[last] != threshscore)
501  last--;
502 
503  LOG(VB_COMMFLAG, LOG_INFO, QString("template_alloc wanted %1, got %2-%3")
504  .arg(MINSCOREPCTILE, 0, 'f', 6)
505  .arg((float)first / nn, 0, 'f', 6)
506  .arg((float)last / nn, 0, 'f', 6));
507 
508  for (ii = 0; ii < nn; ii++)
509  thresh.data[0][ii] = scores[ii] >= threshscore ? UCHAR_MAX : 0;
510 
511  if (debug_edgecounts)
512  {
513  /* Scores, rescaled to [0..UCHAR_MAX]. */
514  AVFrame scored;
515  if (av_image_alloc(scored.data, scored.linesize,
516  width, height, AV_PIX_FMT_GRAY8, IMAGE_ALIGN))
517  {
518  LOG(VB_COMMFLAG, LOG_ERR,
519  QString("template_alloc av_image_alloc scored (%1x%2) failed")
520  .arg(width).arg(height));
521  goto free_thresh;
522  }
523  unsigned int maxscore = sortedscores[nn - 1];
524  for (ii = 0; ii < nn; ii++)
525  scored.data[0][ii] = scores[ii] * UCHAR_MAX / maxscore;
526  bool success = writeJPG(debugdir + "/TemplateFinder-scores", &scored,
527  height);
528  av_freep(&scored.data[0]);
529  if (!success)
530  goto free_thresh;
531 
532  /* Thresholded scores. */
533  if (!writeJPG(debugdir + "/TemplateFinder-edgecounts", &thresh, height))
534  goto free_thresh;
535  }
536 
537  /* Crop to a minimal bounding box. */
538 
539  if (bounding_box(&thresh, height, minrow, mincol, maxrow1, maxcol1,
540  ptmplrow, ptmplcol, ptmplwidth, ptmplheight))
541  goto free_thresh;
542 
543  if ((uint)(*ptmplwidth * *ptmplheight) > USHRT_MAX)
544  {
545  /* Max value of data type of TemplateMatcher::edgematch */
546  LOG(VB_COMMFLAG, LOG_ERR,
547  QString("template_alloc bounding_box too big (%1x%2)")
548  .arg(*ptmplwidth).arg(*ptmplheight));
549  goto free_thresh;
550  }
551 
552  if (av_image_alloc(tmpl->data, tmpl->linesize,
553  *ptmplwidth, *ptmplheight, AV_PIX_FMT_GRAY8, IMAGE_ALIGN))
554  {
555  LOG(VB_COMMFLAG, LOG_ERR,
556  QString("template_alloc av_image_alloc tmpl (%1x%2) failed")
557  .arg(*ptmplwidth).arg(*ptmplheight));
558  goto free_thresh;
559  }
560 
561  if (pgm_crop(tmpl, &thresh, height, *ptmplrow, *ptmplcol,
562  *ptmplwidth, *ptmplheight))
563  goto free_thresh;
564 
565  delete []sortedscores;
566  av_freep(&thresh.data[0]);
567 
568  return true;
569 
570 free_thresh:
571  delete []sortedscores;
572  av_freep(&thresh.data[0]);
573  return false;
574 }
575 
576 bool
577 analyzeFrameDebug(long long frameno, const AVFrame *pgm, int pgmheight,
578  const AVFrame *cropped, const AVFrame *edges, int cropheight,
579  int croprow, int cropcol, bool debug_frames, const QString& debugdir)
580 {
581  static const int delta = 24;
582  static int lastrow, lastcol, lastwidth, lastheight;
583  const int cropwidth = cropped->linesize[0];
584  int rowsame, colsame, widthsame, heightsame;
585 
586  rowsame = abs(lastrow - croprow) <= delta ? 1 : 0;
587  colsame = abs(lastcol - cropcol) <= delta ? 1 : 0;
588  widthsame = abs(lastwidth - cropwidth) <= delta ? 1 : 0;
589  heightsame = abs(lastheight - cropheight) <= delta ? 1 : 0;
590 
591  if (frameno > 0 && rowsame + colsame + widthsame + heightsame >= 3)
592  return true;
593 
594  LOG(VB_COMMFLAG, LOG_INFO,
595  QString("TemplateFinder Frame %1: %2x%3@(%4,%5)")
596  .arg(frameno, 5)
597  .arg(cropwidth).arg(cropheight)
598  .arg(cropcol).arg(croprow));
599 
600  lastrow = croprow;
601  lastcol = cropcol;
602  lastwidth = cropwidth;
603  lastheight = cropheight;
604 
605  if (debug_frames)
606  {
607  QString base = QString("%1/TemplateFinder-%2")
608  .arg(debugdir).arg(frameno, 5, 10, QChar('0'));
609 
610  /* PGM greyscale image of frame. */
611  if (!writeJPG(base, pgm, pgmheight))
612  return false;
613 
614  /* Cropped template area of frame. */
615  if (!writeJPG(base + "-cropped", cropped, cropheight))
616  return false;
617 
618  /* Edges of cropped template area of frame. */
619  if (!writeJPG(base + "-edges", edges, cropheight))
620  return false;
621  }
622 
623  return true;
624 }
625 
626 bool
627 readTemplate(const QString& datafile, int *prow, int *pcol, int *pwidth, int *pheight,
628  const QString& tmplfile, AVFrame *tmpl, bool *pvalid)
629 {
630  QFile dfile(datafile);
631  QFileInfo dfileinfo(dfile);
632 
633  if (!dfile.open(QIODevice::ReadOnly))
634  return false;
635 
636  if (!dfileinfo.size())
637  {
638  /* Dummy file: no template. */
639  *pvalid = false;
640  return true;
641  }
642 
643  QTextStream stream(&dfile);
644  stream >> *prow >> *pcol >> *pwidth >> *pheight;
645  dfile.close();
646 
647  if (av_image_alloc(tmpl->data, tmpl->linesize,
648  *pwidth, *pheight, AV_PIX_FMT_GRAY8, IMAGE_ALIGN))
649  {
650  LOG(VB_COMMFLAG, LOG_ERR,
651  QString("readTemplate av_image_alloc %1 (%2x%3) failed")
652  .arg(tmplfile).arg(*pwidth).arg(*pheight));
653  return false;
654  }
655 
656  QByteArray tmfile = tmplfile.toLatin1();
657  if (pgm_read(tmpl->data[0], *pwidth, *pheight, tmfile.constData()))
658  {
659  av_freep(&tmpl->data[0]);
660  return false;
661  }
662 
663  *pvalid = true;
664  return true;
665 }
666 
667 void
668 writeDummyTemplate(const QString& datafile)
669 {
670  /* Leave a 0-byte file. */
671  QFile dfile(datafile);
672 
673  if (!dfile.open(QIODevice::WriteOnly | QIODevice::Truncate) &&
674  dfile.exists())
675  (void)dfile.remove();
676 }
677 
678 bool
679 writeTemplate(const QString& tmplfile, const AVFrame *tmpl, const QString& datafile,
680  int row, int col, int width, int height)
681 {
682  QFile tfile(tmplfile);
683 
684  QByteArray tmfile = tmplfile.toLatin1();
685  if (pgm_write(tmpl->data[0], width, height, tmfile.constData()))
686  return false;
687 
688  QFile dfile(datafile);
689  if (!dfile.open(QIODevice::WriteOnly))
690  return false;
691 
692  QTextStream stream(&dfile);
693  stream << row << " " << col << "\n" << width << " " << height << "\n";
694  dfile.close();
695  return true;
696 }
697 
698 }; /* namespace */
699 
701  EdgeDetector *ed, MythPlayer *player, int proglen,
702  const QString& debugdir)
703  : m_pgmConverter(pgmc)
704  , m_borderDetector(bd)
705  , m_edgeDetector(ed)
706  , m_debugdir(debugdir)
707  , m_debugdata(debugdir + "/TemplateFinder.txt")
708  , m_debugtmpl(debugdir + "/TemplateFinder.pgm")
709 {
710  /*
711  * TUNABLE:
712  *
713  * The number of frames desired for sampling to build the template.
714  *
715  * Higher values should yield a more accurate template, but requires more
716  * time.
717  */
718  unsigned int samplesNeeded = 300;
719 
720  /*
721  * TUNABLE:
722  *
723  * The leading amount of time (in seconds) to sample frames for building up
724  * the possible template, and the interval between frames for analysis.
725  * This affects how soon flagging can start after a recording has begun
726  * (a.k.a. "real-time flagging").
727  *
728  * Sample half of the program length or 20 minutes, whichever is less.
729  */
730  m_sampleTime = min(proglen / 2, 20 * 60);
731 
732  const float fps = player->GetFrameRate();
733 
734  m_frameInterval = (int)roundf(m_sampleTime * fps / samplesNeeded);
735  m_endFrame = 0 + (long long)m_frameInterval * samplesNeeded - 1;
736 
737  LOG(VB_COMMFLAG, LOG_INFO,
738  QString("TemplateFinder: sampleTime=%1s, samplesNeeded=%2, endFrame=%3")
739  .arg(m_sampleTime).arg(samplesNeeded).arg(m_endFrame));
740 
741  memset(&m_cropped, 0, sizeof(m_cropped));
742  memset(&m_tmpl, 0, sizeof(m_tmpl));
743  memset(&m_analyze_time, 0, sizeof(m_analyze_time));
744 
745  /*
746  * debugLevel:
747  * 0: no extra debugging
748  * 1: cache computations into debugdir [O(1) files]
749  * 2: extra verbosity [O(nframes)]
750  * 3: dump frames into debugdir [O(nframes) files]
751  */
752  m_debugLevel = gCoreContext->GetNumSetting("TemplateFinderDebugLevel", 0);
753 
754  if (m_debugLevel >= 1)
755  {
757  QString("TemplateFinder debugLevel %1").arg(m_debugLevel));
758 
759  m_debug_template = true;
760  m_debug_edgecounts = true;
761 
762  if (m_debugLevel >= 3)
763  m_debug_frames = true;
764  }
765 }
766 
768 {
769  delete []scores;
770  av_freep(&m_tmpl.data[0]);
771  av_freep(&m_cropped.data[0]);
772 }
773 
775 TemplateFinder::MythPlayerInited(MythPlayer *player, long long nframes)
776 {
777  /*
778  * Only detect edges in portions of the frame where we expect to find
779  * a template. This serves two purposes:
780  *
781  * - Speed: reduce search space.
782  * - Correctness (insofar as the assumption of template location is
783  * correct): don't "pollute" the set of candidate template edges with
784  * the "content" edges in the non-template portions of the frame.
785  */
786  QString tmpldims, playerdims;
787 
788  (void)nframes; /* gcc */
789  QSize buf_dim = player->GetVideoBufferSize();
790  m_width = buf_dim.width();
791  m_height = buf_dim.height();
792  playerdims = QString("%1x%2").arg(m_width).arg(m_height);
793 
794  if (m_debug_template)
795  {
796  if ((m_tmpl_done = readTemplate(m_debugdata, &m_tmplrow, &m_tmplcol,
798  &m_tmpl_valid)))
799  {
800  tmpldims = m_tmpl_valid ? QString("%1x%2@(%3,%4)")
801  .arg(m_tmplwidth).arg(m_tmplheight).arg(m_tmplcol).arg(m_tmplrow) :
802  "no template";
803 
804  LOG(VB_COMMFLAG, LOG_INFO,
805  QString("TemplateFinder::MythPlayerInited read %1: %2")
806  .arg(m_debugtmpl)
807  .arg(tmpldims));
808  }
809  }
810 
811  if (m_pgmConverter->MythPlayerInited(player))
812  goto free_tmpl;
813 
814  if (m_borderDetector->MythPlayerInited(player))
815  goto free_tmpl;
816 
817  if (m_tmpl_done)
818  {
819  if (m_tmpl_valid)
820  {
821  LOG(VB_COMMFLAG, LOG_INFO,
822  QString("TemplateFinder::MythPlayerInited %1 of %2 (%3)")
823  .arg(tmpldims).arg(playerdims).arg(m_debugtmpl));
824  }
825  return ANALYZE_FINISHED;
826  }
827 
828  LOG(VB_COMMFLAG, LOG_INFO,
829  QString("TemplateFinder::MythPlayerInited framesize %1")
830  .arg(playerdims));
831  scores = new unsigned int[m_width * m_height];
832 
833  return ANALYZE_OK;
834 
835 free_tmpl:
836  av_freep(&m_tmpl.data[0]);
837  return ANALYZE_FATAL;
838 }
839 
840 int
841 TemplateFinder::resetBuffers(int newwidth, int newheight)
842 {
843  if (m_cwidth == newwidth && m_cheight == newheight)
844  return 0;
845 
846  av_freep(&m_cropped.data[0]);
847 
848  if (av_image_alloc(m_cropped.data, m_cropped.linesize,
849  newwidth, newheight, AV_PIX_FMT_GRAY8, IMAGE_ALIGN))
850  {
851  LOG(VB_COMMFLAG, LOG_ERR,
852  QString("TemplateFinder::resetBuffers "
853  "av_image_alloc cropped (%1x%2) failed")
854  .arg(newwidth).arg(newheight));
855  return -1;
856  }
857 
858  m_cwidth = newwidth;
859  m_cheight = newheight;
860  return 0;
861 }
862 
864 TemplateFinder::analyzeFrame(const VideoFrame *frame, long long frameno,
865  long long *pNextFrame)
866 {
867  /*
868  * TUNABLE:
869  *
870  * When looking for edges in frames, select some percentile of
871  * squared-gradient magnitudes (intensities) as candidate edges. (This
872  * number conventionally should not go any lower than the 95th percentile;
873  * see edge_mark.)
874  *
875  * Higher values result in fewer edges; faint logos might not be picked up.
876  * Lower values result in more edges; non-logo edges might be picked up.
877  *
878  * The TemplateFinder accumulates all its state in the "scores" array to
879  * be processed later by TemplateFinder::finished.
880  */
881  const int FRAMESGMPCTILE = 90;
882 
883  /*
884  * TUNABLE:
885  *
886  * Exclude some portion of the center of the frame from edge analysis.
887  * Elminate false edge-detection logo positives from talking-host types of
888  * shows where the high-contrast host and clothes (e.g., tie against white
889  * shirt against dark jacket) dominates the edges.
890  *
891  * This has a nice side-effect of reducing the area to be examined (speed
892  * optimization).
893  */
894  static const float EXCLUDEWIDTH = 0.5;
895  static const float EXCLUDEHEIGHT = 0.5;
896 
897  const AVFrame *pgm, *edges;
898  int pgmwidth, pgmheight;
899  int croprow, cropcol, cropwidth, cropheight;
900  int excluderow, excludecol, excludewidth, excludeheight;
901  struct timeval start, end, elapsed;
902 
903  if (frameno < m_nextFrame)
904  {
905  *pNextFrame = m_nextFrame;
906  return ANALYZE_OK;
907  }
908 
909  m_nextFrame = frameno + m_frameInterval;
910  *pNextFrame = min(m_endFrame, m_nextFrame);
911 
912  if (!(pgm = m_pgmConverter->getImage(frame, frameno, &pgmwidth, &pgmheight)))
913  goto error;
914 
915  if (!m_borderDetector->getDimensions(pgm, pgmheight, frameno,
916  &croprow, &cropcol, &cropwidth, &cropheight))
917  {
918  /* Not a blank frame. */
919 
920  (void)gettimeofday(&start, nullptr);
921 
922  if (croprow < mincontentrow)
923  mincontentrow = croprow;
924  if (cropcol < mincontentcol)
925  mincontentcol = cropcol;
926  if (cropcol + cropwidth > maxcontentcol1)
927  maxcontentcol1 = cropcol + cropwidth;
928  if (croprow + cropheight > maxcontentrow1)
929  maxcontentrow1 = croprow + cropheight;
930 
931  if (resetBuffers(cropwidth, cropheight))
932  goto error;
933 
934  if (pgm_crop(&m_cropped, pgm, pgmheight, croprow, cropcol,
935  cropwidth, cropheight))
936  goto error;
937 
938  /*
939  * Translate the excluded area of the screen into "cropped"
940  * coordinates.
941  */
942  excludewidth = (int)(pgmwidth * EXCLUDEWIDTH);
943  excludeheight = (int)(pgmheight * EXCLUDEHEIGHT);
944  excluderow = (pgmheight - excludeheight) / 2 - croprow;
945  excludecol = (pgmwidth - excludewidth) / 2 - cropcol;
946  (void)m_edgeDetector->setExcludeArea(excluderow, excludecol,
947  excludewidth, excludeheight);
948 
949  if (!(edges = m_edgeDetector->detectEdges(&m_cropped, cropheight,
950  FRAMESGMPCTILE)))
951  goto error;
952 
953  if (pgm_scorepixels(scores, pgmwidth, croprow, cropcol,
954  edges, cropheight))
955  goto error;
956 
957  if (m_debugLevel >= 2)
958  {
959  if (!analyzeFrameDebug(frameno, pgm, pgmheight, &m_cropped, edges,
960  cropheight, croprow, cropcol, m_debug_frames, m_debugdir))
961  goto error;
962  }
963 
964  (void)gettimeofday(&end, nullptr);
965  timersub(&end, &start, &elapsed);
966  timeradd(&m_analyze_time, &elapsed, &m_analyze_time);
967  }
968 
969  if (m_nextFrame > m_endFrame)
970  return ANALYZE_FINISHED;
971 
972  return ANALYZE_OK;
973 
974 error:
975  LOG(VB_COMMFLAG, LOG_ERR,
976  QString("TemplateFinder::analyzeFrame error at frame %1")
977  .arg(frameno));
978 
979  if (m_nextFrame > m_endFrame)
980  return ANALYZE_FINISHED;
981 
982  return ANALYZE_ERROR;
983 }
984 
985 int
986 TemplateFinder::finished(long long nframes, bool final)
987 {
988  (void)nframes; /* gcc */
989  if (!m_tmpl_done)
990  {
991  if (!template_alloc(scores, m_width, m_height,
996  {
997  if (final)
998  writeDummyTemplate(m_debugdata);
999  }
1000  else
1001  {
1002  if (final && m_debug_template)
1003  {
1004  if (!(m_tmpl_valid = writeTemplate(m_debugtmpl, &m_tmpl, m_debugdata,
1006  goto free_tmpl;
1007 
1008  LOG(VB_COMMFLAG, LOG_INFO,
1009  QString("TemplateFinder::finished wrote %1"
1010  " and %2 [%3x%4@(%5,%6)]")
1011  .arg(m_debugtmpl).arg(m_debugdata)
1012  .arg(m_tmplwidth).arg(m_tmplheight)
1013  .arg(m_tmplcol).arg(m_tmplrow));
1014  }
1015  }
1016 
1017  if (final)
1018  m_tmpl_done = true;
1019  }
1020 
1022 
1023  return 0;
1024 
1025 free_tmpl:
1026  av_freep(&m_tmpl.data[0]);
1027  return -1;
1028 }
1029 
1030 int
1032 {
1033  if (m_pgmConverter->reportTime())
1034  return -1;
1035 
1037  return -1;
1038 
1039  LOG(VB_COMMFLAG, LOG_INFO, QString("TF Time: analyze=%1s")
1040  .arg(strftimeval(&m_analyze_time)));
1041  return 0;
1042 }
1043 
1044 const struct AVFrame *
1045 TemplateFinder::getTemplate(int *prow, int *pcol, int *pwidth, int *pheight)
1046  const
1047 {
1048  if (m_tmpl_valid)
1049  {
1050  *prow = m_tmplrow;
1051  *pcol = m_tmplcol;
1052  *pwidth = m_tmplwidth;
1053  *pheight = m_tmplheight;
1054  return &m_tmpl;
1055  }
1056  return nullptr;
1057 }
1058 
1059 /* vim: set expandtab tabstop=4 shiftwidth=4: */
Definition: cc.h:13
struct exc__state * last
Definition: pxsup2dast.c:98
QString m_debugtmpl
#define GENERIC_EXIT_OK
Exited with no error.
Definition: exitcodes.h:10
enum analyzeFrameResult analyzeFrame(const VideoFrame *frame, long long frameno, long long *pNextFrame) override
static void error(const char *str,...)
Definition: vbi.c:42
virtual int setExcludeArea(int row, int col, int width, int height)
long long m_endFrame
#define timeradd(a, b, result)
Definition: compat.h:300
struct AVFrame AVFrame
PGMConverter * m_pgmConverter
static int sort_ascending(const void *aa, const void *bb)
QString m_debugdata
unsigned int uint
Definition: compat.h:140
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
int finished(long long nframes, bool final) override
TemplateFinder(PGMConverter *pgmc, BorderDetector *bd, EdgeDetector *ed, MythPlayer *player, int proglen, const QString &debugdir)
int pgm_read(unsigned char *buf, int width, int height, const char *filename)
Definition: pgm.cpp:32
int MythPlayerInited(const MythPlayer *player)
int reportTime(void) const override
const struct AVFrame * getTemplate(int *prow, int *pcol, int *pwidth, int *pheight) const
float GetFrameRate(void) const
Definition: mythplayer.h:176
void setLogoState(TemplateFinder *finder)
enum analyzeFrameResult MythPlayerInited(MythPlayer *player, long long nframes) override
uint myth_system(const QString &command, uint flags, uint timeout)
int MythPlayerInited(const MythPlayer *player)
int resetBuffers(int newwidth, int newheight)
EdgeDetector * m_edgeDetector
BorderDetector * m_borderDetector
int GetNumSetting(const QString &key, int defaultval=0)
void createDebugDirectory(const QString &dirname, const QString &comment)
#define LOG(_MASK_, _LEVEL_, _STRING_)
Definition: mythlogging.h:41
QSize GetVideoBufferSize(void) const
Definition: mythplayer.h:173
unsigned int m_sampleTime
struct timeval m_analyze_time
int pgm_crop(AVFrame *dst, const AVFrame *src, int srcheight, int srcrow, int srccol, int cropwidth, int cropheight)
Definition: pgm.cpp:161
int pgm_write(const unsigned char *buf, int width, int height, const char *filename)
Definition: pgm.cpp:79
QString m_debugdir
QString strftimeval(const struct timeval *tv)
int reportTime(void)
#define timersub(a, b, result)
Definition: compat.h:310
int getDimensions(const AVFrame *pgm, int pgmheight, long long frameno, int *prow, int *pcol, int *pwidth, int *pheight)
long long m_nextFrame
virtual const AVFrame * detectEdges(const AVFrame *pgm, int pgmheight, int percentile)=0
unsigned int * scores
const AVFrame * getImage(const VideoFrame *frame, long long frameno, int *pwidth, int *pheight)
int reportTime(void)