MythTV  master
BlankFrameDetector.cpp
Go to the documentation of this file.
1 // ANSI C headers
2 #include <cmath>
3 #include <cstdlib>
4 #include <utility>
5 
6 // MythTV headers
7 #include "mythcorecontext.h" /* gContext */
8 #include "mythplayer.h"
9 
10 // Commercial Flagging headers
11 #include "CommDetector2.h"
12 #include "FrameAnalyzer.h"
13 #include "quickselect.h"
14 #include "HistogramAnalyzer.h"
15 #include "BlankFrameDetector.h"
16 #include "TemplateMatcher.h"
17 
18 using namespace commDetector2;
19 using namespace frameAnalyzer;
20 
21 namespace {
22 
23 bool
24 isBlank(unsigned char median, float stddev, unsigned char maxmedian,
25  float maxstddev)
26 {
27  return ((median < maxmedian) ||
28  ((median == maxmedian) && (stddev <= maxstddev)));
29 }
30 
31 int
32 sort_ascending_uchar(const void *aa, const void *bb)
33 {
34  return *(unsigned char*)aa - *(unsigned char*)bb;
35 }
36 
37 int
38 sort_ascending_float(const void *aa, const void *bb)
39 {
40  float faa = *(float*)aa;
41  float fbb = *(float*)bb;
42  return faa < fbb ? -1 : faa == fbb ? 0 : 1;
43 }
44 
45 bool
46 pickmedian(const unsigned char medianval,
47  unsigned char minval, unsigned char maxval)
48 {
49  return medianval >= minval && medianval <= maxval;
50 }
51 
52 void
53 computeBlankMap(FrameAnalyzer::FrameMap *blankMap, long long nframes,
54  const unsigned char *median, const float *stddev,
55  const unsigned char *monochromatic)
56 {
57  /*
58  * Select a "black" value based on a curve, to deal with varying "black"
59  * levels.
60  */
61  const unsigned char MINBLANKMEDIAN = 1;
62  const unsigned char MAXBLANKMEDIAN = 96;
63  const float MEDIANPCTILE = 0.95;
64  const float STDDEVPCTILE = 0.85;
65 
66  long long frameno = 1;
67  long long segb = 0;
68  long long sege = 0;
69 
70  /* Count and select for monochromatic frames. */
71 
72  long long nblanks = 0;
73  for (frameno = 0; frameno < nframes; frameno++)
74  {
75  if (monochromatic[frameno] && pickmedian(median[frameno],
76  MINBLANKMEDIAN, MAXBLANKMEDIAN))
77  nblanks++;
78  }
79 
80  if (!nblanks)
81  {
82  /* No monochromatic frames. */
83  LOG(VB_COMMFLAG, LOG_INFO,
84  "BlankFrameDetector::computeBlankMap: No blank frames.");
85  return;
86  }
87 
88  /* Select percentile values from monochromatic frames. */
89 
90  auto *blankmedian = new unsigned char[nblanks];
91  auto *blankstddev = new float[nblanks];
92  long long blankno = 0;
93  for (frameno = 0; frameno < nframes; frameno++)
94  {
95  if (monochromatic[frameno] && pickmedian(median[frameno],
96  MINBLANKMEDIAN, MAXBLANKMEDIAN))
97  {
98  blankmedian[blankno] = median[frameno];
99  blankstddev[blankno] = stddev[frameno];
100  blankno++;
101  }
102  }
103 
104  qsort(blankmedian, nblanks, sizeof(*blankmedian), sort_ascending_uchar);
105  blankno = min(nblanks - 1, (long long)roundf(nblanks * MEDIANPCTILE));
106  uchar maxmedian = blankmedian[blankno];
107 
108  qsort(blankstddev, nblanks, sizeof(*blankstddev), sort_ascending_float);
109  long long stddevno = min(nblanks - 1, (long long)roundf(nblanks * STDDEVPCTILE));
110  float maxstddev = blankstddev[stddevno];
111 
112  /* Determine effective percentile ranges (for debugging). */
113 
114  long long blankno1 = blankno;
115  long long blankno2 = blankno;
116  while (blankno1 > 0 && blankmedian[blankno1] == maxmedian)
117  blankno1--;
118  if (blankmedian[blankno1] != maxmedian)
119  blankno1++;
120  while (blankno2 < nblanks && blankmedian[blankno2] == maxmedian)
121  blankno2++;
122  if (blankno2 == nblanks)
123  blankno2--;
124 
125  long long stddevno1 = stddevno;
126  long long stddevno2 = stddevno;
127  while (stddevno1 > 0 && blankstddev[stddevno1] == maxstddev)
128  stddevno1--;
129  if (blankstddev[stddevno1] != maxstddev)
130  stddevno1++;
131  while (stddevno2 < nblanks && blankstddev[stddevno2] == maxstddev)
132  stddevno2++;
133  if (stddevno2 == nblanks)
134  stddevno2--;
135 
136  LOG(VB_COMMFLAG, LOG_INFO,
137  QString("Blanks selecting median<=%1 (%2-%3%), stddev<=%4 (%5-%6%)")
138  .arg(maxmedian)
139  .arg(blankno1 * 100 / nblanks).arg(blankno2 * 100 / nblanks)
140  .arg(maxstddev)
141  .arg(stddevno1 * 100 / nblanks).arg(stddevno2 * 100 / nblanks));
142 
143  delete []blankmedian;
144  delete []blankstddev;
145 
146  blankMap->clear();
147  if (monochromatic[0] && isBlank(median[0], stddev[0], maxmedian, maxstddev))
148  {
149  segb = 0;
150  sege = 0;
151  }
152  else
153  {
154  /* Fake up a dummy blank frame for interval calculations. */
155  blankMap->insert(0, 0);
156  segb = -1;
157  sege = -1;
158  }
159  for (frameno = 1; frameno < nframes; frameno++)
160  {
161  if (monochromatic[frameno] && isBlank(median[frameno], stddev[frameno],
162  maxmedian, maxstddev))
163  {
164  /* Blank frame. */
165  if (sege < frameno - 1)
166  {
167  /* Start counting. */
168  segb = frameno;
169  sege = frameno;
170  }
171  else
172  {
173  /* Continue counting. */
174  sege = frameno;
175  }
176  }
177  else if (sege == frameno - 1)
178  {
179  /* Transition to non-blank frame. */
180  long long seglen = frameno - segb;
181  blankMap->insert(segb, seglen);
182  }
183  }
184  if (sege == frameno - 1)
185  {
186  /* Possibly ending on blank frames. */
187  long long seglen = frameno - segb;
188  blankMap->insert(segb, seglen);
189  }
190 
191  FrameAnalyzer::FrameMap::Iterator iiblank = blankMap->end();
192  --iiblank;
193  if (iiblank.key() + *iiblank < nframes)
194  {
195  /*
196  * Didn't end on blank frames, so add a dummy blank frame at the
197  * end.
198  */
199  blankMap->insert(nframes - 1, 0);
200  }
201 }
202 
203 void
204 computeBreakMap(FrameAnalyzer::FrameMap *breakMap,
205  const FrameAnalyzer::FrameMap *blankMap, float fps,
206  int debugLevel)
207 {
208  /*
209  * TUNABLE:
210  *
211  * Common commercial-break lengths.
212  */
213  static constexpr struct {
214  int m_len; /* seconds */
215  int m_delta; /* seconds */
216  } kBreakType[] = {
217  /* Sort by "len". */
218  { 15, 2 },
219  { 20, 2 },
220  { 30, 5 },
221  { 60, 5 },
222  };
223  static constexpr unsigned int kNBreakTypes =
224  sizeof(kBreakType)/sizeof(*kBreakType);
225 
226  /*
227  * TUNABLE:
228  *
229  * Shortest non-commercial length, used to coalesce consecutive commercial
230  * breaks that are usually identified due to in-commercial cuts.
231  */
232  static const int kMinContentLen = (int)roundf(10 * fps);
233 
234  breakMap->clear();
235  for (FrameAnalyzer::FrameMap::const_iterator iiblank = blankMap->begin();
236  iiblank != blankMap->end();
237  ++iiblank)
238  {
239  long long brkb = iiblank.key();
240  long long iilen = *iiblank;
241  long long start = brkb + iilen / 2;
242 
243  for (unsigned int ii = 0; ii < kNBreakTypes; ii++)
244  {
245  /* Look for next blank frame that is an acceptable distance away. */
246  FrameAnalyzer::FrameMap::const_iterator jjblank = iiblank;
247  for (++jjblank; jjblank != blankMap->end(); ++jjblank)
248  {
249  long long brke = jjblank.key();
250  long long jjlen = *jjblank;
251  long long end = brke + jjlen / 2;
252 
253  auto testlen = (long long)roundf((end - start) / fps);
254  if (testlen > kBreakType[ii].m_len + kBreakType[ii].m_delta)
255  break; /* Too far ahead; break to next break length. */
256 
257  long long delta = testlen - kBreakType[ii].m_len;
258  if (delta < 0)
259  delta = 0 - delta;
260 
261  if (delta > kBreakType[ii].m_delta)
262  continue; /* Outside delta range; try next end-blank. */
263 
264  /* Mark this commercial break. */
265  bool inserted = false;
266  for (unsigned int jj = 0;; jj++)
267  {
268  long long newbrkb = brkb + jj;
269  if (newbrkb >= brke)
270  {
271  LOG(VB_COMMFLAG, LOG_INFO,
272  QString("BF [%1,%2] ran out of slots")
273  .arg(brkb).arg(brke - 1));
274  break;
275  }
276  if (breakMap->find(newbrkb) == breakMap->end())
277  {
278  breakMap->insert(newbrkb, brke - newbrkb);
279  inserted = true;
280  break;
281  }
282  }
283  if (inserted)
284  break; /* next break type */
285  }
286  }
287  }
288 
289  if (debugLevel >= 1)
290  {
291  frameAnalyzerReportMap(breakMap, fps, "BF Break");
292  LOG(VB_COMMFLAG, LOG_INFO,
293  "BF coalescing overlapping/nearby breaks ...");
294  }
295 
296  /*
297  * Coalesce overlapping or very-nearby breaks (handles cut-scenes within a
298  * commercial).
299  */
300  for (;;)
301  {
302  bool coalesced = false;
303  FrameAnalyzer::FrameMap::iterator iibreak = breakMap->begin();
304  while (iibreak != breakMap->end())
305  {
306  long long iib = iibreak.key();
307  long long iie = iib + *iibreak;
308 
309  FrameAnalyzer::FrameMap::iterator jjbreak = iibreak;
310  ++jjbreak;
311  if (jjbreak == breakMap->end())
312  break;
313 
314  long long jjb = jjbreak.key();
315  long long jje = jjb + *jjbreak;
316 
317  if (jjb < iib)
318  {
319  /* (jjb,jje) is behind (iib,iie). */
320  ++iibreak;
321  continue;
322  }
323 
324  if (iie + kMinContentLen < jjb)
325  {
326  /* (jjb,jje) is too far ahead. */
327  ++iibreak;
328  continue;
329  }
330 
331  /* Coalesce. */
332  if (jje > iie)
333  {
334  breakMap->remove(iib); /* overlap */
335  breakMap->insert(iib, jje - iib); /* overlap */
336  }
337  breakMap->erase(jjbreak);
338  coalesced = true;
339  iibreak = breakMap->find(iib);
340  }
341  if (!coalesced)
342  break;
343  }
344 
345  /* Adjust for blank intervals. */
346  FrameAnalyzer::FrameMap::iterator iibreak = breakMap->begin();
347  while (iibreak != breakMap->end())
348  {
349  long long iib = iibreak.key();
350  long long iie = iib + *iibreak;
351  FrameAnalyzer::FrameMap::iterator jjbreak = iibreak;
352  ++jjbreak;
353  breakMap->erase(iibreak);
354 
355  /* Trim leading blanks from commercial break. */
356  long long addb = *blankMap->find(iib);
357  addb = addb / 2;
358  if (addb > MAX_BLANK_FRAMES)
359  addb = MAX_BLANK_FRAMES;
360  iib += addb;
361  /* Add trailing blanks to commercial break. */
362  long long adde = *blankMap->find(iie);
363  iie += adde;
364  long long sube = adde / 2;
365  if (sube > MAX_BLANK_FRAMES)
366  sube = MAX_BLANK_FRAMES;
367  iie -= sube;
368  breakMap->insert(iib, iie - iib);
369  iibreak = jjbreak;
370  }
371 }
372 
373 }; /* namespace */
374 
376  : m_histogramAnalyzer(ha)
377 {
378  /*
379  * debugLevel:
380  * 0: no debugging
381  * 2: extra verbosity [O(nframes)]
382  */
383  m_debugLevel = gCoreContext->GetNumSetting("BlankFrameDetectorDebugLevel", 0);
384 
385  if (m_debugLevel >= 1)
386  createDebugDirectory(debugdir,
387  QString("BlankFrameDetector debugLevel %1").arg(m_debugLevel));
388 }
389 
392 {
394  m_histogramAnalyzer->MythPlayerInited(player, nframes);
395 
396  m_fps = player->GetFrameRate();
397 
398  QSize video_disp_dim = player->GetVideoSize();
399 
400  LOG(VB_COMMFLAG, LOG_INFO,
401  QString("BlankFrameDetector::MythPlayerInited %1x%2")
402  .arg(video_disp_dim.width())
403  .arg(video_disp_dim.height()));
404 
405  return ares;
406 }
407 
409 BlankFrameDetector::analyzeFrame(const VideoFrame *frame, long long frameno,
410  long long *pNextFrame)
411 {
412  *pNextFrame = NEXTFRAME;
413 
414  if (m_histogramAnalyzer->analyzeFrame(frame, frameno) ==
416  return ANALYZE_OK;
417 
418  LOG(VB_COMMFLAG, LOG_INFO,
419  QString("BlankFrameDetector::analyzeFrame error at frame %1")
420  .arg(frameno));
421  return ANALYZE_ERROR;
422 }
423 
424 int
425 BlankFrameDetector::finished(long long nframes, bool final)
426 {
427  if (m_histogramAnalyzer->finished(nframes, final))
428  return -1;
429 
430  LOG(VB_COMMFLAG, LOG_INFO, QString("BlankFrameDetector::finished(%1)")
431  .arg(nframes));
432 
433  /* Identify all sequences of blank frames (blankMap). */
434  computeBlankMap(&m_blankMap, nframes,
437  if (m_debugLevel >= 2)
439 
440  return 0;
441 }
442 
443 int
445  const TemplateMatcher *templateMatcher)
446 {
447  /*
448  * See TemplateMatcher::templateCoverage; some commercial breaks have
449  * logos. Conversely, any logo breaks are probably really breaks, so prefer
450  * those over blank-frame-calculated breaks.
451  */
452  const FrameAnalyzer::FrameMap *logoBreakMap = templateMatcher->getBreaks();
453 
454  /* TUNABLE: see TemplateMatcher::adjustForBlanks */
455  const int MAXBLANKADJUSTMENT = (int)roundf(5 * m_fps); /* frames */
456 
457  LOG(VB_COMMFLAG, LOG_INFO, "BlankFrameDetector adjusting for logo surplus");
458 
459  /*
460  * For each logo break, find the blank frames closest to its beginning and
461  * end. This helps properly support CommSkipAllBlanks.
462  */
463  for (FrameAnalyzer::FrameMap::const_iterator ii =
464  logoBreakMap->constBegin();
465  ii != logoBreakMap->constEnd();
466  ++ii)
467  {
468  /* Get bounds of beginning of logo break. */
469  long long iikey = ii.key();
470  long long iibb = iikey - MAXBLANKADJUSTMENT;
471  long long iiee = iikey + MAXBLANKADJUSTMENT;
472  FrameAnalyzer::FrameMap::Iterator jjfound = m_blankMap.end();
473 
474  /* Look for a blank frame near beginning of logo break. */
475  for (auto jj = m_blankMap.begin(); jj != m_blankMap.end(); ++jj)
476  {
477  long long jjbb = jj.key();
478  long long jjee = jjbb + *jj;
479 
480  if (iiee < jjbb)
481  break; /* No nearby blank frames. */
482 
483  if (jjee < iibb)
484  continue; /* Too early; keep looking. */
485 
486  jjfound = jj;
487  if (iikey <= jjbb)
488  {
489  /*
490  * Prefer the first blank frame beginning after the logo break
491  * begins.
492  */
493  break;
494  }
495  }
496 
497  /* Adjust blank frame to begin with logo break beginning. */
498  if (jjfound != m_blankMap.end())
499  {
500  long long jjee = jjfound.key() + *jjfound;
501  m_blankMap.erase(jjfound);
502  if (jjee <= iikey)
503  {
504  /* Move blank frame to beginning of logo break. */
505  m_blankMap.remove(iikey);
506  m_blankMap.insert(iikey, 1);
507  }
508  else
509  {
510  /* Adjust blank frame to begin with logo break. */
511  m_blankMap.insert(iikey, jjee - iikey);
512  }
513  }
514 
515  /* Get bounds of end of logo break. */
516  long long kkkey = ii.key() + *ii;
517  long long kkbb = kkkey - MAXBLANKADJUSTMENT;
518  long long kkee = kkkey + MAXBLANKADJUSTMENT;
519  FrameAnalyzer::FrameMap::Iterator mmfound = m_blankMap.end();
520 
521  /* Look for a blank frame near end of logo break. */
522  for (auto mm = m_blankMap.begin(); mm != m_blankMap.end(); ++mm)
523  {
524  long long mmbb = mm.key();
525  long long mmee = mmbb + *mm;
526 
527  if (kkee < mmbb)
528  break; /* No nearby blank frames. */
529 
530  if (mmee < kkbb)
531  continue; /* Too early; keep looking. */
532 
533  /* Prefer the last blank frame ending before the logo break ends. */
534  if (mmee < kkkey || mmfound == m_blankMap.end())
535  mmfound = mm;
536  if (mmee >= kkkey)
537  break;
538  }
539 
540  /* Adjust blank frame to end with logo break end. */
541  if (mmfound != m_blankMap.end())
542  {
543  long long mmbb = mmfound.key();
544  if (mmbb < kkkey)
545  {
546  /* Adjust blank frame to end with logo break. */
547  m_blankMap.remove(mmbb);
548  m_blankMap.insert(mmbb, kkkey - mmbb);
549  }
550  else
551  {
552  /* Move blank frame to end of logo break. */
553  m_blankMap.erase(mmfound);
554  m_blankMap.remove(kkkey - 1);
555  m_blankMap.insert(kkkey - 1, 1);
556  }
557  }
558  }
559 
560  /*
561  * Compute breaks (breakMap).
562  */
563  computeBreakMap(&m_breakMap, &m_blankMap, m_fps, m_debugLevel);
564 
565  /*
566  * Expand blank-frame breaks to fully include overlapping logo breaks.
567  * Fully include logo breaks that don't include any blank-frame breaks.
568  */
569  for (FrameAnalyzer::FrameMap::const_iterator ii =
570  logoBreakMap->constBegin();
571  ii != logoBreakMap->constEnd();
572  ++ii)
573  {
574  long long iibb = ii.key();
575  long long iiee = iibb + *ii;
576  bool overlap = false;
577 
578  for (auto jj = m_breakMap.begin(); jj != m_breakMap.end(); )
579  {
580  long long jjbb = jj.key();
581  long long jjee = jjbb + *jj;
582  FrameAnalyzer::FrameMap::Iterator jjnext = jj;
583  ++jjnext;
584 
585  if (iiee < jjbb)
586  {
587  if (!overlap)
588  {
589  /* Fully incorporate logo break */
590  m_breakMap.insert(iibb, iiee - iibb);
591  }
592  break;
593  }
594 
595  if (iibb < jjbb && jjbb < iiee)
596  {
597  /* End of logo break includes beginning of blank-frame break. */
598  overlap = true;
599  m_breakMap.erase(jj);
600  m_breakMap.insert(iibb, max(iiee, jjee) - iibb);
601  }
602  else if (jjbb < iibb && iibb < jjee)
603  {
604  /* End of blank-frame break includes beginning of logo break. */
605  overlap = true;
606  if (jjee < iiee)
607  {
608  m_breakMap.remove(jjbb);
609  m_breakMap.insert(jjbb, iiee - jjbb);
610  }
611  }
612 
613  jj = jjnext;
614  }
615  }
616 
617  frameAnalyzerReportMap(&m_breakMap, m_fps, "BF Break");
618  return 0;
619 }
620 
621 int
623  const TemplateMatcher *templateMatcher)
624 {
625  (void)templateMatcher; /* gcc */
626 
627  LOG(VB_COMMFLAG, LOG_INFO, "BlankFrameDetector adjusting for "
628  "too little logo coverage (unimplemented)");
629  return 0;
630 }
631 
632 int
634 {
635  if (m_breakMap.empty())
636  {
637  /* Compute breaks (m_breakMap). */
638  computeBreakMap(&m_breakMap, &m_blankMap, m_fps, m_debugLevel);
639  frameAnalyzerReportMap(&m_breakMap, m_fps, "BF Break");
640  }
641 
642  breaks->clear();
643  for (auto bb = m_breakMap.begin(); bb != m_breakMap.end(); ++bb)
644  breaks->insert(bb.key(), *bb);
645 
646  return 0;
647 }
648 
649 int
651 {
653 }
654 
655 /* vim: set expandtab tabstop=4 shiftwidth=4: */
#define MAX_BLANK_FRAMES
const unsigned char * getMonochromatics(void) const
int finished(long long nframes, bool final)
int reportTime(void) const override
static const long long NEXTFRAME
Definition: FrameAnalyzer.h:57
int reportTime(void) const
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
BlankFrameDetector(HistogramAnalyzer *ha, const QString &debugdir)
FrameAnalyzer::FrameMap m_blankMap
HistogramAnalyzer * m_histogramAnalyzer
QSize GetVideoSize(void) const
Definition: mythplayer.h:188
const FrameAnalyzer::FrameMap * getBreaks(void) const
int finished(long long nframes, bool final) override
enum analyzeFrameResult MythPlayerInited(MythPlayer *player, long long nframes) override
int computeForLogoSurplus(const TemplateMatcher *templateMatcher)
float GetFrameRate(void) const
Definition: mythplayer.h:190
enum analyzeFrameResult analyzeFrame(const VideoFrame *frame, long long frameno, long long *pNextFrame) override
enum FrameAnalyzer::analyzeFrameResult analyzeFrame(const VideoFrame *frame, long long frameno)
void frameAnalyzerReportMap(const FrameAnalyzer::FrameMap *frameMap, float fps, const char *comment)
int GetNumSetting(const QString &key, int defaultval=0)
QMap< long long, long long > FrameMap
Definition: FrameAnalyzer.h:43
void createDebugDirectory(const QString &dirname, const QString &comment)
#define LOG(_MASK_, _LEVEL_, _STRING_)
Definition: mythlogging.h:41
const unsigned char * getMedians(void) const
enum FrameAnalyzer::analyzeFrameResult MythPlayerInited(MythPlayer *player, long long nframes)
static int computeForLogoDeficit(const TemplateMatcher *templateMatcher)
const float * getStdDevs(void) const
FrameAnalyzer::FrameMap m_breakMap
void frameAnalyzerReportMapms(const FrameAnalyzer::FrameMap *frameMap, float fps, const char *comment)
int computeBreaks(FrameMap *breaks)