MythTV  master
BlankFrameDetector.cpp
Go to the documentation of this file.
1 // ANSI C headers
2 #include <cmath>
3 #include <cstdlib>
4 #include <utility>
5 
6 // MythTV headers
7 #include "mythcorecontext.h" /* gContext */
8 #include "mythplayer.h"
9 
10 // Commercial Flagging headers
11 #include "CommDetector2.h"
12 #include "FrameAnalyzer.h"
13 #include "quickselect.h"
14 #include "HistogramAnalyzer.h"
15 #include "BlankFrameDetector.h"
16 #include "TemplateMatcher.h"
17 
18 using namespace commDetector2;
19 using namespace frameAnalyzer;
20 
21 namespace {
22 
23 bool
24 isBlank(unsigned char median, float stddev, unsigned char maxmedian,
25  float maxstddev)
26 {
27  return ((median < maxmedian) ||
28  ((median == maxmedian) && (stddev <= maxstddev)));
29 }
30 
31 int
32 sort_ascending_uchar(const void *aa, const void *bb)
33 {
34  return *(unsigned char*)aa - *(unsigned char*)bb;
35 }
36 
37 int
38 sort_ascending_float(const void *aa, const void *bb)
39 {
40  float faa = *(float*)aa;
41  float fbb = *(float*)bb;
42  return faa < fbb ? -1 : faa == fbb ? 0 : 1;
43 }
44 
45 bool
46 pickmedian(const unsigned char medianval,
47  unsigned char minval, unsigned char maxval)
48 {
49  return medianval >= minval && medianval <= maxval;
50 }
51 
52 void
53 computeBlankMap(FrameAnalyzer::FrameMap *blankMap, long long nframes,
54  const unsigned char *median, const float *stddev,
55  const unsigned char *monochromatic)
56 {
57  /*
58  * Select a "black" value based on a curve, to deal with varying "black"
59  * levels.
60  */
61  const unsigned char MINBLANKMEDIAN = 1;
62  const unsigned char MAXBLANKMEDIAN = 96;
63  const float MEDIANPCTILE = 0.95;
64  const float STDDEVPCTILE = 0.85;
65 
66  long long frameno, segb, sege, nblanks;
67  long long blankno, blankno1, blankno2;
68  long long stddevno, stddevno1, stddevno2;
69  unsigned char *blankmedian, maxmedian;
70  float *blankstddev, maxstddev;
71 
72  /* Count and select for monochromatic frames. */
73 
74  nblanks = 0;
75  for (frameno = 0; frameno < nframes; frameno++)
76  {
77  if (monochromatic[frameno] && pickmedian(median[frameno],
78  MINBLANKMEDIAN, MAXBLANKMEDIAN))
79  nblanks++;
80  }
81 
82  if (!nblanks)
83  {
84  /* No monochromatic frames. */
85  LOG(VB_COMMFLAG, LOG_INFO,
86  "BlankFrameDetector::computeBlankMap: No blank frames.");
87  return;
88  }
89 
90  /* Select percentile values from monochromatic frames. */
91 
92  blankmedian = new unsigned char[nblanks];
93  blankstddev = new float[nblanks];
94  blankno = 0;
95  for (frameno = 0; frameno < nframes; frameno++)
96  {
97  if (monochromatic[frameno] && pickmedian(median[frameno],
98  MINBLANKMEDIAN, MAXBLANKMEDIAN))
99  {
100  blankmedian[blankno] = median[frameno];
101  blankstddev[blankno] = stddev[frameno];
102  blankno++;
103  }
104  }
105 
106  qsort(blankmedian, nblanks, sizeof(*blankmedian), sort_ascending_uchar);
107  blankno = min(nblanks - 1, (long long)roundf(nblanks * MEDIANPCTILE));
108  maxmedian = blankmedian[blankno];
109 
110  qsort(blankstddev, nblanks, sizeof(*blankstddev), sort_ascending_float);
111  stddevno = min(nblanks - 1, (long long)roundf(nblanks * STDDEVPCTILE));
112  maxstddev = blankstddev[stddevno];
113 
114  /* Determine effective percentile ranges (for debugging). */
115 
116  blankno1 = blankno;
117  blankno2 = blankno;
118  while (blankno1 > 0 && blankmedian[blankno1] == maxmedian)
119  blankno1--;
120  if (blankmedian[blankno1] != maxmedian)
121  blankno1++;
122  while (blankno2 < nblanks && blankmedian[blankno2] == maxmedian)
123  blankno2++;
124  if (blankno2 == nblanks)
125  blankno2--;
126 
127  stddevno1 = stddevno;
128  stddevno2 = stddevno;
129  while (stddevno1 > 0 && blankstddev[stddevno1] == maxstddev)
130  stddevno1--;
131  if (blankstddev[stddevno1] != maxstddev)
132  stddevno1++;
133  while (stddevno2 < nblanks && blankstddev[stddevno2] == maxstddev)
134  stddevno2++;
135  if (stddevno2 == nblanks)
136  stddevno2--;
137 
138  LOG(VB_COMMFLAG, LOG_INFO,
139  QString("Blanks selecting median<=%1 (%2-%3%), stddev<=%4 (%5-%6%)")
140  .arg(maxmedian)
141  .arg(blankno1 * 100 / nblanks).arg(blankno2 * 100 / nblanks)
142  .arg(maxstddev)
143  .arg(stddevno1 * 100 / nblanks).arg(stddevno2 * 100 / nblanks));
144 
145  delete []blankmedian;
146  delete []blankstddev;
147 
148  blankMap->clear();
149  if (monochromatic[0] && isBlank(median[0], stddev[0], maxmedian, maxstddev))
150  {
151  segb = 0;
152  sege = 0;
153  }
154  else
155  {
156  /* Fake up a dummy blank frame for interval calculations. */
157  blankMap->insert(0, 0);
158  segb = -1;
159  sege = -1;
160  }
161  for (frameno = 1; frameno < nframes; frameno++)
162  {
163  if (monochromatic[frameno] && isBlank(median[frameno], stddev[frameno],
164  maxmedian, maxstddev))
165  {
166  /* Blank frame. */
167  if (sege < frameno - 1)
168  {
169  /* Start counting. */
170  segb = frameno;
171  sege = frameno;
172  }
173  else
174  {
175  /* Continue counting. */
176  sege = frameno;
177  }
178  }
179  else if (sege == frameno - 1)
180  {
181  /* Transition to non-blank frame. */
182  long long seglen = frameno - segb;
183  blankMap->insert(segb, seglen);
184  }
185  }
186  if (sege == frameno - 1)
187  {
188  /* Possibly ending on blank frames. */
189  long long seglen = frameno - segb;
190  blankMap->insert(segb, seglen);
191  }
192 
193  FrameAnalyzer::FrameMap::Iterator iiblank = blankMap->end();
194  --iiblank;
195  if (iiblank.key() + *iiblank < nframes)
196  {
197  /*
198  * Didn't end on blank frames, so add a dummy blank frame at the
199  * end.
200  */
201  blankMap->insert(nframes - 1, 0);
202  }
203 }
204 
205 void
206 computeBreakMap(FrameAnalyzer::FrameMap *breakMap,
207  const FrameAnalyzer::FrameMap *blankMap, float fps,
208  int debugLevel)
209 {
210  /*
211  * TUNABLE:
212  *
213  * Common commercial-break lengths.
214  */
215  static const struct {
216  int len; /* seconds */
217  int delta; /* seconds */
218  } breaktype[] = {
219  /* Sort by "len". */
220  { 15, 2 },
221  { 20, 2 },
222  { 30, 5 },
223  { 60, 5 },
224  };
225  static const unsigned int nbreaktypes =
226  sizeof(breaktype)/sizeof(*breaktype);
227 
228  /*
229  * TUNABLE:
230  *
231  * Shortest non-commercial length, used to coalesce consecutive commercial
232  * breaks that are usually identified due to in-commercial cuts.
233  */
234  static const int MINCONTENTLEN = (int)roundf(10 * fps);
235 
236  breakMap->clear();
237  for (FrameAnalyzer::FrameMap::const_iterator iiblank = blankMap->begin();
238  iiblank != blankMap->end();
239  ++iiblank)
240  {
241  long long brkb = iiblank.key();
242  long long iilen = *iiblank;
243  long long start = brkb + iilen / 2;
244 
245  for (unsigned int ii = 0; ii < nbreaktypes; ii++)
246  {
247  /* Look for next blank frame that is an acceptable distance away. */
248  FrameAnalyzer::FrameMap::const_iterator jjblank = iiblank;
249  for (++jjblank; jjblank != blankMap->end(); ++jjblank)
250  {
251  long long brke = jjblank.key();
252  long long jjlen = *jjblank;
253  long long end = brke + jjlen / 2;
254 
255  long long testlen = (long long)roundf((end - start) / fps);
256  if (testlen > breaktype[ii].len + breaktype[ii].delta)
257  break; /* Too far ahead; break to next break length. */
258 
259  long long delta = testlen - breaktype[ii].len;
260  if (delta < 0)
261  delta = 0 - delta;
262 
263  if (delta > breaktype[ii].delta)
264  continue; /* Outside delta range; try next end-blank. */
265 
266  /* Mark this commercial break. */
267  bool inserted = false;
268  for (unsigned int jj = 0;; jj++)
269  {
270  long long newbrkb = brkb + jj;
271  if (newbrkb >= brke)
272  {
273  LOG(VB_COMMFLAG, LOG_INFO,
274  QString("BF [%1,%2] ran out of slots")
275  .arg(brkb).arg(brke - 1));
276  break;
277  }
278  if (breakMap->find(newbrkb) == breakMap->end())
279  {
280  breakMap->insert(newbrkb, brke - newbrkb);
281  inserted = true;
282  break;
283  }
284  }
285  if (inserted)
286  break; /* next break type */
287  }
288  }
289  }
290 
291  if (debugLevel >= 1)
292  {
293  frameAnalyzerReportMap(breakMap, fps, "BF Break");
294  LOG(VB_COMMFLAG, LOG_INFO,
295  "BF coalescing overlapping/nearby breaks ...");
296  }
297 
298  /*
299  * Coalesce overlapping or very-nearby breaks (handles cut-scenes within a
300  * commercial).
301  */
302  for (;;)
303  {
304  bool coalesced = false;
305  FrameAnalyzer::FrameMap::iterator iibreak = breakMap->begin();
306  while (iibreak != breakMap->end())
307  {
308  long long iib = iibreak.key();
309  long long iie = iib + *iibreak;
310 
311  FrameAnalyzer::FrameMap::iterator jjbreak = iibreak;
312  ++jjbreak;
313  if (jjbreak == breakMap->end())
314  break;
315 
316  long long jjb = jjbreak.key();
317  long long jje = jjb + *jjbreak;
318 
319  if (jjb < iib)
320  {
321  /* (jjb,jje) is behind (iib,iie). */
322  ++iibreak;
323  continue;
324  }
325 
326  if (iie + MINCONTENTLEN < jjb)
327  {
328  /* (jjb,jje) is too far ahead. */
329  ++iibreak;
330  continue;
331  }
332 
333  /* Coalesce. */
334  if (jje > iie)
335  {
336  breakMap->remove(iib); /* overlap */
337  breakMap->insert(iib, jje - iib); /* overlap */
338  }
339  breakMap->erase(jjbreak);
340  coalesced = true;
341  iibreak = breakMap->find(iib);
342  }
343  if (!coalesced)
344  break;
345  }
346 
347  /* Adjust for blank intervals. */
348  FrameAnalyzer::FrameMap::iterator iibreak = breakMap->begin();
349  while (iibreak != breakMap->end())
350  {
351  long long iib = iibreak.key();
352  long long iie = iib + *iibreak;
353  FrameAnalyzer::FrameMap::iterator jjbreak = iibreak;
354  ++jjbreak;
355  breakMap->erase(iibreak);
356 
357  /* Trim leading blanks from commercial break. */
358  long long addb = *blankMap->find(iib);
359  addb = addb / 2;
360  if (addb > MAX_BLANK_FRAMES)
361  addb = MAX_BLANK_FRAMES;
362  iib += addb;
363  /* Add trailing blanks to commercial break. */
364  long long adde = *blankMap->find(iie);
365  iie += adde;
366  long long sube = adde / 2;
367  if (sube > MAX_BLANK_FRAMES)
368  sube = MAX_BLANK_FRAMES;
369  iie -= sube;
370  breakMap->insert(iib, iie - iib);
371  iibreak = jjbreak;
372  }
373 }
374 
375 }; /* namespace */
376 
378  : m_histogramAnalyzer(ha)
379 {
380  /*
381  * debugLevel:
382  * 0: no debugging
383  * 2: extra verbosity [O(nframes)]
384  */
385  m_debugLevel = gCoreContext->GetNumSetting("BlankFrameDetectorDebugLevel", 0);
386 
387  if (m_debugLevel >= 1)
388  createDebugDirectory(std::move(debugdir),
389  QString("BlankFrameDetector debugLevel %1").arg(m_debugLevel));
390 }
391 
394 {
396  m_histogramAnalyzer->MythPlayerInited(player, nframes);
397 
398  m_fps = player->GetFrameRate();
399 
400  QSize video_disp_dim = player->GetVideoSize();
401 
402  LOG(VB_COMMFLAG, LOG_INFO,
403  QString("BlankFrameDetector::MythPlayerInited %1x%2")
404  .arg(video_disp_dim.width())
405  .arg(video_disp_dim.height()));
406 
407  return ares;
408 }
409 
411 BlankFrameDetector::analyzeFrame(const VideoFrame *frame, long long frameno,
412  long long *pNextFrame)
413 {
414  *pNextFrame = NEXTFRAME;
415 
416  if (m_histogramAnalyzer->analyzeFrame(frame, frameno) ==
418  return ANALYZE_OK;
419 
420  LOG(VB_COMMFLAG, LOG_INFO,
421  QString("BlankFrameDetector::analyzeFrame error at frame %1")
422  .arg(frameno));
423  return ANALYZE_ERROR;
424 }
425 
426 int
427 BlankFrameDetector::finished(long long nframes, bool final)
428 {
429  if (m_histogramAnalyzer->finished(nframes, final))
430  return -1;
431 
432  LOG(VB_COMMFLAG, LOG_INFO, QString("BlankFrameDetector::finished(%1)")
433  .arg(nframes));
434 
435  /* Identify all sequences of blank frames (blankMap). */
436  computeBlankMap(&m_blankMap, nframes,
439  if (m_debugLevel >= 2)
441 
442  return 0;
443 }
444 
445 int
447  const TemplateMatcher *templateMatcher)
448 {
449  /*
450  * See TemplateMatcher::templateCoverage; some commercial breaks have
451  * logos. Conversely, any logo breaks are probably really breaks, so prefer
452  * those over blank-frame-calculated breaks.
453  */
454  const FrameAnalyzer::FrameMap *logoBreakMap = templateMatcher->getBreaks();
455 
456  /* TUNABLE: see TemplateMatcher::adjustForBlanks */
457  const int MAXBLANKADJUSTMENT = (int)roundf(5 * m_fps); /* frames */
458 
459  LOG(VB_COMMFLAG, LOG_INFO, "BlankFrameDetector adjusting for logo surplus");
460 
461  /*
462  * For each logo break, find the blank frames closest to its beginning and
463  * end. This helps properly support CommSkipAllBlanks.
464  */
465  for (FrameAnalyzer::FrameMap::const_iterator ii =
466  logoBreakMap->constBegin();
467  ii != logoBreakMap->constEnd();
468  ++ii)
469  {
470  /* Get bounds of beginning of logo break. */
471  long long iikey = ii.key();
472  long long iibb = iikey - MAXBLANKADJUSTMENT;
473  long long iiee = iikey + MAXBLANKADJUSTMENT;
474  FrameAnalyzer::FrameMap::Iterator jjfound = m_blankMap.end();
475 
476  /* Look for a blank frame near beginning of logo break. */
477  for (auto jj = m_blankMap.begin(); jj != m_blankMap.end(); ++jj)
478  {
479  long long jjbb = jj.key();
480  long long jjee = jjbb + *jj;
481 
482  if (iiee < jjbb)
483  break; /* No nearby blank frames. */
484 
485  if (jjee < iibb)
486  continue; /* Too early; keep looking. */
487 
488  jjfound = jj;
489  if (iikey <= jjbb)
490  {
491  /*
492  * Prefer the first blank frame beginning after the logo break
493  * begins.
494  */
495  break;
496  }
497  }
498 
499  /* Adjust blank frame to begin with logo break beginning. */
500  if (jjfound != m_blankMap.end())
501  {
502  long long jjee = jjfound.key() + *jjfound;
503  m_blankMap.erase(jjfound);
504  if (jjee <= iikey)
505  {
506  /* Move blank frame to beginning of logo break. */
507  m_blankMap.remove(iikey);
508  m_blankMap.insert(iikey, 1);
509  }
510  else
511  {
512  /* Adjust blank frame to begin with logo break. */
513  m_blankMap.insert(iikey, jjee - iikey);
514  }
515  }
516 
517  /* Get bounds of end of logo break. */
518  long long kkkey = ii.key() + *ii;
519  long long kkbb = kkkey - MAXBLANKADJUSTMENT;
520  long long kkee = kkkey + MAXBLANKADJUSTMENT;
521  FrameAnalyzer::FrameMap::Iterator mmfound = m_blankMap.end();
522 
523  /* Look for a blank frame near end of logo break. */
524  for (auto mm = m_blankMap.begin(); mm != m_blankMap.end(); ++mm)
525  {
526  long long mmbb = mm.key();
527  long long mmee = mmbb + *mm;
528 
529  if (kkee < mmbb)
530  break; /* No nearby blank frames. */
531 
532  if (mmee < kkbb)
533  continue; /* Too early; keep looking. */
534 
535  /* Prefer the last blank frame ending before the logo break ends. */
536  if (mmee < kkkey || mmfound == m_blankMap.end())
537  mmfound = mm;
538  if (mmee >= kkkey)
539  break;
540  }
541 
542  /* Adjust blank frame to end with logo break end. */
543  if (mmfound != m_blankMap.end())
544  {
545  long long mmbb = mmfound.key();
546  if (mmbb < kkkey)
547  {
548  /* Adjust blank frame to end with logo break. */
549  m_blankMap.remove(mmbb);
550  m_blankMap.insert(mmbb, kkkey - mmbb);
551  }
552  else
553  {
554  /* Move blank frame to end of logo break. */
555  m_blankMap.erase(mmfound);
556  m_blankMap.remove(kkkey - 1);
557  m_blankMap.insert(kkkey - 1, 1);
558  }
559  }
560  }
561 
562  /*
563  * Compute breaks (breakMap).
564  */
565  computeBreakMap(&m_breakMap, &m_blankMap, m_fps, m_debugLevel);
566 
567  /*
568  * Expand blank-frame breaks to fully include overlapping logo breaks.
569  * Fully include logo breaks that don't include any blank-frame breaks.
570  */
571  for (FrameAnalyzer::FrameMap::const_iterator ii =
572  logoBreakMap->constBegin();
573  ii != logoBreakMap->constEnd();
574  ++ii)
575  {
576  long long iibb = ii.key();
577  long long iiee = iibb + *ii;
578  bool overlap = false;
579 
580  for (auto jj = m_breakMap.begin(); jj != m_breakMap.end(); )
581  {
582  long long jjbb = jj.key();
583  long long jjee = jjbb + *jj;
584  FrameAnalyzer::FrameMap::Iterator jjnext = jj;
585  ++jjnext;
586 
587  if (iiee < jjbb)
588  {
589  if (!overlap)
590  {
591  /* Fully incorporate logo break */
592  m_breakMap.insert(iibb, iiee - iibb);
593  }
594  break;
595  }
596 
597  if (iibb < jjbb && jjbb < iiee)
598  {
599  /* End of logo break includes beginning of blank-frame break. */
600  overlap = true;
601  m_breakMap.erase(jj);
602  m_breakMap.insert(iibb, max(iiee, jjee) - iibb);
603  }
604  else if (jjbb < iibb && iibb < jjee)
605  {
606  /* End of blank-frame break includes beginning of logo break. */
607  overlap = true;
608  if (jjee < iiee)
609  {
610  m_breakMap.remove(jjbb);
611  m_breakMap.insert(jjbb, iiee - jjbb);
612  }
613  }
614 
615  jj = jjnext;
616  }
617  }
618 
619  frameAnalyzerReportMap(&m_breakMap, m_fps, "BF Break");
620  return 0;
621 }
622 
623 int
625  const TemplateMatcher *templateMatcher)
626 {
627  (void)templateMatcher; /* gcc */
628 
629  LOG(VB_COMMFLAG, LOG_INFO, "BlankFrameDetector adjusting for "
630  "too little logo coverage (unimplemented)");
631  return 0;
632 }
633 
634 int
636 {
637  if (m_breakMap.empty())
638  {
639  /* Compute breaks (m_breakMap). */
640  computeBreakMap(&m_breakMap, &m_blankMap, m_fps, m_debugLevel);
641  frameAnalyzerReportMap(&m_breakMap, m_fps, "BF Break");
642  }
643 
644  breaks->clear();
645  for (auto bb = m_breakMap.begin(); bb != m_breakMap.end(); ++bb)
646  breaks->insert(bb.key(), *bb);
647 
648  return 0;
649 }
650 
651 int
653 {
655 }
656 
657 /* vim: set expandtab tabstop=4 shiftwidth=4: */
#define MAX_BLANK_FRAMES
const unsigned char * getMonochromatics(void) const
int finished(long long nframes, bool final)
int reportTime(void) const override
static const long long NEXTFRAME
Definition: FrameAnalyzer.h:58
int reportTime(void) const
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
FrameAnalyzer::FrameMap m_blankMap
HistogramAnalyzer * m_histogramAnalyzer
QSize GetVideoSize(void) const
Definition: mythplayer.h:174
const FrameAnalyzer::FrameMap * getBreaks(void) const
int finished(long long nframes, bool final) override
enum analyzeFrameResult MythPlayerInited(MythPlayer *player, long long nframes) override
int computeForLogoSurplus(const TemplateMatcher *templateMatcher)
float GetFrameRate(void) const
Definition: mythplayer.h:176
BlankFrameDetector(HistogramAnalyzer *ha, QString debugdir)
enum analyzeFrameResult analyzeFrame(const VideoFrame *frame, long long frameno, long long *pNextFrame) override
enum FrameAnalyzer::analyzeFrameResult analyzeFrame(const VideoFrame *frame, long long frameno)
void frameAnalyzerReportMap(const FrameAnalyzer::FrameMap *frameMap, float fps, const char *comment)
int GetNumSetting(const QString &key, int defaultval=0)
void createDebugDirectory(const QString &dirname, const QString &comment)
#define LOG(_MASK_, _LEVEL_, _STRING_)
Definition: mythlogging.h:41
const unsigned char * getMedians(void) const
enum FrameAnalyzer::analyzeFrameResult MythPlayerInited(MythPlayer *player, long long nframes)
int computeForLogoDeficit(const TemplateMatcher *templateMatcher)
const float * getStdDevs(void) const
QMap< long long, long long > FrameMap
Definition: FrameAnalyzer.h:44
FrameAnalyzer::FrameMap m_breakMap
void frameAnalyzerReportMapms(const FrameAnalyzer::FrameMap *frameMap, float fps, const char *comment)
int computeBreaks(FrameMap *breaks)