28 #include "libavutil/imgutils.h"
36 bool writeJPG(
const QString&
prefix,
const AVFrame *img,
int imgheight)
38 const int imgwidth = img->linesize[0];
39 QFileInfo jpgfi(
prefix +
".jpg");
42 QFile pgmfile(
prefix +
".pgm");
43 if (!pgmfile.exists())
45 QByteArray pfname = pgmfile.fileName().toLocal8Bit();
46 if (
pgm_write(img->data[0], imgwidth, imgheight,
53 QString cmd = QString(
"convert -quality 50 -resize 192x144 %1 %2")
54 .arg(pgmfile.fileName(), jpgfi.filePath());
58 if (!pgmfile.remove())
60 LOG(VB_COMMFLAG, LOG_ERR,
61 QString(
"TemplateFinder.writeJPG error removing %1 (%2)")
62 .arg(pgmfile.fileName(), strerror(errno)));
70 pgm_scorepixels(
unsigned int *scores,
int width,
int row,
int col,
71 const AVFrame *src,
int srcheight)
74 const int srcwidth = src->linesize[0];
76 for (
int rr = 0; rr < srcheight; rr++)
78 for (
int cc = 0;
cc < srcwidth;
cc++)
80 if (src->data[0][rr * srcwidth +
cc])
81 scores[(row + rr) * width + col +
cc]++;
91 return *(
unsigned int*)aa - *(
unsigned int*)bb;
95 bounding_score(
const AVFrame *img,
int row,
int col,
int width,
int height)
98 const int imgwidth = img->linesize[0];
101 int rr2 = row + height;
102 int cc2 = col + width;
103 for (
int rr = row; rr < rr2; rr++)
105 for (
int cc = col;
cc < cc2;
cc++)
107 if (img->data[0][rr * imgwidth +
cc])
111 return (
float)score / (width * height);
115 rowisempty(
const AVFrame *img,
int row,
int col,
int width)
117 const int imgwidth = img->linesize[0];
118 for (
int cc = col;
cc < col + width;
cc++)
119 if (img->data[0][row * imgwidth +
cc])
125 colisempty(
const AVFrame *img,
int col,
int row,
int height)
127 const int imgwidth = img->linesize[0];
128 for (
int rr = row; rr < row + height; rr++)
129 if (img->data[0][rr * imgwidth + col])
135 bounding_box(
const AVFrame *img,
int imgheight,
136 int minrow,
int mincol,
int maxrow1,
int maxcol1,
137 int *prow,
int *pcol,
int *pwidth,
int *pheight)
139 const int imgwidth = img->linesize[0];
146 static constexpr
int kMaxWidthPct = 20;
147 static constexpr
int kMaxHeightPct = 20;
156 const int VERTSLOP = std::max(4, imgheight * 1 / 15);
157 const int HORIZSLOP = std::max(4, imgwidth * 1 / 20);
159 int maxwidth = (maxcol1 - mincol) * kMaxWidthPct / 100;
160 int maxheight = (maxrow1 - minrow) * kMaxHeightPct / 100;
164 int width = maxcol1 - mincol;
165 int height = maxrow1 - minrow;
173 bool improved =
false;
175 LOG(VB_COMMFLAG, LOG_INFO, QString(
"bounding_box %1x%2@(%3,%4)")
176 .arg(width).arg(height).arg(col).arg(row));
179 float score = bounding_score(img, row, col, width, height);
181 for (
int ii = 1; ii < height; ii++)
184 bounding_score(img, row + ii, col, width, height - ii);
185 if (newscore < score)
193 score = bounding_score(img, row, col, width, height);
195 for (
int ii = 1; ii < width; ii++)
198 bounding_score(img, row, col + ii, width - ii, height);
199 if (newscore < score)
207 score = bounding_score(img, row, col, width, height);
208 newbottom = row + height;
209 for (
int ii = 1; ii < height; ii++)
212 bounding_score(img, row, col, width, height - ii);
213 if (newscore < score)
216 newbottom = row + height - ii;
221 score = bounding_score(img, row, col, width, height);
222 newright = col + width;
223 for (
int ii = 1; ii < width; ii++)
226 bounding_score(img, row, col, width - ii, height);
227 if (newscore < score)
230 newright = col + width - ii;
239 width = newright - newcol;
240 height = newbottom - newrow;
261 if (width > maxwidth)
264 int chop = width / 3;
265 int chopwidth = width - chop;
267 float left = bounding_score(img, row, col, chopwidth, height);
268 float right = bounding_score(img, row, col + chop, chopwidth, height);
269 LOG(VB_COMMFLAG, LOG_INFO,
270 QString(
"bounding_box too wide (%1 > %2); left=%3, right=%4")
271 .arg(width).arg(maxwidth)
272 .arg(left, 0,
'f', 3).arg(right, 0,
'f', 3));
273 float minscore = std::min(left, right);
274 float maxscore = std::max(left, right);
275 if (maxscore < 3 * minscore / 2)
282 LOG(VB_COMMFLAG, LOG_ERR,
"bounding_box giving up (edge "
283 "pixels distributed too uniformly)");
293 if (height > maxheight)
296 int chop = height / 3;
297 int chopheight = height - chop;
299 float upper = bounding_score(img, row, col, width, chopheight);
300 float lower = bounding_score(img, row + chop, col, width, chopheight);
301 LOG(VB_COMMFLAG, LOG_INFO,
302 QString(
"bounding_box too tall (%1 > %2); upper=%3, lower=%4")
303 .arg(height).arg(maxheight)
304 .arg(upper, 0,
'f', 3).arg(lower, 0,
'f', 3));
305 float minscore = std::min(upper, lower);
306 float maxscore = std::max(upper, lower);
307 if (maxscore < 3 * minscore / 2)
314 LOG(VB_COMMFLAG, LOG_ERR,
"bounding_box giving up (edge "
315 "pixel distribution too uniform)");
335 LOG(VB_COMMFLAG, LOG_INFO,
336 QString(
"bounding_box %1x%2@(%3,%4); horizslop=%5,vertslop=%6")
337 .arg(width).arg(height).arg(col).arg(row)
338 .arg(HORIZSLOP).arg(VERTSLOP));
344 if (newrow <= minrow)
349 if (row - newrow >= VERTSLOP)
351 newrow = row - VERTSLOP;
354 if (rowisempty(img, newrow, col, width))
361 newrow = std::max(minrow, newrow - 1);
367 if (newcol <= mincol)
372 if (col - newcol >= HORIZSLOP)
374 newcol = col - HORIZSLOP;
377 if (colisempty(img, newcol, row, height))
384 newcol = std::max(mincol, newcol - 1);
387 newright = col + width;
390 if (newright >= maxcol1)
395 if (newright - (col + width) >= HORIZSLOP)
397 newright = col + width + HORIZSLOP;
400 if (colisempty(img, newright, row, height))
404 newright = std::min(maxcol1, newright + 1);
407 newbottom = row + height;
410 if (newbottom >= maxrow1)
415 if (newbottom - (row + height) >= VERTSLOP)
417 newbottom = row + height + VERTSLOP;
420 if (rowisempty(img, newbottom, col, width))
424 newbottom = std::min(maxrow1, newbottom + 1);
428 width = newright - newcol;
429 height = newbottom - newrow;
431 LOG(VB_COMMFLAG, LOG_INFO, QString(
"bounding_box %1x%2@(%3,%4)")
432 .arg(width).arg(height).arg(col).arg(row));
442 template_alloc(
const unsigned int *scores,
int width,
int height,
443 int minrow,
int mincol,
int maxrow1,
int maxcol1,
AVFrame *tmpl,
444 int *ptmplrow,
int *ptmplcol,
int *ptmplwidth,
int *ptmplheight,
445 bool debug_edgecounts,
const QString& debugdir)
456 static constexpr
float kMinScorePctile = 0.998;
458 const int nn = width * height;
462 unsigned int threshscore = 0;
465 if (av_image_alloc(thresh.data, thresh.linesize,
466 width, height, AV_PIX_FMT_GRAY8, IMAGE_ALIGN))
468 LOG(VB_COMMFLAG, LOG_ERR,
469 QString(
"template_alloc av_image_alloc thresh (%1x%2) failed")
470 .arg(width).arg(height));
474 uint *sortedscores =
new unsigned int[nn];
475 memcpy(sortedscores, scores, nn *
sizeof(*sortedscores));
478 if (sortedscores[0] == sortedscores[nn - 1])
481 LOG(VB_COMMFLAG, LOG_ERR,
482 QString(
"template_alloc: %1x%2 pixels all identical!")
483 .arg(width).arg(height));
489 ii = (int)roundf(nn * kMinScorePctile);
490 threshscore = sortedscores[ii];
491 for (first = ii; first > 0 && sortedscores[first] == threshscore; first--)
493 if (sortedscores[first] != threshscore)
495 for (last = ii; last < nn - 1 && sortedscores[last] == threshscore; last++)
497 if (sortedscores[last] != threshscore)
500 LOG(VB_COMMFLAG, LOG_INFO, QString(
"template_alloc wanted %1, got %2-%3")
501 .arg(kMinScorePctile, 0,
'f', 6)
502 .arg((
float)first / nn, 0,
'f', 6)
503 .arg((
float)last / nn, 0,
'f', 6));
505 for (ii = 0; ii < nn; ii++)
506 thresh.data[0][ii] = scores[ii] >= threshscore ? UCHAR_MAX : 0;
508 if (debug_edgecounts)
512 if (av_image_alloc(scored.data, scored.linesize,
513 width, height, AV_PIX_FMT_GRAY8, IMAGE_ALIGN))
515 LOG(VB_COMMFLAG, LOG_ERR,
516 QString(
"template_alloc av_image_alloc scored (%1x%2) failed")
517 .arg(width).arg(height));
520 unsigned int maxscore = sortedscores[nn - 1];
521 for (ii = 0; ii < nn; ii++)
522 scored.data[0][ii] = scores[ii] * UCHAR_MAX / maxscore;
523 bool success = writeJPG(debugdir +
"/TemplateFinder-scores", &scored,
525 av_freep(&scored.data[0]);
530 if (!writeJPG(debugdir +
"/TemplateFinder-edgecounts", &thresh, height))
536 if (bounding_box(&thresh, height, minrow, mincol, maxrow1, maxcol1,
537 ptmplrow, ptmplcol, ptmplwidth, ptmplheight))
540 if ((
uint)(*ptmplwidth * *ptmplheight) > USHRT_MAX)
543 LOG(VB_COMMFLAG, LOG_ERR,
544 QString(
"template_alloc bounding_box too big (%1x%2)")
545 .arg(*ptmplwidth).arg(*ptmplheight));
549 if (av_image_alloc(tmpl->data, tmpl->linesize,
550 *ptmplwidth, *ptmplheight, AV_PIX_FMT_GRAY8, IMAGE_ALIGN))
552 LOG(VB_COMMFLAG, LOG_ERR,
553 QString(
"template_alloc av_image_alloc tmpl (%1x%2) failed")
554 .arg(*ptmplwidth).arg(*ptmplheight));
558 if (
pgm_crop(tmpl, &thresh, height, *ptmplrow, *ptmplcol,
559 *ptmplwidth, *ptmplheight))
562 delete []sortedscores;
563 av_freep(&thresh.data[0]);
568 delete []sortedscores;
569 av_freep(&thresh.data[0]);
574 analyzeFrameDebug(
long long frameno,
const AVFrame *pgm,
int pgmheight,
576 int croprow,
int cropcol,
bool debug_frames,
const QString& debugdir)
578 static constexpr
int kDelta = 24;
579 static int s_lastrow;
580 static int s_lastcol;
581 static int s_lastwidth;
582 static int s_lastheight;
583 const int cropwidth = cropped->linesize[0];
585 int rowsame = abs(s_lastrow - croprow) <= kDelta ? 1 : 0;
586 int colsame = abs(s_lastcol - cropcol) <= kDelta ? 1 : 0;
587 int widthsame = abs(s_lastwidth - cropwidth) <= kDelta ? 1 : 0;
588 int heightsame = abs(s_lastheight - cropheight) <= kDelta ? 1 : 0;
590 if (frameno > 0 && rowsame + colsame + widthsame + heightsame >= 3)
593 LOG(VB_COMMFLAG, LOG_INFO,
594 QString(
"TemplateFinder Frame %1: %2x%3@(%4,%5)")
596 .arg(cropwidth).arg(cropheight)
597 .arg(cropcol).arg(croprow));
601 s_lastwidth = cropwidth;
602 s_lastheight = cropheight;
606 QString base = QString(
"%1/TemplateFinder-%2")
607 .arg(debugdir).arg(frameno, 5, 10, QChar(
'0'));
610 if (!writeJPG(base, pgm, pgmheight))
614 if (!writeJPG(base +
"-cropped", cropped, cropheight))
618 if (!writeJPG(base +
"-edges", edges, cropheight))
626 bool readTemplate(
const QString& datafile,
int *prow,
int *pcol,
int *pwidth,
int *pheight,
627 const QString& tmplfile,
AVFrame *tmpl,
bool *pvalid)
629 QFile dfile(datafile);
630 QFileInfo dfileinfo(dfile);
632 if (!dfile.open(QIODevice::ReadOnly))
635 if (!dfileinfo.size())
642 QTextStream stream(&dfile);
643 stream >> *prow >> *pcol >> *pwidth >> *pheight;
646 if (av_image_alloc(tmpl->data, tmpl->linesize,
647 *pwidth, *pheight, AV_PIX_FMT_GRAY8, IMAGE_ALIGN))
649 LOG(VB_COMMFLAG, LOG_ERR,
650 QString(
"readTemplate av_image_alloc %1 (%2x%3) failed")
651 .arg(tmplfile).arg(*pwidth).arg(*pheight));
655 QByteArray tmfile = tmplfile.toLatin1();
656 if (
pgm_read(tmpl->data[0], *pwidth, *pheight, tmfile.constData()))
658 av_freep(&tmpl->data[0]);
667 writeDummyTemplate(
const QString& datafile)
670 QFile dfile(datafile);
672 if (!dfile.open(QIODevice::WriteOnly | QIODevice::Truncate) &&
674 (void)dfile.remove();
678 writeTemplate(
const QString& tmplfile,
const AVFrame *tmpl,
const QString& datafile,
679 int row,
int col,
int width,
int height)
681 QFile tfile(tmplfile);
683 QByteArray tmfile = tmplfile.toLatin1();
684 if (
pgm_write(tmpl->data[0], width, height, tmfile.constData()))
687 QFile dfile(datafile);
688 if (!dfile.open(QIODevice::WriteOnly))
691 QTextStream stream(&dfile);
692 stream << row <<
" " << col <<
"\n" << width <<
" " << height <<
"\n";
700 std::shared_ptr<BorderDetector> bd,
701 std::shared_ptr<EdgeDetector> ed,
702 MythPlayer *player, std::chrono::seconds proglen,
703 const QString& debugdir)
704 : m_pgmConverter(
std::move(pgmc)),
705 m_borderDetector(
std::move(bd)),
706 m_edgeDetector(
std::move(ed)),
707 m_sampleTime(
std::min(proglen / 2, 20 * 60s)),
708 m_debugDir(debugdir),
709 m_debugData(debugdir +
"/TemplateFinder.txt"),
710 m_debugTmpl(debugdir +
"/TemplateFinder.pgm")
720 unsigned int samplesNeeded = 300;
739 LOG(VB_COMMFLAG, LOG_INFO,
740 QString(
"TemplateFinder: sampleTime=%1s, samplesNeeded=%2, endFrame=%3")
755 QString(
"TemplateFinder debugLevel %1").arg(
m_debugLevel));
768 av_freep(&
m_tmpl.data[0]);
774 [[maybe_unused]]
long long nframes)
804 LOG(VB_COMMFLAG, LOG_INFO,
805 QString(
"TemplateFinder::MythPlayerInited read %1: %2")
820 LOG(VB_COMMFLAG, LOG_INFO,
821 QString(
"TemplateFinder::MythPlayerInited %1 of %2 (%3)")
827 LOG(VB_COMMFLAG, LOG_INFO,
828 QString(
"TemplateFinder::MythPlayerInited framesize %1")
835 av_freep(&
m_tmpl.data[0]);
848 newwidth, newheight, AV_PIX_FMT_GRAY8, IMAGE_ALIGN))
850 LOG(VB_COMMFLAG, LOG_ERR,
851 QString(
"TemplateFinder::resetBuffers "
852 "av_image_alloc cropped (%1x%2) failed")
853 .arg(newwidth).arg(newheight));
864 long long *pNextFrame)
880 const int FRAMESGMPCTILE = 90;
893 static constexpr
float kExcludeWidth = 0.5;
894 static constexpr
float kExcludeHeight = 0.5;
917 &croprow, &cropcol, &cropwidth, &cropheight))
921 auto start = nowAsDuration<std::chrono::microseconds>();
936 cropwidth, cropheight))
943 int excludewidth = (int)(pgmwidth * kExcludeWidth);
944 int excludeheight = (int)(pgmheight * kExcludeHeight);
945 int excluderow = (pgmheight - excludeheight) / 2 - croprow;
946 int excludecol = (pgmwidth - excludewidth) / 2 - cropcol;
948 excludewidth, excludeheight);
952 if (edges ==
nullptr)
955 if (pgm_scorepixels(
m_scores, pgmwidth, croprow, cropcol,
961 if (!analyzeFrameDebug(frameno, pgm, pgmheight, &
m_cropped, edges,
966 auto end = nowAsDuration<std::chrono::microseconds>();
976 LOG(VB_COMMFLAG, LOG_ERR,
977 QString(
"TemplateFinder::analyzeFrame error at frame %1")
1009 LOG(VB_COMMFLAG, LOG_INFO,
1010 QString(
"TemplateFinder::finished wrote %1"
1011 " and %2 [%3x%4@(%5,%6)]")
1027 av_freep(&
m_tmpl.data[0]);
1040 LOG(VB_COMMFLAG, LOG_INFO, QString(
"TF Time: analyze=%1s")