MythTV master
mythmediacodeccontext.cpp
Go to the documentation of this file.
1// Qt
2#include <QtGlobal>
3#if QT_VERSION < QT_VERSION_CHECK(6,0,0)
4#include <QtAndroidExtras>
5#include <QAndroidJniEnvironment>
6#else
7#include <QCoreApplication>
8#include <QJniEnvironment>
9#include <QJniObject>
10#define QAndroidJniEnvironment QJniEnvironment
11#define QAndroidJniObject QJniObject
12#endif
13
14// MythTV
18
19#include "avformatdecoder.h"
21#include "mythplayerui.h"
23
24// FFmpeg
25extern "C" {
26#include "libavutil/pixfmt.h"
27#include "libavutil/hwcontext_mediacodec.h"
28#include "libavcodec/mediacodec.h"
29#include "libavcodec/avcodec.h"
30}
31
32#define LOC QString("MediaCodec: ")
33
34// MedicaCodec profile constants from MediaCodecInfo.CodecProfileLevel
35#define MC_MPEG2_SIMPLE (0x0)
36#define MC_MPEG2_MAIN (0x1)
37#define MC_MPEG2_422 (0x2)
38#define MC_MPEG2_SNR (0x3)
39#define MC_MPEG2_SPATIAL (0x4)
40#define MC_MPEG2_HIGH (0x5)
41#define MC_MPEG4_SIMPLE (0x0001)
42#define MC_MPEG4_SIMPLE_SCALEABLE (0x0002)
43#define MC_MPEG4_CORE (0x0004)
44#define MC_MPEG4_MAIN (0x0008)
45#define MC_MPEG4_NBIT (0x0010)
46#define MC_MPEG4_SCALEABLE_TEX (0x0020)
47#define MC_MPEG4_SIMPLE_FACE (0x0040)
48#define MC_MPEG4_SIMPLE_FBA (0x0080)
49#define MC_MPEG4_BASIC_ANIMATED (0x0100)
50#define MC_MPEG4_HYBRID (0x0200)
51#define MC_MPEG4_ADV_REALTIME (0x0400)
52#define MC_MPEG4_CORE_SCALEABLE (0x0800)
53#define MC_MPEG4_ADV_CODING (0x1000)
54#define MC_MPEG4_ADV_CORE (0x2000)
55#define MC_MPEG4_ADV_SCALEABLE (0x4000)
56#define MC_MPEG4_ADV_SIMPLE (0x8000)
57#define MC_H264_BASELINE (0x00001)
58#define MC_H264_MAIN (0x00002)
59#define MC_H264_EXTENDED (0x00004)
60#define MC_H264_HIGH (0x00008)
61#define MC_H264_HIGH10 (0x00010)
62#define MC_H264_HIGH422 (0x00020)
63#define MC_H264_HIGH444 (0x00040)
64#define MC_H264_CONST_BASELINE (0x10000)
65#define MC_H264_CONST_HIGH (0x80000)
66#define MC_HEVC_MAIN (0x0001)
67#define MC_HEVC_MAIN10 (0x0002)
68#define MC_HEVC_MAIN_STILL (0x0004)
69#define MC_HEVC_MAIN10HDR10 (0x1000)
70#define MC_HEVC_MMAIN10HDR10PLUS (0x2000)
71#define MC_VP8_MAIN (0x0001)
72#define MC_VP9_0 (0x0001)
73#define MC_VP9_1 (0x0002)
74#define MC_VP9_2 (0x0004)
75#define MC_VP9_3 (0x0008)
76#define MC_VP9_2HDR (0x1000)
77#define MC_VP9_3HDR (0x2000)
78#define MC_VP9_2HDRPLUS (0x4000)
79#define MC_VP9_3HDRPLUS (0x8000)
80
82{
83 if (Codec == MythCodecContext::MPEG2)
84 {
85 switch (Profile)
86 {
93 default: return MythCodecContext::MPEG2;
94 }
95 }
96
97 if (Codec == MythCodecContext::MPEG4)
98 {
99 switch (Profile)
100 {
108 case MC_MPEG4_SIMPLE_FBA: return MythCodecContext::MPEG4SimpleStudio; // Is this correct?
117 default: return MythCodecContext::MPEG4;
118 }
119 }
120
121 if (Codec == MythCodecContext::H264)
122 {
123 switch (Profile)
124 {
134 default: return MythCodecContext::H264;
135 }
136 }
137
138 if (Codec == MythCodecContext::HEVC)
139 {
140 switch (Profile)
141 {
147 default: return MythCodecContext::HEVC;
148 }
149 }
150
151 if (Codec == MythCodecContext::VP8)
153
154 if (Codec == MythCodecContext::VP9)
155 {
156 switch (Profile)
157 {
166 default: return MythCodecContext::VP9;
167 }
168 }
169
171}
172
174{
175 if (!Context || !gCoreContext->IsUIThread())
176 return -1;
177
178 // The interop must have a reference to the ui player so it can be deleted
179 // from the main thread.
180 auto * player = GetPlayerUI(Context);
181 if (!player)
182 return -1;
183
184 // Retrieve OpenGL render context
185 auto * render = dynamic_cast<MythRenderOpenGL*>(player->GetRender());
186 if (!render)
187 return -1;
188 OpenGLLocker locker(render);
189
190 // Create interop - NB no interop check here or in MythMediaCodecInterop
191 QSize size(Context->width, Context->height);
192 auto * interop = MythMediaCodecInterop::CreateMediaCodec(player, render, size);
193 if (!interop)
194 return -1;
195 if (!interop->GetSurface())
196 {
197 interop->DecrRef();
198 return -1;
199 }
200
201 // Create the hardware context
202 AVBufferRef *hwdeviceref = av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_MEDIACODEC);
203 AVHWDeviceContext *hwdevicectx = reinterpret_cast<AVHWDeviceContext*>(hwdeviceref->data);
204 hwdevicectx->free = &MythCodecContext::DeviceContextFinished;
205 hwdevicectx->user_opaque = interop;
206 AVMediaCodecDeviceContext *hwctx = reinterpret_cast<AVMediaCodecDeviceContext*>(hwdevicectx->hwctx);
207 hwctx->surface = interop->GetSurface();
208 if (av_hwdevice_ctx_init(hwdeviceref) < 0)
209 {
210 LOG(VB_GENERAL, LOG_ERR, LOC + "av_hwdevice_ctx_init failed");
211 av_buffer_unref(&hwdeviceref);
212 return -1;
213 }
214
215 Context->hw_device_ctx = hwdeviceref;
216 LOG(VB_GENERAL, LOG_INFO, LOC + "Created MediaCodec hardware device context");
217 return 0;
218}
219
221 const AVCodec **Codec,
222 const QString &Decoder,
223 AVStream *Stream,
224 uint StreamType)
225{
226 bool decodeonly = Decoder == "mediacodec-dec";
227 MythCodecID success = static_cast<MythCodecID>((decodeonly ? kCodec_MPEG1_MEDIACODEC_DEC : kCodec_MPEG1_MEDIACODEC) + (StreamType - 1));
228 MythCodecID failure = static_cast<MythCodecID>(kCodec_MPEG1 + (StreamType - 1));
229
230 if (!Decoder.startsWith("mediacodec"))
231 return failure;
232
233 if (!HaveMediaCodec())
234 return failure;
235
236 if (!decodeonly)
237 if (!FrameTypeIsSupported(*Context, FMT_MEDIACODEC))
238 return failure;
239
240 bool found = false;
243 MythCodecContext::FFmpegToMythProfile((*Context)->codec_id, (*Context)->profile);
244 for (auto profile : std::as_const(profiles))
245 {
246 if (profile.first == mythprofile &&
247 profile.second.width() >= (*Context)->width &&
248 profile.second.height() >= (*Context)->height)
249 {
250 found = true;
251 break;
252 }
253 }
254
255 AvFormatDecoder *decoder = dynamic_cast<AvFormatDecoder*>(reinterpret_cast<DecoderBase*>((*Context)->opaque));
256 QString profilestr = MythCodecContext::GetProfileDescription(mythprofile, QSize());
257 if (found && decoder)
258 {
259 QString decodername = QString((*Codec)->name) + "_mediacodec";
260 if (decodername == "mpeg2video_mediacodec")
261 decodername = "mpeg2_mediacodec";
262 const AVCodec *newCodec = avcodec_find_decoder_by_name (decodername.toLocal8Bit());
263 if (newCodec)
264 {
265 *Codec = newCodec;
266 LOG(VB_PLAYBACK, LOG_INFO, LOC + QString("HW device type '%1' supports decoding '%2' (%3)")
267 .arg(av_hwdevice_get_type_name(AV_HWDEVICE_TYPE_MEDIACODEC)).arg((*Codec)->name).arg(profilestr));
268 decoder->CodecMap()->FreeCodecContext(Stream);
269 *Context = decoder->CodecMap()->GetCodecContext(Stream, *Codec);
270 (*Context)->pix_fmt = AV_PIX_FMT_MEDIACODEC;
271 return success;
272 }
273 }
274
275 LOG(VB_PLAYBACK, LOG_INFO, LOC + QString("HW device type '%1' does not support decoding '%2' (%3)")
276 .arg(av_hwdevice_get_type_name(AV_HWDEVICE_TYPE_MEDIACODEC)).arg((*Codec)->name).arg(profilestr));
277 return failure;
278}
279
281 : MythCodecContext(Parent, CodecID)
282{
283}
284
285void MythMediaCodecContext::InitVideoCodec(AVCodecContext *Context, bool SelectedStream, bool &DirectRendering)
286{
287 if (CODEC_IS_MEDIACODEC(Context->codec))
288 {
289 Context->get_format = MythMediaCodecContext::GetFormat;
290 Context->slice_flags = SLICE_FLAG_CODED_ORDER | SLICE_FLAG_ALLOW_FIELD;
291 DirectRendering = false;
292 return;
293 }
294
295 MythCodecContext::InitVideoCodec(Context, SelectedStream, DirectRendering);
296}
297
298int MythMediaCodecContext::HwDecoderInit(AVCodecContext *Context)
299{
301 return 0;
303 return MythCodecContext::InitialiseDecoder2(Context, MythMediaCodecContext::InitialiseDecoder, "Create MediaCodec decoder");
304 return -1;
305}
306
307bool MythMediaCodecContext::RetrieveFrame(AVCodecContext *Context, MythVideoFrame *Frame, AVFrame *AvFrame)
308{
309 if (AvFrame->format != AV_PIX_FMT_MEDIACODEC)
310 return false;
312 return GetBuffer2(Context, Frame, AvFrame, 0);
313 return false;
314}
315
316AVPixelFormat MythMediaCodecContext::GetFormat(AVCodecContext*, const AVPixelFormat *PixFmt)
317{
318 while (*PixFmt != AV_PIX_FMT_NONE)
319 {
320 if (*PixFmt == AV_PIX_FMT_MEDIACODEC)
321 return AV_PIX_FMT_MEDIACODEC;
322 PixFmt++;
323 }
324 return AV_PIX_FMT_NONE;
325}
326
332{
333 if (!Frame)
334 return;
335
336 Frame->m_deinterlaceInuse = DEINT_BASIC | DEINT_DRIVER;
337 Frame->m_deinterlaceInuse2x = false;
338 Frame->m_interlaced = false;
339 Frame->m_interlacedReverse = false;
340 Frame->m_topFieldFirst = false;
341 Frame->m_deinterlaceAllowed = DEINT_NONE;
342 Frame->m_alreadyDeinterlaced = true;
343}
344
349bool MythMediaCodecContext::IsDeinterlacing(bool &DoubleRate, bool)
350{
351 DoubleRate = true;
352 return true;
353}
354
356{
357 // TODO Something tells me this is leakier than a leaky thing
358 static QRecursiveMutex lock;
359 static bool s_initialised = false;
360 static MCProfiles s_profiles;
361
362 QMutexLocker locker(&lock);
363 if (s_initialised)
364 return s_profiles;
365 s_initialised = true;
366
367 static const QPair<QString,QPair<MythCodecContext::CodecProfile, QList<int> > > mimetypes[] =
368 {
369 { "video/mpeg2", { MythCodecContext::MPEG2,
372 { "video/mp4v-es", { MythCodecContext::MPEG4,
379 { "video/avc", { MythCodecContext::H264,
383 { "video/hevc", { MythCodecContext::HEVC,
386 { "video/x-vnd.on2.vp8", { MythCodecContext::VP8, { MC_VP8_MAIN }}},
387 { "video/x-vnd.on2.vp9", { MythCodecContext::VP9,
390 //{ "video/vc1", { MythCodecContext::VC1 , {}}}, // No FFmpeg support
391 //{ "video/3gpp", { MythCodecContext::H263 , {}}}, // No FFmpeg support
392 //{ "video/av01", { MythCodecContext::AV1 , {}}} // No FFmpeg support, API Level 29
393 };
394
395 // Retrieve MediaCodecList
397 QAndroidJniObject list("android/media/MediaCodecList", "(I)V", 0); // 0 = REGULAR_CODECS
398 if (!list.isValid())
399 return s_profiles;
400 // Retrieve array of MediaCodecInfo's
401 QAndroidJniObject qtcodecs = list.callObjectMethod("getCodecInfos", "()[Landroid/media/MediaCodecInfo;");
402 if (!qtcodecs.isValid())
403 return s_profiles;
404
405 // Iterate over MediaCodecInfo's
406 jobjectArray codecs = qtcodecs.object<jobjectArray>();
407 jsize codeccount = env->GetArrayLength(codecs);
408 for (jsize i = 0; i < codeccount; ++i)
409 {
410 QAndroidJniObject codec(env->GetObjectArrayElement(codecs, i));
411 if (!codec.isValid())
412 continue;
413
414 // Ignore encoders
415 if (codec.callMethod<jboolean>("isEncoder"))
416 continue;
417
418 // Ignore software decoders
419 QString name = codec.callObjectMethod<jstring>("getName").toString();
420 if (name.contains("OMX.google", Qt::CaseInsensitive))
421 continue;
422
423 // Retrieve supported mimetypes (there is usually just one)
424 QAndroidJniObject qttypes = codec.callObjectMethod("getSupportedTypes", "()[Ljava/lang/String;");
425 jobjectArray types = qttypes.object<jobjectArray>();
426 jsize typecount = env->GetArrayLength(types);
427 for (jsize j = 0; j < typecount; ++j)
428 {
429 QAndroidJniObject type(env->GetObjectArrayElement(types, j));
430 if (!type.isValid())
431 continue;
432
433 // Match mimetype to types supported by FFmpeg
434 QString typestr = type.toString();
435 for (auto mimetype : mimetypes)
436 {
437 if (mimetype.first != typestr)
438 continue;
439
440 // Retrieve MediaCodecInfo.CodecCapabilities for mimetype
441 QAndroidJniObject caps = codec.callObjectMethod("getCapabilitiesForType",
442 "(Ljava/lang/String;)Landroid/media/MediaCodecInfo$CodecCapabilities;",
443 type.object<jstring>());
444 if (!caps.isValid())
445 continue;
446
447 // Retrieve MediaCodecInfo.VideoCapabilities from CodecCapabilities
448 QAndroidJniObject videocaps = caps.callObjectMethod("getVideoCapabilities",
449 "()Landroid/media/MediaCodecInfo$VideoCapabilities;");
450 QAndroidJniObject widthrange = videocaps.callObjectMethod("getSupportedWidths",
451 "()Landroid/util/Range;");
452 QAndroidJniObject heightrange = videocaps.callObjectMethod("getSupportedHeights",
453 "()Landroid/util/Range;");
454
455 QAndroidJniObject widthqt = widthrange.callObjectMethod("getUpper", "()Ljava/lang/Comparable;");
456 QAndroidJniObject heightqt = heightrange.callObjectMethod("getUpper", "()Ljava/lang/Comparable;");
457 int width = widthqt.callMethod<jint>("intValue", "()I");
458 int height = heightqt.callMethod<jint>("intValue", "()I");
459
460 // Profiles are available from CodecCapabilities.profileLevel field
461 QAndroidJniObject profiles = caps.getObjectField("profileLevels",
462 "[Landroid/media/MediaCodecInfo$CodecProfileLevel;");
463 jobjectArray profilearr = profiles.object<jobjectArray>();
464 jsize profilecount = env->GetArrayLength(profilearr);
465 if (profilecount < 1)
466 {
467 s_profiles.append(QPair<MythCodecContext::CodecProfile,QSize>(mimetype.second.first, QSize(width, height)));
468 continue;
469 }
470
471 for (jsize k = 0; k < profilecount; ++k)
472 {
473 jobject profile = env->GetObjectArrayElement(profilearr, k);
474 jclass objclass = env->GetObjectClass(profile);
475 jfieldID id = env->GetFieldID(objclass, "profile", "I");
476 int value = static_cast<int>(env->GetIntField(profile, id));
477 QList<int>& mcprofiles = mimetype.second.second;
478 auto sameprof = [value](auto mcprofile) { return value == mcprofile; };
479 if (std::any_of(mcprofiles.cbegin(), mcprofiles.cend(), sameprof))
480 {
481 MythCodecContext::CodecProfile p = MediaCodecToMythProfile(mimetype.second.first, value);
482 s_profiles.append(QPair<MythCodecContext::CodecProfile,QSize>(p, QSize(width, height)));
483 }
484 else
485 s_profiles.append(QPair<MythCodecContext::CodecProfile,QSize>(mimetype.second.first, QSize(width, height)));
486 }
487 }
488 }
489 }
490
491 return s_profiles;
492}
493
494void MythMediaCodecContext::GetDecoderList(QStringList &Decoders)
495{
497 if (profiles.isEmpty())
498 return;
499
500 Decoders.append("MediaCodec:");
501 for (auto profile : std::as_const(profiles))
502 Decoders.append(MythCodecContext::GetProfileDescription(profile.first, profile.second));
503}
504
505bool MythMediaCodecContext::HaveMediaCodec(bool Reinit /*=false*/)
506{
507 static QRecursiveMutex lock;
508 static bool s_initialised = false;
509 static bool s_available = false;
510
511 QMutexLocker locker(&lock);
512 if (!s_initialised || Reinit)
513 {
515 if (profiles.isEmpty())
516 {
517 LOG(VB_GENERAL, LOG_INFO, LOC + "No MediaCodec decoders found");
518 }
519 else
520 {
521 s_available = true;
522 LOG(VB_GENERAL, LOG_INFO, LOC + "Supported/available MediaCodec decoders:");
523 for (auto profile : std::as_const(profiles))
524 {
525 LOG(VB_GENERAL, LOG_INFO, LOC +
527 }
528 }
529 }
530 s_initialised = true;
531 return s_available;
532}
533
AVFrame AVFrame
A decoder for media files.
MythCodecMap * CodecMap(void)
static void DeviceContextFinished(AVHWDeviceContext *Context)
static bool FrameTypeIsSupported(AVCodecContext *Context, VideoFrameType Format)
static int InitialiseDecoder2(AVCodecContext *Context, CreateHWDecoder Callback, const QString &Debug)
Initialise a hardware decoder that is NOT expected to use AVHWFramesContext.
virtual void InitVideoCodec(AVCodecContext *Context, bool SelectedStream, bool &DirectRendering)
static QString GetProfileDescription(CodecProfile Profile, QSize Size, VideoFrameType Format=FMT_NONE, uint ColorDepth=0)
static bool GetBuffer2(struct AVCodecContext *Context, MythVideoFrame *Frame, AVFrame *AvFrame, int Flags)
A generic hardware buffer initialisation method when AVHWFramesContext is NOT used.
static MythPlayerUI * GetPlayerUI(AVCodecContext *Context)
MythCodecID m_codecID
static CodecProfile FFmpegToMythProfile(AVCodecID CodecID, int Profile)
AVCodecContext * GetCodecContext(const AVStream *Stream, const AVCodec *Codec=nullptr, bool NullCodec=false)
Definition: mythavutil.cpp:288
void FreeCodecContext(const AVStream *Stream)
Definition: mythavutil.cpp:333
int HwDecoderInit(AVCodecContext *Context) override
static MythCodecID GetBestSupportedCodec(AVCodecContext **Context, const AVCodec **Codec, const QString &Decoder, AVStream *Stream, uint StreamType)
bool RetrieveFrame(AVCodecContext *Context, MythVideoFrame *Frame, AVFrame *AvFrame) override
MythMediaCodecContext(DecoderBase *Parent, MythCodecID CodecID)
static bool HaveMediaCodec(bool Reinit=false)
bool IsDeinterlacing(bool &DoubleRate, bool=false) override
static AVPixelFormat GetFormat(AVCodecContext *, const AVPixelFormat *PixFmt)
void InitVideoCodec(AVCodecContext *Context, bool SelectedStream, bool &DirectRendering) override
static void GetDecoderList(QStringList &Decoders)
void PostProcessFrame(AVCodecContext *, MythVideoFrame *) override
Mark all MediaCodec decoded frames as progressive,.
static int InitialiseDecoder(AVCodecContext *Context)
static MCProfiles & GetProfiles(void)
static MythMediaCodecInterop * CreateMediaCodec(MythPlayerUI *Player, MythRenderOpenGL *Context, QSize Size)
unsigned int uint
Definition: freesurround.h:24
static const struct wl_interface * types[]
static bool codec_is_mediacodec_dec(MythCodecID id)
Definition: mythcodecid.h:340
MythCodecID
Definition: mythcodecid.h:14
@ kCodec_MPEG1_MEDIACODEC_DEC
Definition: mythcodecid.h:136
@ kCodec_MPEG1
Definition: mythcodecid.h:24
@ kCodec_MPEG1_MEDIACODEC
Definition: mythcodecid.h:120
static bool codec_is_mediacodec(MythCodecID id)
Definition: mythcodecid.h:337
static bool CODEC_IS_MEDIACODEC(const struct AVCodec *)
Definition: mythcodecid.h:416
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
@ DEINT_DRIVER
Definition: mythframe.h:74
@ DEINT_BASIC
Definition: mythframe.h:69
@ DEINT_NONE
Definition: mythframe.h:68
@ FMT_MEDIACODEC
Definition: mythframe.h:60
#define LOG(_MASK_, _LEVEL_, _QSTRING_)
Definition: mythlogging.h:39
#define MC_VP8_MAIN
#define QAndroidJniEnvironment
#define LOC
MythCodecContext::CodecProfile MediaCodecToMythProfile(int Codec, int Profile)
#define MC_MPEG4_SIMPLE_FACE
#define MC_VP9_2
#define MC_H264_HIGH10
#define MC_MPEG4_NBIT
#define MC_VP9_3HDRPLUS
#define MC_H264_HIGH444
#define MC_MPEG4_BASIC_ANIMATED
#define MC_VP9_2HDRPLUS
#define MC_MPEG4_SCALEABLE_TEX
#define MC_MPEG4_SIMPLE_SCALEABLE
#define MC_MPEG4_ADV_SIMPLE
#define MC_H264_HIGH422
#define MC_MPEG4_ADV_REALTIME
#define MC_VP9_2HDR
#define MC_H264_EXTENDED
#define MC_MPEG2_SPATIAL
#define MC_VP9_1
#define MC_HEVC_MMAIN10HDR10PLUS
#define MC_HEVC_MAIN10
#define MC_HEVC_MAIN_STILL
#define MC_MPEG4_ADV_CORE
#define MC_MPEG2_422
#define MC_MPEG2_SNR
#define MC_VP9_0
#define MC_MPEG4_SIMPLE
#define MC_H264_HIGH
#define MC_HEVC_MAIN10HDR10
#define MC_H264_CONST_BASELINE
#define MC_VP9_3
#define MC_MPEG4_MAIN
#define MC_MPEG2_MAIN
#define MC_MPEG4_CORE
#define MC_VP9_3HDR
#define MC_MPEG2_SIMPLE
#define MC_MPEG2_HIGH
#define MC_MPEG4_SIMPLE_FBA
#define MC_MPEG4_CORE_SCALEABLE
#define MC_H264_BASELINE
#define MC_MPEG4_HYBRID
#define MC_MPEG4_ADV_CODING
#define QAndroidJniObject
#define MC_H264_CONST_HIGH
#define MC_H264_MAIN
#define MC_HEVC_MAIN
#define MC_MPEG4_ADV_SCALEABLE
QList< QPair< MythCodecContext::CodecProfile, QSize > > MCProfiles
QString toString(const QDateTime &raw_dt, uint format)
Returns formatted string representing the time.
Definition: mythdate.cpp:93