MythTV  master
mythmediacodeccontext.cpp
Go to the documentation of this file.
1 // Qt
2 #include <QtAndroidExtras>
3 #include <QAndroidJniEnvironment>
4 
5 // MythTV
6 #include "mythlogging.h"
7 #include "mythcorecontext.h"
8 #include "mythmainwindow.h"
9 #include "avformatdecoder.h"
10 #include "mythmediacodecinterop.h"
11 #include "mythmediacodeccontext.h"
12 
13 // FFmpeg
14 extern "C" {
15 #include "libavutil/pixfmt.h"
16 #include "libavutil/hwcontext_mediacodec.h"
17 #include "libavcodec/mediacodec.h"
18 #include "libavcodec/avcodec.h"
19 }
20 
21 #define LOC QString("MediaCodec: ")
22 
23 // MedicaCodec profile constants from MediaCodecInfo.CodecProfileLevel
24 #define MC_MPEG2_SIMPLE (0x0)
25 #define MC_MPEG2_MAIN (0x1)
26 #define MC_MPEG2_422 (0x2)
27 #define MC_MPEG2_SNR (0x3)
28 #define MC_MPEG2_SPATIAL (0x4)
29 #define MC_MPEG2_HIGH (0x5)
30 #define MC_MPEG4_SIMPLE (0x0001)
31 #define MC_MPEG4_SIMPLE_SCALEABLE (0x0002)
32 #define MC_MPEG4_CORE (0x0004)
33 #define MC_MPEG4_MAIN (0x0008)
34 #define MC_MPEG4_NBIT (0x0010)
35 #define MC_MPEG4_SCALEABLE_TEX (0x0020)
36 #define MC_MPEG4_SIMPLE_FACE (0x0040)
37 #define MC_MPEG4_SIMPLE_FBA (0x0080)
38 #define MC_MPEG4_BASIC_ANIMATED (0x0100)
39 #define MC_MPEG4_HYBRID (0x0200)
40 #define MC_MPEG4_ADV_REALTIME (0x0400)
41 #define MC_MPEG4_CORE_SCALEABLE (0x0800)
42 #define MC_MPEG4_ADV_CODING (0x1000)
43 #define MC_MPEG4_ADV_CORE (0x2000)
44 #define MC_MPEG4_ADV_SCALEABLE (0x4000)
45 #define MC_MPEG4_ADV_SIMPLE (0x8000)
46 #define MC_H264_BASELINE (0x00001)
47 #define MC_H264_MAIN (0x00002)
48 #define MC_H264_EXTENDED (0x00004)
49 #define MC_H264_HIGH (0x00008)
50 #define MC_H264_HIGH10 (0x00010)
51 #define MC_H264_HIGH422 (0x00020)
52 #define MC_H264_HIGH444 (0x00040)
53 #define MC_H264_CONST_BASELINE (0x10000)
54 #define MC_H264_CONST_HIGH (0x80000)
55 #define MC_HEVC_MAIN (0x0001)
56 #define MC_HEVC_MAIN10 (0x0002)
57 #define MC_HEVC_MAIN_STILL (0x0004)
58 #define MC_HEVC_MAIN10HDR10 (0x1000)
59 #define MC_HEVC_MMAIN10HDR10PLUS (0x2000)
60 #define MC_VP8_MAIN (0x0001)
61 #define MC_VP9_0 (0x0001)
62 #define MC_VP9_1 (0x0002)
63 #define MC_VP9_2 (0x0004)
64 #define MC_VP9_3 (0x0008)
65 #define MC_VP9_2HDR (0x1000)
66 #define MC_VP9_3HDR (0x2000)
67 #define MC_VP9_2HDRPLUS (0x4000)
68 #define MC_VP9_3HDRPLUS (0x8000)
69 
71 {
72  if (Codec == MythCodecContext::MPEG2)
73  {
74  switch (Profile)
75  {
82  default: return MythCodecContext::MPEG2;
83  }
84  }
85 
86  if (Codec == MythCodecContext::MPEG4)
87  {
88  switch (Profile)
89  {
97  case MC_MPEG4_SIMPLE_FBA: return MythCodecContext::MPEG4SimpleStudio; // Is this correct?
106  default: return MythCodecContext::MPEG4;
107  }
108  }
109 
110  if (Codec == MythCodecContext::H264)
111  {
112  switch (Profile)
113  {
123  default: return MythCodecContext::H264;
124  }
125  }
126 
127  if (Codec == MythCodecContext::HEVC)
128  {
129  switch (Profile)
130  {
136  default: return MythCodecContext::HEVC;
137  }
138  }
139 
140  if (Codec == MythCodecContext::VP8)
141  return MythCodecContext::VP8;
142 
143  if (Codec == MythCodecContext::VP9)
144  {
145  switch (Profile)
146  {
147  case MC_VP9_0: return MythCodecContext::VP9_0;
148  case MC_VP9_1: return MythCodecContext::VP9_1;
149  case MC_VP9_2: return MythCodecContext::VP9_2;
150  case MC_VP9_3: return MythCodecContext::VP9_3;
155  default: return MythCodecContext::VP9;
156  }
157  }
158 
160 }
161 
163 {
164  if (!Context || !gCoreContext->IsUIThread())
165  return -1;
166 
167  // We need a player to release the interop
168  MythPlayer *player = nullptr;
169  auto *decoder = reinterpret_cast<AvFormatDecoder*>(Context->opaque);
170  if (decoder)
171  player = decoder->GetPlayer();
172  if (!player)
173  return -1;
174 
175  // Retrieve OpenGL render context
177  if (!render)
178  return -1;
179  OpenGLLocker locker(render);
180 
181  // Create interop - NB no interop check here or in MythMediaCodecInterop
182  QSize size(Context->width, Context->height);
183  MythMediaCodecInterop *interop = MythMediaCodecInterop::Create(render, size);
184  if (!interop)
185  return -1;
186  if (!interop->GetSurface())
187  {
188  interop->DecrRef();
189  return -1;
190  }
191 
192  // Set player
193  interop->SetPlayer(player);
194 
195  // Create the hardware context
196  AVBufferRef *hwdeviceref = av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_MEDIACODEC);
197  AVHWDeviceContext *hwdevicectx = reinterpret_cast<AVHWDeviceContext*>(hwdeviceref->data);
198  hwdevicectx->free = &MythCodecContext::DeviceContextFinished;
199  hwdevicectx->user_opaque = interop;
200  AVMediaCodecDeviceContext *hwctx = reinterpret_cast<AVMediaCodecDeviceContext*>(hwdevicectx->hwctx);
201  hwctx->surface = interop->GetSurface();
202  if (av_hwdevice_ctx_init(hwdeviceref) < 0)
203  {
204  LOG(VB_GENERAL, LOG_ERR, LOC + "av_hwdevice_ctx_init failed");
205  av_buffer_unref(&hwdeviceref);
206  return -1;
207  }
208 
209  Context->hw_device_ctx = hwdeviceref;
210  LOG(VB_GENERAL, LOG_INFO, LOC + "Created MediaCodec hardware device context");
211  return 0;
212 }
213 
215  AVCodec **Codec,
216  const QString &Decoder,
217  AVStream *Stream,
218  uint StreamType)
219 {
220  bool decodeonly = Decoder == "mediacodec-dec";
221  MythCodecID success = static_cast<MythCodecID>((decodeonly ? kCodec_MPEG1_MEDIACODEC_DEC : kCodec_MPEG1_MEDIACODEC) + (StreamType - 1));
222  MythCodecID failure = static_cast<MythCodecID>(kCodec_MPEG1 + (StreamType - 1));
223 
224  if (!Decoder.startsWith("mediacodec"))
225  return failure;
226 
227  if (!HaveMediaCodec())
228  return failure;
229 
230  bool found = false;
232  MythCodecContext::CodecProfile mythprofile =
233  MythCodecContext::FFmpegToMythProfile((*Context)->codec_id, (*Context)->profile);
234  foreach (auto profile, profiles)
235  {
236  if (profile.first == mythprofile &&
237  profile.second.width() >= (*Context)->width &&
238  profile.second.height() >= (*Context)->height)
239  {
240  found = true;
241  break;
242  }
243  }
244 
245  AvFormatDecoder *decoder = dynamic_cast<AvFormatDecoder*>(reinterpret_cast<DecoderBase*>((*Context)->opaque));
246  QString profilestr = MythCodecContext::GetProfileDescription(mythprofile, QSize());
247  if (found && decoder)
248  {
249  QString decodername = QString((*Codec)->name) + "_mediacodec";
250  if (decodername == "mpeg2video_mediacodec")
251  decodername = "mpeg2_mediacodec";
252  AVCodec *newCodec = avcodec_find_decoder_by_name (decodername.toLocal8Bit());
253  if (newCodec)
254  {
255  *Codec = newCodec;
256  LOG(VB_PLAYBACK, LOG_INFO, LOC + QString("HW device type '%1' supports decoding '%2' (%3)")
257  .arg(av_hwdevice_get_type_name(AV_HWDEVICE_TYPE_MEDIACODEC)).arg((*Codec)->name).arg(profilestr));
258  decoder->CodecMap()->freeCodecContext(Stream);
259  *Context = decoder->CodecMap()->getCodecContext(Stream, *Codec);
260  (*Context)->pix_fmt = AV_PIX_FMT_MEDIACODEC;
261  return success;
262  }
263  }
264 
265  LOG(VB_PLAYBACK, LOG_INFO, LOC + QString("HW device type '%1' does not support decoding '%2' (%3)")
266  .arg(av_hwdevice_get_type_name(AV_HWDEVICE_TYPE_MEDIACODEC)).arg((*Codec)->name).arg(profilestr));
267  return failure;
268 }
269 
271  : MythCodecContext(Parent, CodecID)
272 {
273 }
274 
275 void MythMediaCodecContext::InitVideoCodec(AVCodecContext *Context, bool SelectedStream, bool &DirectRendering)
276 {
277  if (CODEC_IS_MEDIACODEC(Context->codec))
278  {
280  Context->slice_flags = SLICE_FLAG_CODED_ORDER | SLICE_FLAG_ALLOW_FIELD;
281  DirectRendering = false;
282  return;
283  }
284 
285  MythCodecContext::InitVideoCodec(Context, SelectedStream, DirectRendering);
286 }
287 
289 {
291  return 0;
292  else if (codec_is_mediacodec(m_codecID))
294  return -1;
295 }
296 
298 {
299  if (AvFrame->format != AV_PIX_FMT_MEDIACODEC)
300  return false;
302  return GetBuffer2(Context, Frame, AvFrame, 0);
303  return false;
304 }
305 
306 AVPixelFormat MythMediaCodecContext::GetFormat(AVCodecContext*, const AVPixelFormat *PixFmt)
307 {
308  while (*PixFmt != AV_PIX_FMT_NONE)
309  {
310  if (*PixFmt == AV_PIX_FMT_MEDIACODEC)
311  return AV_PIX_FMT_MEDIACODEC;
312  PixFmt++;
313  }
314  return AV_PIX_FMT_NONE;
315 }
316 
322 {
323  if (!Frame)
324  return;
325 
326  Frame->deinterlace_inuse = DEINT_BASIC | DEINT_DRIVER;
327  Frame->deinterlace_inuse2x = 0;
328  Frame->interlaced_frame = 0;
329  Frame->interlaced_reversed = 0;
330  Frame->top_field_first = 0;
331  Frame->deinterlace_allowed = DEINT_NONE;
332  Frame->already_deinterlaced = 1;
333 }
334 
339 bool MythMediaCodecContext::IsDeinterlacing(bool &DoubleRate, bool)
340 {
341  DoubleRate = true;
342  return true;
343 }
344 
346 {
347  // TODO Something tells me this is leakier than a leaky thing
348  static QMutex lock(QMutex::Recursive);
349  static bool s_initialised = false;
350  static MCProfiles s_profiles;
351 
352  QMutexLocker locker(&lock);
353  if (s_initialised)
354  return s_profiles;
355  s_initialised = true;
356 
357  static const QPair<QString,QPair<MythCodecContext::CodecProfile, QList<int> > > mimetypes[] =
358  {
359  { "video/mpeg2", { MythCodecContext::MPEG2,
362  { "video/mp4v-es", { MythCodecContext::MPEG4,
369  { "video/avc", { MythCodecContext::H264,
373  { "video/hevc", { MythCodecContext::HEVC,
376  { "video/x-vnd.on2.vp8", { MythCodecContext::VP8, { MC_VP8_MAIN }}},
377  { "video/x-vnd.on2.vp9", { MythCodecContext::VP9,
380  //{ "video/vc1", { MythCodecContext::VC1 , {}}}, // No FFmpeg support
381  //{ "video/3gpp", { MythCodecContext::H263 , {}}}, // No FFmpeg support
382  //{ "video/av01", { MythCodecContext::AV1 , {}}} // No FFmpeg support, API Level 29
383  };
384 
385  // Retrieve MediaCodecList
386  QAndroidJniEnvironment env;
387  QAndroidJniObject list("android/media/MediaCodecList", "(I)V", 0); // 0 = REGULAR_CODECS
388  if (!list.isValid())
389  return s_profiles;
390  // Retrieve array of MediaCodecInfo's
391  QAndroidJniObject qtcodecs = list.callObjectMethod("getCodecInfos", "()[Landroid/media/MediaCodecInfo;");
392  if (!qtcodecs.isValid())
393  return s_profiles;
394 
395  // Iterate over MediaCodecInfo's
396  jobjectArray codecs = qtcodecs.object<jobjectArray>();
397  jsize codeccount = env->GetArrayLength(codecs);
398  for (jsize i = 0; i < codeccount; ++i)
399  {
400  QAndroidJniObject codec(env->GetObjectArrayElement(codecs, i));
401  if (!codec.isValid())
402  continue;
403 
404  // Ignore encoders
405  if (codec.callMethod<jboolean>("isEncoder"))
406  continue;
407 
408  // Ignore software decoders
409  QString name = codec.callObjectMethod<jstring>("getName").toString();
410  if (name.contains("OMX.google", Qt::CaseInsensitive))
411  continue;
412 
413  // Retrieve supported mimetypes (there is usually just one)
414  QAndroidJniObject qttypes = codec.callObjectMethod("getSupportedTypes", "()[Ljava/lang/String;");
415  jobjectArray types = qttypes.object<jobjectArray>();
416  jsize typecount = env->GetArrayLength(types);
417  for (jsize j = 0; j < typecount; ++j)
418  {
419  QAndroidJniObject type(env->GetObjectArrayElement(types, j));
420  if (!type.isValid())
421  continue;
422 
423  // Match mimetype to types supported by FFmpeg
424  QString typestr = type.toString();
425  for (auto mimetype : mimetypes)
426  {
427  if (mimetype.first != typestr)
428  continue;
429 
430  // Retrieve MediaCodecInfo.CodecCapabilities for mimetype
431  QAndroidJniObject caps = codec.callObjectMethod("getCapabilitiesForType",
432  "(Ljava/lang/String;)Landroid/media/MediaCodecInfo$CodecCapabilities;",
433  type.object<jstring>());
434  if (!caps.isValid())
435  continue;
436 
437  // Retrieve MediaCodecInfo.VideoCapabilities from CodecCapabilities
438  QAndroidJniObject videocaps = caps.callObjectMethod("getVideoCapabilities",
439  "()Landroid/media/MediaCodecInfo$VideoCapabilities;");
440  QAndroidJniObject widthrange = videocaps.callObjectMethod("getSupportedWidths",
441  "()Landroid/util/Range;");
442  QAndroidJniObject heightrange = videocaps.callObjectMethod("getSupportedHeights",
443  "()Landroid/util/Range;");
444 
445  QAndroidJniObject widthqt = widthrange.callObjectMethod("getUpper", "()Ljava/lang/Comparable;");
446  QAndroidJniObject heightqt = heightrange.callObjectMethod("getUpper", "()Ljava/lang/Comparable;");
447  int width = widthqt.callMethod<jint>("intValue", "()I");
448  int height = heightqt.callMethod<jint>("intValue", "()I");
449 
450  // Profiles are available from CodecCapabilities.profileLevel field
451  QAndroidJniObject profiles = caps.getObjectField("profileLevels",
452  "[Landroid/media/MediaCodecInfo$CodecProfileLevel;");
453  jobjectArray profilearr = profiles.object<jobjectArray>();
454  jsize profilecount = env->GetArrayLength(profilearr);
455  if (profilecount < 1)
456  {
457  s_profiles.append(QPair<MythCodecContext::CodecProfile,QSize>(mimetype.second.first, QSize(width, height)));
458  continue;
459  }
460 
461  for (jsize k = 0; k < profilecount; ++k)
462  {
463  jobject profile = env->GetObjectArrayElement(profilearr, k);
464  jclass objclass = env->GetObjectClass(profile);
465  jfieldID id = env->GetFieldID(objclass, "profile", "I");
466  int value = static_cast<int>(env->GetIntField(profile, id));
467  QList<int>& mcprofiles = mimetype.second.second;
468  bool found = false;
469  foreach (auto mcprofile, mcprofiles)
470  {
471  if (value == mcprofile)
472  {
473  found = true;
474  MythCodecContext::CodecProfile p = MediaCodecToMythProfile(mimetype.second.first, value);
475  s_profiles.append(QPair<MythCodecContext::CodecProfile,QSize>(p, QSize(width, height)));
476  break;
477  }
478  }
479 
480  if (!found)
481  s_profiles.append(QPair<MythCodecContext::CodecProfile,QSize>(mimetype.second.first, QSize(width, height)));
482  }
483  }
484  }
485  }
486 
487  return s_profiles;
488 }
489 
490 void MythMediaCodecContext::GetDecoderList(QStringList &Decoders)
491 {
493  if (profiles.isEmpty())
494  return;
495 
496  Decoders.append("MediaCodec:");
497  foreach (auto profile, profiles)
498  Decoders.append(MythCodecContext::GetProfileDescription(profile.first, profile.second));
499 }
500 
502 {
503  static QMutex lock(QMutex::Recursive);
504  static bool s_initialised = false;
505  static bool s_available = false;
506 
507  QMutexLocker locker(&lock);
508  if (!s_initialised)
509  {
511  if (profiles.isEmpty())
512  {
513  LOG(VB_GENERAL, LOG_INFO, LOC + "No MediaCodec decoders found");
514  }
515  else
516  {
517  s_available = true;
518  LOG(VB_GENERAL, LOG_INFO, LOC + "Supported/available MediaCodec decoders:");
519  foreach (auto profile, profiles)
520  {
521  LOG(VB_GENERAL, LOG_INFO, LOC +
523  }
524  }
525  }
526  s_initialised = true;
527  return s_available;
528 }
529 
#define MC_VP9_1
MythCodecContext::CodecProfile MediaCodecToMythProfile(int Codec, int Profile)
#define MC_H264_CONST_HIGH
static AVPixelFormat GetFormat(AVCodecContext *, const AVPixelFormat *PixFmt)
int HwDecoderInit(AVCodecContext *Context) override
#define MC_VP9_0
#define MC_H264_EXTENDED
static int InitialiseDecoder(AVCodecContext *Context)
#define MC_HEVC_MAIN10
void SetPlayer(MythPlayer *Player)
static MCProfiles & GetProfiles(void)
QHash< QString, Action * > Context
Definition: action.h:77
MythMediaCodecContext(DecoderBase *Parent, MythCodecID CodecID)
QString toString(MarkTypes type)
#define MC_MPEG4_SCALEABLE_TEX
MythCodecID
Definition: mythcodecid.h:10
#define MC_VP9_3HDR
#define MC_MPEG2_HIGH
#define MC_MPEG4_SIMPLE_SCALEABLE
#define MC_H264_MAIN
#define MC_VP9_2HDR
void PostProcessFrame(AVCodecContext *, VideoFrame *) override
Mark all MediaCodec decoded frames as progressive,.
static QString GetProfileDescription(CodecProfile Profile, QSize Size, VideoFrameType Format=FMT_NONE, uint ColorDepth=0)
struct AVFrame AVFrame
#define MC_VP8_MAIN
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
#define MC_MPEG2_MAIN
#define MC_HEVC_MMAIN10HDR10PLUS
#define MC_MPEG2_SIMPLE
void freeCodecContext(const AVStream *stream)
Definition: mythavutil.cpp:554
#define MC_H264_HIGH10
MythCodecID m_codecID
#define MC_MPEG4_ADV_REALTIME
#define MC_MPEG4_BASIC_ANIMATED
#define CODEC_IS_MEDIACODEC(codec)
Definition: mythcodecid.h:382
#define MC_MPEG4_SIMPLE_FACE
#define LOC
#define MC_MPEG4_ADV_CORE
static int InitialiseDecoder2(AVCodecContext *Context, CreateHWDecoder Callback, const QString &Debug)
Initialise a hardware decoder that is NOT expected to use AVHWFramesContext.
static MythCodecID GetBestSupportedCodec(AVCodecContext **Context, AVCodec **Codec, const QString &Decoder, AVStream *Stream, uint StreamType)
#define MC_VP9_2
AVCodecContext * getCodecContext(const AVStream *stream, const AVCodec *pCodec=nullptr, bool nullCodec=false)
Definition: mythavutil.cpp:512
static void GetDecoderList(QStringList &Decoders)
virtual void InitVideoCodec(AVCodecContext *Context, bool SelectedStream, bool &DirectRendering)
static bool HaveMediaCodec(void)
virtual int DecrRef(void)
Decrements reference count and deletes on 0.
A decoder for video files.
#define MC_HEVC_MAIN10HDR10
#define MC_MPEG4_SIMPLE_FBA
#define MC_MPEG4_HYBRID
unsigned int uint
Definition: compat.h:140
#define MC_MPEG2_422
#define codec_is_mediacodec(id)
Definition: mythcodecid.h:323
static CodecProfile FFmpegToMythProfile(AVCodecID CodecID, int Profile)
bool RetrieveFrame(AVCodecContext *Context, VideoFrame *Frame, AVFrame *AvFrame) override
#define MC_MPEG2_SPATIAL
#define MC_MPEG2_SNR
#define MC_H264_HIGH444
#define MC_MPEG4_ADV_CODING
#define LOG(_MASK_, _LEVEL_, _STRING_)
Definition: mythlogging.h:41
bool IsDeinterlacing(bool &DoubleRate, bool=false) override
#define MC_VP9_3HDRPLUS
#define MC_MPEG4_ADV_SIMPLE
#define MC_HEVC_MAIN
static bool GetBuffer2(struct AVCodecContext *Context, VideoFrame *Frame, AVFrame *AvFrame, int Flags)
A generic hardware buffer initialisation method when AVHWFramesContext is NOT used.
#define MC_HEVC_MAIN_STILL
#define MC_MPEG4_NBIT
static void DeviceContextFinished(AVHWDeviceContext *Context)
static MythRenderOpenGL * GetOpenGLRender(void)
#define MC_MPEG4_ADV_SCALEABLE
#define MC_MPEG4_MAIN
QList< QPair< MythCodecContext::CodecProfile, QSize > > MCProfiles
#define MC_H264_HIGH
#define MC_MPEG4_CORE_SCALEABLE
#define MC_MPEG4_CORE
MythCodecMap * CodecMap(void)
#define MC_H264_BASELINE
#define MC_H264_CONST_BASELINE
#define MC_H264_HIGH422
#define MC_MPEG4_SIMPLE
static MythMediaCodecInterop * Create(MythRenderOpenGL *Context, QSize Size)
void InitVideoCodec(AVCodecContext *Context, bool SelectedStream, bool &DirectRendering) override
#define codec_is_mediacodec_dec(id)
Definition: mythcodecid.h:325
#define MC_VP9_3
#define MC_VP9_2HDRPLUS