MythTV  master
mythmediacodeccontext.cpp
Go to the documentation of this file.
1 // Qt
2 #include <QtAndroidExtras>
3 #include <QAndroidJniEnvironment>
4 
5 // MythTV
6 #include "mythlogging.h"
7 #include "mythcorecontext.h"
8 #include "mythmainwindow.h"
9 #include "avformatdecoder.h"
10 #include "mythmediacodecinterop.h"
11 #include "mythmediacodeccontext.h"
12 
13 // FFmpeg
14 extern "C" {
15 #include "libavutil/pixfmt.h"
16 #include "libavutil/hwcontext_mediacodec.h"
17 #include "libavcodec/mediacodec.h"
18 #include "libavcodec/avcodec.h"
19 }
20 
21 #define LOC QString("MediaCodec: ")
22 
23 // MedicaCodec profile constants from MediaCodecInfo.CodecProfileLevel
24 #define MC_MPEG2_SIMPLE (0x0)
25 #define MC_MPEG2_MAIN (0x1)
26 #define MC_MPEG2_422 (0x2)
27 #define MC_MPEG2_SNR (0x3)
28 #define MC_MPEG2_SPATIAL (0x4)
29 #define MC_MPEG2_HIGH (0x5)
30 #define MC_MPEG4_SIMPLE (0x0001)
31 #define MC_MPEG4_SIMPLE_SCALEABLE (0x0002)
32 #define MC_MPEG4_CORE (0x0004)
33 #define MC_MPEG4_MAIN (0x0008)
34 #define MC_MPEG4_NBIT (0x0010)
35 #define MC_MPEG4_SCALEABLE_TEX (0x0020)
36 #define MC_MPEG4_SIMPLE_FACE (0x0040)
37 #define MC_MPEG4_SIMPLE_FBA (0x0080)
38 #define MC_MPEG4_BASIC_ANIMATED (0x0100)
39 #define MC_MPEG4_HYBRID (0x0200)
40 #define MC_MPEG4_ADV_REALTIME (0x0400)
41 #define MC_MPEG4_CORE_SCALEABLE (0x0800)
42 #define MC_MPEG4_ADV_CODING (0x1000)
43 #define MC_MPEG4_ADV_CORE (0x2000)
44 #define MC_MPEG4_ADV_SCALEABLE (0x4000)
45 #define MC_MPEG4_ADV_SIMPLE (0x8000)
46 #define MC_H264_BASELINE (0x00001)
47 #define MC_H264_MAIN (0x00002)
48 #define MC_H264_EXTENDED (0x00004)
49 #define MC_H264_HIGH (0x00008)
50 #define MC_H264_HIGH10 (0x00010)
51 #define MC_H264_HIGH422 (0x00020)
52 #define MC_H264_HIGH444 (0x00040)
53 #define MC_H264_CONST_BASELINE (0x10000)
54 #define MC_H264_CONST_HIGH (0x80000)
55 #define MC_HEVC_MAIN (0x0001)
56 #define MC_HEVC_MAIN10 (0x0002)
57 #define MC_HEVC_MAIN_STILL (0x0004)
58 #define MC_HEVC_MAIN10HDR10 (0x1000)
59 #define MC_HEVC_MMAIN10HDR10PLUS (0x2000)
60 #define MC_VP8_MAIN (0x0001)
61 #define MC_VP9_0 (0x0001)
62 #define MC_VP9_1 (0x0002)
63 #define MC_VP9_2 (0x0004)
64 #define MC_VP9_3 (0x0008)
65 #define MC_VP9_2HDR (0x1000)
66 #define MC_VP9_3HDR (0x2000)
67 #define MC_VP9_2HDRPLUS (0x4000)
68 #define MC_VP9_3HDRPLUS (0x8000)
69 
71 {
72  if (Codec == MythCodecContext::MPEG2)
73  {
74  switch (Profile)
75  {
82  default: return MythCodecContext::MPEG2;
83  }
84  }
85 
86  if (Codec == MythCodecContext::MPEG4)
87  {
88  switch (Profile)
89  {
97  case MC_MPEG4_SIMPLE_FBA: return MythCodecContext::MPEG4SimpleStudio; // Is this correct?
106  default: return MythCodecContext::MPEG4;
107  }
108  }
109 
110  if (Codec == MythCodecContext::H264)
111  {
112  switch (Profile)
113  {
123  default: return MythCodecContext::H264;
124  }
125  }
126 
127  if (Codec == MythCodecContext::HEVC)
128  {
129  switch (Profile)
130  {
136  default: return MythCodecContext::HEVC;
137  }
138  }
139 
140  if (Codec == MythCodecContext::VP8)
141  return MythCodecContext::VP8;
142 
143  if (Codec == MythCodecContext::VP9)
144  {
145  switch (Profile)
146  {
147  case MC_VP9_0: return MythCodecContext::VP9_0;
148  case MC_VP9_1: return MythCodecContext::VP9_1;
149  case MC_VP9_2: return MythCodecContext::VP9_2;
150  case MC_VP9_3: return MythCodecContext::VP9_3;
155  default: return MythCodecContext::VP9;
156  }
157  }
158 
160 }
161 
163 {
164  if (!Context || !gCoreContext->IsUIThread())
165  return -1;
166 
167  // We need a player to release the interop
168  MythPlayer *player = nullptr;
169  auto *decoder = reinterpret_cast<AvFormatDecoder*>(Context->opaque);
170  if (decoder)
171  player = decoder->GetPlayer();
172  if (!player)
173  return -1;
174 
175  // Retrieve OpenGL render context
177  if (!render)
178  return -1;
179  OpenGLLocker locker(render);
180 
181  // Create interop - NB no interop check here or in MythMediaCodecInterop
182  QSize size(Context->width, Context->height);
183  MythMediaCodecInterop *interop = MythMediaCodecInterop::Create(render, size);
184  if (!interop)
185  return -1;
186  if (!interop->GetSurface())
187  {
188  interop->DecrRef();
189  return -1;
190  }
191 
192  // Set player
193  interop->SetPlayer(player);
194 
195  // Create the hardware context
196  AVBufferRef *hwdeviceref = av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_MEDIACODEC);
197  AVHWDeviceContext *hwdevicectx = reinterpret_cast<AVHWDeviceContext*>(hwdeviceref->data);
198  hwdevicectx->free = &MythCodecContext::DeviceContextFinished;
199  hwdevicectx->user_opaque = interop;
200  AVMediaCodecDeviceContext *hwctx = reinterpret_cast<AVMediaCodecDeviceContext*>(hwdevicectx->hwctx);
201  hwctx->surface = interop->GetSurface();
202  if (av_hwdevice_ctx_init(hwdeviceref) < 0)
203  {
204  LOG(VB_GENERAL, LOG_ERR, LOC + "av_hwdevice_ctx_init failed");
205  av_buffer_unref(&hwdeviceref);
206  return -1;
207  }
208 
209  Context->hw_device_ctx = hwdeviceref;
210  LOG(VB_GENERAL, LOG_INFO, LOC + "Created MediaCodec hardware device context");
211  return 0;
212 }
213 
215  AVCodec **Codec,
216  const QString &Decoder,
217  AVStream *Stream,
218  uint StreamType)
219 {
220  bool decodeonly = Decoder == "mediacodec-dec";
221  MythCodecID success = static_cast<MythCodecID>((decodeonly ? kCodec_MPEG1_MEDIACODEC_DEC : kCodec_MPEG1_MEDIACODEC) + (StreamType - 1));
222  MythCodecID failure = static_cast<MythCodecID>(kCodec_MPEG1 + (StreamType - 1));
223 
224  if (!Decoder.startsWith("mediacodec"))
225  return failure;
226 
227  if (!HaveMediaCodec())
228  return failure;
229 
230  bool found = false;
232  MythCodecContext::CodecProfile mythprofile =
233  MythCodecContext::FFmpegToMythProfile((*Context)->codec_id, (*Context)->profile);
234  foreach (auto profile, profiles)
235  {
236  if (profile.first == mythprofile &&
237  profile.second.width() >= (*Context)->width &&
238  profile.second.height() >= (*Context)->height)
239  {
240  found = true;
241  break;
242  }
243  }
244 
245  QString profilestr = MythCodecContext::GetProfileDescription(mythprofile, QSize());
246  if (found)
247  {
248  QString decodername = QString((*Codec)->name) + "_mediacodec";
249  if (decodername == "mpeg2video_mediacodec")
250  decodername = "mpeg2_mediacodec";
251  AVCodec *newCodec = avcodec_find_decoder_by_name (decodername.toLocal8Bit());
252  if (newCodec)
253  {
254  *Codec = newCodec;
255  LOG(VB_PLAYBACK, LOG_INFO, LOC + QString("HW device type '%1' supports decoding '%2' (%3)")
256  .arg(av_hwdevice_get_type_name(AV_HWDEVICE_TYPE_MEDIACODEC)).arg((*Codec)->name).arg(profilestr));
257  gCodecMap->freeCodecContext(Stream);
258  *Context = gCodecMap->getCodecContext(Stream, *Codec);
259  (*Context)->pix_fmt = AV_PIX_FMT_MEDIACODEC;
260  return success;
261  }
262  }
263 
264  LOG(VB_PLAYBACK, LOG_INFO, LOC + QString("HW device type '%1' does not support decoding '%2' (%3)")
265  .arg(av_hwdevice_get_type_name(AV_HWDEVICE_TYPE_MEDIACODEC)).arg((*Codec)->name).arg(profilestr));
266  return failure;
267 }
268 
270  : MythCodecContext(Parent, CodecID)
271 {
272 }
273 
274 void MythMediaCodecContext::InitVideoCodec(AVCodecContext *Context, bool SelectedStream, bool &DirectRendering)
275 {
276  if (CODEC_IS_MEDIACODEC(Context->codec))
277  {
279  Context->slice_flags = SLICE_FLAG_CODED_ORDER | SLICE_FLAG_ALLOW_FIELD;
280  DirectRendering = false;
281  return;
282  }
283 
284  MythCodecContext::InitVideoCodec(Context, SelectedStream, DirectRendering);
285 }
286 
288 {
290  return 0;
291  else if (codec_is_mediacodec(m_codecID))
293  return -1;
294 }
295 
297 {
298  if (AvFrame->format != AV_PIX_FMT_MEDIACODEC)
299  return false;
301  return GetBuffer2(Context, Frame, AvFrame, 0);
302  return false;
303 }
304 
305 AVPixelFormat MythMediaCodecContext::GetFormat(AVCodecContext*, const AVPixelFormat *PixFmt)
306 {
307  while (*PixFmt != AV_PIX_FMT_NONE)
308  {
309  if (*PixFmt == AV_PIX_FMT_MEDIACODEC)
310  return AV_PIX_FMT_MEDIACODEC;
311  PixFmt++;
312  }
313  return AV_PIX_FMT_NONE;
314 }
315 
321 {
322  if (!Frame)
323  return;
324 
325  Frame->deinterlace_inuse = DEINT_BASIC | DEINT_DRIVER;
326  Frame->deinterlace_inuse2x = 0;
327  Frame->interlaced_frame = 0;
328  Frame->interlaced_reversed = 0;
329  Frame->top_field_first = 0;
330  Frame->deinterlace_allowed = DEINT_NONE;
331  Frame->already_deinterlaced = 1;
332 }
333 
338 bool MythMediaCodecContext::IsDeinterlacing(bool &DoubleRate, bool)
339 {
340  DoubleRate = true;
341  return true;
342 }
343 
345 {
346  // TODO Something tells me this is leakier than a leaky thing
347  static QMutex lock(QMutex::Recursive);
348  static bool s_initialised = false;
349  static MCProfiles s_profiles;
350 
351  QMutexLocker locker(&lock);
352  if (s_initialised)
353  return s_profiles;
354  s_initialised = true;
355 
356  static const QPair<QString,QPair<MythCodecContext::CodecProfile, QList<int> > > mimetypes[] =
357  {
358  { "video/mpeg2", { MythCodecContext::MPEG2,
361  { "video/mp4v-es", { MythCodecContext::MPEG4,
368  { "video/avc", { MythCodecContext::H264,
372  { "video/hevc", { MythCodecContext::HEVC,
375  { "video/x-vnd.on2.vp8", { MythCodecContext::VP8, { MC_VP8_MAIN }}},
376  { "video/x-vnd.on2.vp9", { MythCodecContext::VP9,
379  //{ "video/vc1", { MythCodecContext::VC1 , {}}}, // No FFmpeg support
380  //{ "video/3gpp", { MythCodecContext::H263 , {}}}, // No FFmpeg support
381  //{ "video/av01", { MythCodecContext::AV1 , {}}} // No FFmpeg support, API Level 29
382  };
383 
384  // Retrieve MediaCodecList
385  QAndroidJniEnvironment env;
386  QAndroidJniObject list("android/media/MediaCodecList", "(I)V", 0); // 0 = REGULAR_CODECS
387  if (!list.isValid())
388  return s_profiles;
389  // Retrieve array of MediaCodecInfo's
390  QAndroidJniObject qtcodecs = list.callObjectMethod("getCodecInfos", "()[Landroid/media/MediaCodecInfo;");
391  if (!qtcodecs.isValid())
392  return s_profiles;
393 
394  // Iterate over MediaCodecInfo's
395  jobjectArray codecs = qtcodecs.object<jobjectArray>();
396  jsize codeccount = env->GetArrayLength(codecs);
397  for (jsize i = 0; i < codeccount; ++i)
398  {
399  QAndroidJniObject codec(env->GetObjectArrayElement(codecs, i));
400  if (!codec.isValid())
401  continue;
402 
403  // Ignore encoders
404  if (codec.callMethod<jboolean>("isEncoder"))
405  continue;
406 
407  // Ignore software decoders
408  QString name = codec.callObjectMethod<jstring>("getName").toString();
409  if (name.contains("OMX.google", Qt::CaseInsensitive))
410  continue;
411 
412  // Retrieve supported mimetypes (there is usually just one)
413  QAndroidJniObject qttypes = codec.callObjectMethod("getSupportedTypes", "()[Ljava/lang/String;");
414  jobjectArray types = qttypes.object<jobjectArray>();
415  jsize typecount = env->GetArrayLength(types);
416  for (jsize j = 0; j < typecount; ++j)
417  {
418  QAndroidJniObject type(env->GetObjectArrayElement(types, j));
419  if (!type.isValid())
420  continue;
421 
422  // Match mimetype to types supported by FFmpeg
423  QString typestr = type.toString();
424  for (auto mimetype : mimetypes)
425  {
426  if (mimetype.first != typestr)
427  continue;
428 
429  // Retrieve MediaCodecInfo.CodecCapabilities for mimetype
430  QAndroidJniObject caps = codec.callObjectMethod("getCapabilitiesForType",
431  "(Ljava/lang/String;)Landroid/media/MediaCodecInfo$CodecCapabilities;",
432  type.object<jstring>());
433  if (!caps.isValid())
434  continue;
435 
436  // Retrieve MediaCodecInfo.VideoCapabilities from CodecCapabilities
437  QAndroidJniObject videocaps = caps.callObjectMethod("getVideoCapabilities",
438  "()Landroid/media/MediaCodecInfo$VideoCapabilities;");
439  QAndroidJniObject widthrange = videocaps.callObjectMethod("getSupportedWidths",
440  "()Landroid/util/Range;");
441  QAndroidJniObject heightrange = videocaps.callObjectMethod("getSupportedHeights",
442  "()Landroid/util/Range;");
443 
444  QAndroidJniObject widthqt = widthrange.callObjectMethod("getUpper", "()Ljava/lang/Comparable;");
445  QAndroidJniObject heightqt = heightrange.callObjectMethod("getUpper", "()Ljava/lang/Comparable;");
446  int width = widthqt.callMethod<jint>("intValue", "()I");
447  int height = heightqt.callMethod<jint>("intValue", "()I");
448 
449  // Profiles are available from CodecCapabilities.profileLevel field
450  QAndroidJniObject profiles = caps.getObjectField("profileLevels",
451  "[Landroid/media/MediaCodecInfo$CodecProfileLevel;");
452  jobjectArray profilearr = profiles.object<jobjectArray>();
453  jsize profilecount = env->GetArrayLength(profilearr);
454  if (profilecount < 1)
455  {
456  s_profiles.append(QPair<MythCodecContext::CodecProfile,QSize>(mimetype.second.first, QSize(width, height)));
457  continue;
458  }
459 
460  for (jsize k = 0; k < profilecount; ++k)
461  {
462  jobject profile = env->GetObjectArrayElement(profilearr, k);
463  jclass objclass = env->GetObjectClass(profile);
464  jfieldID id = env->GetFieldID(objclass, "profile", "I");
465  int value = static_cast<int>(env->GetIntField(profile, id));
466  QList<int>& mcprofiles = mimetype.second.second;
467  bool found = false;
468  foreach (auto mcprofile, mcprofiles)
469  {
470  if (value == mcprofile)
471  {
472  found = true;
473  MythCodecContext::CodecProfile p = MediaCodecToMythProfile(mimetype.second.first, value);
474  s_profiles.append(QPair<MythCodecContext::CodecProfile,QSize>(p, QSize(width, height)));
475  break;
476  }
477  }
478 
479  if (!found)
480  s_profiles.append(QPair<MythCodecContext::CodecProfile,QSize>(mimetype.second.first, QSize(width, height)));
481  }
482  }
483  }
484  }
485 
486  return s_profiles;
487 }
488 
489 void MythMediaCodecContext::GetDecoderList(QStringList &Decoders)
490 {
492  if (profiles.isEmpty())
493  return;
494 
495  Decoders.append("MediaCodec:");
496  foreach (auto profile, profiles)
497  Decoders.append(MythCodecContext::GetProfileDescription(profile.first, profile.second));
498 }
499 
501 {
502  static QMutex lock(QMutex::Recursive);
503  static bool s_initialised = false;
504  static bool s_available = false;
505 
506  QMutexLocker locker(&lock);
507  if (!s_initialised)
508  {
510  if (profiles.isEmpty())
511  {
512  LOG(VB_GENERAL, LOG_INFO, LOC + "No MediaCodec decoders found");
513  }
514  else
515  {
516  s_available = true;
517  LOG(VB_GENERAL, LOG_INFO, LOC + "Supported/available MediaCodec decoders:");
518  foreach (auto profile, profiles)
519  {
520  LOG(VB_GENERAL, LOG_INFO, LOC +
522  }
523  }
524  }
525  s_initialised = true;
526  return s_available;
527 }
528 
#define MC_VP9_1
MythCodecContext::CodecProfile MediaCodecToMythProfile(int Codec, int Profile)
#define MC_H264_CONST_HIGH
static AVPixelFormat GetFormat(AVCodecContext *, const AVPixelFormat *PixFmt)
int HwDecoderInit(AVCodecContext *Context) override
#define MC_VP9_0
#define MC_H264_EXTENDED
static int InitialiseDecoder(AVCodecContext *Context)
#define MC_HEVC_MAIN10
void SetPlayer(MythPlayer *Player)
static MCProfiles & GetProfiles(void)
QHash< QString, Action * > Context
Definition: action.h:77
MythMediaCodecContext(DecoderBase *Parent, MythCodecID CodecID)
QString toString(MarkTypes type)
#define MC_MPEG4_SCALEABLE_TEX
MythCodecID
Definition: mythcodecid.h:10
#define MC_VP9_3HDR
#define MC_MPEG2_HIGH
#define MC_MPEG4_SIMPLE_SCALEABLE
#define MC_H264_MAIN
#define MC_VP9_2HDR
void PostProcessFrame(AVCodecContext *, VideoFrame *) override
Mark all MediaCodec decoded frames as progressive,.
static QString GetProfileDescription(CodecProfile Profile, QSize Size, VideoFrameType Format=FMT_NONE, uint ColorDepth=0)
struct AVFrame AVFrame
#define MC_VP8_MAIN
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
#define MC_MPEG2_MAIN
#define MC_HEVC_MMAIN10HDR10PLUS
#define MC_MPEG2_SIMPLE
void freeCodecContext(const AVStream *stream)
Definition: mythavutil.cpp:557
#define MC_H264_HIGH10
MythCodecID m_codecID
#define MC_MPEG4_ADV_REALTIME
#define MC_MPEG4_BASIC_ANIMATED
#define CODEC_IS_MEDIACODEC(codec)
Definition: mythcodecid.h:382
#define MC_MPEG4_SIMPLE_FACE
#define LOC
#define MC_MPEG4_ADV_CORE
static int InitialiseDecoder2(AVCodecContext *Context, CreateHWDecoder Callback, const QString &Debug)
Initialise a hardware decoder that is NOT expected to use AVHWFramesContext.
static MythCodecID GetBestSupportedCodec(AVCodecContext **Context, AVCodec **Codec, const QString &Decoder, AVStream *Stream, uint StreamType)
MythCodecMap * gCodecMap
This global variable contains the MythCodecMap instance for the app.
Definition: mythavutil.cpp:508
#define MC_VP9_2
AVCodecContext * getCodecContext(const AVStream *stream, const AVCodec *pCodec=nullptr, bool nullCodec=false)
Definition: mythavutil.cpp:515
static void GetDecoderList(QStringList &Decoders)
virtual void InitVideoCodec(AVCodecContext *Context, bool SelectedStream, bool &DirectRendering)
static bool HaveMediaCodec(void)
virtual int DecrRef(void)
Decrements reference count and deletes on 0.
#define MC_HEVC_MAIN10HDR10
#define MC_MPEG4_SIMPLE_FBA
#define MC_MPEG4_HYBRID
unsigned int uint
Definition: compat.h:140
#define MC_MPEG2_422
#define codec_is_mediacodec(id)
Definition: mythcodecid.h:323
static CodecProfile FFmpegToMythProfile(AVCodecID CodecID, int Profile)
bool RetrieveFrame(AVCodecContext *Context, VideoFrame *Frame, AVFrame *AvFrame) override
#define MC_MPEG2_SPATIAL
#define MC_MPEG2_SNR
#define MC_H264_HIGH444
#define MC_MPEG4_ADV_CODING
#define LOG(_MASK_, _LEVEL_, _STRING_)
Definition: mythlogging.h:41
bool IsDeinterlacing(bool &DoubleRate, bool=false) override
#define MC_VP9_3HDRPLUS
#define MC_MPEG4_ADV_SIMPLE
#define MC_HEVC_MAIN
static bool GetBuffer2(struct AVCodecContext *Context, VideoFrame *Frame, AVFrame *AvFrame, int Flags)
A generic hardware buffer initialisation method when AVHWFramesContext is NOT used.
#define MC_HEVC_MAIN_STILL
#define MC_MPEG4_NBIT
static void DeviceContextFinished(AVHWDeviceContext *Context)
static MythRenderOpenGL * GetOpenGLRender(void)
#define MC_MPEG4_ADV_SCALEABLE
#define MC_MPEG4_MAIN
QList< QPair< MythCodecContext::CodecProfile, QSize > > MCProfiles
#define MC_H264_HIGH
#define MC_MPEG4_CORE_SCALEABLE
#define MC_MPEG4_CORE
#define MC_H264_BASELINE
#define MC_H264_CONST_BASELINE
#define MC_H264_HIGH422
#define MC_MPEG4_SIMPLE
static MythMediaCodecInterop * Create(MythRenderOpenGL *Context, QSize Size)
void InitVideoCodec(AVCodecContext *Context, bool SelectedStream, bool &DirectRendering) override
#define codec_is_mediacodec_dec(id)
Definition: mythcodecid.h:325
#define MC_VP9_3
#define MC_VP9_2HDRPLUS