MythTV  master
mythvtbinterop.cpp
Go to the documentation of this file.
1 // MythTV
2 #include "mythmainwindow.h"
3 #include "mythcorecontext.h"
4 #include "videocolourspace.h"
5 #include "mythvtbinterop.h"
6 
7 #define LOC QString("VTBInterop: ")
8 
10 {
11  if ((FMT_VTB != Format) || !gCoreContext->IsUIThread())
12  return Unsupported;
13 
15  if (!context)
16  return Unsupported;
17 
18  if (context->isOpenGLES() || context->IsEGL())
19  return Unsupported;
20 
21  if (context->hasExtension("GL_ARB_texture_rg"))
22  return VTBSURFACE;
23  return VTBOPENGL;
24 }
25 
27 {
28  if (Context)
29  {
30  if (Type == VTBSURFACE)
31  return new MythVTBSurfaceInterop(Context);
32  else if (Type == VTBOPENGL)
33  return new MythVTBInterop(Context, VTBOPENGL);
34  }
35  return nullptr;
36 }
37 
40 {
41 }
42 
44 {
45 }
46 
48 {
49  if (!Frame)
50  return nullptr;
51 
52  if (Context && (Context != m_context))
53  LOG(VB_GENERAL, LOG_WARNING, LOC + "Mismatched OpenGL contexts");
54 
55  // Check size
56  QSize surfacesize(Frame->width, Frame->height);
57  if (m_openglTextureSize != surfacesize)
58  {
59  if (!m_openglTextureSize.isEmpty())
60  {
61  LOG(VB_GENERAL, LOG_WARNING, LOC + QString("Video texture size changed! %1x%2->%3x%4")
62  .arg(m_openglTextureSize.width()).arg(m_openglTextureSize.height())
63  .arg(Frame->width).arg(Frame->height));
64  }
66  m_openglTextureSize = surfacesize;
67  }
68 
69  // Update colourspace and initialise on first frame
70  if (ColourSpace)
71  {
72  if (m_openglTextures.isEmpty())
74  ColourSpace->UpdateColourSpace(Frame);
75  }
76 
77  // Retrieve pixel buffer
78  // N.B. Buffer was retained in MythVTBContext and will be released in VideoBuffers
79  return reinterpret_cast<CVPixelBufferRef>(Frame->buf);
80 }
81 
83  VideoColourSpace *ColourSpace,
86 {
87  vector<MythVideoTexture*> result;
88  OpenGLLocker locker(m_context);
89 
90  CVPixelBufferRef buffer = Verify(Context, ColourSpace, Frame);
91  if (!buffer)
92  return result;
93 
94  // There are only ever one set of textures which are updated on each pass.
95  // Hence we cannot use reference frames without significant additional work here -
96  // but MythVTBSurfaceInterop will almost certainly always be used - so just drop back
97  // to Linearblend
98  Frame->deinterlace_allowed = (Frame->deinterlace_allowed & ~DEINT_HIGH) | DEINT_MEDIUM;
99 
100  int planes = CVPixelBufferGetPlaneCount(buffer);
101  CVPixelBufferLockBaseAddress(buffer, kCVPixelBufferLock_ReadOnly);
102 
103  // FIXME? There is no P010 support here as I cannot test it - but SurfaceInterop
104  // should be used at all times - so this code is not going to be hit.
105  // Note - and the reason this code is retained is because it may at some point
106  // be useful for an iOS port (iOS has no surace interop)
107 
108  // Create necessary textures
109  if (m_openglTextures.isEmpty())
110  {
111  for (int plane = 0; plane < planes; ++plane)
112  {
113  int width = CVPixelBufferGetWidthOfPlane(buffer, plane);
114  int height = CVPixelBufferGetHeightOfPlane(buffer, plane);
115  QSize size(width, height);
117  if (texture)
118  {
119  texture->m_frameType = FMT_VTB;
120  texture->m_frameFormat = FMT_NV12;
121  texture->m_plane = static_cast<uint>(plane);
122  texture->m_planeCount = static_cast<uint>(planes);
123  texture->m_allowGLSLDeint = true;
124  result.push_back(texture);
125  }
126  }
127 
128  if (result.size() != static_cast<uint>(planes))
129  LOG(VB_GENERAL, LOG_ERR, LOC + "Failed to create all textures");
130  m_openglTextures.insert(DUMMY_INTEROP_ID, result);
131  }
132 
133  if (m_openglTextures.contains(DUMMY_INTEROP_ID))
134  {
136  m_context->glEnable(QOpenGLTexture::Target2D);
137 
138  // Update textures
139  for (int plane = 0; plane < planes; ++plane)
140  {
141  if (static_cast<uint>(plane) >= result.size())
142  continue;
143 
144  int width = CVPixelBufferGetWidthOfPlane(buffer, plane);
145  int height = CVPixelBufferGetHeightOfPlane(buffer, plane);
146  int bytes = CVPixelBufferGetBytesPerRowOfPlane(buffer, plane) / width;
147  void* buf = CVPixelBufferGetBaseAddressOfPlane(buffer, plane);
148  result[plane]->m_texture->bind();
149  if ((bytes == 1) && buf)
150  {
151  m_context->glTexImage2D(QOpenGLTexture::Target2D, 0, QOpenGLTexture::RGBA, width, height,
152  0, QOpenGLTexture::Red, QOpenGLTexture::UInt8, buf);
153  }
154  else if ((bytes == 2) && buf)
155  {
156  m_context->glTexImage2D(QOpenGLTexture::Target2D, 0, QOpenGLTexture::RGBA, width, height,
157  0, QOpenGLTexture::RG, QOpenGLTexture::UInt8, buf);
158  }
159  result[plane]->m_texture->release();
160  }
161  }
162  CVPixelBufferUnlockBaseAddress(buffer, kCVPixelBufferLock_ReadOnly);
163  return result;
164 }
165 
167  : MythVTBInterop(Context, VTBSURFACE)
168 {
169 }
170 
172 {
173 }
174 
176  VideoColourSpace *ColourSpace,
177  VideoFrame *Frame,
178  FrameScanType Scan)
179 {
180  vector<MythVideoTexture*> result;
181  OpenGLLocker locker(m_context);
182 
183  CVPixelBufferRef buffer = Verify(Context, ColourSpace, Frame);
184  if (!buffer)
185  return result;
186 
187  IOSurfaceRef surface = CVPixelBufferGetIOSurface(buffer);
188  if (!surface)
189  {
190  LOG(VB_GENERAL, LOG_ERR, LOC + "Failed to retrieve IOSurface from CV buffer");
191  return result;
192  }
193 
194  IOSurfaceID surfaceid = IOSurfaceGetID(surface);
195  if (!surfaceid)
196  return result;
197 
198  // check whether we need to return multiple frames for GLSL kernel deinterlacing
199  bool needreferences = false;
200  if (is_interlaced(Scan))
201  {
203  if (shader)
204  needreferences = shader == DEINT_HIGH;
205  else
206  needreferences = GetSingleRateOption(Frame, DEINT_SHADER) == DEINT_HIGH;
207  }
208 
209  if (needreferences)
210  {
211  if (abs(Frame->frameCounter - m_discontinuityCounter) > 1)
212  m_referenceFrames.clear();
213  RotateReferenceFrames(surfaceid);
214  }
215  else
216  {
217  m_referenceFrames.clear();
218  }
219  m_discontinuityCounter = Frame->frameCounter;
220 
221  // return cached textures if available
222  if (m_openglTextures.contains(surfaceid))
223  {
224  if (needreferences)
225  return GetReferenceFrames();
226  return m_openglTextures[surfaceid];
227  }
228 
229  // Create new and attach to IOSurface
230  int planes = IOSurfaceGetPlaneCount(surface);
231  IOSurfaceLock(surface, kIOSurfaceLockReadOnly, nullptr);
232 
233  // Create raw rectangular textures
234  vector<QSize> sizes;
235  for (int plane = 0; plane < planes; ++plane)
236  {
237  int width = IOSurfaceGetWidthOfPlane(surface, plane);
238  int height = IOSurfaceGetHeightOfPlane(surface, plane);
239  sizes.push_back(QSize(width, height));
240  }
241  // NB P010 support is untested
242  // NB P010 support was added to FFmpeg in https://github.com/FFmpeg/FFmpeg/commit/036b4b0f85933f49a709
243  // which has not yet been merged into the MythTV FFmpeg version (as of 22/6/19)
244  VideoFrameType frameformat = PixelFormatToFrameType(static_cast<AVPixelFormat>(Frame->sw_pix_fmt));
245  if ((frameformat != FMT_NV12) && (frameformat != FMT_P010))
246  {
247  IOSurfaceUnlock(surface, kIOSurfaceLockReadOnly, nullptr);
248  LOG(VB_GENERAL, LOG_ERR, LOC + QString("Unsupported frame format %1")
249  .arg(format_description(frameformat)));
250  return result;
251  }
252 
253  result = MythVideoTexture::CreateTextures(m_context, FMT_VTB, frameformat, sizes, QOpenGLTexture::TargetRectangle);
254 
255  for (uint plane = 0; plane < result.size(); ++plane)
256  {
257  MythVideoTexture* texture = result[plane];
258  if (!texture)
259  continue;
260  texture->m_allowGLSLDeint = true;
261  m_context->glBindTexture(texture->m_target, texture->m_textureId);
262 
263  GLenum format = (plane == 0) ? QOpenGLTexture::Red : QOpenGLTexture::RG;;
264  GLenum dataformat = (frameformat == FMT_NV12) ? QOpenGLTexture::UInt8 : QOpenGLTexture::UInt16;
265 
266  CGLError error = CGLTexImageIOSurface2D(
267  CGLGetCurrentContext(), QOpenGLTexture::TargetRectangle, format,
268  texture->m_size.width(), texture->m_size.height(),
269  format, dataformat, surface, plane);
270  if (error != kCGLNoError)
271  LOG(VB_GENERAL, LOG_ERR, LOC + QString("CGLTexImageIOSurface2D error %1").arg(error));
272  m_context->glBindTexture(QOpenGLTexture::TargetRectangle, 0);
273  }
274 
275  IOSurfaceUnlock(surface, kIOSurfaceLockReadOnly, nullptr);
276  m_openglTextures.insert(surfaceid, result);
277 
278  if (needreferences)
279  return GetReferenceFrames();
280  return result;
281 }
282 
284 {
285  if (!Buffer)
286  return;
287 
288  // don't retain twice for double rate
289  if ((m_referenceFrames.size() > 0) && (m_referenceFrames[0] == Buffer))
290  return;
291 
292  m_referenceFrames.push_front(Buffer);
293 
294  // release old frames
295  while (m_referenceFrames.size() > 3)
296  m_referenceFrames.pop_back();
297 }
298 
299 vector<MythVideoTexture*> MythVTBSurfaceInterop::GetReferenceFrames(void)
300 {
301  vector<MythVideoTexture*> result;
302  int size = m_referenceFrames.size();
303  if (size < 1)
304  return result;
305 
306  IOSurfaceID next = m_referenceFrames[0];
307  IOSurfaceID current = m_referenceFrames[size > 1 ? 1 : 0];
308  IOSurfaceID last = m_referenceFrames[size > 2 ? 2 : 0];
309 
310  if (!m_openglTextures.contains(next) || !m_openglTextures.contains(current) ||
311  !m_openglTextures.contains(last))
312  {
313  LOG(VB_GENERAL, LOG_ERR, LOC + "Reference frame error");
314  return result;
315  }
316 
317  result = m_openglTextures[last];
318  for (MythVideoTexture* tex : qAsConst(m_openglTextures[current]))
319  result.push_back(tex);
320  for (MythVideoTexture* tex : qAsConst(m_openglTextures[next]))
321  result.push_back(tex);
322  return result;
323 }
GetSingleRateOption
MythDeintType GetSingleRateOption(const VideoFrame *Frame, MythDeintType Type, MythDeintType Override)
Definition: mythframe.cpp:834
MythVTBSurfaceInterop::m_referenceFrames
QVector< IOSurfaceID > m_referenceFrames
Definition: mythvtbinterop.h:44
MythVTBInterop::Verify
CVPixelBufferRef Verify(MythRenderOpenGL *Context, VideoColourSpace *ColourSpace, VideoFrame *Frame)
Definition: mythvtbinterop.cpp:47
DEINT_MEDIUM
@ DEINT_MEDIUM
Definition: mythframe.h:125
error
static void error(const char *str,...)
Definition: vbi.cpp:42
MythVTBInterop::GetInteropType
static Type GetInteropType(VideoFrameType Format)
Definition: mythvtbinterop.cpp:9
MythVideoTexture::m_frameType
VideoFrameType m_frameType
Definition: mythvideotexture.h:49
FMT_VTB
@ FMT_VTB
Definition: mythframe.h:66
DEINT_SHADER
@ DEINT_SHADER
Definition: mythframe.h:128
MythVTBSurfaceInterop::MythVTBSurfaceInterop
MythVTBSurfaceInterop(MythRenderOpenGL *Context)
Definition: mythvtbinterop.cpp:166
Frame
Definition: zmdefines.h:93
FMT_P010
@ FMT_P010
Definition: mythframe.h:58
MythVideoTexture::CreateTextures
static vector< MythVideoTexture * > CreateTextures(MythRenderOpenGL *Context, VideoFrameType Type, VideoFrameType Format, vector< QSize > Sizes, GLenum Target=QOpenGLTexture::Target2D)
Create a set of textures suitable for the given Type and Format.
Definition: mythvideotexture.cpp:57
arg
arg(title).arg(filename).arg(doDelete))
PixelFormatToFrameType
VideoFrameType PixelFormatToFrameType(AVPixelFormat fmt)
Definition: mythavutil.cpp:69
FrameScanType
FrameScanType
Definition: videoouttypes.h:78
GetDoubleRateOption
MythDeintType GetDoubleRateOption(const VideoFrame *Frame, MythDeintType Type, MythDeintType Override)
Definition: mythframe.cpp:847
MythCoreContext::IsUIThread
bool IsUIThread(void)
Definition: mythcorecontext.cpp:1329
Context
QHash< QString, Action * > Context
Definition: action.h:77
MythDate::Format
Format
Definition: mythdate.h:12
LOG
#define LOG(_MASK_, _LEVEL_, _QSTRING_)
Definition: mythlogging.h:23
VideoFrame
Definition: mythframe.h:137
DUMMY_INTEROP_ID
#define DUMMY_INTEROP_ID
Definition: mythopenglinterop.h:21
MythDate::current
QDateTime current(bool stripped)
Returns current Date and Time in UTC.
Definition: mythdate.cpp:10
LOC
#define LOC
Definition: mythvtbinterop.cpp:7
MythOpenGLInterop::m_discontinuityCounter
long long m_discontinuityCounter
Definition: mythopenglinterop.h:77
MythVTBInterop::~MythVTBInterop
~MythVTBInterop() override
Definition: mythvtbinterop.cpp:43
MythOpenGLInterop
Definition: mythopenglinterop.h:23
MythVTBInterop::MythVTBInterop
MythVTBInterop(MythRenderOpenGL *Context, MythOpenGLInterop::Type Type)
Definition: mythvtbinterop.cpp:38
MythOpenGLInterop::Unsupported
@ Unsupported
Definition: mythopenglinterop.h:31
MythVTBSurfaceInterop::Acquire
vector< MythVideoTexture * > Acquire(MythRenderOpenGL *Context, VideoColourSpace *ColourSpace, VideoFrame *Frame, FrameScanType Scan) override
Definition: mythvtbinterop.cpp:175
VideoColourSpace::SetSupportedAttributes
void SetSupportedAttributes(PictureAttributeSupported Supported)
Enable the given set of picture attributes.
Definition: videocolourspace.cpp:140
MythEGL::IsEGL
bool IsEGL(void)
Definition: mythegl.cpp:29
mythvtbinterop.h
uint
unsigned int uint
Definition: compat.h:140
MythOpenGLInterop::m_context
MythRenderOpenGL * m_context
Definition: mythopenglinterop.h:73
gCoreContext
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
Definition: mythcorecontext.cpp:57
MythVideoTexture::CreateTexture
static MythVideoTexture * CreateTexture(MythRenderOpenGL *Context, QSize Size, GLenum Target=QOpenGLTexture::Target2D, QOpenGLTexture::PixelType PixelType=QOpenGLTexture::UInt8, QOpenGLTexture::PixelFormat PixelFormat=QOpenGLTexture::RGBA, QOpenGLTexture::TextureFormat Format=QOpenGLTexture::NoFormat, QOpenGLTexture::Filter Filter=QOpenGLTexture::Linear, QOpenGLTexture::WrapMode Wrap=QOpenGLTexture::ClampToEdge)
Create and initialise a MythVideoTexture that is backed by a QOpenGLTexture.
Definition: mythvideotexture.cpp:431
videocolourspace.h
MythOpenGLInterop::Type
Type
Definition: mythopenglinterop.h:29
ALL_PICTURE_ATTRIBUTES
#define ALL_PICTURE_ATTRIBUTES
Definition: videoouttypes.h:111
MythOpenGLInterop::VTBOPENGL
@ VTBOPENGL
Definition: mythopenglinterop.h:35
MythVTBSurfaceInterop::~MythVTBSurfaceInterop
~MythVTBSurfaceInterop() override
Definition: mythvtbinterop.cpp:171
MythGLTexture::m_target
GLenum m_target
Definition: mythrenderopengl.h:65
MythOpenGLInterop::VTBSURFACE
@ VTBSURFACE
Definition: mythopenglinterop.h:36
MythRenderOpenGL
Definition: mythrenderopengl.h:86
MythVTBSurfaceInterop::GetReferenceFrames
vector< MythVideoTexture * > GetReferenceFrames(void)
Definition: mythvtbinterop.cpp:299
VideoColourSpace
VideoColourSpace contains a QMatrix4x4 that can convert YCbCr data to RGB.
Definition: videocolourspace.h:17
MythDeintType
MythDeintType
Definition: mythframe.h:121
next
PictureAttribute next(PictureAttributeSupported Supported, PictureAttribute Attribute)
Definition: videoouttypes.h:350
Buffer
Definition: MythExternControl.h:36
MythVideoTexture::m_frameFormat
VideoFrameType m_frameFormat
Definition: mythvideotexture.h:50
musicbrainzngs.compat.bytes
bytes
Definition: compat.py:49
planes
static uint planes(VideoFrameType Type)
Definition: mythframe.h:570
MythGLTexture::m_textureId
GLuint m_textureId
Definition: mythrenderopengl.h:53
mythcorecontext.h
VideoColourSpace::UpdateColourSpace
bool UpdateColourSpace(const VideoFrame *Frame)
Set the current colourspace to use.
Definition: videocolourspace.cpp:332
DEINT_HIGH
@ DEINT_HIGH
Definition: mythframe.h:126
MythGLTexture::m_size
QSize m_size
Definition: mythrenderopengl.h:58
MythRenderOpenGL::GetOpenGLRender
static MythRenderOpenGL * GetOpenGLRender(void)
Definition: mythrenderopengl.cpp:63
MythVTBSurfaceInterop
Definition: mythvtbinterop.h:29
MythVTBSurfaceInterop::RotateReferenceFrames
void RotateReferenceFrames(IOSurfaceID Buffer)
Definition: mythvtbinterop.cpp:283
MythVideoTexture::m_plane
uint m_plane
Definition: mythvideotexture.h:51
MythVTBInterop::Create
static MythVTBInterop * Create(MythRenderOpenGL *Context, MythOpenGLInterop::Type Type)
Definition: mythvtbinterop.cpp:26
MythVTBInterop::Acquire
vector< MythVideoTexture * > Acquire(MythRenderOpenGL *Context, VideoColourSpace *ColourSpace, VideoFrame *Frame, FrameScanType Scan) override
Definition: mythvtbinterop.cpp:82
VideoFrameType
VideoFrameType
Definition: mythframe.h:24
MythVideoTexture::m_planeCount
uint m_planeCount
Definition: mythvideotexture.h:52
MythOpenGLInterop::DeleteTextures
virtual void DeleteTextures(void)
Definition: mythopenglinterop.cpp:246
FMT_NV12
@ FMT_NV12
Definition: mythframe.h:57
MythOpenGLInterop::m_openglTextures
QHash< unsigned long long, vector< MythVideoTexture * > > m_openglTextures
Definition: mythopenglinterop.h:75
MythVideoTexture
Definition: mythvideotexture.h:20
format_description
const char * format_description(VideoFrameType Type)
Definition: mythframe.cpp:33
mythmainwindow.h
is_interlaced
bool is_interlaced(FrameScanType Scan)
Definition: videoouttypes.h:174
MythVTBInterop
Definition: mythvtbinterop.h:10
MythOpenGLInterop::m_openglTextureSize
QSize m_openglTextureSize
Definition: mythopenglinterop.h:76
MythVideoTexture::m_allowGLSLDeint
bool m_allowGLSLDeint
Definition: mythvideotexture.h:53
OpenGLLocker
Definition: mythrenderopengl.h:244