MythTV  master
mythnvdecinterop.cpp
Go to the documentation of this file.
1 // MythTV
2 #include "mythconfig.h"
3 #include "mythcorecontext.h"
4 #include "mythvideocolourspace.h"
6 
7 // Std
8 #include <chrono>
9 #include <thread>
10 
11 #define LOC QString("NVDECInterop: ")
12 
13 #define CUDA_CHECK(CUDA_FUNCS, CUDA_CALL) \
14 { \
15  CUresult res = (CUDA_FUNCS)->CUDA_CALL; \
16  if (res != CUDA_SUCCESS) { \
17  const char * desc; \
18  (CUDA_FUNCS)->cuGetErrorString(res, &desc); \
19  LOG(VB_GENERAL, LOG_ERR, LOC + QString("CUDA error %1 (%2)").arg(res).arg(desc)); \
20  } \
21 }
22 
24  : MythOpenGLInterop(Context, GL_NVDEC, Player),
25  m_cudaContext()
26 {
28 }
29 
31 {
32  m_referenceFrames.clear();
35 }
36 
38 {
39  if (!(m_cudaContext && m_cudaFuncs))
40  return;
41 
43  CUDA_CHECK(m_cudaFuncs, cuCtxPushCurrent(m_cudaContext))
44 
45  if (!m_openglTextures.isEmpty())
46  {
47  LOG(VB_PLAYBACK, LOG_INFO, LOC + "Deleting CUDA resources");
48  for (auto it = m_openglTextures.constBegin(); it != m_openglTextures.constEnd(); ++it)
49  {
50  std::vector<MythVideoTextureOpenGL*> textures = it.value();
51  for (auto & texture : textures)
52  {
53  auto *data = reinterpret_cast<QPair<CUarray,CUgraphicsResource>*>(texture->m_data);
54  if (data && data->second)
55  CUDA_CHECK(m_cudaFuncs, cuGraphicsUnregisterResource(data->second))
56  delete data;
57  texture->m_data = nullptr;
58  }
59  }
60  }
61 
62  CUcontext dummy = nullptr;
63  CUDA_CHECK(m_cudaFuncs, cuCtxPopCurrent(&dummy))
64 
66 }
67 
69 {
70  return m_cudaFuncs && m_cudaContext;
71 }
72 
74 {
75  return m_cudaContext;
76 }
77 
79 {
80  if (!(Context && Player))
81  return nullptr;
82 
84  GetNVDECTypes(Context, types);
85  if (auto nvdec = types.find(FMT_NVDEC); nvdec != types.end())
86  {
87  auto matchType = [](auto type){ return (type == GL_NVDEC); };
88  if (std::any_of(nvdec->second.cbegin(), nvdec->second.cend(), matchType))
89  return new MythNVDECInterop(Player, Context);
90  }
91  return nullptr;
92 }
93 
95 {
96  if (Render)
97  Types[FMT_NVDEC] = { GL_NVDEC };
98 }
99 
107 std::vector<MythVideoTextureOpenGL*>
109  MythVideoColourSpace* ColourSpace,
111  FrameScanType Scan)
112 {
113  std::vector<MythVideoTextureOpenGL*> result;
114  if (!Frame || !m_cudaContext || !m_cudaFuncs)
115  return result;
116 
117  if (Context && (Context != m_openglContext))
118  LOG(VB_GENERAL, LOG_WARNING, LOC + "Mismatched OpenGL contexts");
119 
120  // Check size
121  QSize surfacesize(Frame->m_width, Frame->m_height);
122  if (m_textureSize != surfacesize)
123  {
124  if (!m_textureSize.isEmpty())
125  {
126  LOG(VB_GENERAL, LOG_WARNING, LOC + QString("Video texture size changed! %1x%2->%3x%4")
127  .arg(m_textureSize.width()).arg(m_textureSize.height())
128  .arg(Frame->m_width).arg(Frame->m_height));
129  }
130  DeleteTextures();
131  m_textureSize = surfacesize;
132  }
133 
134  // Lock
136 
137  // Update colourspace and initialise on first frame
138  if (ColourSpace)
139  {
140  if (m_openglTextures.isEmpty())
142  ColourSpace->UpdateColourSpace(Frame);
143  }
144 
145  // Retrieve hardware frames context and AVCUDADeviceContext
146  if ((Frame->m_pixFmt != AV_PIX_FMT_CUDA) || (Frame->m_type != FMT_NVDEC) ||
147  !Frame->m_buffer || !Frame->m_priv[0] || !Frame->m_priv[1])
148  {
149  return result;
150  }
151 
152  auto cudabuffer = reinterpret_cast<CUdeviceptr>(Frame->m_buffer);
153  if (!cudabuffer)
154  return result;
155 
156  // make the CUDA context current
157  CUcontext dummy = nullptr;
158  CUDA_CHECK(m_cudaFuncs, cuCtxPushCurrent(m_cudaContext))
159 
160  // create and map textures for a new buffer
161  VideoFrameType type = (Frame->m_swPixFmt == AV_PIX_FMT_NONE) ? FMT_NV12 :
162  MythAVUtil::PixelFormatToFrameType(static_cast<AVPixelFormat>(Frame->m_swPixFmt));
163  bool p010 = MythVideoFrame::ColorDepth(type) > 8;
164  if (!m_openglTextures.contains(cudabuffer))
165  {
166  std::vector<QSize> sizes;
167  sizes.emplace_back(QSize(Frame->m_width, Frame->m_height));
168  sizes.emplace_back(QSize(Frame->m_width, Frame->m_height >> 1));
169  std::vector<MythVideoTextureOpenGL*> textures =
171  if (textures.empty())
172  {
173  CUDA_CHECK(m_cudaFuncs, cuCtxPopCurrent(&dummy))
174  return result;
175  }
176 
177  bool success = true;
178  for (uint plane = 0; plane < textures.size(); ++plane)
179  {
180  // N.B. I think the texture formats for P010 are not strictly compliant
181  // with OpenGL ES 3.X but the Nvidia driver does not complain.
182  MythVideoTextureOpenGL *tex = textures[plane];
183  tex->m_allowGLSLDeint = true;
184  m_openglContext->glBindTexture(tex->m_target, tex->m_textureId);
185  QOpenGLTexture::PixelFormat format = QOpenGLTexture::Red;
186  QOpenGLTexture::PixelType pixtype = p010 ? QOpenGLTexture::UInt16 : QOpenGLTexture::UInt8;
187  QOpenGLTexture::TextureFormat internal = p010 ? QOpenGLTexture::R16_UNorm : QOpenGLTexture::R8_UNorm;
188  int width = tex->m_size.width();
189 
190  if (plane)
191  {
192  internal = p010 ? QOpenGLTexture::RG16_UNorm : QOpenGLTexture::RG8_UNorm;
193  format = QOpenGLTexture::RG;
194  width /= 2;
195  }
196 
197  m_openglContext->glTexImage2D(tex->m_target, 0, internal, width, tex->m_size.height(),
198  0, format, pixtype, nullptr);
199 
200  CUarray array = nullptr;
201  CUgraphicsResource graphicsResource = nullptr;
202  CUDA_CHECK(m_cudaFuncs, cuGraphicsGLRegisterImage(&graphicsResource, tex->m_textureId,
203  QOpenGLTexture::Target2D, CU_GRAPHICS_REGISTER_FLAGS_WRITE_DISCARD))
204  if (graphicsResource)
205  {
206  CUDA_CHECK(m_cudaFuncs, cuGraphicsMapResources(1, &graphicsResource, nullptr))
207  CUDA_CHECK(m_cudaFuncs, cuGraphicsSubResourceGetMappedArray(&array, graphicsResource, 0, 0))
208  CUDA_CHECK(m_cudaFuncs, cuGraphicsUnmapResources(1, &graphicsResource, nullptr))
209  tex->m_data = reinterpret_cast<unsigned char*>(new QPair<CUarray,CUgraphicsResource>(array, graphicsResource));
210  }
211  else
212  {
213  success = false;
214  break;
215  }
216  }
217 
218  if (success)
219  {
220  m_openglTextures.insert(cudabuffer, textures);
221  }
222  else
223  {
224  for (auto & texture : textures)
225  {
226  auto *data = reinterpret_cast<QPair<CUarray,CUgraphicsResource>*>(texture->m_data);
227  if (data && data->second)
228  CUDA_CHECK(m_cudaFuncs, cuGraphicsUnregisterResource(data->second))
229  delete data;
230  texture->m_data = nullptr;
231  if (texture->m_textureId)
232  m_openglContext->glDeleteTextures(1, &texture->m_textureId);
234  }
235  }
236  }
237 
238  if (!m_openglTextures.contains(cudabuffer))
239  {
240  CUDA_CHECK(m_cudaFuncs, cuCtxPopCurrent(&dummy))
241  return result;
242  }
243 
244  // Copy device data to array data (i.e. texture) - surely this can be avoided?
245  // In theory, asynchronous copies should not be required but we use async
246  // followed by stream synchronisation to ensure CUDA and OpenGL are in sync
247  // which avoids presenting old/stale frames when the GPU is under load.
248  result = m_openglTextures[cudabuffer];
249  for (uint i = 0; i < result.size(); ++i)
250  {
251  auto *data = reinterpret_cast<QPair<CUarray,CUgraphicsResource>*>(result[i]->m_data);
252  CUDA_MEMCPY2D cpy;
253  memset(&cpy, 0, sizeof(cpy));
254  cpy.srcMemoryType = CU_MEMORYTYPE_DEVICE;
255  cpy.srcDevice = cudabuffer + static_cast<CUdeviceptr>(Frame->m_offsets[i]);
256  cpy.srcPitch = static_cast<size_t>(Frame->m_pitches[i]);
257  cpy.dstMemoryType = CU_MEMORYTYPE_ARRAY;
258  cpy.dstArray = data->first;
259  cpy.WidthInBytes = static_cast<size_t>(result[i]->m_size.width()) * (p010 ? 2 : 1);
260  cpy.Height = static_cast<size_t>(result[i]->m_size.height());
261  CUDA_CHECK(m_cudaFuncs, cuMemcpy2DAsync(&cpy, nullptr))
262  }
263 
264  CUDA_CHECK(m_cudaFuncs, cuStreamSynchronize(nullptr))
265  CUDA_CHECK(m_cudaFuncs, cuCtxPopCurrent(&dummy))
266 
267  // GLSL deinterlacing. The decoder will pick up any CPU or driver preference
268  // and return a stream of deinterlaced frames. Just check for GLSL here.
269  bool needreferences = false;
270  if (is_interlaced(Scan) && !Frame->m_alreadyDeinterlaced)
271  {
272  MythDeintType shader = Frame->GetDoubleRateOption(DEINT_SHADER);
273  if (shader)
274  needreferences = shader == DEINT_HIGH;
275  else
276  needreferences = Frame->GetSingleRateOption(DEINT_SHADER) == DEINT_HIGH;
277  }
278 
279  if (needreferences)
280  {
281  if (qAbs(Frame->m_frameCounter - m_discontinuityCounter) > 1)
282  m_referenceFrames.clear();
283 
284  RotateReferenceFrames(cudabuffer);
285  int size = m_referenceFrames.size();
286 
287  CUdeviceptr next = m_referenceFrames[0];
288  CUdeviceptr current = m_referenceFrames[size > 1 ? 1 : 0];
289  CUdeviceptr last = m_referenceFrames[size > 2 ? 2 : 0];
290 
291  if (!m_openglTextures.contains(next) || !m_openglTextures.contains(current) ||
292  !m_openglTextures.contains(last))
293  {
294  LOG(VB_GENERAL, LOG_ERR, LOC + "Reference frame error");
295  return result;
296  }
297 
298  result = m_openglTextures[last];
299  std::copy(m_openglTextures[current].cbegin(), m_openglTextures[current].cend(), std::back_inserter(result));
300  std::copy(m_openglTextures[next].cbegin(), m_openglTextures[next].cend(), std::back_inserter(result));
301  return result;
302  }
303  m_referenceFrames.clear();
304  m_discontinuityCounter = Frame->m_frameCounter;
305 
306  return result;
307 }
308 
314 {
316 }
317 
318 bool MythNVDECInterop::CreateCUDAPriv(MythRenderOpenGL* GLContext, CudaFunctions*& CudaFuncs,
319  CUcontext& CudaContext, bool& Retry)
320 {
321  Retry = false;
322  if (!GLContext)
323  return false;
324 
325  // Make OpenGL context current
326  OpenGLLocker locker(GLContext);
327 
328  // retrieve CUDA entry points
329  if (cuda_load_functions(&CudaFuncs, nullptr) != 0)
330  {
331  LOG(VB_PLAYBACK, LOG_ERR, LOC + "Failed to load functions");
332  return false;
333  }
334 
335  // create a CUDA context for the current device
336  CUdevice cudevice = 0;
337  CUcontext dummy = nullptr;
338  CUresult res = CudaFuncs->cuInit(0);
339  if (res != CUDA_SUCCESS)
340  {
341  LOG(VB_GENERAL, LOG_ERR, LOC + "Failed to initialise CUDA API");
342  return false;
343  }
344 
345  unsigned int devicecount = 0;
346  res = CudaFuncs->cuGLGetDevices(&devicecount, &cudevice, 1, CU_GL_DEVICE_LIST_ALL);
347  if (res != CUDA_SUCCESS)
348  {
349  LOG(VB_GENERAL, LOG_ERR, LOC + "Failed to get CUDA device");
350  return false;
351  }
352 
353  if (devicecount < 1)
354  {
355  LOG(VB_GENERAL, LOG_ERR, LOC + "No CUDA devices");
356  return false;
357  }
358 
359  res = CudaFuncs->cuCtxCreate(&CudaContext, CU_CTX_SCHED_BLOCKING_SYNC, cudevice);
360  if (res != CUDA_SUCCESS)
361  {
362  LOG(VB_GENERAL, LOG_ERR, LOC + QString("Failed to create CUDA context (Err: %1)")
363  .arg(res));
364  Retry = true;
365  return false;
366  }
367 
368  CudaFuncs->cuCtxPopCurrent(&dummy);
369  LOG(VB_PLAYBACK, LOG_INFO, LOC + "Created CUDA context");
370  return true;
371 }
372 
373 bool MythNVDECInterop::CreateCUDAContext(MythRenderOpenGL* GLContext, CudaFunctions*& CudaFuncs,
374  CUcontext& CudaContext)
375 {
376  if (!gCoreContext->IsUIThread())
377  {
378  LOG(VB_GENERAL, LOG_ERR, LOC + "Must create CUDA context from main thread");
379  return false;
380  }
381 
382  int retries = 0;
383  bool retry = false;
384  while (retries++ < 5)
385  {
386  if (CreateCUDAPriv(GLContext, CudaFuncs, CudaContext, retry))
387  return true;
388  CleanupContext(GLContext, CudaFuncs, CudaContext);
389  if (!retry)
390  break;
391  LOG(VB_GENERAL, LOG_WARNING, LOC + "Will retry in 50ms");
392  std::this_thread::sleep_for(50ms);
393  }
394  return false;
395 }
396 
397 void MythNVDECInterop::CleanupContext(MythRenderOpenGL* GLContext, CudaFunctions*& CudaFuncs,
398  CUcontext& CudaContext)
399 {
400  if (!GLContext)
401  return;
402 
403  OpenGLLocker locker(GLContext);
404  if (CudaFuncs)
405  {
406  if (CudaContext)
407  CUDA_CHECK(CudaFuncs, cuCtxDestroy(CudaContext))
408  cuda_free_functions(&CudaFuncs);
409  }
410 }
411 
413 {
414  if (!Buffer)
415  return;
416 
417  // don't retain twice for double rate
418  if (!m_referenceFrames.empty() && (m_referenceFrames[0] == Buffer))
419  return;
420 
421  m_referenceFrames.push_front(Buffer);
422 
423  // release old frames
424  while (m_referenceFrames.size() > 3)
425  m_referenceFrames.pop_back();
426 }
MythNVDECInterop::GetNVDECTypes
static void GetNVDECTypes(MythRenderOpenGL *Render, MythInteropGPU::InteropMap &Types)
Definition: mythnvdecinterop.cpp:94
MythNVDECInterop::m_referenceFrames
QVector< CUdeviceptr > m_referenceFrames
Definition: mythnvdecinterop.h:42
MythNVDECInterop::m_cudaContext
CUcontext m_cudaContext
Definition: mythnvdecinterop.h:40
DEINT_SHADER
@ DEINT_SHADER
Definition: mythframe.h:74
MythVideoTextureOpenGL
Definition: mythvideotextureopengl.h:21
Frame
Definition: zmdefines.h:93
MythVideoColourSpace::UpdateColourSpace
bool UpdateColourSpace(const MythVideoFrame *Frame)
Set the current colourspace to use.
Definition: mythvideocolourspace.cpp:321
MythGLTexture::m_data
unsigned char * m_data
Definition: mythrenderopengl.h:67
FrameScanType
FrameScanType
Definition: videoouttypes.h:94
MythCoreContext::IsUIThread
bool IsUIThread(void)
Definition: mythcorecontext.cpp:1374
types
static const struct wl_interface * types[]
Definition: idle_inhibit_unstable_v1.c:39
CUDA_CHECK
#define CUDA_CHECK(CUDA_FUNCS, CUDA_CALL)
Definition: mythnvdecinterop.cpp:13
MythVideoColourSpace::SetSupportedAttributes
void SetSupportedAttributes(PictureAttributeSupported Supported)
Enable the given set of picture attributes.
Definition: mythvideocolourspace.cpp:111
LOG
#define LOG(_MASK_, _LEVEL_, _QSTRING_)
Definition: mythlogging.h:23
MythNVDECInterop
Definition: mythnvdecinterop.h:13
MythNVDECInterop::RotateReferenceFrames
void RotateReferenceFrames(CUdeviceptr Buffer)
Definition: mythnvdecinterop.cpp:412
MythDate::current
QDateTime current(bool stripped)
Returns current Date and Time in UTC.
Definition: mythdate.cpp:10
MythInteropGPU::InteropMap
std::map< VideoFrameType, InteropTypes > InteropMap
Definition: mythinteropgpu.h:44
MythPlayerUI
Definition: mythplayerui.h:10
mythvideocolourspace.h
MythOpenGLInterop
Definition: mythopenglinterop.h:17
MythOpenGLInterop::m_openglContext
MythRenderOpenGL * m_openglContext
Definition: mythopenglinterop.h:41
MythFile::copy
MBASE_PUBLIC long long copy(QFile &dst, QFile &src, uint block_size=0)
Copies src file to dst file.
Definition: mythmiscutil.cpp:314
MythNVDECInterop::Acquire
std::vector< MythVideoTextureOpenGL * > Acquire(MythRenderOpenGL *Context, MythVideoColourSpace *ColourSpace, MythVideoFrame *Frame, FrameScanType Scan) override
Map CUDA video memory to OpenGL textures.
Definition: mythnvdecinterop.cpp:108
MythNVDECInterop::CreateCUDAContext
static bool CreateCUDAContext(MythRenderOpenGL *GLContext, CudaFunctions *&CudaFuncs, CUcontext &CudaContext)
Definition: mythnvdecinterop.cpp:373
MythVideoTextureOpenGL::m_allowGLSLDeint
bool m_allowGLSLDeint
Definition: mythvideotextureopengl.h:55
MythNVDECInterop::IsValid
bool IsValid()
Definition: mythnvdecinterop.cpp:68
MythVideoFrame::ColorDepth
static int ColorDepth(int Format)
Definition: mythframe.h:399
MythVideoTextureOpenGL::DeleteTexture
static void DeleteTexture(MythRenderOpenGL *Context, MythVideoTextureOpenGL *Texture)
Definition: mythvideotextureopengl.cpp:17
uint
unsigned int uint
Definition: compat.h:144
gCoreContext
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
Definition: mythcorecontext.cpp:61
MythInteropGPU::GL_NVDEC
@ GL_NVDEC
Definition: mythinteropgpu.h:35
ALL_PICTURE_ATTRIBUTES
#define ALL_PICTURE_ATTRIBUTES
Definition: videoouttypes.h:127
MythAVUtil::PixelFormatToFrameType
static VideoFrameType PixelFormatToFrameType(AVPixelFormat Fmt)
Definition: mythavutil.cpp:70
MythOpenGLInterop::m_openglTextures
QHash< unsigned long long, std::vector< MythVideoTextureOpenGL * > > m_openglTextures
Definition: mythopenglinterop.h:42
MythNVDECInterop::CreateCUDAPriv
static bool CreateCUDAPriv(MythRenderOpenGL *GLContext, CudaFunctions *&CudaFuncs, CUcontext &CudaContext, bool &Retry)
Definition: mythnvdecinterop.cpp:318
MythGLTexture::m_target
GLenum m_target
Definition: mythrenderopengl.h:81
FMT_NVDEC
@ FMT_NVDEC
Definition: mythframe.h:63
MythNVDECInterop::DeleteTextures
void DeleteTextures() override
Definition: mythnvdecinterop.cpp:37
MythRenderOpenGL
Definition: mythrenderopengl.h:100
MythDeintType
MythDeintType
Definition: mythframe.h:67
Buffer
Definition: MythExternControl.h:36
MythGLTexture::m_textureId
GLuint m_textureId
Definition: mythrenderopengl.h:69
mythcorecontext.h
MythNVDECInterop::GetCUDAContext
CUcontext GetCUDAContext()
Definition: mythnvdecinterop.cpp:73
MythOpenGLInterop::DeleteTextures
virtual void DeleteTextures()
Definition: mythopenglinterop.cpp:137
DEINT_HIGH
@ DEINT_HIGH
Definition: mythframe.h:72
MythGLTexture::m_size
QSize m_size
Definition: mythrenderopengl.h:74
MythNVDECInterop::InitialiseCuda
bool InitialiseCuda()
Initialise a CUDA context.
Definition: mythnvdecinterop.cpp:313
MythInteropGPU::m_discontinuityCounter
uint64_t m_discontinuityCounter
Definition: mythinteropgpu.h:63
MythNVDECInterop::CreateNVDEC
static MythNVDECInterop * CreateNVDEC(MythPlayerUI *Player, MythRenderOpenGL *Context)
Definition: mythnvdecinterop.cpp:78
MythNVDECInterop::CleanupContext
static void CleanupContext(MythRenderOpenGL *GLContext, CudaFunctions *&CudaFuncs, CUcontext &CudaContext)
Definition: mythnvdecinterop.cpp:397
MythNVDECInterop::~MythNVDECInterop
~MythNVDECInterop() override
Definition: mythnvdecinterop.cpp:30
LOC
#define LOC
Definition: mythnvdecinterop.cpp:11
mythnvdecinterop.h
VideoFrameType
VideoFrameType
Definition: mythframe.h:20
Player
Definition: zmliveplayer.h:34
MythVideoFrame
Definition: mythframe.h:88
FMT_NV12
@ FMT_NV12
Definition: mythframe.h:53
MythInteropGPU::m_textureSize
QSize m_textureSize
Definition: mythinteropgpu.h:62
is_interlaced
bool is_interlaced(FrameScanType Scan)
Definition: videoouttypes.h:188
MythVideoColourSpace
MythVideoColourSpace contains a QMatrix4x4 that can convert YCbCr data to RGB.
Definition: mythvideocolourspace.h:18
MythNVDECInterop::m_cudaFuncs
CudaFunctions * m_cudaFuncs
Definition: mythnvdecinterop.h:41
MythVideoTextureOpenGL::CreateTextures
static std::vector< MythVideoTextureOpenGL * > CreateTextures(MythRenderOpenGL *Context, VideoFrameType Type, VideoFrameType Format, std::vector< QSize > Sizes, GLenum Target=QOpenGLTexture::Target2D)
Create a set of textures suitable for the given Type and Format.
Definition: mythvideotextureopengl.cpp:57
MythNVDECInterop::MythNVDECInterop
MythNVDECInterop(MythPlayerUI *Player, MythRenderOpenGL *Context)
Definition: mythnvdecinterop.cpp:23
OpenGLLocker
Definition: mythrenderopengl.h:263