1// Copyright (C) 2021 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
4#include "qffmpeghwaccel_vaapi_p.h"
5
6#if !QT_CONFIG(vaapi)
7#error "Configuration error"
8#endif
9
10#include <va/va.h>
11
12#include <qvideoframeformat.h>
13#include "qffmpegvideobuffer_p.h"
14#include "private/qvideotexturehelper_p.h"
15
16#include <rhi/qrhi.h>
17
18#include <qguiapplication.h>
19#include <qpa/qplatformnativeinterface.h>
20
21#include <qopenglfunctions.h>
22
23//#define VA_EXPORT_USE_LAYERS
24
25#if __has_include("drm/drm_fourcc.h")
26#include <drm/drm_fourcc.h>
27#elif __has_include("libdrm/drm_fourcc.h")
28#include <libdrm/drm_fourcc.h>
29#else
30// keep things building without drm_fourcc.h
31#define fourcc_code(a, b, c, d) ((uint32_t)(a) | ((uint32_t)(b) << 8) | \
32 ((uint32_t)(c) << 16) | ((uint32_t)(d) << 24))
33
34#define DRM_FORMAT_RGBA8888 fourcc_code('R', 'A', '2', '4') /* [31:0] R:G:B:A 8:8:8:8 little endian */
35#define DRM_FORMAT_RGB888 fourcc_code('R', 'G', '2', '4') /* [23:0] R:G:B little endian */
36#define DRM_FORMAT_RG88 fourcc_code('R', 'G', '8', '8') /* [15:0] R:G 8:8 little endian */
37#define DRM_FORMAT_ABGR8888 fourcc_code('A', 'B', '2', '4') /* [31:0] A:B:G:R 8:8:8:8 little endian */
38#define DRM_FORMAT_BGR888 fourcc_code('B', 'G', '2', '4') /* [23:0] B:G:R little endian */
39#define DRM_FORMAT_GR88 fourcc_code('G', 'R', '8', '8') /* [15:0] G:R 8:8 little endian */
40#define DRM_FORMAT_R8 fourcc_code('R', '8', ' ', ' ') /* [7:0] R */
41#define DRM_FORMAT_R16 fourcc_code('R', '1', '6', ' ') /* [15:0] R little endian */
42#define DRM_FORMAT_RGB565 fourcc_code('R', 'G', '1', '6') /* [15:0] R:G:B 5:6:5 little endian */
43#define DRM_FORMAT_RG1616 fourcc_code('R', 'G', '3', '2') /* [31:0] R:G 16:16 little endian */
44#define DRM_FORMAT_GR1616 fourcc_code('G', 'R', '3', '2') /* [31:0] G:R 16:16 little endian */
45#define DRM_FORMAT_BGRA1010102 fourcc_code('B', 'A', '3', '0') /* [31:0] B:G:R:A 10:10:10:2 little endian */
46#endif
47
48extern "C" {
49#include <libavutil/hwcontext_vaapi.h>
50}
51
52#include <va/va_drm.h>
53#include <va/va_drmcommon.h>
54
55#include <EGL/egl.h>
56#include <EGL/eglext.h>
57
58#include <unistd.h>
59
60#include <qloggingcategory.h>
61
62QT_BEGIN_NAMESPACE
63
64static Q_LOGGING_CATEGORY(qLHWAccelVAAPI, "qt.multimedia.ffmpeg.hwaccelvaapi");
65
66namespace QFFmpeg {
67
68static const quint32 *fourccFromPixelFormat(const QVideoFrameFormat::PixelFormat format)
69{
70#if G_BYTE_ORDER == G_LITTLE_ENDIAN
71 const quint32 rgba_fourcc = DRM_FORMAT_ABGR8888;
72 const quint32 rg_fourcc = DRM_FORMAT_GR88;
73 const quint32 rg16_fourcc = DRM_FORMAT_GR1616;
74#else
75 const quint32 rgba_fourcc = DRM_FORMAT_RGBA8888;
76 const quint32 rg_fourcc = DRM_FORMAT_RG88;
77 const quint32 rg16_fourcc = DRM_FORMAT_RG1616;
78#endif
79
80// qCDebug(qLHWAccelVAAPI) << "Getting DRM fourcc for pixel format" << format;
81
82 switch (format) {
83 case QVideoFrameFormat::Format_Invalid:
84 case QVideoFrameFormat::Format_IMC1:
85 case QVideoFrameFormat::Format_IMC2:
86 case QVideoFrameFormat::Format_IMC3:
87 case QVideoFrameFormat::Format_IMC4:
88 case QVideoFrameFormat::Format_SamplerExternalOES:
89 case QVideoFrameFormat::Format_Jpeg:
90 case QVideoFrameFormat::Format_SamplerRect:
91 return nullptr;
92
93 case QVideoFrameFormat::Format_ARGB8888:
94 case QVideoFrameFormat::Format_ARGB8888_Premultiplied:
95 case QVideoFrameFormat::Format_XRGB8888:
96 case QVideoFrameFormat::Format_BGRA8888:
97 case QVideoFrameFormat::Format_BGRA8888_Premultiplied:
98 case QVideoFrameFormat::Format_BGRX8888:
99 case QVideoFrameFormat::Format_ABGR8888:
100 case QVideoFrameFormat::Format_XBGR8888:
101 case QVideoFrameFormat::Format_RGBA8888:
102 case QVideoFrameFormat::Format_RGBX8888:
103 case QVideoFrameFormat::Format_AYUV:
104 case QVideoFrameFormat::Format_AYUV_Premultiplied:
105 case QVideoFrameFormat::Format_UYVY:
106 case QVideoFrameFormat::Format_YUYV:
107 {
108 static constexpr quint32 format[] = { rgba_fourcc, 0, 0, 0 };
109 return format;
110 }
111
112 case QVideoFrameFormat::Format_Y8:
113 {
114 static constexpr quint32 format[] = { DRM_FORMAT_R8, 0, 0, 0 };
115 return format;
116 }
117 case QVideoFrameFormat::Format_Y16:
118 {
119 static constexpr quint32 format[] = { DRM_FORMAT_R16, 0, 0, 0 };
120 return format;
121 }
122
123 case QVideoFrameFormat::Format_YUV420P:
124 case QVideoFrameFormat::Format_YUV422P:
125 case QVideoFrameFormat::Format_YV12:
126 {
127 static constexpr quint32 format[] = { DRM_FORMAT_R8, DRM_FORMAT_R8, DRM_FORMAT_R8, 0 };
128 return format;
129 }
130 case QVideoFrameFormat::Format_YUV420P10:
131 {
132 static constexpr quint32 format[] = { DRM_FORMAT_R16, DRM_FORMAT_R16, DRM_FORMAT_R16, 0 };
133 return format;
134 }
135
136 case QVideoFrameFormat::Format_NV12:
137 case QVideoFrameFormat::Format_NV21:
138 {
139 static constexpr quint32 format[] = { DRM_FORMAT_R8, rg_fourcc, 0, 0 };
140 return format;
141 }
142
143 case QVideoFrameFormat::Format_P010:
144 case QVideoFrameFormat::Format_P016:
145 {
146 static constexpr quint32 format[] = { DRM_FORMAT_R16, rg16_fourcc, 0, 0 };
147 return format;
148 }
149 }
150 return nullptr;
151}
152
153namespace {
154class VAAPITextureHandles : public QVideoFrameTexturesHandles
155{
156public:
157 ~VAAPITextureHandles() override;
158 quint64 textureHandle(QRhi &, int plane) override {
159 return textures[plane];
160 }
161
162 TextureConverterBackendPtr parentConverterBackend; // ensures the backend is deleted after the texture
163 QRhi *rhi = nullptr;
164 QOpenGLContext *glContext = nullptr;
165 int nPlanes = 0;
166 GLuint textures[4] = {};
167};
168} // namespace
169
170VAAPITextureConverter::VAAPITextureConverter(QRhi *rhi)
171 : TextureConverterBackend(nullptr)
172{
173 qCDebug(qLHWAccelVAAPI) << ">>>> Creating VAAPI HW accelerator";
174
175 if (!rhi || rhi->backend() != QRhi::OpenGLES2) {
176 qWarning() << "VAAPITextureConverter: No rhi or non openGL based RHI";
177 this->rhi = nullptr;
178 return;
179 }
180
181 auto *nativeHandles = static_cast<const QRhiGles2NativeHandles *>(rhi->nativeHandles());
182 glContext = nativeHandles->context;
183 if (!glContext) {
184 qCDebug(qLHWAccelVAAPI) << " no GL context, disabling";
185 return;
186 }
187 const QString platform = QGuiApplication::platformName();
188 QPlatformNativeInterface *pni = QGuiApplication::platformNativeInterface();
189 eglDisplay = pni->nativeResourceForIntegration(QByteArrayLiteral("egldisplay"));
190 qCDebug(qLHWAccelVAAPI) << " platform is" << platform << eglDisplay;
191
192 if (!eglDisplay) {
193 qCDebug(qLHWAccelVAAPI) << " no egl display, disabling";
194 return;
195 }
196 eglImageTargetTexture2D = eglGetProcAddress(procname: "glEGLImageTargetTexture2DOES");
197 if (!eglDisplay) {
198 qCDebug(qLHWAccelVAAPI) << " no eglImageTargetTexture2D, disabling";
199 return;
200 }
201
202 // everything ok, indicate that we can do zero copy
203 this->rhi = rhi;
204}
205
206VAAPITextureConverter::~VAAPITextureConverter() = default;
207
208//#define VA_EXPORT_USE_LAYERS
209QVideoFrameTexturesHandlesUPtr
210VAAPITextureConverter::createTextureHandles(AVFrame *frame,
211 QVideoFrameTexturesHandlesUPtr /*oldHandles*/)
212{
213 // qCDebug(qLHWAccelVAAPI) << "VAAPIAccel::createTextureHandles";
214 if (frame->format != AV_PIX_FMT_VAAPI || !eglDisplay) {
215 qCDebug(qLHWAccelVAAPI) << "format/egl error" << frame->format << eglDisplay;
216 return nullptr;
217 }
218
219 if (!frame->hw_frames_ctx)
220 return nullptr;
221
222 auto *ctx = avFrameDeviceContext(frame);
223 if (!ctx)
224 return nullptr;
225
226 auto *vaCtx = (AVVAAPIDeviceContext *)ctx->hwctx;
227 auto vaDisplay = vaCtx->display;
228 if (!vaDisplay) {
229 qCDebug(qLHWAccelVAAPI) << " no VADisplay, disabling";
230 return nullptr;
231 }
232
233 VASurfaceID vaSurface = (uintptr_t)frame->data[3];
234
235 VADRMPRIMESurfaceDescriptor prime = {};
236 if (vaExportSurfaceHandle(dpy: vaDisplay, surface_id: vaSurface,
237 VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME_2,
238 VA_EXPORT_SURFACE_READ_ONLY |
239#ifdef VA_EXPORT_USE_LAYERS
240 VA_EXPORT_SURFACE_SEPARATE_LAYERS,
241#else
242 VA_EXPORT_SURFACE_COMPOSED_LAYERS,
243#endif
244 descriptor: &prime) != VA_STATUS_SUCCESS)
245 {
246 qWarning() << "vaExportSurfaceHandle failed";
247 return nullptr;
248 }
249
250 // Make sure all fd's in 'prime' are closed when we return from this function
251 QScopeGuard closeObjectsGuard([&prime]() {
252 for (uint32_t i = 0; i < prime.num_objects; ++i)
253 close(fd: prime.objects[i].fd);
254 });
255
256 // ### Check that prime.fourcc is what we expect
257 vaSyncSurface(dpy: vaDisplay, render_target: vaSurface);
258
259// qCDebug(qLHWAccelVAAPI) << "VAAPIAccel: vaSufraceDesc: width/height" << prime.width << prime.height << "num objects"
260// << prime.num_objects << "num layers" << prime.num_layers;
261
262 QOpenGLFunctions functions(glContext);
263
264 AVPixelFormat fmt = HWAccel::format(frame);
265 bool needsConversion;
266 auto qtFormat = QFFmpegVideoBuffer::toQtPixelFormat(avPixelFormat: fmt, needsConversion: &needsConversion);
267 auto *drm_formats = fourccFromPixelFormat(format: qtFormat);
268 if (!drm_formats || needsConversion) {
269 qWarning() << "can't use DMA transfer for pixel format" << fmt << qtFormat;
270 return nullptr;
271 }
272
273 auto *desc = QVideoTextureHelper::textureDescription(format: qtFormat);
274 int nPlanes = 0;
275 for (; nPlanes < 5; ++nPlanes) {
276 if (drm_formats[nPlanes] == 0)
277 break;
278 }
279 Q_ASSERT(nPlanes == desc->nplanes);
280 nPlanes = desc->nplanes;
281// qCDebug(qLHWAccelVAAPI) << "VAAPIAccel: nPlanes" << nPlanes;
282
283 rhi->makeThreadLocalNativeContextCurrent();
284
285 EGLImage images[4];
286 GLuint glTextures[4] = {};
287 functions.glGenTextures(n: nPlanes, textures: glTextures);
288 for (int i = 0; i < nPlanes; ++i) {
289#ifdef VA_EXPORT_USE_LAYERS
290#define LAYER i
291#define PLANE 0
292 if (prime.layers[i].drm_format != drm_formats[i]) {
293 qWarning() << "expected DRM format check failed expected"
294 << Qt::hex << drm_formats[i] << "got" << prime.layers[i].drm_format;
295 }
296#else
297#define LAYER 0
298#define PLANE i
299#endif
300
301 QSize planeSize = desc->rhiPlaneSize(frameSize: QSize(frame->width, frame->height), plane: i, rhi);
302 EGLAttrib img_attr[] = {
303 EGL_LINUX_DRM_FOURCC_EXT, (EGLint)drm_formats[i],
304 EGL_WIDTH, planeSize.width(),
305 EGL_HEIGHT, planeSize.height(),
306 EGL_DMA_BUF_PLANE0_FD_EXT, prime.objects[prime.layers[LAYER].object_index[PLANE]].fd,
307 EGL_DMA_BUF_PLANE0_OFFSET_EXT, (EGLint)prime.layers[LAYER].offset[PLANE],
308 EGL_DMA_BUF_PLANE0_PITCH_EXT, (EGLint)prime.layers[LAYER].pitch[PLANE],
309 EGL_NONE
310 };
311 images[i] = eglCreateImage(dpy: eglDisplay, EGL_NO_CONTEXT, EGL_LINUX_DMA_BUF_EXT, buffer: nullptr, attrib_list: img_attr);
312 if (!images[i]) {
313 const GLenum error = eglGetError();
314 if (error == EGL_BAD_MATCH) {
315 qWarning() << "eglCreateImage failed for plane" << i << "with error code EGL_BAD_MATCH, "
316 "disabling hardware acceleration. This could indicate an EGL implementation issue."
317 "\nVAAPI driver: " << vaQueryVendorString(dpy: vaDisplay)
318 << "\nEGL vendor:" << eglQueryString(dpy: eglDisplay, EGL_VENDOR);
319 this->rhi = nullptr; // Disabling texture conversion here to fix QTBUG-112312
320 return nullptr;
321 }
322 if (error) {
323 qWarning() << "eglCreateImage failed for plane" << i << "with error code" << error;
324 return nullptr;
325 }
326 }
327 functions.glActiveTexture(GL_TEXTURE0 + i);
328 functions.glBindTexture(GL_TEXTURE_2D, texture: glTextures[i]);
329
330 PFNGLEGLIMAGETARGETTEXTURE2DOESPROC eglImageTargetTexture2D = (PFNGLEGLIMAGETARGETTEXTURE2DOESPROC)this->eglImageTargetTexture2D;
331 eglImageTargetTexture2D(GL_TEXTURE_2D, images[i]);
332 GLenum error = glGetError();
333 if (error)
334 qWarning() << "eglImageTargetTexture2D failed with error code" << error;
335 }
336
337 for (int i = 0; i < nPlanes; ++i) {
338 functions.glActiveTexture(GL_TEXTURE0 + i);
339 functions.glBindTexture(GL_TEXTURE_2D, texture: 0);
340 eglDestroyImage(dpy: eglDisplay, image: images[i]);
341 }
342
343 auto textureHandles = std::make_unique<VAAPITextureHandles>();
344 textureHandles->parentConverterBackend = shared_from_this();
345 textureHandles->nPlanes = nPlanes;
346 textureHandles->rhi = rhi;
347 textureHandles->glContext = glContext;
348
349 for (int i = 0; i < 4; ++i)
350 textureHandles->textures[i] = glTextures[i];
351// qCDebug(qLHWAccelVAAPI) << "VAAPIAccel: got textures" << textures[0] << textures[1] << textures[2] << textures[3];
352
353 return textureHandles;
354}
355
356VAAPITextureHandles::~VAAPITextureHandles()
357{
358 if (rhi) {
359 rhi->makeThreadLocalNativeContextCurrent();
360 QOpenGLFunctions functions(glContext);
361 functions.glDeleteTextures(n: nPlanes, textures);
362 }
363}
364
365} // namespace QFFmpeg
366
367QT_END_NAMESPACE
368

source code of qtmultimedia/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_vaapi.cpp