FreeRDP
Loading...
Searching...
No Matches
h264_ffmpeg.c
1
21#include <freerdp/config.h>
22
23#include <winpr/wlog.h>
24#include <winpr/json.h>
25
26#include <freerdp/log.h>
27#include <freerdp/codec/h264.h>
28#include <freerdp/utils/helpers.h>
29#include <libavcodec/avcodec.h>
30#include <libavutil/opt.h>
31#include <libswscale/swscale.h>
32
33#include "h264.h"
34
35#ifdef WITH_FFMPEG_HWACCEL
36#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(55, 9, 0)
37#include <libavutil/hwcontext.h>
38#else
39#pragma warning You have asked for VA - API decoding, \
40 but your version of libavutil is too old !Disabling.
41#undef WITH_FFMPEG_HWACCEL
42#endif
43#endif
44
45/* Fallback support for older libavcodec versions */
46#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(54, 59, 100)
47#define AV_CODEC_ID_H264 CODEC_ID_H264
48#endif
49
50#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(56, 34, 2)
51#define AV_CODEC_FLAG_LOOP_FILTER CODEC_FLAG_LOOP_FILTER
52#define AV_CODEC_CAP_TRUNCATED CODEC_CAP_TRUNCATED
53#define AV_CODEC_FLAG_TRUNCATED CODEC_FLAG_TRUNCATED
54#endif
55
56#if LIBAVUTIL_VERSION_MAJOR < 52
57#define AV_PIX_FMT_YUV420P PIX_FMT_YUV420P
58#endif
59
60#if defined(WITH_FFMPEG_HWACCEL)
61static const char* av_format_str(int32_t format)
62{
63#define EVCASE(x) \
64 case x: \
65 return #x
66
67 switch (format)
68 {
69 EVCASE(AV_PIX_FMT_NONE);
70 EVCASE(AV_PIX_FMT_YUV420P);
71 EVCASE(AV_PIX_FMT_YUYV422);
72 EVCASE(AV_PIX_FMT_RGB24);
73 EVCASE(AV_PIX_FMT_BGR24);
74 EVCASE(AV_PIX_FMT_YUV422P);
75 EVCASE(AV_PIX_FMT_YUV444P);
76 EVCASE(AV_PIX_FMT_YUV410P);
77 EVCASE(AV_PIX_FMT_YUV411P);
78 EVCASE(AV_PIX_FMT_GRAY8);
79 EVCASE(AV_PIX_FMT_MONOWHITE);
80 EVCASE(AV_PIX_FMT_MONOBLACK);
81 EVCASE(AV_PIX_FMT_PAL8);
82 EVCASE(AV_PIX_FMT_YUVJ420P);
83 EVCASE(AV_PIX_FMT_YUVJ422P);
84 EVCASE(AV_PIX_FMT_YUVJ444P);
85 EVCASE(AV_PIX_FMT_UYVY422);
86 EVCASE(AV_PIX_FMT_UYYVYY411);
87 EVCASE(AV_PIX_FMT_BGR8);
88 EVCASE(AV_PIX_FMT_BGR4);
89 EVCASE(AV_PIX_FMT_BGR4_BYTE);
90 EVCASE(AV_PIX_FMT_RGB8);
91 EVCASE(AV_PIX_FMT_RGB4);
92 EVCASE(AV_PIX_FMT_RGB4_BYTE);
93 EVCASE(AV_PIX_FMT_NV12);
94 EVCASE(AV_PIX_FMT_NV21);
95 EVCASE(AV_PIX_FMT_ARGB);
96 EVCASE(AV_PIX_FMT_RGBA);
97 EVCASE(AV_PIX_FMT_ABGR);
98 EVCASE(AV_PIX_FMT_BGRA);
99 EVCASE(AV_PIX_FMT_GRAY16BE);
100 EVCASE(AV_PIX_FMT_GRAY16LE);
101 EVCASE(AV_PIX_FMT_YUV440P);
102 EVCASE(AV_PIX_FMT_YUVJ440P);
103 EVCASE(AV_PIX_FMT_YUVA420P);
104 EVCASE(AV_PIX_FMT_RGB48BE);
105 EVCASE(AV_PIX_FMT_RGB48LE);
106 EVCASE(AV_PIX_FMT_RGB565BE);
107 EVCASE(AV_PIX_FMT_RGB565LE);
108 EVCASE(AV_PIX_FMT_RGB555BE);
109 EVCASE(AV_PIX_FMT_RGB555LE);
110 EVCASE(AV_PIX_FMT_BGR565BE);
111 EVCASE(AV_PIX_FMT_BGR565LE);
112 EVCASE(AV_PIX_FMT_BGR555BE);
113 EVCASE(AV_PIX_FMT_BGR555LE);
114 EVCASE(AV_PIX_FMT_VAAPI);
115 EVCASE(AV_PIX_FMT_YUV420P16LE);
116 EVCASE(AV_PIX_FMT_YUV420P16BE);
117 EVCASE(AV_PIX_FMT_YUV422P16LE);
118 EVCASE(AV_PIX_FMT_YUV422P16BE);
119 EVCASE(AV_PIX_FMT_YUV444P16LE);
120 EVCASE(AV_PIX_FMT_YUV444P16BE);
121 EVCASE(AV_PIX_FMT_DXVA2_VLD);
122 EVCASE(AV_PIX_FMT_RGB444LE);
123 EVCASE(AV_PIX_FMT_RGB444BE);
124 EVCASE(AV_PIX_FMT_BGR444LE);
125 EVCASE(AV_PIX_FMT_BGR444BE);
126 EVCASE(AV_PIX_FMT_GRAY8A);
127 EVCASE(AV_PIX_FMT_BGR48BE);
128 EVCASE(AV_PIX_FMT_BGR48LE);
129 EVCASE(AV_PIX_FMT_YUV420P9BE);
130 EVCASE(AV_PIX_FMT_YUV420P9LE);
131 EVCASE(AV_PIX_FMT_YUV420P10BE);
132 EVCASE(AV_PIX_FMT_YUV420P10LE);
133 EVCASE(AV_PIX_FMT_YUV422P10BE);
134 EVCASE(AV_PIX_FMT_YUV422P10LE);
135 EVCASE(AV_PIX_FMT_YUV444P9BE);
136 EVCASE(AV_PIX_FMT_YUV444P9LE);
137 EVCASE(AV_PIX_FMT_YUV444P10BE);
138 EVCASE(AV_PIX_FMT_YUV444P10LE);
139 EVCASE(AV_PIX_FMT_YUV422P9BE);
140 EVCASE(AV_PIX_FMT_YUV422P9LE);
141 EVCASE(AV_PIX_FMT_GBR24P);
142 EVCASE(AV_PIX_FMT_GBRP9BE);
143 EVCASE(AV_PIX_FMT_GBRP9LE);
144 EVCASE(AV_PIX_FMT_GBRP10BE);
145 EVCASE(AV_PIX_FMT_GBRP10LE);
146 EVCASE(AV_PIX_FMT_GBRP16BE);
147 EVCASE(AV_PIX_FMT_GBRP16LE);
148 EVCASE(AV_PIX_FMT_YUVA422P);
149 EVCASE(AV_PIX_FMT_YUVA444P);
150 EVCASE(AV_PIX_FMT_YUVA420P9BE);
151 EVCASE(AV_PIX_FMT_YUVA420P9LE);
152 EVCASE(AV_PIX_FMT_YUVA422P9BE);
153 EVCASE(AV_PIX_FMT_YUVA422P9LE);
154 EVCASE(AV_PIX_FMT_YUVA444P9BE);
155 EVCASE(AV_PIX_FMT_YUVA444P9LE);
156 EVCASE(AV_PIX_FMT_YUVA420P10BE);
157 EVCASE(AV_PIX_FMT_YUVA420P10LE);
158 EVCASE(AV_PIX_FMT_YUVA422P10BE);
159 EVCASE(AV_PIX_FMT_YUVA422P10LE);
160 EVCASE(AV_PIX_FMT_YUVA444P10BE);
161 EVCASE(AV_PIX_FMT_YUVA444P10LE);
162 EVCASE(AV_PIX_FMT_YUVA420P16BE);
163 EVCASE(AV_PIX_FMT_YUVA420P16LE);
164 EVCASE(AV_PIX_FMT_YUVA422P16BE);
165 EVCASE(AV_PIX_FMT_YUVA422P16LE);
166 EVCASE(AV_PIX_FMT_YUVA444P16BE);
167 EVCASE(AV_PIX_FMT_YUVA444P16LE);
168 EVCASE(AV_PIX_FMT_VDPAU);
169 EVCASE(AV_PIX_FMT_XYZ12LE);
170 EVCASE(AV_PIX_FMT_XYZ12BE);
171 EVCASE(AV_PIX_FMT_NV16);
172 EVCASE(AV_PIX_FMT_NV20LE);
173 EVCASE(AV_PIX_FMT_NV20BE);
174 EVCASE(AV_PIX_FMT_RGBA64BE);
175 EVCASE(AV_PIX_FMT_RGBA64LE);
176 EVCASE(AV_PIX_FMT_BGRA64BE);
177 EVCASE(AV_PIX_FMT_BGRA64LE);
178 EVCASE(AV_PIX_FMT_YVYU422);
179 EVCASE(AV_PIX_FMT_YA16BE);
180 EVCASE(AV_PIX_FMT_YA16LE);
181 EVCASE(AV_PIX_FMT_GBRAP);
182 EVCASE(AV_PIX_FMT_GBRAP16BE);
183 EVCASE(AV_PIX_FMT_GBRAP16LE);
184 EVCASE(AV_PIX_FMT_QSV);
185 EVCASE(AV_PIX_FMT_MMAL);
186 EVCASE(AV_PIX_FMT_D3D11VA_VLD);
187 EVCASE(AV_PIX_FMT_CUDA);
188 EVCASE(AV_PIX_FMT_0RGB);
189 EVCASE(AV_PIX_FMT_RGB0);
190 EVCASE(AV_PIX_FMT_0BGR);
191 EVCASE(AV_PIX_FMT_BGR0);
192 EVCASE(AV_PIX_FMT_YUV420P12BE);
193 EVCASE(AV_PIX_FMT_YUV420P12LE);
194 EVCASE(AV_PIX_FMT_YUV420P14BE);
195 EVCASE(AV_PIX_FMT_YUV420P14LE);
196 EVCASE(AV_PIX_FMT_YUV422P12BE);
197 EVCASE(AV_PIX_FMT_YUV422P12LE);
198 EVCASE(AV_PIX_FMT_YUV422P14BE);
199 EVCASE(AV_PIX_FMT_YUV422P14LE);
200 EVCASE(AV_PIX_FMT_YUV444P12BE);
201 EVCASE(AV_PIX_FMT_YUV444P12LE);
202 EVCASE(AV_PIX_FMT_YUV444P14BE);
203 EVCASE(AV_PIX_FMT_YUV444P14LE);
204 EVCASE(AV_PIX_FMT_GBRP12BE);
205 EVCASE(AV_PIX_FMT_GBRP12LE);
206 EVCASE(AV_PIX_FMT_GBRP14BE);
207 EVCASE(AV_PIX_FMT_GBRP14LE);
208 EVCASE(AV_PIX_FMT_YUVJ411P);
209 EVCASE(AV_PIX_FMT_BAYER_BGGR8);
210 EVCASE(AV_PIX_FMT_BAYER_RGGB8);
211 EVCASE(AV_PIX_FMT_BAYER_GBRG8);
212 EVCASE(AV_PIX_FMT_BAYER_GRBG8);
213 EVCASE(AV_PIX_FMT_BAYER_BGGR16LE);
214 EVCASE(AV_PIX_FMT_BAYER_BGGR16BE);
215 EVCASE(AV_PIX_FMT_BAYER_RGGB16LE);
216 EVCASE(AV_PIX_FMT_BAYER_RGGB16BE);
217 EVCASE(AV_PIX_FMT_BAYER_GBRG16LE);
218 EVCASE(AV_PIX_FMT_BAYER_GBRG16BE);
219 EVCASE(AV_PIX_FMT_BAYER_GRBG16LE);
220 EVCASE(AV_PIX_FMT_BAYER_GRBG16BE);
221 EVCASE(AV_PIX_FMT_YUV440P10LE);
222 EVCASE(AV_PIX_FMT_YUV440P10BE);
223 EVCASE(AV_PIX_FMT_YUV440P12LE);
224 EVCASE(AV_PIX_FMT_YUV440P12BE);
225 EVCASE(AV_PIX_FMT_AYUV64LE);
226 EVCASE(AV_PIX_FMT_AYUV64BE);
227 EVCASE(AV_PIX_FMT_VIDEOTOOLBOX);
228 EVCASE(AV_PIX_FMT_P010LE);
229 EVCASE(AV_PIX_FMT_P010BE);
230 EVCASE(AV_PIX_FMT_GBRAP12BE);
231 EVCASE(AV_PIX_FMT_GBRAP12LE);
232 EVCASE(AV_PIX_FMT_GBRAP10BE);
233 EVCASE(AV_PIX_FMT_GBRAP10LE);
234 EVCASE(AV_PIX_FMT_MEDIACODEC);
235 EVCASE(AV_PIX_FMT_GRAY12BE);
236 EVCASE(AV_PIX_FMT_GRAY12LE);
237 EVCASE(AV_PIX_FMT_GRAY10BE);
238 EVCASE(AV_PIX_FMT_GRAY10LE);
239 EVCASE(AV_PIX_FMT_P016LE);
240 EVCASE(AV_PIX_FMT_P016BE);
241 EVCASE(AV_PIX_FMT_D3D11);
242 EVCASE(AV_PIX_FMT_GRAY9BE);
243 EVCASE(AV_PIX_FMT_GRAY9LE);
244 EVCASE(AV_PIX_FMT_GBRPF32BE);
245 EVCASE(AV_PIX_FMT_GBRPF32LE);
246 EVCASE(AV_PIX_FMT_GBRAPF32BE);
247 EVCASE(AV_PIX_FMT_GBRAPF32LE);
248 EVCASE(AV_PIX_FMT_DRM_PRIME);
249 EVCASE(AV_PIX_FMT_OPENCL);
250 EVCASE(AV_PIX_FMT_GRAY14BE);
251 EVCASE(AV_PIX_FMT_GRAY14LE);
252 EVCASE(AV_PIX_FMT_GRAYF32BE);
253 EVCASE(AV_PIX_FMT_GRAYF32LE);
254 EVCASE(AV_PIX_FMT_YUVA422P12BE);
255 EVCASE(AV_PIX_FMT_YUVA422P12LE);
256 EVCASE(AV_PIX_FMT_YUVA444P12BE);
257 EVCASE(AV_PIX_FMT_YUVA444P12LE);
258 EVCASE(AV_PIX_FMT_NV24);
259 EVCASE(AV_PIX_FMT_NV42);
260 EVCASE(AV_PIX_FMT_VULKAN);
261 EVCASE(AV_PIX_FMT_Y210BE);
262 EVCASE(AV_PIX_FMT_Y210LE);
263#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(57, 17, 100)
264 EVCASE(AV_PIX_FMT_P210BE);
265 EVCASE(AV_PIX_FMT_P210LE);
266 EVCASE(AV_PIX_FMT_P216BE);
267 EVCASE(AV_PIX_FMT_P216LE);
268 EVCASE(AV_PIX_FMT_P410BE);
269 EVCASE(AV_PIX_FMT_P410LE);
270 EVCASE(AV_PIX_FMT_X2RGB10LE);
271 EVCASE(AV_PIX_FMT_X2RGB10BE);
272 EVCASE(AV_PIX_FMT_X2BGR10LE);
273 EVCASE(AV_PIX_FMT_X2BGR10BE);
274 EVCASE(AV_PIX_FMT_P416BE);
275 EVCASE(AV_PIX_FMT_P416LE);
276#endif
277#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(58, 2, 100)
278 EVCASE(AV_PIX_FMT_VUYA);
279 EVCASE(AV_PIX_FMT_VUYX);
280#endif
281#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(57, 33, 101)
282 EVCASE(AV_PIX_FMT_RGBAF16BE);
283 EVCASE(AV_PIX_FMT_RGBAF16LE);
284#endif
285#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(57, 37, 100)
286 EVCASE(AV_PIX_FMT_P012LE);
287 EVCASE(AV_PIX_FMT_P012BE);
288 EVCASE(AV_PIX_FMT_Y212BE);
289 EVCASE(AV_PIX_FMT_Y212LE);
290 EVCASE(AV_PIX_FMT_XV30BE);
291 EVCASE(AV_PIX_FMT_XV30LE);
292 EVCASE(AV_PIX_FMT_XV36BE);
293 EVCASE(AV_PIX_FMT_XV36LE);
294 EVCASE(AV_PIX_FMT_RGBF32BE);
295 EVCASE(AV_PIX_FMT_RGBF32LE);
296 EVCASE(AV_PIX_FMT_RGBAF32BE);
297 EVCASE(AV_PIX_FMT_RGBAF32LE);
298#endif
299#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(59, 10, 100)
300 EVCASE(AV_PIX_FMT_P212BE);
301 EVCASE(AV_PIX_FMT_P212LE);
302 EVCASE(AV_PIX_FMT_P412BE);
303 EVCASE(AV_PIX_FMT_P412LE);
304#endif
305#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(59, 14, 100)
306 EVCASE(AV_PIX_FMT_GBRAP14BE);
307 EVCASE(AV_PIX_FMT_GBRAP14LE);
308#endif
309#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(59,16,100)
310 EVCASE(AV_PIX_FMT_D3D12);
311#endif
312 EVCASE(AV_PIX_FMT_NB);
313 default:
314 return "AV_PIX_FMT_UNKNOWN";
315 }
316}
317#endif
318
319/* Ubuntu 14.04 ships without the functions provided by avutil,
320 * so define error to string methods here. */
321#if !defined(av_err2str)
322static inline char* error_string(char* errbuf, size_t errbuf_size, int errnum)
323{
324 av_strerror(errnum, errbuf, errbuf_size);
325 return errbuf;
326}
327
328#define av_err2str(errnum) error_string((char[64])WINPR_C_ARRAY_INIT, 64, errnum)
329#endif
330
331typedef struct
332{
333 const AVCodec* codecDecoder;
334 AVCodecContext* codecDecoderContext;
335 const AVCodec* codecEncoder;
336 AVCodecContext* codecEncoderContext;
337 AVCodecParserContext* codecParser;
338 AVFrame* videoFrame;
339#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 133, 100)
340 AVPacket bufferpacket;
341#endif
342 AVPacket* packet;
343#if defined(WITH_FFMPEG_HWACCEL)
344 AVBufferRef* hwctx;
345 struct SwsContext* swsctx;
346 int hwFrameSupportsNativeFormat;
347 AVFrame* hwVideoFrame;
348 AVFrame* cpuVideoFrame;
349 enum AVPixelFormat hw_pix_fmt;
350#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(57, 80, 100)
351 AVBufferRef* hw_frames_ctx;
352#endif
353
354 enum AVHWDeviceType type;
355 char* device;
356 BOOL useHwDecoder;
357 BOOL useHwEncoder;
358#endif
359} H264_CONTEXT_LIBAVCODEC;
360
361static void libavcodec_destroy_encoder_context(H264_CONTEXT* WINPR_RESTRICT h264)
362{
363 if (!h264 || !h264->subsystem)
364 return;
365
366 H264_CONTEXT_LIBAVCODEC* sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
367
368 if (sys->codecEncoderContext)
369 {
370#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 69, 100)
371 avcodec_free_context(&sys->codecEncoderContext);
372#else
373 avcodec_close(sys->codecEncoderContext);
374 av_free(sys->codecEncoderContext);
375#endif
376 }
377
378 sys->codecEncoderContext = nullptr;
379}
380
381#if defined(WITH_FFMPEG_HWACCEL)
382WINPR_ATTR_NODISCARD
383static enum AVPixelFormat get_hwctx_image_format(H264_CONTEXT_LIBAVCODEC* sys, wLog* log)
384{
385 WINPR_ASSERT(sys);
386 enum AVHWDeviceType type = sys->type;
387 switch (type)
388 {
389 case AV_HWDEVICE_TYPE_VAAPI:
390 return AV_PIX_FMT_VAAPI;
391 case AV_HWDEVICE_TYPE_VIDEOTOOLBOX:
392 return AV_PIX_FMT_VIDEOTOOLBOX;
393 case AV_HWDEVICE_TYPE_VDPAU:
394 return AV_PIX_FMT_VDPAU;
395 case AV_HWDEVICE_TYPE_CUDA:
396 return AV_PIX_FMT_CUDA;
397 case AV_HWDEVICE_TYPE_DXVA2:
398 return AV_PIX_FMT_DXVA2_VLD;
399 case AV_HWDEVICE_TYPE_QSV:
400 return AV_PIX_FMT_QSV;
401#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(59, 16, 100)
402 case AV_HWDEVICE_TYPE_D3D12VA:
403 return AV_PIX_FMT_D3D12;
404#endif
405 case AV_HWDEVICE_TYPE_D3D11VA:
406 return AV_PIX_FMT_D3D11VA_VLD;
407 case AV_HWDEVICE_TYPE_DRM:
408 return AV_PIX_FMT_DRM_PRIME;
409 case AV_HWDEVICE_TYPE_OPENCL:
410 return AV_PIX_FMT_OPENCL;
411 case AV_HWDEVICE_TYPE_MEDIACODEC:
412 return AV_PIX_FMT_MEDIACODEC;
413 case AV_HWDEVICE_TYPE_VULKAN:
414 return AV_PIX_FMT_VULKAN;
415 default:
416 WLog_Print(log, WLOG_INFO, "Missing pixel format mapping for %s",
417 av_hwdevice_get_type_name(type));
418 return AV_PIX_FMT_NONE;
419 }
420}
421
422WINPR_ATTR_NODISCARD
423static const AVCodec* get_hwctx_codec(H264_CONTEXT_LIBAVCODEC* sys, wLog* log,
424 enum AVHWDeviceType type)
425{
426 WINPR_ASSERT(sys);
427 switch (type)
428 {
429 case AV_HWDEVICE_TYPE_VAAPI:
430 return avcodec_find_encoder_by_name("h264_vaapi");
431 case AV_HWDEVICE_TYPE_VULKAN:
432 return avcodec_find_encoder_by_name("h264_vulkan");
433 case AV_HWDEVICE_TYPE_D3D11VA:
434 return avcodec_find_encoder_by_name("h264_d3d11va");
435#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(59, 16, 100)
436 case AV_HWDEVICE_TYPE_D3D12VA:
437 return avcodec_find_encoder_by_name("h264_d3d12va");
438#endif
439 case AV_HWDEVICE_TYPE_VIDEOTOOLBOX:
440 return avcodec_find_encoder_by_name("h264_videotoolbox");
441 case AV_HWDEVICE_TYPE_VDPAU:
442 return avcodec_find_encoder_by_name("h264_vdpau");
443 default:
444 WLog_Print(log, WLOG_ERROR, "could not map %s to h264 encoder",
445 av_hwdevice_get_type_name(sys->type));
446 return nullptr;
447 }
448}
449
450WINPR_ATTR_NODISCARD
451static const char* get_hwctx_device(H264_CONTEXT_LIBAVCODEC* sys)
452{
453 WINPR_ASSERT(sys);
454
455 switch (sys->type)
456 {
457 case AV_HWDEVICE_TYPE_VAAPI:
458 break;
459 default:
460 return nullptr;
461 }
462
463 return sys->device;
464}
465#endif
466
467#ifdef WITH_FFMPEG_HWACCEL
468static int set_hw_frames_ctx(H264_CONTEXT* WINPR_RESTRICT h264)
469{
470 H264_CONTEXT_LIBAVCODEC* sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
471 AVBufferRef* hw_frames_ref = nullptr;
472 AVHWFramesContext* frames_ctx = nullptr;
473 int err = 0;
474
475 if (!(hw_frames_ref = av_hwframe_ctx_alloc(sys->hwctx)))
476 {
477 WLog_Print(h264->log, WLOG_ERROR, "Failed to create %s frame context",
478 av_hwdevice_get_type_name(sys->type));
479 return -1;
480 }
481 frames_ctx = (AVHWFramesContext*)(hw_frames_ref->data);
482 frames_ctx->format = get_hwctx_image_format(sys, h264->log);
483 frames_ctx->sw_format = AV_PIX_FMT_NV12;
484 frames_ctx->width = sys->codecEncoderContext->width;
485 frames_ctx->height = sys->codecEncoderContext->height;
486 frames_ctx->initial_pool_size = 20;
487 if ((err = av_hwframe_ctx_init(hw_frames_ref)) < 0)
488 {
489 WLog_Print(h264->log, WLOG_ERROR,
490 "Failed to initialize %s frame context."
491 "Error code: %s",
492 av_hwdevice_get_type_name(sys->type), av_err2str(err));
493 av_buffer_unref(&hw_frames_ref);
494 return err;
495 }
496 sys->codecEncoderContext->hw_frames_ctx = av_buffer_ref(hw_frames_ref);
497 if (!sys->codecEncoderContext->hw_frames_ctx)
498 err = AVERROR(ENOMEM);
499
500 av_buffer_unref(&hw_frames_ref);
501 return err;
502}
503#endif
504
505static BOOL libavcodec_create_encoder_context(H264_CONTEXT* WINPR_RESTRICT h264)
506{
507 BOOL recreate = FALSE;
508 H264_CONTEXT_LIBAVCODEC* sys = nullptr;
509
510 if (!h264 || !h264->subsystem)
511 return FALSE;
512
513 if ((h264->width > INT_MAX) || (h264->height > INT_MAX))
514 return FALSE;
515
516 sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
517 if (!sys || !sys->codecEncoder)
518 return FALSE;
519
520 recreate = !sys->codecEncoderContext;
521
522 if (sys->codecEncoderContext)
523 {
524 if ((sys->codecEncoderContext->width != (int)h264->width) ||
525 (sys->codecEncoderContext->height != (int)h264->height))
526 recreate = TRUE;
527 }
528
529 if (!recreate)
530 return TRUE;
531
532 libavcodec_destroy_encoder_context(h264);
533
534 sys->codecEncoderContext = avcodec_alloc_context3(sys->codecEncoder);
535
536 if (!sys->codecEncoderContext)
537 goto EXCEPTION;
538
539 switch (h264->RateControlMode)
540 {
541 case H264_RATECONTROL_VBR:
542 sys->codecEncoderContext->bit_rate = h264->BitRate;
543 break;
544
545 case H264_RATECONTROL_CQP:
546 if (av_opt_set_int(sys->codecEncoderContext, "qp", h264->QP, AV_OPT_SEARCH_CHILDREN) <
547 0)
548 {
549 WLog_Print(h264->log, WLOG_ERROR, "av_opt_set_int failed");
550 }
551 break;
552
553 default:
554 break;
555 }
556
557 sys->codecEncoderContext->width = (int)MIN(INT32_MAX, h264->width);
558 sys->codecEncoderContext->height = (int)MIN(INT32_MAX, h264->height);
559 sys->codecEncoderContext->delay = 0;
560#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(56, 13, 100)
561 sys->codecEncoderContext->framerate =
562 (AVRational){ WINPR_ASSERTING_INT_CAST(int, h264->FrameRate), 1 };
563#endif
564 sys->codecEncoderContext->time_base =
565 (AVRational){ 1, WINPR_ASSERTING_INT_CAST(int, h264->FrameRate) };
566 av_opt_set(sys->codecEncoderContext, "tune", "zerolatency", AV_OPT_SEARCH_CHILDREN);
567
568 sys->codecEncoderContext->flags |= AV_CODEC_FLAG_LOOP_FILTER;
569
570#ifdef WITH_FFMPEG_HWACCEL
571 if (sys->hwctx)
572 {
573 av_opt_set(sys->codecEncoderContext, "preset", "veryslow", AV_OPT_SEARCH_CHILDREN);
574
575 sys->codecEncoderContext->pix_fmt = get_hwctx_image_format(sys, h264->log);
576 /* set hw_frames_ctx for encoder's AVCodecContext */
577 if (set_hw_frames_ctx(h264) < 0)
578 goto EXCEPTION;
579 }
580 else
581#endif
582 {
583 av_opt_set(sys->codecEncoderContext, "preset", "medium", AV_OPT_SEARCH_CHILDREN);
584 sys->codecEncoderContext->pix_fmt = AV_PIX_FMT_YUV420P;
585 }
586
587 if (avcodec_open2(sys->codecEncoderContext, sys->codecEncoder, nullptr) < 0)
588 goto EXCEPTION;
589
590 return TRUE;
591EXCEPTION:
592 libavcodec_destroy_encoder_context(h264);
593 return FALSE;
594}
595
596#if defined(WITH_FFMPEG_HWACCEL)
597static int hw_supports_target_format(AVBufferRef* hwframectx, enum AVPixelFormat target)
598{
599 WINPR_ASSERT(hwframectx);
600
601 enum AVPixelFormat* formats = nullptr;
602 int rc = av_hwframe_transfer_get_formats(hwframectx, AV_HWFRAME_TRANSFER_DIRECTION_FROM,
603 &formats, 0);
604 if (rc == 0)
605 {
606 enum AVPixelFormat* it = formats;
607 while ((rc == 0) && (*it != AV_PIX_FMT_NONE))
608 {
609 enum AVPixelFormat cur = *it++;
610 if (cur == target)
611 rc = 1;
612 }
613 }
614
615 av_free(formats);
616 return rc;
617}
618#endif
619
620static int libavcodec_decompress(H264_CONTEXT* WINPR_RESTRICT h264,
621 const BYTE* WINPR_RESTRICT pSrcData, UINT32 SrcSize)
622{
623 union
624 {
625 const BYTE* cpv;
626 BYTE* pv;
627 } cnv;
628 int rc = -1;
629 int status = 0;
630 int gotFrame = 0;
631 AVPacket* packet = nullptr;
632
633 WINPR_ASSERT(h264);
634 WINPR_ASSERT(pSrcData || (SrcSize == 0));
635
636 H264_CONTEXT_LIBAVCODEC* sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
637 BYTE** pYUVData = h264->pYUVData;
638 UINT32* iStride = h264->iStride;
639
640 WINPR_ASSERT(sys);
641
642#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 133, 100)
643 packet = &sys->bufferpacket;
644 WINPR_ASSERT(packet);
645 av_init_packet(packet);
646#else
647 packet = av_packet_alloc();
648#endif
649 if (!packet)
650 {
651 WLog_Print(h264->log, WLOG_ERROR, "Failed to allocate AVPacket");
652 goto fail;
653 }
654
655 cnv.cpv = pSrcData;
656 packet->data = cnv.pv;
657 packet->size = (int)MIN(SrcSize, INT32_MAX);
658
659 WINPR_ASSERT(sys->codecDecoderContext);
660 /* avcodec_decode_video2 is deprecated with libavcodec 57.48.101 */
661#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 48, 101)
662 status = avcodec_send_packet(sys->codecDecoderContext, packet);
663
664 if (status < 0)
665 {
666 WLog_Print(h264->log, WLOG_ERROR, "Failed to decode video frame (status=%d)", status);
667 goto fail;
668 }
669
670 sys->videoFrame->format = AV_PIX_FMT_YUV420P;
671
672#if defined(WITH_FFMPEG_HWACCEL)
673 status = avcodec_receive_frame(sys->codecDecoderContext, (sys->useHwDecoder && sys->hwctx)
674 ? sys->hwVideoFrame
675 : sys->videoFrame);
676#else
677 status = avcodec_receive_frame(sys->codecDecoderContext, sys->videoFrame);
678#endif
679 if (status == AVERROR(EAGAIN))
680 {
681 rc = 0;
682 goto fail;
683 }
684
685 if (status == 0)
686 gotFrame = 1;
687#else
688#if defined(WITH_FFMPEG_HWACCEL)
689 status =
690 avcodec_decode_video2(sys->codecDecoderContext,
691 sys->hwctx ? sys->hwVideoFrame : sys->videoFrame, &gotFrame, packet);
692#else
693 status = avcodec_decode_video2(sys->codecDecoderContext, sys->videoFrame, &gotFrame, packet);
694#endif
695#endif
696 if (status < 0)
697 {
698 WLog_Print(h264->log, WLOG_ERROR, "Failed to decode video frame (status=%d)", status);
699 goto fail;
700 }
701
702#if defined(WITH_FFMPEG_HWACCEL)
703
704 if (sys->hwctx)
705 {
706 AVFrame* target = sys->videoFrame;
707 if (sys->hwFrameSupportsNativeFormat < 0)
708 {
709 const int res = hw_supports_target_format(sys->codecDecoderContext->hw_frames_ctx,
710 sys->videoFrame->format);
711 if (res < 0)
712 status = res;
713 else
714 {
715 sys->hwFrameSupportsNativeFormat = res;
716 const char* msg = "hardware supported copy";
717 if (res == 0)
718 msg = "swscale format conversion";
719 WLog_Print(h264->log, WLOG_DEBUG, "formats: { src:%s, dst:%s } using %s",
720 av_format_str(sys->hwVideoFrame->format),
721 av_format_str(sys->videoFrame->format), msg);
722 }
723 }
724
725 if (sys->hwFrameSupportsNativeFormat == 0)
726 target = sys->cpuVideoFrame;
727
728 if (sys->hwFrameSupportsNativeFormat >= 0)
729 {
730 if (sys->hwVideoFrame->format == sys->hw_pix_fmt)
731 {
732 target->width = sys->hwVideoFrame->width;
733 target->height = sys->hwVideoFrame->height;
734 status = av_hwframe_transfer_data(target, sys->hwVideoFrame, 0);
735 }
736 else
737 status = av_frame_copy(target, sys->hwVideoFrame);
738 }
739
740 gotFrame = (status == 0);
741
742 if (status < 0)
743 {
744 WLog_Print(h264->log, WLOG_ERROR, "Failed to transfer video frame (status=%d) (%s)",
745 status, av_err2str(status));
746 goto fail;
747 }
748
749 if (gotFrame && (target != sys->videoFrame))
750 {
751 if (!sys->swsctx)
752 {
753 sys->swsctx = sws_getContext(target->width, target->height, target->format,
754 target->width, target->height, sys->videoFrame->format,
755 SWS_BILINEAR, nullptr, nullptr, nullptr);
756 if (!sys->swsctx)
757 goto fail;
758 }
759
760#if LIBSWSCALE_VERSION_INT >= AV_VERSION_INT(6, 4, 100)
761 status = sws_scale_frame(sys->swsctx, sys->videoFrame, target);
762#else
763#error "Need SWScale >= 6.4.100"
764#endif
765
766 if (status < 0)
767 {
768 WLog_Print(h264->log, WLOG_ERROR, "Failed to convert video frame (status=%d) (%s)",
769 status, av_err2str(status));
770 goto fail;
771 }
772 }
773 }
774
775#endif
776
777 if (gotFrame)
778 {
779 WINPR_ASSERT(sys->videoFrame);
780
781 pYUVData[0] = sys->videoFrame->data[0];
782 pYUVData[1] = sys->videoFrame->data[1];
783 pYUVData[2] = sys->videoFrame->data[2];
784 iStride[0] = (UINT32)MAX(0, sys->videoFrame->linesize[0]);
785 iStride[1] = (UINT32)MAX(0, sys->videoFrame->linesize[1]);
786 iStride[2] = (UINT32)MAX(0, sys->videoFrame->linesize[2]);
787
788 h264->YUVWidth = WINPR_ASSERTING_INT_CAST(UINT32, sys->videoFrame->width);
789 h264->YUVHeight = WINPR_ASSERTING_INT_CAST(UINT32, sys->videoFrame->height);
790 rc = 1;
791 }
792 else
793 rc = -2;
794
795fail:
796#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 133, 100)
797 av_packet_unref(packet);
798#else
799 av_packet_free(&packet);
800#endif
801
802 return rc;
803}
804
805static int libavcodec_compress(H264_CONTEXT* WINPR_RESTRICT h264,
806 const BYTE** WINPR_RESTRICT pSrcYuv,
807 const UINT32* WINPR_RESTRICT pStride,
808 BYTE** WINPR_RESTRICT ppDstData, UINT32* WINPR_RESTRICT pDstSize)
809{
810 union
811 {
812 const BYTE* cpv;
813 uint8_t* pv;
814 } cnv;
815 int rc = -1;
816 int status = 0;
817 int gotFrame = 0;
818
819 WINPR_ASSERT(h264);
820
821 H264_CONTEXT_LIBAVCODEC* sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
822 WINPR_ASSERT(sys);
823
824 if (!libavcodec_create_encoder_context(h264))
825 return -1;
826
827#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 133, 100)
828 sys->packet = &sys->bufferpacket;
829 av_packet_unref(sys->packet);
830 av_init_packet(sys->packet);
831#else
832 av_packet_free(&sys->packet);
833 sys->packet = av_packet_alloc();
834#endif
835 if (!sys->packet)
836 {
837 WLog_Print(h264->log, WLOG_ERROR, "Failed to allocate AVPacket");
838 goto fail;
839 }
840
841 WINPR_ASSERT(sys->packet);
842 sys->packet->data = nullptr;
843 sys->packet->size = 0;
844
845 WINPR_ASSERT(sys->videoFrame);
846 WINPR_ASSERT(sys->codecEncoderContext);
847 sys->videoFrame->format = AV_PIX_FMT_YUV420P;
848 sys->videoFrame->width = sys->codecEncoderContext->width;
849 sys->videoFrame->height = sys->codecEncoderContext->height;
850#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(52, 48, 100)
851 sys->videoFrame->colorspace = AVCOL_SPC_BT709;
852#endif
853#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(52, 92, 100)
854 sys->videoFrame->chroma_location = AVCHROMA_LOC_LEFT;
855#endif
856 cnv.cpv = pSrcYuv[0];
857 sys->videoFrame->data[0] = cnv.pv;
858
859 cnv.cpv = pSrcYuv[1];
860 sys->videoFrame->data[1] = cnv.pv;
861
862 cnv.cpv = pSrcYuv[2];
863 sys->videoFrame->data[2] = cnv.pv;
864
865 sys->videoFrame->linesize[0] = (int)pStride[0];
866 sys->videoFrame->linesize[1] = (int)pStride[1];
867 sys->videoFrame->linesize[2] = (int)pStride[2];
868 sys->videoFrame->pts++;
869
870#ifdef WITH_FFMPEG_HWACCEL
871 if (sys->useHwEncoder && sys->hwctx)
872 {
873 av_frame_unref(sys->hwVideoFrame);
874 if ((status = av_hwframe_get_buffer(sys->codecEncoderContext->hw_frames_ctx,
875 sys->hwVideoFrame, 0)) < 0 ||
876 !sys->hwVideoFrame->hw_frames_ctx)
877 {
878 WLog_Print(h264->log, WLOG_ERROR, "av_hwframe_get_buffer failed (%s [%d])",
879 av_err2str(status), status);
880 goto fail;
881 }
882 sys->videoFrame->format = AV_PIX_FMT_NV12;
883 if ((status = av_hwframe_transfer_data(sys->hwVideoFrame, sys->videoFrame, 0)) < 0)
884 {
885 WLog_Print(h264->log, WLOG_ERROR, "av_hwframe_transfer_data failed (%s [%d])",
886 av_err2str(status), status);
887 goto fail;
888 }
889 }
890#endif
891
892 /* avcodec_encode_video2 is deprecated with libavcodec 57.48.101 */
893#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 48, 101)
894#ifdef WITH_FFMPEG_HWACCEL
895 status = avcodec_send_frame(sys->codecEncoderContext,
896 sys->hwctx ? sys->hwVideoFrame : sys->videoFrame);
897#else
898 status = avcodec_send_frame(sys->codecEncoderContext, sys->videoFrame);
899#endif
900
901 if (status < 0)
902 {
903 WLog_Print(h264->log, WLOG_ERROR, "Failed to encode video frame (%s [%d])",
904 av_err2str(status), status);
905 goto fail;
906 }
907
908 status = avcodec_receive_packet(sys->codecEncoderContext, sys->packet);
909
910 if (status < 0)
911 {
912 WLog_Print(h264->log, WLOG_ERROR, "Failed to encode video frame (%s [%d])",
913 av_err2str(status), status);
914 goto fail;
915 }
916
917 gotFrame = (status == 0);
918#elif LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(54, 59, 100)
919
920 do
921 {
922 status = avcodec_encode_video2(sys->codecEncoderContext, sys->packet, sys->videoFrame,
923 &gotFrame);
924 } while ((status >= 0) && (gotFrame == 0));
925
926#else
927 sys->packet->size =
928 avpicture_get_size(sys->codecDecoderContext->pix_fmt, sys->codecDecoderContext->width,
929 sys->codecDecoderContext->height);
930 sys->packet->data = av_malloc(sys->packet->size);
931
932 if (!sys->packet->data)
933 status = -1;
934 else
935 {
936 status = avcodec_encode_video(sys->codecDecoderContext, sys->packet->data,
937 sys->packet->size, sys->videoFrame);
938 }
939
940#endif
941
942 if (status < 0)
943 {
944 WLog_Print(h264->log, WLOG_ERROR, "Failed to encode video frame (%s [%d])",
945 av_err2str(status), status);
946 goto fail;
947 }
948
949 WINPR_ASSERT(sys->packet);
950 *ppDstData = sys->packet->data;
951 *pDstSize = (UINT32)MAX(0, sys->packet->size);
952
953 if (!gotFrame)
954 {
955 WLog_Print(h264->log, WLOG_ERROR, "Did not get frame! (%s [%d])", av_err2str(status),
956 status);
957 rc = -2;
958 }
959 else
960 rc = 1;
961fail:
962 return rc;
963}
964
965static void libavcodec_uninit(H264_CONTEXT* h264)
966{
967 WINPR_ASSERT(h264);
968
969 H264_CONTEXT_LIBAVCODEC* sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
970
971 if (!sys)
972 return;
973
974 if (sys->packet)
975 {
976#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 133, 100)
977 av_packet_unref(sys->packet);
978#else
979 av_packet_free(&sys->packet);
980#endif
981 }
982
983 if (sys->videoFrame)
984 {
985#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 18, 102)
986 av_frame_free(&sys->videoFrame);
987#else
988 av_free(sys->videoFrame);
989#endif
990 }
991
992#if defined(WITH_FFMPEG_HWACCEL)
993 if (sys->hwVideoFrame)
994 {
995#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 18, 102)
996 av_frame_free(&sys->hwVideoFrame);
997#else
998 av_free(sys->hwVideoFrame);
999 sys->hwVideoFrame = nullptr;
1000#endif
1001 }
1002
1003 if (sys->cpuVideoFrame)
1004 {
1005#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 18, 102)
1006 av_frame_free(&sys->cpuVideoFrame);
1007#else
1008 av_free(sys->cpuVideoFrame);
1009#endif
1010 }
1011
1012 if (sys->hwctx)
1013 av_buffer_unref(&sys->hwctx);
1014 sws_freeContext(sys->swsctx);
1015
1016#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(57, 80, 100)
1017
1018 if (sys->hw_frames_ctx)
1019 av_buffer_unref(&sys->hw_frames_ctx);
1020
1021#endif
1022
1023#endif
1024
1025 if (sys->codecParser)
1026 av_parser_close(sys->codecParser);
1027
1028 if (sys->codecDecoderContext)
1029 {
1030#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 69, 100)
1031 avcodec_free_context(&sys->codecDecoderContext);
1032#else
1033 avcodec_close(sys->codecDecoderContext);
1034 av_free(sys->codecDecoderContext);
1035#endif
1036 }
1037
1038 libavcodec_destroy_encoder_context(h264);
1039#if defined(WITH_FFMPEG_HWACCEL)
1040 free(sys->device);
1041#endif
1042 free(sys);
1043 h264->pSystemData = nullptr;
1044}
1045
1046#if defined(WITH_FFMPEG_HWACCEL)
1047static BOOL hwctx_supports_dimensions(H264_CONTEXT_LIBAVCODEC* sys, int width, int height)
1048{
1049 WINPR_ASSERT(sys);
1050
1051 AVHWFramesConstraints* constraints = av_hwdevice_get_hwframe_constraints(sys->hwctx, nullptr);
1052 if (!constraints)
1053 return TRUE;
1054
1055 const BOOL supported = (width >= constraints->min_width) &&
1056 (height >= constraints->min_height) &&
1057 ((constraints->max_width == 0) || (width <= constraints->max_width)) &&
1058 ((constraints->max_height == 0) || (height <= constraints->max_height));
1059 av_hwframe_constraints_free(&constraints);
1060 return supported;
1061}
1062
1063WINPR_ATTR_NODISCARD
1064static const char* vaapi_vendor(H264_CONTEXT_LIBAVCODEC* sys)
1065{
1066 WINPR_ASSERT(sys);
1067
1068 const char* vendor = "UNKNOWN";
1069#if 0
1070 if (!hwctx)
1071 return vendor;
1072
1073 const AVHWDeviceContext* devctx = (const AVHWDeviceContext*)hwctx->data;
1074 if (!devctx)
1075 return vendor;
1076
1077 const AVVAAPIDeviceContext* ctx = devctx->hwctx;
1078 if (!ctx)
1079 return vendor;
1080
1081 const char* vavendor = vaQueryVendorString(ctx->display);
1082 if (vavendor)
1083 return vavendor;
1084#endif
1085 return vendor;
1086}
1087#endif
1088#if defined(WITH_FFMPEG_HWACCEL)
1089WINPR_ATTR_NODISCARD
1090static enum AVPixelFormat libavcodec_get_format(struct AVCodecContext* ctx,
1091 const enum AVPixelFormat* fmts)
1092{
1093 WINPR_ASSERT(ctx);
1094
1095 H264_CONTEXT* h264 = (H264_CONTEXT*)ctx->opaque;
1096 WINPR_ASSERT(h264);
1097
1098 H264_CONTEXT_LIBAVCODEC* sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
1099 WINPR_ASSERT(sys);
1100 if ((ctx->coded_width > 0) && (ctx->coded_height > 0) &&
1101 !hwctx_supports_dimensions(sys, ctx->coded_width, ctx->coded_height))
1102 {
1103 WLog_Print(h264->log, WLOG_WARN,
1104 "Hardware decoder does not support %dx%d, falling back to software",
1105 ctx->coded_width, ctx->coded_height);
1106 sys->useHwDecoder = FALSE;
1107 av_buffer_unref(&ctx->hw_device_ctx);
1108 for (const enum AVPixelFormat* p = fmts; *p != AV_PIX_FMT_NONE; p++)
1109 {
1110 if (*p != sys->hw_pix_fmt)
1111 return *p;
1112 }
1113 return AV_PIX_FMT_NONE;
1114 }
1115
1116 for (const enum AVPixelFormat* p = fmts; *p != AV_PIX_FMT_NONE; p++)
1117 {
1118 if (*p == sys->hw_pix_fmt)
1119 {
1120#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(57, 80, 100)
1121 if (sys->hw_frames_ctx)
1122 av_buffer_unref(&sys->hw_frames_ctx);
1123
1124 sys->hw_frames_ctx = av_hwframe_ctx_alloc(sys->hwctx);
1125
1126 if (!sys->hw_frames_ctx)
1127 {
1128 return AV_PIX_FMT_NONE;
1129 }
1130
1131 sys->codecDecoderContext->pix_fmt = *p;
1132 AVHWFramesContext* frames = (AVHWFramesContext*)sys->hw_frames_ctx->data;
1133 frames->format = *p;
1134 frames->height = sys->codecDecoderContext->coded_height;
1135 frames->width = sys->codecDecoderContext->coded_width;
1136#ifdef WITH_VIDEOTOOLBOX
1137 frames->sw_format = AV_PIX_FMT_YUV420P;
1138#else
1139 frames->sw_format =
1140 (sys->codecDecoderContext->sw_pix_fmt == AV_PIX_FMT_YUV420P10 ? AV_PIX_FMT_P010
1141 : AV_PIX_FMT_NV12);
1142#endif
1143 frames->initial_pool_size = 20;
1144
1145 if (sys->codecDecoderContext->active_thread_type & FF_THREAD_FRAME)
1146 frames->initial_pool_size += sys->codecDecoderContext->thread_count;
1147
1148 int err = av_hwframe_ctx_init(sys->hw_frames_ctx);
1149
1150 if (err < 0)
1151 {
1152 WLog_Print(h264->log, WLOG_ERROR, "Could not init hwframes context: %s",
1153 av_err2str(err));
1154 return AV_PIX_FMT_NONE;
1155 }
1156
1157 sys->codecDecoderContext->hw_frames_ctx = av_buffer_ref(sys->hw_frames_ctx);
1158#endif
1159 return *p;
1160 }
1161 }
1162
1163 return AV_PIX_FMT_NONE;
1164}
1165
1166WINPR_ATTR_MALLOC(WINPR_JSON_Delete, 1)
1167static WINPR_JSON* load(BOOL system)
1168{
1169 return freerdp_GetJSONConfigFile(system, "ffmpeg.json");
1170}
1171
1172static BOOL read_json_bool(WINPR_JSON* json, const char* what, BOOL* pValue)
1173{
1174 WINPR_ASSERT(json);
1175 WINPR_ASSERT(what);
1176 WINPR_ASSERT(pValue);
1177
1178 WINPR_JSON* val = WINPR_JSON_GetObjectItemCaseSensitive(json, what);
1179 if (!val)
1180 return FALSE;
1181 if (!WINPR_JSON_IsBool(val))
1182 return FALSE;
1183 *pValue = WINPR_JSON_IsTrue(val);
1184 return TRUE;
1185}
1186
1187static BOOL read_json_const_string(WINPR_JSON* json, const char* what, const char** pValue)
1188{
1189 WINPR_ASSERT(json);
1190 WINPR_ASSERT(what);
1191 WINPR_ASSERT(pValue);
1192
1193 WINPR_JSON* val = WINPR_JSON_GetObjectItemCaseSensitive(json, what);
1194 if (!val)
1195 return FALSE;
1196 if (!WINPR_JSON_IsString(val))
1197 return FALSE;
1198 *pValue = WINPR_JSON_GetStringValue(val);
1199 return *pValue != nullptr;
1200}
1201
1202static BOOL read_json_string(WINPR_JSON* json, const char* what, char** pValue)
1203{
1204 WINPR_ASSERT(json);
1205 WINPR_ASSERT(what);
1206 WINPR_ASSERT(pValue);
1207
1208 WINPR_JSON* val = WINPR_JSON_GetObjectItemCaseSensitive(json, what);
1209 if (!val)
1210 return FALSE;
1211 if (!WINPR_JSON_IsString(val))
1212 return FALSE;
1213 const char* str = nullptr;
1214 const BOOL rc = read_json_const_string(json, what, &str);
1215 if (!rc)
1216 return FALSE;
1217 if (!str)
1218 return FALSE;
1219 free(*pValue);
1220 *pValue = _strdup(str);
1221 return *pValue != nullptr;
1222}
1223
1224static BOOL load_config_json(BOOL system, H264_CONTEXT_LIBAVCODEC* sys, BOOL* pAllowUserConfig,
1225 BOOL* pAllowEnvOverride)
1226{
1227 WINPR_ASSERT(sys);
1228 WINPR_ASSERT(pAllowUserConfig);
1229 WINPR_ASSERT(pAllowEnvOverride);
1230
1231 WINPR_JSON* syscfg = load(system);
1232 if (syscfg)
1233 {
1234 read_json_bool(syscfg, "allowUserConfig", pAllowUserConfig);
1235 read_json_bool(syscfg, "allowEnvOverrides", pAllowEnvOverride);
1236
1237 read_json_string(syscfg, "hwDeviceName", &sys->device);
1238
1239 const char* str = nullptr;
1240 if (read_json_const_string(syscfg, "hwDeviceType", &str))
1241 sys->type = av_hwdevice_find_type_by_name(str);
1242 read_json_bool(syscfg, "useHwDecoder", &sys->useHwDecoder);
1243 read_json_bool(syscfg, "useHwEncoder", &sys->useHwEncoder);
1244 WINPR_JSON_Delete(syscfg);
1245 }
1246 return TRUE;
1247}
1248#endif
1249
1250WINPR_ATTR_NODISCARD
1251static BOOL load_config(H264_CONTEXT_LIBAVCODEC* sys, BOOL enabled, wLog* log)
1252{
1253 WINPR_ASSERT(sys);
1254
1255#if defined(WITH_FFMPEG_HWACCEL)
1256 sys->useHwDecoder = TRUE;
1257 sys->useHwEncoder = TRUE;
1258#if defined(__APPLE__)
1259 sys->type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX;
1260#elif defined(__ANDROID__)
1261 sys->type = AV_HWDEVICE_TYPE_MEDIACODEC;
1262#elif defined(_WIN32)
1263 sys->type = AV_HWDEVICE_TYPE_DXVA2;
1264#elif defined(__linux__)
1265 sys->type = AV_HWDEVICE_TYPE_VAAPI;
1266 sys->device = _strdup("/dev/dri/renderD128");
1267 if (!sys->device)
1268 return FALSE;
1269#else
1270 sys->type = AV_HWDEVICE_TYPE_NONE;
1271#endif
1272
1273 BOOL allowUserConfig = TRUE;
1274 BOOL allowEnvOverrides = TRUE;
1275
1276 load_config_json(TRUE, sys, &allowUserConfig, &allowEnvOverrides);
1277 if (allowUserConfig)
1278 load_config_json(FALSE, sys, &allowUserConfig, &allowEnvOverrides);
1279
1280 if (allowEnvOverrides)
1281 {
1282 // NOLINTNEXTLINE(concurrency-mt-unsafe)
1283 const char* env = getenv("FREERDP_HWCTX_DEVICE");
1284 if (!env)
1285 env = getenv("FREERDP_VAAPI_DEVICE");
1286
1287 if (env)
1288 {
1289 free(sys->device);
1290 sys->device = _strdup(env);
1291 if (!sys->device)
1292 {
1293 WLog_Print(log, WLOG_ERROR,
1294 "Failed to copy value of FREERDP_HWCTX_TYPE=%s, aborting", env);
1295 return FALSE;
1296 }
1297 }
1298
1299 env = getenv("FREERDP_HWCTX_TYPE");
1300 if (env)
1301 {
1302 sys->type = av_hwdevice_find_type_by_name(env);
1303 if (sys->type == AV_HWDEVICE_TYPE_NONE)
1304 {
1305 WLog_Print(log, WLOG_ERROR, "Unsupported value for FREERDP_HWCTX_TYPE=%s, aborting",
1306 env);
1307 return FALSE;
1308 }
1309 }
1310 }
1311
1312 // Fallback in case of an invalid type
1313 if (!enabled || (sys->type == AV_HWDEVICE_TYPE_NONE))
1314 {
1315 sys->useHwDecoder = FALSE;
1316 sys->useHwEncoder = FALSE;
1317 }
1318#else
1319 WINPR_UNUSED(enabled);
1320 WINPR_UNUSED(log);
1321 WINPR_UNUSED(sys);
1322#endif
1323 return TRUE;
1324}
1325
1326#if defined(WITH_FFMPEG_HWACCEL)
1327WINPR_ATTR_NODISCARD
1328static const char* bool_to_string(BOOL val)
1329{
1330 if (val)
1331 return "true";
1332 return "false";
1333}
1334#endif
1335
1336static BOOL libavcodec_init(H264_CONTEXT* h264)
1337{
1338 WINPR_ASSERT(h264);
1339 H264_CONTEXT_LIBAVCODEC* sys =
1340 (H264_CONTEXT_LIBAVCODEC*)calloc(1, sizeof(H264_CONTEXT_LIBAVCODEC));
1341
1342 if (!sys)
1343 goto EXCEPTION;
1344
1345 if (!load_config(sys, h264->hwAccel > 0, h264->log))
1346 goto EXCEPTION;
1347
1348 h264->pSystemData = (void*)sys;
1349
1350#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 10, 100)
1351 avcodec_register_all();
1352#endif
1353
1354 if (!h264->Compressor)
1355 {
1356 sys->codecDecoder = avcodec_find_decoder(AV_CODEC_ID_H264);
1357
1358 if (!sys->codecDecoder)
1359 {
1360 WLog_Print(h264->log, WLOG_ERROR, "Failed to find libav H.264 codec");
1361 goto EXCEPTION;
1362 }
1363
1364 sys->codecDecoderContext = avcodec_alloc_context3(sys->codecDecoder);
1365
1366 if (!sys->codecDecoderContext)
1367 {
1368 WLog_Print(h264->log, WLOG_ERROR, "Failed to allocate libav codec context");
1369 goto EXCEPTION;
1370 }
1371
1372#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(59, 18, 100)
1373 if (sys->codecDecoder->capabilities & AV_CODEC_CAP_TRUNCATED)
1374 {
1375 sys->codecDecoderContext->flags |= AV_CODEC_FLAG_TRUNCATED;
1376 }
1377#endif
1378
1379#ifdef WITH_FFMPEG_HWACCEL
1380 if (sys->useHwDecoder && (sys->type != AV_HWDEVICE_TYPE_NONE))
1381 WLog_Print(h264->log, WLOG_DEBUG,
1382 "Trying to initialize hardware backed decoder of type %s",
1383 av_hwdevice_get_type_name(sys->type));
1384 else
1385 WLog_Print(h264->log, WLOG_DEBUG,
1386 "Skipping hardware backed decoder (useHwDecoder=%s, type=%s)",
1387 bool_to_string(sys->useHwDecoder), av_hwdevice_get_type_name(sys->type));
1388
1389 if (sys->useHwDecoder && (sys->type != AV_HWDEVICE_TYPE_NONE))
1390 {
1391 sys->hwFrameSupportsNativeFormat = -1;
1392 enum AVHWDeviceType type = sys->type;
1393 if (!sys->hwctx)
1394 {
1395 int ret =
1396 av_hwdevice_ctx_create(&sys->hwctx, type, get_hwctx_device(sys), nullptr, 0);
1397
1398 if (ret < 0)
1399 {
1400 WLog_Print(h264->log, WLOG_ERROR,
1401 "Could not initialize hardware decoder for %s, falling back to "
1402 "software: %s",
1403 get_hwctx_device(sys), av_err2str(ret));
1404 sys->hwctx = nullptr;
1405 goto fail_hwdevice_create;
1406 }
1407 }
1408 WLog_Print(h264->log, WLOG_INFO, "Using %s [%s|%s] for accelerated H264 decoding",
1409 av_hwdevice_get_type_name(type), get_hwctx_device(sys), vaapi_vendor(sys));
1410
1411 sys->codecDecoderContext->get_format = libavcodec_get_format;
1412 sys->hw_pix_fmt = get_hwctx_image_format(sys, h264->log);
1413
1414#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 80, 100)
1415 sys->codecDecoderContext->hw_device_ctx = av_buffer_ref(sys->hwctx);
1416#endif
1417 sys->codecDecoderContext->opaque = (void*)h264;
1418
1419 fail_hwdevice_create:
1420 }
1421#endif
1422
1423 if (avcodec_open2(sys->codecDecoderContext, sys->codecDecoder, nullptr) < 0)
1424 {
1425 WLog_Print(h264->log, WLOG_ERROR, "Failed to open libav codec");
1426 goto EXCEPTION;
1427 }
1428
1429 sys->codecParser = av_parser_init(AV_CODEC_ID_H264);
1430
1431 if (!sys->codecParser)
1432 {
1433 WLog_Print(h264->log, WLOG_ERROR, "Failed to initialize libav parser");
1434 goto EXCEPTION;
1435 }
1436 }
1437 else
1438 {
1439#ifdef WITH_FFMPEG_HWACCEL
1440 if (sys->useHwEncoder && (sys->type != AV_HWDEVICE_TYPE_NONE))
1441 WLog_Print(h264->log, WLOG_DEBUG,
1442 "Trying to initialize hardware backed encoder of type %s",
1443 av_hwdevice_get_type_name(sys->type));
1444 else
1445 WLog_Print(h264->log, WLOG_DEBUG,
1446 "Skipping hardware backed encoder (useHwEncoder=%s, type=%s)",
1447 bool_to_string(sys->useHwEncoder), av_hwdevice_get_type_name(sys->type));
1448
1449 if (sys->useHwEncoder) /* user requested hw accel */
1450 {
1451 enum AVHWDeviceType type = sys->type;
1452
1453 sys->codecEncoder = get_hwctx_codec(sys, h264->log, type);
1454 if (!sys->codecEncoder)
1455 {
1456 WLog_Print(h264->log, WLOG_ERROR, "H264 hardware encoder not found");
1457 }
1458 else if (av_hwdevice_ctx_create(&sys->hwctx, type, get_hwctx_device(sys), nullptr, 0) <
1459 0)
1460 {
1461 WLog_Print(h264->log, WLOG_ERROR, "av_hwdevice_ctx_create(%s) failed",
1462 get_hwctx_device(sys));
1463 sys->codecEncoder = nullptr;
1464 sys->hwctx = nullptr;
1465 }
1466 else
1467 {
1468 WLog_Print(h264->log, WLOG_INFO, "Using %s[%s|%s] for accelerated H264 encoding",
1469 av_hwdevice_get_type_name(sys->type), get_hwctx_device(sys),
1470 vaapi_vendor(sys));
1471 }
1472 }
1473#endif
1474 if (!sys->codecEncoder)
1475 {
1476 sys->codecEncoder = avcodec_find_encoder(AV_CODEC_ID_H264);
1477 h264->hwAccel = FALSE; /* not supported */
1478 }
1479
1480 if (!sys->codecEncoder)
1481 {
1482 WLog_Print(h264->log, WLOG_ERROR, "Failed to initialize H264 encoder");
1483 goto EXCEPTION;
1484 }
1485 }
1486
1487#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 18, 102)
1488 sys->videoFrame = av_frame_alloc();
1489#if defined(WITH_FFMPEG_HWACCEL)
1490 sys->hwVideoFrame = av_frame_alloc();
1491 sys->cpuVideoFrame = av_frame_alloc();
1492#endif
1493#else
1494 sys->videoFrame = avcodec_alloc_frame();
1495#endif
1496
1497 if (!sys->videoFrame)
1498 {
1499 WLog_Print(h264->log, WLOG_ERROR, "Failed to allocate libav frame");
1500 goto EXCEPTION;
1501 }
1502
1503#if defined(WITH_FFMPEG_HWACCEL)
1504 if (!sys->hwVideoFrame || !sys->cpuVideoFrame)
1505 {
1506 WLog_Print(h264->log, WLOG_ERROR, "Failed to allocate libav hw frame");
1507 goto EXCEPTION;
1508 }
1509
1510#endif
1511 sys->videoFrame->pts = 0;
1512 return TRUE;
1513EXCEPTION:
1514 libavcodec_uninit(h264);
1515 return FALSE;
1516}
1517
1518const H264_CONTEXT_SUBSYSTEM g_Subsystem_libavcodec = { "libavcodec", libavcodec_init,
1519 libavcodec_uninit, libavcodec_decompress,
1520 libavcodec_compress };
WINPR_JSON_IsBool
WINPR_ATTR_NODISCARD WINPR_API BOOL WINPR_JSON_IsBool(const WINPR_JSON *item)
Check if JSON item is of type BOOL.
Definition c-json.c:167
WINPR_JSON_IsTrue
WINPR_ATTR_NODISCARD WINPR_API BOOL WINPR_JSON_IsTrue(const WINPR_JSON *item)
Check if JSON item is BOOL value True.
Definition c-json.c:162
WINPR_JSON_GetObjectItemCaseSensitive
WINPR_ATTR_NODISCARD WINPR_API WINPR_JSON * WINPR_JSON_GetObjectItemCaseSensitive(const WINPR_JSON *object, const char *string)
Same as WINPR_JSON_GetObjectItem but with case sensitive matching.
Definition c-json.c:127
WINPR_JSON_IsString
WINPR_ATTR_NODISCARD WINPR_API BOOL WINPR_JSON_IsString(const WINPR_JSON *item)
Check if JSON item is of type String.
Definition c-json.c:182
WINPR_JSON_Delete
WINPR_API void WINPR_JSON_Delete(WINPR_JSON *item)
Delete a WinPR JSON wrapper object.
Definition c-json.c:103
WINPR_JSON_GetStringValue
WINPR_ATTR_NODISCARD WINPR_API const char * WINPR_JSON_GetStringValue(WINPR_JSON *item)
Return the String value of a JSON item.
Definition c-json.c:142