FreeRDP
Loading...
Searching...
No Matches
h264_ffmpeg.c
1
21#include <freerdp/config.h>
22
23#include <winpr/wlog.h>
24#include <winpr/json.h>
25
26#include <freerdp/log.h>
27#include <freerdp/codec/h264.h>
28#include <freerdp/utils/helpers.h>
29#include <libavcodec/avcodec.h>
30#include <libavutil/opt.h>
31#include <libswscale/swscale.h>
32
33#include "h264.h"
34
35#ifdef WITH_FFMPEG_HWACCEL
36#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(55, 9, 0)
37#include <libavutil/hwcontext.h>
38#else
39#pragma warning You have asked for VA - API decoding, \
40 but your version of libavutil is too old !Disabling.
41#undef WITH_FFMPEG_HWACCEL
42#endif
43#endif
44
45/* Fallback support for older libavcodec versions */
46#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(54, 59, 100)
47#define AV_CODEC_ID_H264 CODEC_ID_H264
48#endif
49
50#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(56, 34, 2)
51#define AV_CODEC_FLAG_LOOP_FILTER CODEC_FLAG_LOOP_FILTER
52#define AV_CODEC_CAP_TRUNCATED CODEC_CAP_TRUNCATED
53#define AV_CODEC_FLAG_TRUNCATED CODEC_FLAG_TRUNCATED
54#endif
55
56#if LIBAVUTIL_VERSION_MAJOR < 52
57#define AV_PIX_FMT_YUV420P PIX_FMT_YUV420P
58#endif
59
60#if defined(WITH_FFMPEG_HWACCEL)
61static const char* av_format_str(int32_t format)
62{
63#define EVCASE(x) \
64 case x: \
65 return #x
66
67 switch (format)
68 {
69 EVCASE(AV_PIX_FMT_NONE);
70 EVCASE(AV_PIX_FMT_YUV420P);
71 EVCASE(AV_PIX_FMT_YUYV422);
72 EVCASE(AV_PIX_FMT_RGB24);
73 EVCASE(AV_PIX_FMT_BGR24);
74 EVCASE(AV_PIX_FMT_YUV422P);
75 EVCASE(AV_PIX_FMT_YUV444P);
76 EVCASE(AV_PIX_FMT_YUV410P);
77 EVCASE(AV_PIX_FMT_YUV411P);
78 EVCASE(AV_PIX_FMT_GRAY8);
79 EVCASE(AV_PIX_FMT_MONOWHITE);
80 EVCASE(AV_PIX_FMT_MONOBLACK);
81 EVCASE(AV_PIX_FMT_PAL8);
82 EVCASE(AV_PIX_FMT_YUVJ420P);
83 EVCASE(AV_PIX_FMT_YUVJ422P);
84 EVCASE(AV_PIX_FMT_YUVJ444P);
85 EVCASE(AV_PIX_FMT_UYVY422);
86 EVCASE(AV_PIX_FMT_UYYVYY411);
87 EVCASE(AV_PIX_FMT_BGR8);
88 EVCASE(AV_PIX_FMT_BGR4);
89 EVCASE(AV_PIX_FMT_BGR4_BYTE);
90 EVCASE(AV_PIX_FMT_RGB8);
91 EVCASE(AV_PIX_FMT_RGB4);
92 EVCASE(AV_PIX_FMT_RGB4_BYTE);
93 EVCASE(AV_PIX_FMT_NV12);
94 EVCASE(AV_PIX_FMT_NV21);
95 EVCASE(AV_PIX_FMT_ARGB);
96 EVCASE(AV_PIX_FMT_RGBA);
97 EVCASE(AV_PIX_FMT_ABGR);
98 EVCASE(AV_PIX_FMT_BGRA);
99 EVCASE(AV_PIX_FMT_GRAY16BE);
100 EVCASE(AV_PIX_FMT_GRAY16LE);
101 EVCASE(AV_PIX_FMT_YUV440P);
102 EVCASE(AV_PIX_FMT_YUVJ440P);
103 EVCASE(AV_PIX_FMT_YUVA420P);
104 EVCASE(AV_PIX_FMT_RGB48BE);
105 EVCASE(AV_PIX_FMT_RGB48LE);
106 EVCASE(AV_PIX_FMT_RGB565BE);
107 EVCASE(AV_PIX_FMT_RGB565LE);
108 EVCASE(AV_PIX_FMT_RGB555BE);
109 EVCASE(AV_PIX_FMT_RGB555LE);
110 EVCASE(AV_PIX_FMT_BGR565BE);
111 EVCASE(AV_PIX_FMT_BGR565LE);
112 EVCASE(AV_PIX_FMT_BGR555BE);
113 EVCASE(AV_PIX_FMT_BGR555LE);
114 EVCASE(AV_PIX_FMT_VAAPI);
115 EVCASE(AV_PIX_FMT_YUV420P16LE);
116 EVCASE(AV_PIX_FMT_YUV420P16BE);
117 EVCASE(AV_PIX_FMT_YUV422P16LE);
118 EVCASE(AV_PIX_FMT_YUV422P16BE);
119 EVCASE(AV_PIX_FMT_YUV444P16LE);
120 EVCASE(AV_PIX_FMT_YUV444P16BE);
121 EVCASE(AV_PIX_FMT_DXVA2_VLD);
122 EVCASE(AV_PIX_FMT_RGB444LE);
123 EVCASE(AV_PIX_FMT_RGB444BE);
124 EVCASE(AV_PIX_FMT_BGR444LE);
125 EVCASE(AV_PIX_FMT_BGR444BE);
126 EVCASE(AV_PIX_FMT_GRAY8A);
127 EVCASE(AV_PIX_FMT_BGR48BE);
128 EVCASE(AV_PIX_FMT_BGR48LE);
129 EVCASE(AV_PIX_FMT_YUV420P9BE);
130 EVCASE(AV_PIX_FMT_YUV420P9LE);
131 EVCASE(AV_PIX_FMT_YUV420P10BE);
132 EVCASE(AV_PIX_FMT_YUV420P10LE);
133 EVCASE(AV_PIX_FMT_YUV422P10BE);
134 EVCASE(AV_PIX_FMT_YUV422P10LE);
135 EVCASE(AV_PIX_FMT_YUV444P9BE);
136 EVCASE(AV_PIX_FMT_YUV444P9LE);
137 EVCASE(AV_PIX_FMT_YUV444P10BE);
138 EVCASE(AV_PIX_FMT_YUV444P10LE);
139 EVCASE(AV_PIX_FMT_YUV422P9BE);
140 EVCASE(AV_PIX_FMT_YUV422P9LE);
141 EVCASE(AV_PIX_FMT_GBR24P);
142 EVCASE(AV_PIX_FMT_GBRP9BE);
143 EVCASE(AV_PIX_FMT_GBRP9LE);
144 EVCASE(AV_PIX_FMT_GBRP10BE);
145 EVCASE(AV_PIX_FMT_GBRP10LE);
146 EVCASE(AV_PIX_FMT_GBRP16BE);
147 EVCASE(AV_PIX_FMT_GBRP16LE);
148 EVCASE(AV_PIX_FMT_YUVA422P);
149 EVCASE(AV_PIX_FMT_YUVA444P);
150 EVCASE(AV_PIX_FMT_YUVA420P9BE);
151 EVCASE(AV_PIX_FMT_YUVA420P9LE);
152 EVCASE(AV_PIX_FMT_YUVA422P9BE);
153 EVCASE(AV_PIX_FMT_YUVA422P9LE);
154 EVCASE(AV_PIX_FMT_YUVA444P9BE);
155 EVCASE(AV_PIX_FMT_YUVA444P9LE);
156 EVCASE(AV_PIX_FMT_YUVA420P10BE);
157 EVCASE(AV_PIX_FMT_YUVA420P10LE);
158 EVCASE(AV_PIX_FMT_YUVA422P10BE);
159 EVCASE(AV_PIX_FMT_YUVA422P10LE);
160 EVCASE(AV_PIX_FMT_YUVA444P10BE);
161 EVCASE(AV_PIX_FMT_YUVA444P10LE);
162 EVCASE(AV_PIX_FMT_YUVA420P16BE);
163 EVCASE(AV_PIX_FMT_YUVA420P16LE);
164 EVCASE(AV_PIX_FMT_YUVA422P16BE);
165 EVCASE(AV_PIX_FMT_YUVA422P16LE);
166 EVCASE(AV_PIX_FMT_YUVA444P16BE);
167 EVCASE(AV_PIX_FMT_YUVA444P16LE);
168 EVCASE(AV_PIX_FMT_VDPAU);
169 EVCASE(AV_PIX_FMT_XYZ12LE);
170 EVCASE(AV_PIX_FMT_XYZ12BE);
171 EVCASE(AV_PIX_FMT_NV16);
172 EVCASE(AV_PIX_FMT_NV20LE);
173 EVCASE(AV_PIX_FMT_NV20BE);
174 EVCASE(AV_PIX_FMT_RGBA64BE);
175 EVCASE(AV_PIX_FMT_RGBA64LE);
176 EVCASE(AV_PIX_FMT_BGRA64BE);
177 EVCASE(AV_PIX_FMT_BGRA64LE);
178 EVCASE(AV_PIX_FMT_YVYU422);
179 EVCASE(AV_PIX_FMT_YA16BE);
180 EVCASE(AV_PIX_FMT_YA16LE);
181 EVCASE(AV_PIX_FMT_GBRAP);
182 EVCASE(AV_PIX_FMT_GBRAP16BE);
183 EVCASE(AV_PIX_FMT_GBRAP16LE);
184 EVCASE(AV_PIX_FMT_QSV);
185 EVCASE(AV_PIX_FMT_MMAL);
186 EVCASE(AV_PIX_FMT_D3D11VA_VLD);
187 EVCASE(AV_PIX_FMT_CUDA);
188 EVCASE(AV_PIX_FMT_0RGB);
189 EVCASE(AV_PIX_FMT_RGB0);
190 EVCASE(AV_PIX_FMT_0BGR);
191 EVCASE(AV_PIX_FMT_BGR0);
192 EVCASE(AV_PIX_FMT_YUV420P12BE);
193 EVCASE(AV_PIX_FMT_YUV420P12LE);
194 EVCASE(AV_PIX_FMT_YUV420P14BE);
195 EVCASE(AV_PIX_FMT_YUV420P14LE);
196 EVCASE(AV_PIX_FMT_YUV422P12BE);
197 EVCASE(AV_PIX_FMT_YUV422P12LE);
198 EVCASE(AV_PIX_FMT_YUV422P14BE);
199 EVCASE(AV_PIX_FMT_YUV422P14LE);
200 EVCASE(AV_PIX_FMT_YUV444P12BE);
201 EVCASE(AV_PIX_FMT_YUV444P12LE);
202 EVCASE(AV_PIX_FMT_YUV444P14BE);
203 EVCASE(AV_PIX_FMT_YUV444P14LE);
204 EVCASE(AV_PIX_FMT_GBRP12BE);
205 EVCASE(AV_PIX_FMT_GBRP12LE);
206 EVCASE(AV_PIX_FMT_GBRP14BE);
207 EVCASE(AV_PIX_FMT_GBRP14LE);
208 EVCASE(AV_PIX_FMT_YUVJ411P);
209 EVCASE(AV_PIX_FMT_BAYER_BGGR8);
210 EVCASE(AV_PIX_FMT_BAYER_RGGB8);
211 EVCASE(AV_PIX_FMT_BAYER_GBRG8);
212 EVCASE(AV_PIX_FMT_BAYER_GRBG8);
213 EVCASE(AV_PIX_FMT_BAYER_BGGR16LE);
214 EVCASE(AV_PIX_FMT_BAYER_BGGR16BE);
215 EVCASE(AV_PIX_FMT_BAYER_RGGB16LE);
216 EVCASE(AV_PIX_FMT_BAYER_RGGB16BE);
217 EVCASE(AV_PIX_FMT_BAYER_GBRG16LE);
218 EVCASE(AV_PIX_FMT_BAYER_GBRG16BE);
219 EVCASE(AV_PIX_FMT_BAYER_GRBG16LE);
220 EVCASE(AV_PIX_FMT_BAYER_GRBG16BE);
221 EVCASE(AV_PIX_FMT_YUV440P10LE);
222 EVCASE(AV_PIX_FMT_YUV440P10BE);
223 EVCASE(AV_PIX_FMT_YUV440P12LE);
224 EVCASE(AV_PIX_FMT_YUV440P12BE);
225 EVCASE(AV_PIX_FMT_AYUV64LE);
226 EVCASE(AV_PIX_FMT_AYUV64BE);
227 EVCASE(AV_PIX_FMT_VIDEOTOOLBOX);
228 EVCASE(AV_PIX_FMT_P010LE);
229 EVCASE(AV_PIX_FMT_P010BE);
230 EVCASE(AV_PIX_FMT_GBRAP12BE);
231 EVCASE(AV_PIX_FMT_GBRAP12LE);
232 EVCASE(AV_PIX_FMT_GBRAP10BE);
233 EVCASE(AV_PIX_FMT_GBRAP10LE);
234 EVCASE(AV_PIX_FMT_MEDIACODEC);
235 EVCASE(AV_PIX_FMT_GRAY12BE);
236 EVCASE(AV_PIX_FMT_GRAY12LE);
237 EVCASE(AV_PIX_FMT_GRAY10BE);
238 EVCASE(AV_PIX_FMT_GRAY10LE);
239 EVCASE(AV_PIX_FMT_P016LE);
240 EVCASE(AV_PIX_FMT_P016BE);
241 EVCASE(AV_PIX_FMT_D3D11);
242 EVCASE(AV_PIX_FMT_GRAY9BE);
243 EVCASE(AV_PIX_FMT_GRAY9LE);
244 EVCASE(AV_PIX_FMT_GBRPF32BE);
245 EVCASE(AV_PIX_FMT_GBRPF32LE);
246 EVCASE(AV_PIX_FMT_GBRAPF32BE);
247 EVCASE(AV_PIX_FMT_GBRAPF32LE);
248 EVCASE(AV_PIX_FMT_DRM_PRIME);
249 EVCASE(AV_PIX_FMT_OPENCL);
250 EVCASE(AV_PIX_FMT_GRAY14BE);
251 EVCASE(AV_PIX_FMT_GRAY14LE);
252 EVCASE(AV_PIX_FMT_GRAYF32BE);
253 EVCASE(AV_PIX_FMT_GRAYF32LE);
254 EVCASE(AV_PIX_FMT_YUVA422P12BE);
255 EVCASE(AV_PIX_FMT_YUVA422P12LE);
256 EVCASE(AV_PIX_FMT_YUVA444P12BE);
257 EVCASE(AV_PIX_FMT_YUVA444P12LE);
258 EVCASE(AV_PIX_FMT_NV24);
259 EVCASE(AV_PIX_FMT_NV42);
260 EVCASE(AV_PIX_FMT_VULKAN);
261 EVCASE(AV_PIX_FMT_Y210BE);
262 EVCASE(AV_PIX_FMT_Y210LE);
263#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(57, 17, 100)
264 EVCASE(AV_PIX_FMT_P210BE);
265 EVCASE(AV_PIX_FMT_P210LE);
266 EVCASE(AV_PIX_FMT_P216BE);
267 EVCASE(AV_PIX_FMT_P216LE);
268 EVCASE(AV_PIX_FMT_P410BE);
269 EVCASE(AV_PIX_FMT_P410LE);
270 EVCASE(AV_PIX_FMT_X2RGB10LE);
271 EVCASE(AV_PIX_FMT_X2RGB10BE);
272 EVCASE(AV_PIX_FMT_X2BGR10LE);
273 EVCASE(AV_PIX_FMT_X2BGR10BE);
274 EVCASE(AV_PIX_FMT_P416BE);
275 EVCASE(AV_PIX_FMT_P416LE);
276#endif
277#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(58, 2, 100)
278 EVCASE(AV_PIX_FMT_VUYA);
279 EVCASE(AV_PIX_FMT_VUYX);
280#endif
281#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(57, 33, 101)
282 EVCASE(AV_PIX_FMT_RGBAF16BE);
283 EVCASE(AV_PIX_FMT_RGBAF16LE);
284#endif
285#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(57, 37, 100)
286 EVCASE(AV_PIX_FMT_P012LE);
287 EVCASE(AV_PIX_FMT_P012BE);
288 EVCASE(AV_PIX_FMT_Y212BE);
289 EVCASE(AV_PIX_FMT_Y212LE);
290 EVCASE(AV_PIX_FMT_XV30BE);
291 EVCASE(AV_PIX_FMT_XV30LE);
292 EVCASE(AV_PIX_FMT_XV36BE);
293 EVCASE(AV_PIX_FMT_XV36LE);
294 EVCASE(AV_PIX_FMT_RGBF32BE);
295 EVCASE(AV_PIX_FMT_RGBF32LE);
296 EVCASE(AV_PIX_FMT_RGBAF32BE);
297 EVCASE(AV_PIX_FMT_RGBAF32LE);
298#endif
299#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(59, 10, 100)
300 EVCASE(AV_PIX_FMT_P212BE);
301 EVCASE(AV_PIX_FMT_P212LE);
302 EVCASE(AV_PIX_FMT_P412BE);
303 EVCASE(AV_PIX_FMT_P412LE);
304#endif
305#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(59, 14, 100)
306 EVCASE(AV_PIX_FMT_GBRAP14BE);
307 EVCASE(AV_PIX_FMT_GBRAP14LE);
308#endif
309#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(59,16,100)
310 EVCASE(AV_PIX_FMT_D3D12);
311#endif
312 EVCASE(AV_PIX_FMT_NB);
313 default:
314 return "AV_PIX_FMT_UNKNOWN";
315 }
316}
317#endif
318
319/* Ubuntu 14.04 ships without the functions provided by avutil,
320 * so define error to string methods here. */
321#if !defined(av_err2str)
322static inline char* error_string(char* errbuf, size_t errbuf_size, int errnum)
323{
324 av_strerror(errnum, errbuf, errbuf_size);
325 return errbuf;
326}
327
328#define av_err2str(errnum) error_string((char[64])WINPR_C_ARRAY_INIT, 64, errnum)
329#endif
330
331typedef struct
332{
333 const AVCodec* codecDecoder;
334 AVCodecContext* codecDecoderContext;
335 const AVCodec* codecEncoder;
336 AVCodecContext* codecEncoderContext;
337 AVCodecParserContext* codecParser;
338 AVFrame* videoFrame;
339#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 133, 100)
340 AVPacket bufferpacket;
341#endif
342 AVPacket* packet;
343#if defined(WITH_FFMPEG_HWACCEL)
344 AVBufferRef* hwctx;
345 struct SwsContext* swsctx;
346 int hwFrameSupportsNativeFormat;
347 AVFrame* hwVideoFrame;
348 AVFrame* cpuVideoFrame;
349 enum AVPixelFormat hw_pix_fmt;
350#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(57, 80, 100)
351 AVBufferRef* hw_frames_ctx;
352#endif
353
354 enum AVHWDeviceType type;
355 char* device;
356 BOOL useHwDecoder;
357 BOOL useHwEncoder;
358#endif
359} H264_CONTEXT_LIBAVCODEC;
360
361static void libavcodec_destroy_encoder_context(H264_CONTEXT* WINPR_RESTRICT h264)
362{
363 if (!h264 || !h264->subsystem)
364 return;
365
366 H264_CONTEXT_LIBAVCODEC* sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
367
368 if (sys->codecEncoderContext)
369 {
370#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 69, 100)
371 avcodec_free_context(&sys->codecEncoderContext);
372#else
373 avcodec_close(sys->codecEncoderContext);
374 av_free(sys->codecEncoderContext);
375#endif
376 }
377
378 sys->codecEncoderContext = nullptr;
379}
380
381#if defined(WITH_FFMPEG_HWACCEL)
382WINPR_ATTR_NODISCARD
383static enum AVPixelFormat get_hwctx_image_format(H264_CONTEXT_LIBAVCODEC* sys, wLog* log)
384{
385 WINPR_ASSERT(sys);
386 enum AVHWDeviceType type = sys->type;
387 switch (type)
388 {
389 case AV_HWDEVICE_TYPE_VAAPI:
390 return AV_PIX_FMT_VAAPI;
391 case AV_HWDEVICE_TYPE_VIDEOTOOLBOX:
392 return AV_PIX_FMT_VIDEOTOOLBOX;
393 case AV_HWDEVICE_TYPE_VDPAU:
394 return AV_PIX_FMT_VDPAU;
395 case AV_HWDEVICE_TYPE_CUDA:
396 return AV_PIX_FMT_CUDA;
397 case AV_HWDEVICE_TYPE_DXVA2:
398 return AV_PIX_FMT_DXVA2_VLD;
399 case AV_HWDEVICE_TYPE_QSV:
400 return AV_PIX_FMT_QSV;
401#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(59, 16, 100)
402 case AV_HWDEVICE_TYPE_D3D12VA:
403 return AV_PIX_FMT_D3D12;
404#endif
405 case AV_HWDEVICE_TYPE_D3D11VA:
406 return AV_PIX_FMT_D3D11VA_VLD;
407 case AV_HWDEVICE_TYPE_DRM:
408 return AV_PIX_FMT_DRM_PRIME;
409 case AV_HWDEVICE_TYPE_OPENCL:
410 return AV_PIX_FMT_OPENCL;
411 case AV_HWDEVICE_TYPE_MEDIACODEC:
412 return AV_PIX_FMT_MEDIACODEC;
413 case AV_HWDEVICE_TYPE_VULKAN:
414 return AV_PIX_FMT_VULKAN;
415 default:
416 WLog_Print(log, WLOG_INFO, "Missing pixel format mapping for %s",
417 av_hwdevice_get_type_name(type));
418 return AV_PIX_FMT_NONE;
419 }
420}
421
422WINPR_ATTR_NODISCARD
423static const AVCodec* get_hwctx_codec(H264_CONTEXT_LIBAVCODEC* sys, wLog* log,
424 enum AVHWDeviceType type)
425{
426 WINPR_ASSERT(sys);
427 switch (type)
428 {
429 case AV_HWDEVICE_TYPE_VAAPI:
430 return avcodec_find_encoder_by_name("h264_vaapi");
431 case AV_HWDEVICE_TYPE_VULKAN:
432 return avcodec_find_encoder_by_name("h264_vulkan");
433 case AV_HWDEVICE_TYPE_D3D11VA:
434 return avcodec_find_encoder_by_name("h264_d3d11va");
435#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(59, 16, 100)
436 case AV_HWDEVICE_TYPE_D3D12VA:
437 return avcodec_find_encoder_by_name("h264_d3d12va");
438#endif
439 case AV_HWDEVICE_TYPE_VIDEOTOOLBOX:
440 return avcodec_find_encoder_by_name("h264_videotoolbox");
441 case AV_HWDEVICE_TYPE_VDPAU:
442 return avcodec_find_encoder_by_name("h264_vdpau");
443 default:
444 WLog_Print(log, WLOG_ERROR, "could not map %s to h264 encoder",
445 av_hwdevice_get_type_name(sys->type));
446 return nullptr;
447 }
448}
449
450WINPR_ATTR_NODISCARD
451static const char* get_hwctx_device(H264_CONTEXT_LIBAVCODEC* sys)
452{
453 WINPR_ASSERT(sys);
454
455 switch (sys->type)
456 {
457 case AV_HWDEVICE_TYPE_VAAPI:
458 break;
459 default:
460 return nullptr;
461 }
462
463 return sys->device;
464}
465#endif
466
467#ifdef WITH_FFMPEG_HWACCEL
468static int set_hw_frames_ctx(H264_CONTEXT* WINPR_RESTRICT h264)
469{
470 H264_CONTEXT_LIBAVCODEC* sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
471 AVBufferRef* hw_frames_ref = nullptr;
472 AVHWFramesContext* frames_ctx = nullptr;
473 int err = 0;
474
475 if (!(hw_frames_ref = av_hwframe_ctx_alloc(sys->hwctx)))
476 {
477 WLog_Print(h264->log, WLOG_ERROR, "Failed to create %s frame context",
478 av_hwdevice_get_type_name(sys->type));
479 return -1;
480 }
481 frames_ctx = (AVHWFramesContext*)(hw_frames_ref->data);
482 frames_ctx->format = get_hwctx_image_format(sys, h264->log);
483 frames_ctx->sw_format = AV_PIX_FMT_NV12;
484 frames_ctx->width = sys->codecEncoderContext->width;
485 frames_ctx->height = sys->codecEncoderContext->height;
486 frames_ctx->initial_pool_size = 20;
487 if ((err = av_hwframe_ctx_init(hw_frames_ref)) < 0)
488 {
489 WLog_Print(h264->log, WLOG_ERROR,
490 "Failed to initialize %s frame context."
491 "Error code: %s",
492 av_hwdevice_get_type_name(sys->type), av_err2str(err));
493 av_buffer_unref(&hw_frames_ref);
494 return err;
495 }
496 sys->codecEncoderContext->hw_frames_ctx = av_buffer_ref(hw_frames_ref);
497 if (!sys->codecEncoderContext->hw_frames_ctx)
498 err = AVERROR(ENOMEM);
499
500 av_buffer_unref(&hw_frames_ref);
501 return err;
502}
503#endif
504
505static BOOL libavcodec_create_encoder_context(H264_CONTEXT* WINPR_RESTRICT h264)
506{
507 BOOL recreate = FALSE;
508 H264_CONTEXT_LIBAVCODEC* sys = nullptr;
509
510 if (!h264 || !h264->subsystem)
511 return FALSE;
512
513 if ((h264->width > INT_MAX) || (h264->height > INT_MAX))
514 return FALSE;
515
516 sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
517 if (!sys || !sys->codecEncoder)
518 return FALSE;
519
520 recreate = !sys->codecEncoderContext;
521
522 if (sys->codecEncoderContext)
523 {
524 if ((sys->codecEncoderContext->width != (int)h264->width) ||
525 (sys->codecEncoderContext->height != (int)h264->height))
526 recreate = TRUE;
527 }
528
529 if (!recreate)
530 return TRUE;
531
532 libavcodec_destroy_encoder_context(h264);
533
534 sys->codecEncoderContext = avcodec_alloc_context3(sys->codecEncoder);
535
536 if (!sys->codecEncoderContext)
537 goto EXCEPTION;
538
539 switch (h264->RateControlMode)
540 {
541 case H264_RATECONTROL_VBR:
542 sys->codecEncoderContext->bit_rate = h264->BitRate;
543 break;
544
545 case H264_RATECONTROL_CQP:
546 if (av_opt_set_int(sys->codecEncoderContext, "qp", h264->QP, AV_OPT_SEARCH_CHILDREN) <
547 0)
548 {
549 WLog_Print(h264->log, WLOG_ERROR, "av_opt_set_int failed");
550 }
551 break;
552
553 default:
554 break;
555 }
556
557 sys->codecEncoderContext->width = (int)MIN(INT32_MAX, h264->width);
558 sys->codecEncoderContext->height = (int)MIN(INT32_MAX, h264->height);
559 sys->codecEncoderContext->delay = 0;
560#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(56, 13, 100)
561 sys->codecEncoderContext->framerate =
562 (AVRational){ WINPR_ASSERTING_INT_CAST(int, h264->FrameRate), 1 };
563#endif
564 sys->codecEncoderContext->time_base =
565 (AVRational){ 1, WINPR_ASSERTING_INT_CAST(int, h264->FrameRate) };
566 av_opt_set(sys->codecEncoderContext, "tune", "zerolatency", AV_OPT_SEARCH_CHILDREN);
567
568 sys->codecEncoderContext->flags |= AV_CODEC_FLAG_LOOP_FILTER;
569
570#ifdef WITH_FFMPEG_HWACCEL
571 if (sys->hwctx)
572 {
573 av_opt_set(sys->codecEncoderContext, "preset", "veryslow", AV_OPT_SEARCH_CHILDREN);
574
575 sys->codecEncoderContext->pix_fmt = get_hwctx_image_format(sys, h264->log);
576 /* set hw_frames_ctx for encoder's AVCodecContext */
577 if (set_hw_frames_ctx(h264) < 0)
578 goto EXCEPTION;
579 }
580 else
581#endif
582 {
583 av_opt_set(sys->codecEncoderContext, "preset", "medium", AV_OPT_SEARCH_CHILDREN);
584 sys->codecEncoderContext->pix_fmt = AV_PIX_FMT_YUV420P;
585 }
586
587 if (avcodec_open2(sys->codecEncoderContext, sys->codecEncoder, nullptr) < 0)
588 goto EXCEPTION;
589
590 return TRUE;
591EXCEPTION:
592 libavcodec_destroy_encoder_context(h264);
593 return FALSE;
594}
595
596#if defined(WITH_FFMPEG_HWACCEL)
597static int hw_supports_target_format(AVBufferRef* hwframectx, enum AVPixelFormat target)
598{
599 WINPR_ASSERT(hwframectx);
600
601 enum AVPixelFormat* formats = nullptr;
602 int rc = av_hwframe_transfer_get_formats(hwframectx, AV_HWFRAME_TRANSFER_DIRECTION_FROM,
603 &formats, 0);
604 if (rc == 0)
605 {
606 enum AVPixelFormat* it = formats;
607 while ((rc == 0) && (*it != AV_PIX_FMT_NONE))
608 {
609 enum AVPixelFormat cur = *it++;
610 if (cur == target)
611 rc = 1;
612 }
613 }
614
615 av_free(formats);
616 return rc;
617}
618#endif
619
620static int libavcodec_decompress(H264_CONTEXT* WINPR_RESTRICT h264,
621 const BYTE* WINPR_RESTRICT pSrcData, UINT32 SrcSize)
622{
623 union
624 {
625 const BYTE* cpv;
626 BYTE* pv;
627 } cnv;
628 int rc = -1;
629 int status = 0;
630 int gotFrame = 0;
631 AVPacket* packet = nullptr;
632
633 WINPR_ASSERT(h264);
634 WINPR_ASSERT(pSrcData || (SrcSize == 0));
635
636 H264_CONTEXT_LIBAVCODEC* sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
637 BYTE** pYUVData = h264->pYUVData;
638 UINT32* iStride = h264->iStride;
639
640 WINPR_ASSERT(sys);
641
642#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 133, 100)
643 packet = &sys->bufferpacket;
644 WINPR_ASSERT(packet);
645 av_init_packet(packet);
646#else
647 packet = av_packet_alloc();
648#endif
649 if (!packet)
650 {
651 WLog_Print(h264->log, WLOG_ERROR, "Failed to allocate AVPacket");
652 goto fail;
653 }
654
655 cnv.cpv = pSrcData;
656 packet->data = cnv.pv;
657 packet->size = (int)MIN(SrcSize, INT32_MAX);
658
659 WINPR_ASSERT(sys->codecDecoderContext);
660 /* avcodec_decode_video2 is deprecated with libavcodec 57.48.101 */
661#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 48, 101)
662 status = avcodec_send_packet(sys->codecDecoderContext, packet);
663
664 if (status < 0)
665 {
666 WLog_Print(h264->log, WLOG_ERROR, "Failed to decode video frame (status=%d)", status);
667 goto fail;
668 }
669
670 sys->videoFrame->format = AV_PIX_FMT_YUV420P;
671
672#if defined(WITH_FFMPEG_HWACCEL)
673 status = avcodec_receive_frame(sys->codecDecoderContext, (sys->useHwDecoder && sys->hwctx)
674 ? sys->hwVideoFrame
675 : sys->videoFrame);
676#else
677 status = avcodec_receive_frame(sys->codecDecoderContext, sys->videoFrame);
678#endif
679 if (status == AVERROR(EAGAIN))
680 {
681 rc = 0;
682 goto fail;
683 }
684
685 if (status == 0)
686 gotFrame = 1;
687#else
688#if defined(WITH_FFMPEG_HWACCEL)
689 status =
690 avcodec_decode_video2(sys->codecDecoderContext,
691 sys->hwctx ? sys->hwVideoFrame : sys->videoFrame, &gotFrame, packet);
692#else
693 status = avcodec_decode_video2(sys->codecDecoderContext, sys->videoFrame, &gotFrame, packet);
694#endif
695#endif
696 if (status < 0)
697 {
698 WLog_Print(h264->log, WLOG_ERROR, "Failed to decode video frame (status=%d)", status);
699 goto fail;
700 }
701
702#if defined(WITH_FFMPEG_HWACCEL)
703
704 if (sys->hwctx)
705 {
706 AVFrame* target = sys->videoFrame;
707 if (sys->hwFrameSupportsNativeFormat < 0)
708 {
709 const int res = hw_supports_target_format(sys->codecDecoderContext->hw_frames_ctx,
710 sys->videoFrame->format);
711 if (res < 0)
712 status = res;
713 else
714 {
715 sys->hwFrameSupportsNativeFormat = res;
716 const char* msg = "hardware supported copy";
717 if (res == 0)
718 msg = "swscale format conversion";
719 WLog_Print(h264->log, WLOG_DEBUG, "formats: { src:%s, dst:%s } using %s",
720 av_format_str(sys->hwVideoFrame->format),
721 av_format_str(sys->videoFrame->format), msg);
722 }
723 }
724
725 if (sys->hwFrameSupportsNativeFormat == 0)
726 target = sys->cpuVideoFrame;
727
728 if (sys->hwFrameSupportsNativeFormat >= 0)
729 {
730 if (sys->hwVideoFrame->format == sys->hw_pix_fmt)
731 {
732 target->width = sys->hwVideoFrame->width;
733 target->height = sys->hwVideoFrame->height;
734 status = av_hwframe_transfer_data(target, sys->hwVideoFrame, 0);
735 }
736 else
737 status = av_frame_copy(target, sys->hwVideoFrame);
738 }
739
740 gotFrame = (status == 0);
741
742 if (status < 0)
743 {
744 WLog_Print(h264->log, WLOG_ERROR, "Failed to transfer video frame (status=%d) (%s)",
745 status, av_err2str(status));
746 goto fail;
747 }
748
749 if (gotFrame && (target != sys->videoFrame))
750 {
751 if (!sys->swsctx)
752 {
753 sys->swsctx = sws_getContext(target->width, target->height, target->format,
754 target->width, target->height, sys->videoFrame->format,
755 SWS_BILINEAR, nullptr, nullptr, nullptr);
756 if (!sys->swsctx)
757 goto fail;
758 }
759
760#if LIBSWSCALE_VERSION_INT >= AV_VERSION_INT(6, 4, 100)
761 status = sws_scale_frame(sys->swsctx, sys->videoFrame, target);
762#else
763#error "Need SWScale >= 6.4.100"
764#endif
765
766 if (status < 0)
767 {
768 WLog_Print(h264->log, WLOG_ERROR, "Failed to convert video frame (status=%d) (%s)",
769 status, av_err2str(status));
770 goto fail;
771 }
772 }
773 }
774
775#endif
776
777 if (gotFrame)
778 {
779 WINPR_ASSERT(sys->videoFrame);
780
781 pYUVData[0] = sys->videoFrame->data[0];
782 pYUVData[1] = sys->videoFrame->data[1];
783 pYUVData[2] = sys->videoFrame->data[2];
784 iStride[0] = (UINT32)MAX(0, sys->videoFrame->linesize[0]);
785 iStride[1] = (UINT32)MAX(0, sys->videoFrame->linesize[1]);
786 iStride[2] = (UINT32)MAX(0, sys->videoFrame->linesize[2]);
787
788 h264->YUVWidth = WINPR_ASSERTING_INT_CAST(UINT32, sys->videoFrame->width);
789 h264->YUVHeight = WINPR_ASSERTING_INT_CAST(UINT32, sys->videoFrame->height);
790 rc = 1;
791 }
792 else
793 rc = -2;
794
795fail:
796#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 133, 100)
797 av_packet_unref(packet);
798#else
799 av_packet_free(&packet);
800#endif
801
802 return rc;
803}
804
805static int libavcodec_compress(H264_CONTEXT* WINPR_RESTRICT h264,
806 const BYTE** WINPR_RESTRICT pSrcYuv,
807 const UINT32* WINPR_RESTRICT pStride,
808 BYTE** WINPR_RESTRICT ppDstData, UINT32* WINPR_RESTRICT pDstSize)
809{
810 union
811 {
812 const BYTE* cpv;
813 uint8_t* pv;
814 } cnv;
815 int rc = -1;
816 int status = 0;
817 int gotFrame = 0;
818
819 WINPR_ASSERT(h264);
820
821 H264_CONTEXT_LIBAVCODEC* sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
822 WINPR_ASSERT(sys);
823
824 if (!libavcodec_create_encoder_context(h264))
825 return -1;
826
827#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 133, 100)
828 sys->packet = &sys->bufferpacket;
829 av_packet_unref(sys->packet);
830 av_init_packet(sys->packet);
831#else
832 av_packet_free(&sys->packet);
833 sys->packet = av_packet_alloc();
834#endif
835 if (!sys->packet)
836 {
837 WLog_Print(h264->log, WLOG_ERROR, "Failed to allocate AVPacket");
838 goto fail;
839 }
840
841 WINPR_ASSERT(sys->packet);
842 sys->packet->data = nullptr;
843 sys->packet->size = 0;
844
845 WINPR_ASSERT(sys->videoFrame);
846 WINPR_ASSERT(sys->codecEncoderContext);
847 sys->videoFrame->format = AV_PIX_FMT_YUV420P;
848 sys->videoFrame->width = sys->codecEncoderContext->width;
849 sys->videoFrame->height = sys->codecEncoderContext->height;
850#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(52, 48, 100)
851 sys->videoFrame->colorspace = AVCOL_SPC_BT709;
852#endif
853#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(52, 92, 100)
854 sys->videoFrame->chroma_location = AVCHROMA_LOC_LEFT;
855#endif
856 cnv.cpv = pSrcYuv[0];
857 sys->videoFrame->data[0] = cnv.pv;
858
859 cnv.cpv = pSrcYuv[1];
860 sys->videoFrame->data[1] = cnv.pv;
861
862 cnv.cpv = pSrcYuv[2];
863 sys->videoFrame->data[2] = cnv.pv;
864
865 sys->videoFrame->linesize[0] = (int)pStride[0];
866 sys->videoFrame->linesize[1] = (int)pStride[1];
867 sys->videoFrame->linesize[2] = (int)pStride[2];
868 sys->videoFrame->pts++;
869
870#ifdef WITH_FFMPEG_HWACCEL
871 if (sys->useHwEncoder && sys->hwctx)
872 {
873 av_frame_unref(sys->hwVideoFrame);
874 if ((status = av_hwframe_get_buffer(sys->codecEncoderContext->hw_frames_ctx,
875 sys->hwVideoFrame, 0)) < 0 ||
876 !sys->hwVideoFrame->hw_frames_ctx)
877 {
878 WLog_Print(h264->log, WLOG_ERROR, "av_hwframe_get_buffer failed (%s [%d])",
879 av_err2str(status), status);
880 goto fail;
881 }
882 sys->videoFrame->format = AV_PIX_FMT_NV12;
883 if ((status = av_hwframe_transfer_data(sys->hwVideoFrame, sys->videoFrame, 0)) < 0)
884 {
885 WLog_Print(h264->log, WLOG_ERROR, "av_hwframe_transfer_data failed (%s [%d])",
886 av_err2str(status), status);
887 goto fail;
888 }
889 }
890#endif
891
892 /* avcodec_encode_video2 is deprecated with libavcodec 57.48.101 */
893#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 48, 101)
894#ifdef WITH_FFMPEG_HWACCEL
895 status = avcodec_send_frame(sys->codecEncoderContext,
896 sys->hwctx ? sys->hwVideoFrame : sys->videoFrame);
897#else
898 status = avcodec_send_frame(sys->codecEncoderContext, sys->videoFrame);
899#endif
900
901 if (status < 0)
902 {
903 WLog_Print(h264->log, WLOG_ERROR, "Failed to encode video frame (%s [%d])",
904 av_err2str(status), status);
905 goto fail;
906 }
907
908 status = avcodec_receive_packet(sys->codecEncoderContext, sys->packet);
909
910 if (status < 0)
911 {
912 WLog_Print(h264->log, WLOG_ERROR, "Failed to encode video frame (%s [%d])",
913 av_err2str(status), status);
914 goto fail;
915 }
916
917 gotFrame = (status == 0);
918#elif LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(54, 59, 100)
919
920 do
921 {
922 status = avcodec_encode_video2(sys->codecEncoderContext, sys->packet, sys->videoFrame,
923 &gotFrame);
924 } while ((status >= 0) && (gotFrame == 0));
925
926#else
927 sys->packet->size =
928 avpicture_get_size(sys->codecDecoderContext->pix_fmt, sys->codecDecoderContext->width,
929 sys->codecDecoderContext->height);
930 sys->packet->data = av_malloc(sys->packet->size);
931
932 if (!sys->packet->data)
933 status = -1;
934 else
935 {
936 status = avcodec_encode_video(sys->codecDecoderContext, sys->packet->data,
937 sys->packet->size, sys->videoFrame);
938 }
939
940#endif
941
942 if (status < 0)
943 {
944 WLog_Print(h264->log, WLOG_ERROR, "Failed to encode video frame (%s [%d])",
945 av_err2str(status), status);
946 goto fail;
947 }
948
949 WINPR_ASSERT(sys->packet);
950 *ppDstData = sys->packet->data;
951 *pDstSize = (UINT32)MAX(0, sys->packet->size);
952
953 if (!gotFrame)
954 {
955 WLog_Print(h264->log, WLOG_ERROR, "Did not get frame! (%s [%d])", av_err2str(status),
956 status);
957 rc = -2;
958 }
959 else
960 rc = 1;
961fail:
962 return rc;
963}
964
965static void libavcodec_uninit(H264_CONTEXT* h264)
966{
967 WINPR_ASSERT(h264);
968
969 H264_CONTEXT_LIBAVCODEC* sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
970
971 if (!sys)
972 return;
973
974 if (sys->packet)
975 {
976#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 133, 100)
977 av_packet_unref(sys->packet);
978#else
979 av_packet_free(&sys->packet);
980#endif
981 }
982
983 if (sys->videoFrame)
984 {
985#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 18, 102)
986 av_frame_free(&sys->videoFrame);
987#else
988 av_free(sys->videoFrame);
989#endif
990 }
991
992#if defined(WITH_FFMPEG_HWACCEL)
993 if (sys->hwVideoFrame)
994 {
995#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 18, 102)
996 av_frame_free(&sys->hwVideoFrame);
997#else
998 av_free(sys->hwVideoFrame);
999 sys->hwVideoFrame = nullptr;
1000#endif
1001 }
1002
1003 if (sys->cpuVideoFrame)
1004 {
1005#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 18, 102)
1006 av_frame_free(&sys->cpuVideoFrame);
1007#else
1008 av_free(sys->cpuVideoFrame);
1009#endif
1010 }
1011
1012 if (sys->hwctx)
1013 av_buffer_unref(&sys->hwctx);
1014 sws_freeContext(sys->swsctx);
1015
1016#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(57, 80, 100)
1017
1018 if (sys->hw_frames_ctx)
1019 av_buffer_unref(&sys->hw_frames_ctx);
1020
1021#endif
1022
1023#endif
1024
1025 if (sys->codecParser)
1026 av_parser_close(sys->codecParser);
1027
1028 if (sys->codecDecoderContext)
1029 {
1030#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 69, 100)
1031 avcodec_free_context(&sys->codecDecoderContext);
1032#else
1033 avcodec_close(sys->codecDecoderContext);
1034 av_free(sys->codecDecoderContext);
1035#endif
1036 }
1037
1038 libavcodec_destroy_encoder_context(h264);
1039#if defined(WITH_FFMPEG_HWACCEL)
1040 free(sys->device);
1041#endif
1042 free(sys);
1043 h264->pSystemData = nullptr;
1044}
1045
1046#if defined(WITH_FFMPEG_HWACCEL)
1047WINPR_ATTR_NODISCARD
1048static const char* vaapi_vendor(H264_CONTEXT_LIBAVCODEC* sys)
1049{
1050 WINPR_ASSERT(sys);
1051
1052 const char* vendor = "UNKNOWN";
1053#if 0
1054 if (!hwctx)
1055 return vendor;
1056
1057 const AVHWDeviceContext* devctx = (const AVHWDeviceContext*)hwctx->data;
1058 if (!devctx)
1059 return vendor;
1060
1061 const AVVAAPIDeviceContext* ctx = devctx->hwctx;
1062 if (!ctx)
1063 return vendor;
1064
1065 const char* vavendor = vaQueryVendorString(ctx->display);
1066 if (vavendor)
1067 return vavendor;
1068#endif
1069 return vendor;
1070}
1071#endif
1072#if defined(WITH_FFMPEG_HWACCEL)
1073WINPR_ATTR_NODISCARD
1074static enum AVPixelFormat libavcodec_get_format(struct AVCodecContext* ctx,
1075 const enum AVPixelFormat* fmts)
1076{
1077 WINPR_ASSERT(ctx);
1078
1079 H264_CONTEXT* h264 = (H264_CONTEXT*)ctx->opaque;
1080 WINPR_ASSERT(h264);
1081
1082 H264_CONTEXT_LIBAVCODEC* sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
1083 WINPR_ASSERT(sys);
1084
1085 for (const enum AVPixelFormat* p = fmts; *p != AV_PIX_FMT_NONE; p++)
1086 {
1087 if (*p == sys->hw_pix_fmt)
1088 {
1089#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(57, 80, 100)
1090 if (sys->hw_frames_ctx)
1091 av_buffer_unref(&sys->hw_frames_ctx);
1092
1093 sys->hw_frames_ctx = av_hwframe_ctx_alloc(sys->hwctx);
1094
1095 if (!sys->hw_frames_ctx)
1096 {
1097 return AV_PIX_FMT_NONE;
1098 }
1099
1100 sys->codecDecoderContext->pix_fmt = *p;
1101 AVHWFramesContext* frames = (AVHWFramesContext*)sys->hw_frames_ctx->data;
1102 frames->format = *p;
1103 frames->height = sys->codecDecoderContext->coded_height;
1104 frames->width = sys->codecDecoderContext->coded_width;
1105#ifdef WITH_VIDEOTOOLBOX
1106 frames->sw_format = AV_PIX_FMT_YUV420P;
1107#else
1108 frames->sw_format =
1109 (sys->codecDecoderContext->sw_pix_fmt == AV_PIX_FMT_YUV420P10 ? AV_PIX_FMT_P010
1110 : AV_PIX_FMT_NV12);
1111#endif
1112 frames->initial_pool_size = 20;
1113
1114 if (sys->codecDecoderContext->active_thread_type & FF_THREAD_FRAME)
1115 frames->initial_pool_size += sys->codecDecoderContext->thread_count;
1116
1117 int err = av_hwframe_ctx_init(sys->hw_frames_ctx);
1118
1119 if (err < 0)
1120 {
1121 WLog_Print(h264->log, WLOG_ERROR, "Could not init hwframes context: %s",
1122 av_err2str(err));
1123 return AV_PIX_FMT_NONE;
1124 }
1125
1126 sys->codecDecoderContext->hw_frames_ctx = av_buffer_ref(sys->hw_frames_ctx);
1127#endif
1128 return *p;
1129 }
1130 }
1131
1132 return AV_PIX_FMT_NONE;
1133}
1134
1135WINPR_ATTR_MALLOC(WINPR_JSON_Delete, 1)
1136static WINPR_JSON* load(BOOL system)
1137{
1138 return freerdp_GetJSONConfigFile(system, "ffmpeg.json");
1139}
1140
1141static BOOL read_json_bool(WINPR_JSON* json, const char* what, BOOL* pValue)
1142{
1143 WINPR_ASSERT(json);
1144 WINPR_ASSERT(what);
1145 WINPR_ASSERT(pValue);
1146
1147 WINPR_JSON* val = WINPR_JSON_GetObjectItemCaseSensitive(json, what);
1148 if (!val)
1149 return FALSE;
1150 if (!WINPR_JSON_IsBool(val))
1151 return FALSE;
1152 *pValue = WINPR_JSON_IsTrue(val);
1153 return TRUE;
1154}
1155
1156static BOOL read_json_const_string(WINPR_JSON* json, const char* what, const char** pValue)
1157{
1158 WINPR_ASSERT(json);
1159 WINPR_ASSERT(what);
1160 WINPR_ASSERT(pValue);
1161
1162 WINPR_JSON* val = WINPR_JSON_GetObjectItemCaseSensitive(json, what);
1163 if (!val)
1164 return FALSE;
1165 if (!WINPR_JSON_IsString(val))
1166 return FALSE;
1167 *pValue = WINPR_JSON_GetStringValue(val);
1168 return *pValue != nullptr;
1169}
1170
1171static BOOL read_json_string(WINPR_JSON* json, const char* what, char** pValue)
1172{
1173 WINPR_ASSERT(json);
1174 WINPR_ASSERT(what);
1175 WINPR_ASSERT(pValue);
1176
1177 WINPR_JSON* val = WINPR_JSON_GetObjectItemCaseSensitive(json, what);
1178 if (!val)
1179 return FALSE;
1180 if (!WINPR_JSON_IsString(val))
1181 return FALSE;
1182 const char* str = nullptr;
1183 const BOOL rc = read_json_const_string(json, what, &str);
1184 if (!rc)
1185 return FALSE;
1186 if (!str)
1187 return FALSE;
1188 free(*pValue);
1189 *pValue = _strdup(str);
1190 return *pValue != nullptr;
1191}
1192
1193static BOOL load_config_json(BOOL system, H264_CONTEXT_LIBAVCODEC* sys, BOOL* pAllowUserConfig,
1194 BOOL* pAllowEnvOverride)
1195{
1196 WINPR_ASSERT(sys);
1197 WINPR_ASSERT(pAllowUserConfig);
1198 WINPR_ASSERT(pAllowEnvOverride);
1199
1200 WINPR_JSON* syscfg = load(system);
1201 if (syscfg)
1202 {
1203 read_json_bool(syscfg, "allowUserConfig", pAllowUserConfig);
1204 read_json_bool(syscfg, "allowEnvOverrides", pAllowEnvOverride);
1205
1206 read_json_string(syscfg, "hwDeviceName", &sys->device);
1207
1208 const char* str = nullptr;
1209 if (read_json_const_string(syscfg, "hwDeviceType", &str))
1210 sys->type = av_hwdevice_find_type_by_name(str);
1211 read_json_bool(syscfg, "useHwDecoder", &sys->useHwDecoder);
1212 read_json_bool(syscfg, "useHwEncoder", &sys->useHwEncoder);
1213 WINPR_JSON_Delete(syscfg);
1214 }
1215 return TRUE;
1216}
1217#endif
1218
1219WINPR_ATTR_NODISCARD
1220static BOOL load_config(H264_CONTEXT_LIBAVCODEC* sys, BOOL enabled, wLog* log)
1221{
1222 WINPR_ASSERT(sys);
1223
1224#if defined(WITH_FFMPEG_HWACCEL)
1225 sys->useHwDecoder = TRUE;
1226 sys->useHwEncoder = TRUE;
1227#if defined(__APPLE__)
1228 sys->type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX;
1229#elif defined(__ANDROID__)
1230 sys->type = AV_HWDEVICE_TYPE_MEDIACODEC;
1231#elif defined(_WIN32)
1232 sys->type = AV_HWDEVICE_TYPE_DXVA2;
1233#elif defined(__linux__)
1234 sys->type = AV_HWDEVICE_TYPE_VAAPI;
1235 sys->device = _strdup("/dev/dri/renderD128");
1236 if (!sys->device)
1237 return FALSE;
1238#else
1239 sys->type = AV_HWDEVICE_TYPE_NONE;
1240#endif
1241
1242 BOOL allowUserConfig = TRUE;
1243 BOOL allowEnvOverrides = TRUE;
1244
1245 load_config_json(TRUE, sys, &allowUserConfig, &allowEnvOverrides);
1246 if (allowUserConfig)
1247 load_config_json(FALSE, sys, &allowUserConfig, &allowEnvOverrides);
1248
1249 if (allowEnvOverrides)
1250 {
1251 // NOLINTNEXTLINE(concurrency-mt-unsafe)
1252 const char* env = getenv("FREERDP_HWCTX_DEVICE");
1253 if (!env)
1254 env = getenv("FREERDP_VAAPI_DEVICE");
1255
1256 if (env)
1257 {
1258 free(sys->device);
1259 sys->device = _strdup(env);
1260 if (!sys->device)
1261 {
1262 WLog_Print(log, WLOG_ERROR,
1263 "Failed to copy value of FREERDP_HWCTX_TYPE=%s, aborting", env);
1264 return FALSE;
1265 }
1266 }
1267
1268 env = getenv("FREERDP_HWCTX_TYPE");
1269 if (env)
1270 {
1271 sys->type = av_hwdevice_find_type_by_name(env);
1272 if (sys->type == AV_HWDEVICE_TYPE_NONE)
1273 {
1274 WLog_Print(log, WLOG_ERROR, "Unsupported value for FREERDP_HWCTX_TYPE=%s, aborting",
1275 env);
1276 return FALSE;
1277 }
1278 }
1279 }
1280
1281 // Fallback in case of an invalid type
1282 if (!enabled || (sys->type == AV_HWDEVICE_TYPE_NONE))
1283 {
1284 sys->useHwDecoder = FALSE;
1285 sys->useHwEncoder = FALSE;
1286 }
1287#else
1288 WINPR_UNUSED(enabled);
1289 WINPR_UNUSED(log);
1290 WINPR_UNUSED(sys);
1291#endif
1292 return TRUE;
1293}
1294
1295#if defined(WITH_FFMPEG_HWACCEL)
1296WINPR_ATTR_NODISCARD
1297static const char* bool_to_string(BOOL val)
1298{
1299 if (val)
1300 return "true";
1301 return "false";
1302}
1303#endif
1304
1305static BOOL libavcodec_init(H264_CONTEXT* h264)
1306{
1307 WINPR_ASSERT(h264);
1308 H264_CONTEXT_LIBAVCODEC* sys =
1309 (H264_CONTEXT_LIBAVCODEC*)calloc(1, sizeof(H264_CONTEXT_LIBAVCODEC));
1310
1311 if (!sys)
1312 goto EXCEPTION;
1313
1314 if (!load_config(sys, h264->hwAccel > 0, h264->log))
1315 goto EXCEPTION;
1316
1317 h264->pSystemData = (void*)sys;
1318
1319#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 10, 100)
1320 avcodec_register_all();
1321#endif
1322
1323 if (!h264->Compressor)
1324 {
1325 sys->codecDecoder = avcodec_find_decoder(AV_CODEC_ID_H264);
1326
1327 if (!sys->codecDecoder)
1328 {
1329 WLog_Print(h264->log, WLOG_ERROR, "Failed to find libav H.264 codec");
1330 goto EXCEPTION;
1331 }
1332
1333 sys->codecDecoderContext = avcodec_alloc_context3(sys->codecDecoder);
1334
1335 if (!sys->codecDecoderContext)
1336 {
1337 WLog_Print(h264->log, WLOG_ERROR, "Failed to allocate libav codec context");
1338 goto EXCEPTION;
1339 }
1340
1341#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(59, 18, 100)
1342 if (sys->codecDecoder->capabilities & AV_CODEC_CAP_TRUNCATED)
1343 {
1344 sys->codecDecoderContext->flags |= AV_CODEC_FLAG_TRUNCATED;
1345 }
1346#endif
1347
1348#ifdef WITH_FFMPEG_HWACCEL
1349 if (sys->useHwDecoder && (sys->type != AV_HWDEVICE_TYPE_NONE))
1350 WLog_Print(h264->log, WLOG_DEBUG,
1351 "Trying to initialize hardware backed decoder of type %s",
1352 av_hwdevice_get_type_name(sys->type));
1353 else
1354 WLog_Print(h264->log, WLOG_DEBUG,
1355 "Skipping hardware backed decoder (useHwDecoder=%s, type=%s)",
1356 bool_to_string(sys->useHwDecoder), av_hwdevice_get_type_name(sys->type));
1357
1358 if (sys->useHwDecoder && (sys->type != AV_HWDEVICE_TYPE_NONE))
1359 {
1360 sys->hwFrameSupportsNativeFormat = -1;
1361 enum AVHWDeviceType type = sys->type;
1362 if (!sys->hwctx)
1363 {
1364 int ret =
1365 av_hwdevice_ctx_create(&sys->hwctx, type, get_hwctx_device(sys), nullptr, 0);
1366
1367 if (ret < 0)
1368 {
1369 WLog_Print(h264->log, WLOG_ERROR,
1370 "Could not initialize hardware decoder for %s, falling back to "
1371 "software: %s",
1372 get_hwctx_device(sys), av_err2str(ret));
1373 sys->hwctx = nullptr;
1374 goto fail_hwdevice_create;
1375 }
1376 }
1377 WLog_Print(h264->log, WLOG_INFO, "Using %s [%s|%s] for accelerated H264 decoding",
1378 av_hwdevice_get_type_name(type), get_hwctx_device(sys), vaapi_vendor(sys));
1379
1380 sys->codecDecoderContext->get_format = libavcodec_get_format;
1381 sys->hw_pix_fmt = get_hwctx_image_format(sys, h264->log);
1382
1383#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 80, 100)
1384 sys->codecDecoderContext->hw_device_ctx = av_buffer_ref(sys->hwctx);
1385#endif
1386 sys->codecDecoderContext->opaque = (void*)h264;
1387
1388 fail_hwdevice_create:
1389 }
1390#endif
1391
1392 if (avcodec_open2(sys->codecDecoderContext, sys->codecDecoder, nullptr) < 0)
1393 {
1394 WLog_Print(h264->log, WLOG_ERROR, "Failed to open libav codec");
1395 goto EXCEPTION;
1396 }
1397
1398 sys->codecParser = av_parser_init(AV_CODEC_ID_H264);
1399
1400 if (!sys->codecParser)
1401 {
1402 WLog_Print(h264->log, WLOG_ERROR, "Failed to initialize libav parser");
1403 goto EXCEPTION;
1404 }
1405 }
1406 else
1407 {
1408#ifdef WITH_FFMPEG_HWACCEL
1409 if (sys->useHwEncoder && (sys->type != AV_HWDEVICE_TYPE_NONE))
1410 WLog_Print(h264->log, WLOG_DEBUG,
1411 "Trying to initialize hardware backed encoder of type %s",
1412 av_hwdevice_get_type_name(sys->type));
1413 else
1414 WLog_Print(h264->log, WLOG_DEBUG,
1415 "Skipping hardware backed encoder (useHwEncoder=%s, type=%s)",
1416 bool_to_string(sys->useHwEncoder), av_hwdevice_get_type_name(sys->type));
1417
1418 if (sys->useHwEncoder) /* user requested hw accel */
1419 {
1420 enum AVHWDeviceType type = sys->type;
1421
1422 sys->codecEncoder = get_hwctx_codec(sys, h264->log, type);
1423 if (!sys->codecEncoder)
1424 {
1425 WLog_Print(h264->log, WLOG_ERROR, "H264 hardware encoder not found");
1426 }
1427 else if (av_hwdevice_ctx_create(&sys->hwctx, type, get_hwctx_device(sys), nullptr, 0) <
1428 0)
1429 {
1430 WLog_Print(h264->log, WLOG_ERROR, "av_hwdevice_ctx_create(%s) failed",
1431 get_hwctx_device(sys));
1432 sys->codecEncoder = nullptr;
1433 sys->hwctx = nullptr;
1434 }
1435 else
1436 {
1437 WLog_Print(h264->log, WLOG_INFO, "Using %s[%s|%s] for accelerated H264 encoding",
1438 av_hwdevice_get_type_name(sys->type), get_hwctx_device(sys),
1439 vaapi_vendor(sys));
1440 }
1441 }
1442#endif
1443 if (!sys->codecEncoder)
1444 {
1445 sys->codecEncoder = avcodec_find_encoder(AV_CODEC_ID_H264);
1446 h264->hwAccel = FALSE; /* not supported */
1447 }
1448
1449 if (!sys->codecEncoder)
1450 {
1451 WLog_Print(h264->log, WLOG_ERROR, "Failed to initialize H264 encoder");
1452 goto EXCEPTION;
1453 }
1454 }
1455
1456#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 18, 102)
1457 sys->videoFrame = av_frame_alloc();
1458#if defined(WITH_FFMPEG_HWACCEL)
1459 sys->hwVideoFrame = av_frame_alloc();
1460 sys->cpuVideoFrame = av_frame_alloc();
1461#endif
1462#else
1463 sys->videoFrame = avcodec_alloc_frame();
1464#endif
1465
1466 if (!sys->videoFrame)
1467 {
1468 WLog_Print(h264->log, WLOG_ERROR, "Failed to allocate libav frame");
1469 goto EXCEPTION;
1470 }
1471
1472#if defined(WITH_FFMPEG_HWACCEL)
1473 if (!sys->hwVideoFrame || !sys->cpuVideoFrame)
1474 {
1475 WLog_Print(h264->log, WLOG_ERROR, "Failed to allocate libav hw frame");
1476 goto EXCEPTION;
1477 }
1478
1479#endif
1480 sys->videoFrame->pts = 0;
1481 return TRUE;
1482EXCEPTION:
1483 libavcodec_uninit(h264);
1484 return FALSE;
1485}
1486
1487const H264_CONTEXT_SUBSYSTEM g_Subsystem_libavcodec = { "libavcodec", libavcodec_init,
1488 libavcodec_uninit, libavcodec_decompress,
1489 libavcodec_compress };
WINPR_JSON_IsBool
WINPR_ATTR_NODISCARD WINPR_API BOOL WINPR_JSON_IsBool(const WINPR_JSON *item)
Check if JSON item is of type BOOL.
Definition c-json.c:167
WINPR_JSON_IsTrue
WINPR_ATTR_NODISCARD WINPR_API BOOL WINPR_JSON_IsTrue(const WINPR_JSON *item)
Check if JSON item is BOOL value True.
Definition c-json.c:162
WINPR_JSON_GetObjectItemCaseSensitive
WINPR_ATTR_NODISCARD WINPR_API WINPR_JSON * WINPR_JSON_GetObjectItemCaseSensitive(const WINPR_JSON *object, const char *string)
Same as WINPR_JSON_GetObjectItem but with case sensitive matching.
Definition c-json.c:127
WINPR_JSON_IsString
WINPR_ATTR_NODISCARD WINPR_API BOOL WINPR_JSON_IsString(const WINPR_JSON *item)
Check if JSON item is of type String.
Definition c-json.c:182
WINPR_JSON_Delete
WINPR_API void WINPR_JSON_Delete(WINPR_JSON *item)
Delete a WinPR JSON wrapper object.
Definition c-json.c:103
WINPR_JSON_GetStringValue
WINPR_ATTR_NODISCARD WINPR_API const char * WINPR_JSON_GetStringValue(WINPR_JSON *item)
Return the String value of a JSON item.
Definition c-json.c:142