FreeRDP
Loading...
Searching...
No Matches
h264_ffmpeg.c
1
21#include <freerdp/config.h>
22
23#include <winpr/wlog.h>
24#include <freerdp/log.h>
25#include <freerdp/codec/h264.h>
26#include <libavcodec/avcodec.h>
27#include <libavutil/opt.h>
28
29#include "h264.h"
30
31#ifdef WITH_VIDEOTOOLBOX
32#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(55, 9, 0)
33#include <libavutil/hwcontext.h>
34#else
35#pragma warning You have asked for VideoToolbox decoding, \
36 but your version of libavutil is too old !Disabling.
37#undef WITH_VIDEOTOOLBOX
38#endif
39#endif
40
41#ifdef WITH_VAAPI
42#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(55, 9, 0)
43#include <libavutil/hwcontext.h>
44#else
45#pragma warning You have asked for VA - API decoding, \
46 but your version of libavutil is too old !Disabling.
47#undef WITH_VAAPI
48#endif
49#endif
50
51/* Fallback support for older libavcodec versions */
52#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(54, 59, 100)
53#define AV_CODEC_ID_H264 CODEC_ID_H264
54#endif
55
56#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(56, 34, 2)
57#define AV_CODEC_FLAG_LOOP_FILTER CODEC_FLAG_LOOP_FILTER
58#define AV_CODEC_CAP_TRUNCATED CODEC_CAP_TRUNCATED
59#define AV_CODEC_FLAG_TRUNCATED CODEC_FLAG_TRUNCATED
60#endif
61
62#if LIBAVUTIL_VERSION_MAJOR < 52
63#define AV_PIX_FMT_YUV420P PIX_FMT_YUV420P
64#endif
65
66/* Ubuntu 14.04 ships without the functions provided by avutil,
67 * so define error to string methods here. */
68#if !defined(av_err2str)
69static inline char* error_string(char* errbuf, size_t errbuf_size, int errnum)
70{
71 av_strerror(errnum, errbuf, errbuf_size);
72 return errbuf;
73}
74
75#define av_err2str(errnum) error_string((char[64])WINPR_C_ARRAY_INIT, 64, errnum)
76#endif
77
78#if defined(WITH_VAAPI) || defined(WITH_VAAPI_H264_ENCODING)
79static const char* get_vaapi_device(void)
80{
81 static char device[MAX_PATH] = WINPR_C_ARRAY_INIT;
82 static bool initialized = false;
83 if (!initialized)
84 {
85 initialized = true;
86 // NOLINTNEXTLINE(concurrency-mt-unsafe)
87 const char* env = getenv("FREERDP_VAAPI_DEVICE");
88 if (env)
89 (void)_snprintf(device, sizeof(device), "%s", env);
90 else
91 (void)_snprintf(device, sizeof(device), "/dev/dri/renderD128");
92 }
93 return device;
94}
95#endif
96
97typedef struct
98{
99 const AVCodec* codecDecoder;
100 AVCodecContext* codecDecoderContext;
101 const AVCodec* codecEncoder;
102 AVCodecContext* codecEncoderContext;
103 AVCodecParserContext* codecParser;
104 AVFrame* videoFrame;
105#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 133, 100)
106 AVPacket bufferpacket;
107#endif
108 AVPacket* packet;
109#if defined(WITH_VAAPI) || defined(WITH_VAAPI_H264_ENCODING) || defined(WITH_VIDEOTOOLBOX)
110 AVBufferRef* hwctx;
111 AVFrame* hwVideoFrame;
112 enum AVPixelFormat hw_pix_fmt;
113#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(57, 80, 100) || defined(WITH_VIDEOTOOLBOX)
114 AVBufferRef* hw_frames_ctx;
115#endif
116
117#endif
118} H264_CONTEXT_LIBAVCODEC;
119
120static void libavcodec_destroy_encoder_context(H264_CONTEXT* WINPR_RESTRICT h264)
121{
122 H264_CONTEXT_LIBAVCODEC* sys = nullptr;
123
124 if (!h264 || !h264->subsystem)
125 return;
126
127 sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
128
129 if (sys->codecEncoderContext)
130 {
131#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 69, 100)
132 avcodec_free_context(&sys->codecEncoderContext);
133#else
134 avcodec_close(sys->codecEncoderContext);
135 av_free(sys->codecEncoderContext);
136#endif
137 }
138
139 sys->codecEncoderContext = nullptr;
140}
141
142#ifdef WITH_VAAPI_H264_ENCODING
143static int set_hw_frames_ctx(H264_CONTEXT* WINPR_RESTRICT h264)
144{
145 H264_CONTEXT_LIBAVCODEC* sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
146 AVBufferRef* hw_frames_ref = nullptr;
147 AVHWFramesContext* frames_ctx = nullptr;
148 int err = 0;
149
150 if (!(hw_frames_ref = av_hwframe_ctx_alloc(sys->hwctx)))
151 {
152 WLog_Print(h264->log, WLOG_ERROR, "Failed to create VAAPI frame context");
153 return -1;
154 }
155 frames_ctx = (AVHWFramesContext*)(hw_frames_ref->data);
156 frames_ctx->format = AV_PIX_FMT_VAAPI;
157 frames_ctx->sw_format = AV_PIX_FMT_NV12;
158 frames_ctx->width = sys->codecEncoderContext->width;
159 frames_ctx->height = sys->codecEncoderContext->height;
160 frames_ctx->initial_pool_size = 20;
161 if ((err = av_hwframe_ctx_init(hw_frames_ref)) < 0)
162 {
163 WLog_Print(h264->log, WLOG_ERROR,
164 "Failed to initialize VAAPI frame context."
165 "Error code: %s",
166 av_err2str(err));
167 av_buffer_unref(&hw_frames_ref);
168 return err;
169 }
170 sys->codecEncoderContext->hw_frames_ctx = av_buffer_ref(hw_frames_ref);
171 if (!sys->codecEncoderContext->hw_frames_ctx)
172 err = AVERROR(ENOMEM);
173
174 av_buffer_unref(&hw_frames_ref);
175 return err;
176}
177#endif
178
179static BOOL libavcodec_create_encoder_context(H264_CONTEXT* WINPR_RESTRICT h264)
180{
181 BOOL recreate = FALSE;
182 H264_CONTEXT_LIBAVCODEC* sys = nullptr;
183
184 if (!h264 || !h264->subsystem)
185 return FALSE;
186
187 if ((h264->width > INT_MAX) || (h264->height > INT_MAX))
188 return FALSE;
189
190 sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
191 if (!sys || !sys->codecEncoder)
192 return FALSE;
193
194 recreate = !sys->codecEncoderContext;
195
196 if (sys->codecEncoderContext)
197 {
198 if ((sys->codecEncoderContext->width != (int)h264->width) ||
199 (sys->codecEncoderContext->height != (int)h264->height))
200 recreate = TRUE;
201 }
202
203 if (!recreate)
204 return TRUE;
205
206 libavcodec_destroy_encoder_context(h264);
207
208 sys->codecEncoderContext = avcodec_alloc_context3(sys->codecEncoder);
209
210 if (!sys->codecEncoderContext)
211 goto EXCEPTION;
212
213 switch (h264->RateControlMode)
214 {
215 case H264_RATECONTROL_VBR:
216 sys->codecEncoderContext->bit_rate = h264->BitRate;
217 break;
218
219 case H264_RATECONTROL_CQP:
220 if (av_opt_set_int(sys->codecEncoderContext, "qp", h264->QP, AV_OPT_SEARCH_CHILDREN) <
221 0)
222 {
223 WLog_Print(h264->log, WLOG_ERROR, "av_opt_set_int failed");
224 }
225 break;
226
227 default:
228 break;
229 }
230
231 sys->codecEncoderContext->width = (int)MIN(INT32_MAX, h264->width);
232 sys->codecEncoderContext->height = (int)MIN(INT32_MAX, h264->height);
233 sys->codecEncoderContext->delay = 0;
234#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(56, 13, 100)
235 sys->codecEncoderContext->framerate =
236 (AVRational){ WINPR_ASSERTING_INT_CAST(int, h264->FrameRate), 1 };
237#endif
238 sys->codecEncoderContext->time_base =
239 (AVRational){ 1, WINPR_ASSERTING_INT_CAST(int, h264->FrameRate) };
240 av_opt_set(sys->codecEncoderContext, "tune", "zerolatency", AV_OPT_SEARCH_CHILDREN);
241
242 sys->codecEncoderContext->flags |= AV_CODEC_FLAG_LOOP_FILTER;
243
244#ifdef WITH_VAAPI_H264_ENCODING
245 if (sys->hwctx)
246 {
247 av_opt_set(sys->codecEncoderContext, "preset", "veryslow", AV_OPT_SEARCH_CHILDREN);
248
249 sys->codecEncoderContext->pix_fmt = AV_PIX_FMT_VAAPI;
250 /* set hw_frames_ctx for encoder's AVCodecContext */
251 if (set_hw_frames_ctx(h264) < 0)
252 goto EXCEPTION;
253 }
254 else
255#endif
256 {
257 av_opt_set(sys->codecEncoderContext, "preset", "medium", AV_OPT_SEARCH_CHILDREN);
258 sys->codecEncoderContext->pix_fmt = AV_PIX_FMT_YUV420P;
259 }
260
261 if (avcodec_open2(sys->codecEncoderContext, sys->codecEncoder, nullptr) < 0)
262 goto EXCEPTION;
263
264 return TRUE;
265EXCEPTION:
266 libavcodec_destroy_encoder_context(h264);
267 return FALSE;
268}
269
270static int libavcodec_decompress(H264_CONTEXT* WINPR_RESTRICT h264,
271 const BYTE* WINPR_RESTRICT pSrcData, UINT32 SrcSize)
272{
273 union
274 {
275 const BYTE* cpv;
276 BYTE* pv;
277 } cnv;
278 int rc = -1;
279 int status = 0;
280 int gotFrame = 0;
281 AVPacket* packet = nullptr;
282
283 WINPR_ASSERT(h264);
284 WINPR_ASSERT(pSrcData || (SrcSize == 0));
285
286 H264_CONTEXT_LIBAVCODEC* sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
287 BYTE** pYUVData = h264->pYUVData;
288 UINT32* iStride = h264->iStride;
289
290 WINPR_ASSERT(sys);
291
292#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 133, 100)
293 packet = &sys->bufferpacket;
294 WINPR_ASSERT(packet);
295 av_init_packet(packet);
296#else
297 packet = av_packet_alloc();
298#endif
299 if (!packet)
300 {
301 WLog_Print(h264->log, WLOG_ERROR, "Failed to allocate AVPacket");
302 goto fail;
303 }
304
305 cnv.cpv = pSrcData;
306 packet->data = cnv.pv;
307 packet->size = (int)MIN(SrcSize, INT32_MAX);
308
309 WINPR_ASSERT(sys->codecDecoderContext);
310 /* avcodec_decode_video2 is deprecated with libavcodec 57.48.101 */
311#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 48, 101)
312 status = avcodec_send_packet(sys->codecDecoderContext, packet);
313
314 if (status < 0)
315 {
316 WLog_Print(h264->log, WLOG_ERROR, "Failed to decode video frame (status=%d)", status);
317 goto fail;
318 }
319
320 sys->videoFrame->format = AV_PIX_FMT_YUV420P;
321
322#if defined(WITH_VAAPI) || defined(WITH_VIDEOTOOLBOX)
323 status = avcodec_receive_frame(sys->codecDecoderContext,
324 sys->hwctx ? sys->hwVideoFrame : sys->videoFrame);
325#else
326 status = avcodec_receive_frame(sys->codecDecoderContext, sys->videoFrame);
327#endif
328 if (status == AVERROR(EAGAIN))
329 {
330 rc = 0;
331 goto fail;
332 }
333
334 gotFrame = (status == 0);
335#else
336#if defined(WITH_VAAPI) || defined(WITH_VIDEOTOOLBOX)
337 status =
338 avcodec_decode_video2(sys->codecDecoderContext,
339 sys->hwctx ? sys->hwVideoFrame : sys->videoFrame, &gotFrame, packet);
340#else
341 status = avcodec_decode_video2(sys->codecDecoderContext, sys->videoFrame, &gotFrame, packet);
342#endif
343#endif
344 if (status < 0)
345 {
346 WLog_Print(h264->log, WLOG_ERROR, "Failed to decode video frame (status=%d)", status);
347 goto fail;
348 }
349
350#if defined(WITH_VAAPI) || defined(WITH_VIDEOTOOLBOX)
351
352 if (sys->hwctx)
353 {
354 if (sys->hwVideoFrame->format == sys->hw_pix_fmt)
355 {
356 sys->videoFrame->width = sys->hwVideoFrame->width;
357 sys->videoFrame->height = sys->hwVideoFrame->height;
358 status = av_hwframe_transfer_data(sys->videoFrame, sys->hwVideoFrame, 0);
359 }
360 else
361 {
362 status = av_frame_copy(sys->videoFrame, sys->hwVideoFrame);
363 }
364 }
365
366 gotFrame = (status == 0);
367
368 if (status < 0)
369 {
370 WLog_Print(h264->log, WLOG_ERROR, "Failed to transfer video frame (status=%d) (%s)", status,
371 av_err2str(status));
372 goto fail;
373 }
374
375#endif
376
377 if (gotFrame)
378 {
379 WINPR_ASSERT(sys->videoFrame);
380
381 pYUVData[0] = sys->videoFrame->data[0];
382 pYUVData[1] = sys->videoFrame->data[1];
383 pYUVData[2] = sys->videoFrame->data[2];
384 iStride[0] = (UINT32)MAX(0, sys->videoFrame->linesize[0]);
385 iStride[1] = (UINT32)MAX(0, sys->videoFrame->linesize[1]);
386 iStride[2] = (UINT32)MAX(0, sys->videoFrame->linesize[2]);
387 rc = 1;
388 }
389 else
390 rc = -2;
391
392fail:
393#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 133, 100)
394 av_packet_unref(packet);
395#else
396 av_packet_free(&packet);
397#endif
398
399 return rc;
400}
401
402static int libavcodec_compress(H264_CONTEXT* WINPR_RESTRICT h264,
403 const BYTE** WINPR_RESTRICT pSrcYuv,
404 const UINT32* WINPR_RESTRICT pStride,
405 BYTE** WINPR_RESTRICT ppDstData, UINT32* WINPR_RESTRICT pDstSize)
406{
407 union
408 {
409 const BYTE* cpv;
410 uint8_t* pv;
411 } cnv;
412 int rc = -1;
413 int status = 0;
414 int gotFrame = 0;
415
416 WINPR_ASSERT(h264);
417
418 H264_CONTEXT_LIBAVCODEC* sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
419 WINPR_ASSERT(sys);
420
421 if (!libavcodec_create_encoder_context(h264))
422 return -1;
423
424#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 133, 100)
425 sys->packet = &sys->bufferpacket;
426 av_packet_unref(sys->packet);
427 av_init_packet(sys->packet);
428#else
429 av_packet_free(&sys->packet);
430 sys->packet = av_packet_alloc();
431#endif
432 if (!sys->packet)
433 {
434 WLog_Print(h264->log, WLOG_ERROR, "Failed to allocate AVPacket");
435 goto fail;
436 }
437
438 WINPR_ASSERT(sys->packet);
439 sys->packet->data = nullptr;
440 sys->packet->size = 0;
441
442 WINPR_ASSERT(sys->videoFrame);
443 WINPR_ASSERT(sys->codecEncoderContext);
444 sys->videoFrame->format = AV_PIX_FMT_YUV420P;
445 sys->videoFrame->width = sys->codecEncoderContext->width;
446 sys->videoFrame->height = sys->codecEncoderContext->height;
447#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(52, 48, 100)
448 sys->videoFrame->colorspace = AVCOL_SPC_BT709;
449#endif
450#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(52, 92, 100)
451 sys->videoFrame->chroma_location = AVCHROMA_LOC_LEFT;
452#endif
453 cnv.cpv = pSrcYuv[0];
454 sys->videoFrame->data[0] = cnv.pv;
455
456 cnv.cpv = pSrcYuv[1];
457 sys->videoFrame->data[1] = cnv.pv;
458
459 cnv.cpv = pSrcYuv[2];
460 sys->videoFrame->data[2] = cnv.pv;
461
462 sys->videoFrame->linesize[0] = (int)pStride[0];
463 sys->videoFrame->linesize[1] = (int)pStride[1];
464 sys->videoFrame->linesize[2] = (int)pStride[2];
465 sys->videoFrame->pts++;
466
467#ifdef WITH_VAAPI_H264_ENCODING
468 if (sys->hwctx)
469 {
470 av_frame_unref(sys->hwVideoFrame);
471 if ((status = av_hwframe_get_buffer(sys->codecEncoderContext->hw_frames_ctx,
472 sys->hwVideoFrame, 0)) < 0 ||
473 !sys->hwVideoFrame->hw_frames_ctx)
474 {
475 WLog_Print(h264->log, WLOG_ERROR, "av_hwframe_get_buffer failed (%s [%d])",
476 av_err2str(status), status);
477 goto fail;
478 }
479 sys->videoFrame->format = AV_PIX_FMT_NV12;
480 if ((status = av_hwframe_transfer_data(sys->hwVideoFrame, sys->videoFrame, 0)) < 0)
481 {
482 WLog_Print(h264->log, WLOG_ERROR, "av_hwframe_transfer_data failed (%s [%d])",
483 av_err2str(status), status);
484 goto fail;
485 }
486 }
487#endif
488
489 /* avcodec_encode_video2 is deprecated with libavcodec 57.48.101 */
490#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 48, 101)
491#ifdef WITH_VAAPI_H264_ENCODING
492 status = avcodec_send_frame(sys->codecEncoderContext,
493 sys->hwctx ? sys->hwVideoFrame : sys->videoFrame);
494#else
495 status = avcodec_send_frame(sys->codecEncoderContext, sys->videoFrame);
496#endif
497
498 if (status < 0)
499 {
500 WLog_Print(h264->log, WLOG_ERROR, "Failed to encode video frame (%s [%d])",
501 av_err2str(status), status);
502 goto fail;
503 }
504
505 status = avcodec_receive_packet(sys->codecEncoderContext, sys->packet);
506
507 if (status < 0)
508 {
509 WLog_Print(h264->log, WLOG_ERROR, "Failed to encode video frame (%s [%d])",
510 av_err2str(status), status);
511 goto fail;
512 }
513
514 gotFrame = (status == 0);
515#elif LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(54, 59, 100)
516
517 do
518 {
519 status = avcodec_encode_video2(sys->codecEncoderContext, sys->packet, sys->videoFrame,
520 &gotFrame);
521 } while ((status >= 0) && (gotFrame == 0));
522
523#else
524 sys->packet->size =
525 avpicture_get_size(sys->codecDecoderContext->pix_fmt, sys->codecDecoderContext->width,
526 sys->codecDecoderContext->height);
527 sys->packet->data = av_malloc(sys->packet->size);
528
529 if (!sys->packet->data)
530 status = -1;
531 else
532 {
533 status = avcodec_encode_video(sys->codecDecoderContext, sys->packet->data,
534 sys->packet->size, sys->videoFrame);
535 }
536
537#endif
538
539 if (status < 0)
540 {
541 WLog_Print(h264->log, WLOG_ERROR, "Failed to encode video frame (%s [%d])",
542 av_err2str(status), status);
543 goto fail;
544 }
545
546 WINPR_ASSERT(sys->packet);
547 *ppDstData = sys->packet->data;
548 *pDstSize = (UINT32)MAX(0, sys->packet->size);
549
550 if (!gotFrame)
551 {
552 WLog_Print(h264->log, WLOG_ERROR, "Did not get frame! (%s [%d])", av_err2str(status),
553 status);
554 rc = -2;
555 }
556 else
557 rc = 1;
558fail:
559 return rc;
560}
561
562static void libavcodec_uninit(H264_CONTEXT* h264)
563{
564 WINPR_ASSERT(h264);
565
566 H264_CONTEXT_LIBAVCODEC* sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
567
568 if (!sys)
569 return;
570
571 if (sys->packet)
572 {
573#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 133, 100)
574 av_packet_unref(sys->packet);
575#else
576 av_packet_free(&sys->packet);
577#endif
578 }
579
580 if (sys->videoFrame)
581 {
582#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 18, 102)
583 av_frame_free(&sys->videoFrame);
584#else
585 av_free(sys->videoFrame);
586#endif
587 }
588
589#if defined(WITH_VAAPI) || defined(WITH_VAAPI_H264_ENCODING) || defined(WITH_VIDEOTOOLBOX)
590 if (sys->hwVideoFrame)
591 {
592#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 18, 102)
593 av_frame_free(&sys->hwVideoFrame);
594#else
595 av_free(sys->hwVideoFrame);
596#endif
597 }
598
599 if (sys->hwctx)
600 av_buffer_unref(&sys->hwctx);
601
602#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(57, 80, 100) || defined(WITH_VIDEOTOOLBOX)
603
604 if (sys->hw_frames_ctx)
605 av_buffer_unref(&sys->hw_frames_ctx);
606
607#endif
608
609#endif
610
611 if (sys->codecParser)
612 av_parser_close(sys->codecParser);
613
614 if (sys->codecDecoderContext)
615 {
616#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 69, 100)
617 avcodec_free_context(&sys->codecDecoderContext);
618#else
619 avcodec_close(sys->codecDecoderContext);
620 av_free(sys->codecDecoderContext);
621#endif
622 }
623
624 libavcodec_destroy_encoder_context(h264);
625 free(sys);
626 h264->pSystemData = nullptr;
627}
628
629#if defined(WITH_VAAPI) || defined(WITH_VIDEOTOOLBOX)
630static enum AVPixelFormat libavcodec_get_format(struct AVCodecContext* ctx,
631 const enum AVPixelFormat* fmts)
632{
633 WINPR_ASSERT(ctx);
634
635 H264_CONTEXT* h264 = (H264_CONTEXT*)ctx->opaque;
636 WINPR_ASSERT(h264);
637
638 H264_CONTEXT_LIBAVCODEC* sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
639 WINPR_ASSERT(sys);
640
641 for (const enum AVPixelFormat* p = fmts; *p != AV_PIX_FMT_NONE; p++)
642 {
643 if (*p == sys->hw_pix_fmt)
644 {
645#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(57, 80, 100) || defined(WITH_VIDEOTOOLBOX)
646 if (sys->hw_frames_ctx)
647 av_buffer_unref(&sys->hw_frames_ctx);
648
649 sys->hw_frames_ctx = av_hwframe_ctx_alloc(sys->hwctx);
650
651 if (!sys->hw_frames_ctx)
652 {
653 return AV_PIX_FMT_NONE;
654 }
655
656 sys->codecDecoderContext->pix_fmt = *p;
657 AVHWFramesContext* frames = (AVHWFramesContext*)sys->hw_frames_ctx->data;
658 frames->format = *p;
659 frames->height = sys->codecDecoderContext->coded_height;
660 frames->width = sys->codecDecoderContext->coded_width;
661#ifdef WITH_VIDEOTOOLBOX
662 frames->sw_format = AV_PIX_FMT_YUV420P;
663#else
664 frames->sw_format =
665 (sys->codecDecoderContext->sw_pix_fmt == AV_PIX_FMT_YUV420P10 ? AV_PIX_FMT_P010
666 : AV_PIX_FMT_NV12);
667#endif
668 frames->initial_pool_size = 20;
669
670 if (sys->codecDecoderContext->active_thread_type & FF_THREAD_FRAME)
671 frames->initial_pool_size += sys->codecDecoderContext->thread_count;
672
673 int err = av_hwframe_ctx_init(sys->hw_frames_ctx);
674
675 if (err < 0)
676 {
677 WLog_Print(h264->log, WLOG_ERROR, "Could not init hwframes context: %s",
678 av_err2str(err));
679 return AV_PIX_FMT_NONE;
680 }
681
682 sys->codecDecoderContext->hw_frames_ctx = av_buffer_ref(sys->hw_frames_ctx);
683#endif
684 return *p;
685 }
686 }
687
688 return AV_PIX_FMT_NONE;
689}
690#endif
691
692static BOOL libavcodec_init(H264_CONTEXT* h264)
693{
694 WINPR_ASSERT(h264);
695 H264_CONTEXT_LIBAVCODEC* sys =
696 (H264_CONTEXT_LIBAVCODEC*)calloc(1, sizeof(H264_CONTEXT_LIBAVCODEC));
697
698 if (!sys)
699 {
700 goto EXCEPTION;
701 }
702
703 h264->pSystemData = (void*)sys;
704
705#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 10, 100)
706 avcodec_register_all();
707#endif
708
709 if (!h264->Compressor)
710 {
711 sys->codecDecoder = avcodec_find_decoder(AV_CODEC_ID_H264);
712
713 if (!sys->codecDecoder)
714 {
715 WLog_Print(h264->log, WLOG_ERROR, "Failed to find libav H.264 codec");
716 goto EXCEPTION;
717 }
718
719 sys->codecDecoderContext = avcodec_alloc_context3(sys->codecDecoder);
720
721 if (!sys->codecDecoderContext)
722 {
723 WLog_Print(h264->log, WLOG_ERROR, "Failed to allocate libav codec context");
724 goto EXCEPTION;
725 }
726
727#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(59, 18, 100)
728 if (sys->codecDecoder->capabilities & AV_CODEC_CAP_TRUNCATED)
729 {
730 sys->codecDecoderContext->flags |= AV_CODEC_FLAG_TRUNCATED;
731 }
732#endif
733
734#ifdef WITH_VAAPI
735
736 if (!sys->hwctx)
737 {
738 int ret = av_hwdevice_ctx_create(&sys->hwctx, AV_HWDEVICE_TYPE_VAAPI,
739 get_vaapi_device(), nullptr, 0);
740
741 if (ret < 0)
742 {
743 WLog_Print(h264->log, WLOG_ERROR,
744 "Could not initialize hardware decoder, falling back to software: %s",
745 av_err2str(ret));
746 sys->hwctx = nullptr;
747 goto fail_hwdevice_create;
748 }
749 }
750 WLog_Print(h264->log, WLOG_INFO, "Using VAAPI for accelerated H264 decoding");
751
752 sys->codecDecoderContext->get_format = libavcodec_get_format;
753 sys->hw_pix_fmt = AV_PIX_FMT_VAAPI;
754#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 80, 100)
755 sys->codecDecoderContext->hw_device_ctx = av_buffer_ref(sys->hwctx);
756#endif
757 sys->codecDecoderContext->opaque = (void*)h264;
758 fail_hwdevice_create:
759#endif
760
761#ifdef WITH_VIDEOTOOLBOX
762
763 if (!sys->hwctx)
764 {
765 int ret = av_hwdevice_ctx_create(&sys->hwctx, AV_HWDEVICE_TYPE_VIDEOTOOLBOX, nullptr,
766 nullptr, 0);
767
768 if (ret < 0)
769 {
770 WLog_Print(
771 h264->log, WLOG_ERROR,
772 "Could not initialize VideoToolbox decoder, falling back to software: %s",
773 av_err2str(ret));
774 sys->hwctx = nullptr;
775 goto fail_vt_create;
776 }
777 }
778 WLog_Print(h264->log, WLOG_INFO, "Using VideoToolbox for accelerated H264 decoding");
779
780 sys->codecDecoderContext->get_format = libavcodec_get_format;
781 sys->hw_pix_fmt = AV_PIX_FMT_VIDEOTOOLBOX;
782 sys->codecDecoderContext->hw_device_ctx = av_buffer_ref(sys->hwctx);
783 sys->codecDecoderContext->opaque = (void*)h264;
784 fail_vt_create:
785#endif
786
787 if (avcodec_open2(sys->codecDecoderContext, sys->codecDecoder, nullptr) < 0)
788 {
789 WLog_Print(h264->log, WLOG_ERROR, "Failed to open libav codec");
790 goto EXCEPTION;
791 }
792
793 sys->codecParser = av_parser_init(AV_CODEC_ID_H264);
794
795 if (!sys->codecParser)
796 {
797 WLog_Print(h264->log, WLOG_ERROR, "Failed to initialize libav parser");
798 goto EXCEPTION;
799 }
800 }
801 else
802 {
803#ifdef WITH_VAAPI_H264_ENCODING
804 if (h264->hwAccel) /* user requested hw accel */
805 {
806 sys->codecEncoder = avcodec_find_encoder_by_name("h264_vaapi");
807 if (!sys->codecEncoder)
808 {
809 WLog_Print(h264->log, WLOG_ERROR, "H264 VAAPI encoder not found");
810 }
811 else if (av_hwdevice_ctx_create(&sys->hwctx, AV_HWDEVICE_TYPE_VAAPI, get_vaapi_device(),
812 nullptr, 0) < 0)
813 {
814 WLog_Print(h264->log, WLOG_ERROR, "av_hwdevice_ctx_create failed");
815 sys->codecEncoder = nullptr;
816 sys->hwctx = nullptr;
817 }
818 else
819 {
820 WLog_Print(h264->log, WLOG_INFO, "Using VAAPI for accelerated H264 encoding");
821 }
822 }
823#endif
824 if (!sys->codecEncoder)
825 {
826 sys->codecEncoder = avcodec_find_encoder(AV_CODEC_ID_H264);
827 h264->hwAccel = FALSE; /* not supported */
828 }
829
830 if (!sys->codecEncoder)
831 {
832 WLog_Print(h264->log, WLOG_ERROR, "Failed to initialize H264 encoder");
833 goto EXCEPTION;
834 }
835 }
836
837#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 18, 102)
838 sys->videoFrame = av_frame_alloc();
839#if defined(WITH_VAAPI) || defined(WITH_VAAPI_H264_ENCODING) || defined(WITH_VIDEOTOOLBOX)
840 sys->hwVideoFrame = av_frame_alloc();
841#endif
842#else
843 sys->videoFrame = avcodec_alloc_frame();
844#endif
845
846 if (!sys->videoFrame)
847 {
848 WLog_Print(h264->log, WLOG_ERROR, "Failed to allocate libav frame");
849 goto EXCEPTION;
850 }
851
852#if defined(WITH_VAAPI) || defined(WITH_VAAPI_H264_ENCODING) || defined(WITH_VIDEOTOOLBOX)
853 if (!sys->hwVideoFrame)
854 {
855 WLog_Print(h264->log, WLOG_ERROR, "Failed to allocate libav hw frame");
856 goto EXCEPTION;
857 }
858
859#endif
860 sys->videoFrame->pts = 0;
861 return TRUE;
862EXCEPTION:
863 libavcodec_uninit(h264);
864 return FALSE;
865}
866
867const H264_CONTEXT_SUBSYSTEM g_Subsystem_libavcodec = { "libavcodec", libavcodec_init,
868 libavcodec_uninit, libavcodec_decompress,
869 libavcodec_compress };