FreeRDP
Loading...
Searching...
No Matches
h264_ffmpeg.c
1
21#include <freerdp/config.h>
22
23#include <winpr/wlog.h>
24#include <freerdp/log.h>
25#include <freerdp/codec/h264.h>
26#include <libavcodec/avcodec.h>
27#include <libavutil/opt.h>
28
29#include "h264.h"
30
31#ifdef WITH_VIDEOTOOLBOX
32#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(55, 9, 0)
33#include <libavutil/hwcontext.h>
34#else
35#pragma warning You have asked for VideoToolbox decoding, \
36 but your version of libavutil is too old !Disabling.
37#undef WITH_VIDEOTOOLBOX
38#endif
39#endif
40
41#ifdef WITH_VAAPI
42#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(55, 9, 0)
43#include <libavutil/hwcontext.h>
44#else
45#pragma warning You have asked for VA - API decoding, \
46 but your version of libavutil is too old !Disabling.
47#undef WITH_VAAPI
48#endif
49#endif
50
51/* Fallback support for older libavcodec versions */
52#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(54, 59, 100)
53#define AV_CODEC_ID_H264 CODEC_ID_H264
54#endif
55
56#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(56, 34, 2)
57#define AV_CODEC_FLAG_LOOP_FILTER CODEC_FLAG_LOOP_FILTER
58#define AV_CODEC_CAP_TRUNCATED CODEC_CAP_TRUNCATED
59#define AV_CODEC_FLAG_TRUNCATED CODEC_FLAG_TRUNCATED
60#endif
61
62#if LIBAVUTIL_VERSION_MAJOR < 52
63#define AV_PIX_FMT_YUV420P PIX_FMT_YUV420P
64#endif
65
66/* Ubuntu 14.04 ships without the functions provided by avutil,
67 * so define error to string methods here. */
68#if !defined(av_err2str)
69static inline char* error_string(char* errbuf, size_t errbuf_size, int errnum)
70{
71 av_strerror(errnum, errbuf, errbuf_size);
72 return errbuf;
73}
74
75#define av_err2str(errnum) error_string((char[64])WINPR_C_ARRAY_INIT, 64, errnum)
76#endif
77
78#if defined(WITH_VAAPI) || defined(WITH_VAAPI_H264_ENCODING)
79#define VAAPI_DEVICE "/dev/dri/renderD128"
80#endif
81
82typedef struct
83{
84 const AVCodec* codecDecoder;
85 AVCodecContext* codecDecoderContext;
86 const AVCodec* codecEncoder;
87 AVCodecContext* codecEncoderContext;
88 AVCodecParserContext* codecParser;
89 AVFrame* videoFrame;
90#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 133, 100)
91 AVPacket bufferpacket;
92#endif
93 AVPacket* packet;
94#if defined(WITH_VAAPI) || defined(WITH_VAAPI_H264_ENCODING) || defined(WITH_VIDEOTOOLBOX)
95 AVBufferRef* hwctx;
96 AVFrame* hwVideoFrame;
97 enum AVPixelFormat hw_pix_fmt;
98#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(57, 80, 100) || defined(WITH_VIDEOTOOLBOX)
99 AVBufferRef* hw_frames_ctx;
100#endif
101
102#endif
103} H264_CONTEXT_LIBAVCODEC;
104
105static void libavcodec_destroy_encoder_context(H264_CONTEXT* WINPR_RESTRICT h264)
106{
107 H264_CONTEXT_LIBAVCODEC* sys = nullptr;
108
109 if (!h264 || !h264->subsystem)
110 return;
111
112 sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
113
114 if (sys->codecEncoderContext)
115 {
116#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 69, 100)
117 avcodec_free_context(&sys->codecEncoderContext);
118#else
119 avcodec_close(sys->codecEncoderContext);
120 av_free(sys->codecEncoderContext);
121#endif
122 }
123
124 sys->codecEncoderContext = nullptr;
125}
126
127#ifdef WITH_VAAPI_H264_ENCODING
128static int set_hw_frames_ctx(H264_CONTEXT* WINPR_RESTRICT h264)
129{
130 H264_CONTEXT_LIBAVCODEC* sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
131 AVBufferRef* hw_frames_ref = nullptr;
132 AVHWFramesContext* frames_ctx = nullptr;
133 int err = 0;
134
135 if (!(hw_frames_ref = av_hwframe_ctx_alloc(sys->hwctx)))
136 {
137 WLog_Print(h264->log, WLOG_ERROR, "Failed to create VAAPI frame context");
138 return -1;
139 }
140 frames_ctx = (AVHWFramesContext*)(hw_frames_ref->data);
141 frames_ctx->format = AV_PIX_FMT_VAAPI;
142 frames_ctx->sw_format = AV_PIX_FMT_NV12;
143 frames_ctx->width = sys->codecEncoderContext->width;
144 frames_ctx->height = sys->codecEncoderContext->height;
145 frames_ctx->initial_pool_size = 20;
146 if ((err = av_hwframe_ctx_init(hw_frames_ref)) < 0)
147 {
148 WLog_Print(h264->log, WLOG_ERROR,
149 "Failed to initialize VAAPI frame context."
150 "Error code: %s",
151 av_err2str(err));
152 av_buffer_unref(&hw_frames_ref);
153 return err;
154 }
155 sys->codecEncoderContext->hw_frames_ctx = av_buffer_ref(hw_frames_ref);
156 if (!sys->codecEncoderContext->hw_frames_ctx)
157 err = AVERROR(ENOMEM);
158
159 av_buffer_unref(&hw_frames_ref);
160 return err;
161}
162#endif
163
164static BOOL libavcodec_create_encoder_context(H264_CONTEXT* WINPR_RESTRICT h264)
165{
166 BOOL recreate = FALSE;
167 H264_CONTEXT_LIBAVCODEC* sys = nullptr;
168
169 if (!h264 || !h264->subsystem)
170 return FALSE;
171
172 if ((h264->width > INT_MAX) || (h264->height > INT_MAX))
173 return FALSE;
174
175 sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
176 if (!sys || !sys->codecEncoder)
177 return FALSE;
178
179 recreate = !sys->codecEncoderContext;
180
181 if (sys->codecEncoderContext)
182 {
183 if ((sys->codecEncoderContext->width != (int)h264->width) ||
184 (sys->codecEncoderContext->height != (int)h264->height))
185 recreate = TRUE;
186 }
187
188 if (!recreate)
189 return TRUE;
190
191 libavcodec_destroy_encoder_context(h264);
192
193 sys->codecEncoderContext = avcodec_alloc_context3(sys->codecEncoder);
194
195 if (!sys->codecEncoderContext)
196 goto EXCEPTION;
197
198 switch (h264->RateControlMode)
199 {
200 case H264_RATECONTROL_VBR:
201 sys->codecEncoderContext->bit_rate = h264->BitRate;
202 break;
203
204 case H264_RATECONTROL_CQP:
205 if (av_opt_set_int(sys->codecEncoderContext, "qp", h264->QP, AV_OPT_SEARCH_CHILDREN) <
206 0)
207 {
208 WLog_Print(h264->log, WLOG_ERROR, "av_opt_set_int failed");
209 }
210 break;
211
212 default:
213 break;
214 }
215
216 sys->codecEncoderContext->width = (int)MIN(INT32_MAX, h264->width);
217 sys->codecEncoderContext->height = (int)MIN(INT32_MAX, h264->height);
218 sys->codecEncoderContext->delay = 0;
219#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(56, 13, 100)
220 sys->codecEncoderContext->framerate =
221 (AVRational){ WINPR_ASSERTING_INT_CAST(int, h264->FrameRate), 1 };
222#endif
223 sys->codecEncoderContext->time_base =
224 (AVRational){ 1, WINPR_ASSERTING_INT_CAST(int, h264->FrameRate) };
225 av_opt_set(sys->codecEncoderContext, "tune", "zerolatency", AV_OPT_SEARCH_CHILDREN);
226
227 sys->codecEncoderContext->flags |= AV_CODEC_FLAG_LOOP_FILTER;
228
229#ifdef WITH_VAAPI_H264_ENCODING
230 if (sys->hwctx)
231 {
232 av_opt_set(sys->codecEncoderContext, "preset", "veryslow", AV_OPT_SEARCH_CHILDREN);
233
234 sys->codecEncoderContext->pix_fmt = AV_PIX_FMT_VAAPI;
235 /* set hw_frames_ctx for encoder's AVCodecContext */
236 if (set_hw_frames_ctx(h264) < 0)
237 goto EXCEPTION;
238 }
239 else
240#endif
241 {
242 av_opt_set(sys->codecEncoderContext, "preset", "medium", AV_OPT_SEARCH_CHILDREN);
243 sys->codecEncoderContext->pix_fmt = AV_PIX_FMT_YUV420P;
244 }
245
246 if (avcodec_open2(sys->codecEncoderContext, sys->codecEncoder, nullptr) < 0)
247 goto EXCEPTION;
248
249 return TRUE;
250EXCEPTION:
251 libavcodec_destroy_encoder_context(h264);
252 return FALSE;
253}
254
255static int libavcodec_decompress(H264_CONTEXT* WINPR_RESTRICT h264,
256 const BYTE* WINPR_RESTRICT pSrcData, UINT32 SrcSize)
257{
258 union
259 {
260 const BYTE* cpv;
261 BYTE* pv;
262 } cnv;
263 int rc = -1;
264 int status = 0;
265 int gotFrame = 0;
266 AVPacket* packet = nullptr;
267
268 WINPR_ASSERT(h264);
269 WINPR_ASSERT(pSrcData || (SrcSize == 0));
270
271 H264_CONTEXT_LIBAVCODEC* sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
272 BYTE** pYUVData = h264->pYUVData;
273 UINT32* iStride = h264->iStride;
274
275 WINPR_ASSERT(sys);
276
277#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 133, 100)
278 packet = &sys->bufferpacket;
279 WINPR_ASSERT(packet);
280 av_init_packet(packet);
281#else
282 packet = av_packet_alloc();
283#endif
284 if (!packet)
285 {
286 WLog_Print(h264->log, WLOG_ERROR, "Failed to allocate AVPacket");
287 goto fail;
288 }
289
290 cnv.cpv = pSrcData;
291 packet->data = cnv.pv;
292 packet->size = (int)MIN(SrcSize, INT32_MAX);
293
294 WINPR_ASSERT(sys->codecDecoderContext);
295 /* avcodec_decode_video2 is deprecated with libavcodec 57.48.101 */
296#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 48, 101)
297 status = avcodec_send_packet(sys->codecDecoderContext, packet);
298
299 if (status < 0)
300 {
301 WLog_Print(h264->log, WLOG_ERROR, "Failed to decode video frame (status=%d)", status);
302 goto fail;
303 }
304
305 sys->videoFrame->format = AV_PIX_FMT_YUV420P;
306
307#if defined(WITH_VAAPI) || defined(WITH_VIDEOTOOLBOX)
308 status = avcodec_receive_frame(sys->codecDecoderContext,
309 sys->hwctx ? sys->hwVideoFrame : sys->videoFrame);
310#else
311 status = avcodec_receive_frame(sys->codecDecoderContext, sys->videoFrame);
312#endif
313 if (status == AVERROR(EAGAIN))
314 {
315 rc = 0;
316 goto fail;
317 }
318
319 gotFrame = (status == 0);
320#else
321#if defined(WITH_VAAPI) || defined(WITH_VIDEOTOOLBOX)
322 status =
323 avcodec_decode_video2(sys->codecDecoderContext,
324 sys->hwctx ? sys->hwVideoFrame : sys->videoFrame, &gotFrame, packet);
325#else
326 status = avcodec_decode_video2(sys->codecDecoderContext, sys->videoFrame, &gotFrame, packet);
327#endif
328#endif
329 if (status < 0)
330 {
331 WLog_Print(h264->log, WLOG_ERROR, "Failed to decode video frame (status=%d)", status);
332 goto fail;
333 }
334
335#if defined(WITH_VAAPI) || defined(WITH_VIDEOTOOLBOX)
336
337 if (sys->hwctx)
338 {
339 if (sys->hwVideoFrame->format == sys->hw_pix_fmt)
340 {
341 sys->videoFrame->width = sys->hwVideoFrame->width;
342 sys->videoFrame->height = sys->hwVideoFrame->height;
343 status = av_hwframe_transfer_data(sys->videoFrame, sys->hwVideoFrame, 0);
344 }
345 else
346 {
347 status = av_frame_copy(sys->videoFrame, sys->hwVideoFrame);
348 }
349 }
350
351 gotFrame = (status == 0);
352
353 if (status < 0)
354 {
355 WLog_Print(h264->log, WLOG_ERROR, "Failed to transfer video frame (status=%d) (%s)", status,
356 av_err2str(status));
357 goto fail;
358 }
359
360#endif
361
362 if (gotFrame)
363 {
364 WINPR_ASSERT(sys->videoFrame);
365
366 pYUVData[0] = sys->videoFrame->data[0];
367 pYUVData[1] = sys->videoFrame->data[1];
368 pYUVData[2] = sys->videoFrame->data[2];
369 iStride[0] = (UINT32)MAX(0, sys->videoFrame->linesize[0]);
370 iStride[1] = (UINT32)MAX(0, sys->videoFrame->linesize[1]);
371 iStride[2] = (UINT32)MAX(0, sys->videoFrame->linesize[2]);
372 rc = 1;
373 }
374 else
375 rc = -2;
376
377fail:
378#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 133, 100)
379 av_packet_unref(packet);
380#else
381 av_packet_free(&packet);
382#endif
383
384 return rc;
385}
386
387static int libavcodec_compress(H264_CONTEXT* WINPR_RESTRICT h264,
388 const BYTE** WINPR_RESTRICT pSrcYuv,
389 const UINT32* WINPR_RESTRICT pStride,
390 BYTE** WINPR_RESTRICT ppDstData, UINT32* WINPR_RESTRICT pDstSize)
391{
392 union
393 {
394 const BYTE* cpv;
395 uint8_t* pv;
396 } cnv;
397 int rc = -1;
398 int status = 0;
399 int gotFrame = 0;
400
401 WINPR_ASSERT(h264);
402
403 H264_CONTEXT_LIBAVCODEC* sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
404 WINPR_ASSERT(sys);
405
406 if (!libavcodec_create_encoder_context(h264))
407 return -1;
408
409#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 133, 100)
410 sys->packet = &sys->bufferpacket;
411 av_packet_unref(sys->packet);
412 av_init_packet(sys->packet);
413#else
414 av_packet_free(&sys->packet);
415 sys->packet = av_packet_alloc();
416#endif
417 if (!sys->packet)
418 {
419 WLog_Print(h264->log, WLOG_ERROR, "Failed to allocate AVPacket");
420 goto fail;
421 }
422
423 WINPR_ASSERT(sys->packet);
424 sys->packet->data = nullptr;
425 sys->packet->size = 0;
426
427 WINPR_ASSERT(sys->videoFrame);
428 WINPR_ASSERT(sys->codecEncoderContext);
429 sys->videoFrame->format = AV_PIX_FMT_YUV420P;
430 sys->videoFrame->width = sys->codecEncoderContext->width;
431 sys->videoFrame->height = sys->codecEncoderContext->height;
432#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(52, 48, 100)
433 sys->videoFrame->colorspace = AVCOL_SPC_BT709;
434#endif
435#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(52, 92, 100)
436 sys->videoFrame->chroma_location = AVCHROMA_LOC_LEFT;
437#endif
438 cnv.cpv = pSrcYuv[0];
439 sys->videoFrame->data[0] = cnv.pv;
440
441 cnv.cpv = pSrcYuv[1];
442 sys->videoFrame->data[1] = cnv.pv;
443
444 cnv.cpv = pSrcYuv[2];
445 sys->videoFrame->data[2] = cnv.pv;
446
447 sys->videoFrame->linesize[0] = (int)pStride[0];
448 sys->videoFrame->linesize[1] = (int)pStride[1];
449 sys->videoFrame->linesize[2] = (int)pStride[2];
450 sys->videoFrame->pts++;
451
452#ifdef WITH_VAAPI_H264_ENCODING
453 if (sys->hwctx)
454 {
455 av_frame_unref(sys->hwVideoFrame);
456 if ((status = av_hwframe_get_buffer(sys->codecEncoderContext->hw_frames_ctx,
457 sys->hwVideoFrame, 0)) < 0 ||
458 !sys->hwVideoFrame->hw_frames_ctx)
459 {
460 WLog_Print(h264->log, WLOG_ERROR, "av_hwframe_get_buffer failed (%s [%d])",
461 av_err2str(status), status);
462 goto fail;
463 }
464 sys->videoFrame->format = AV_PIX_FMT_NV12;
465 if ((status = av_hwframe_transfer_data(sys->hwVideoFrame, sys->videoFrame, 0)) < 0)
466 {
467 WLog_Print(h264->log, WLOG_ERROR, "av_hwframe_transfer_data failed (%s [%d])",
468 av_err2str(status), status);
469 goto fail;
470 }
471 }
472#endif
473
474 /* avcodec_encode_video2 is deprecated with libavcodec 57.48.101 */
475#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 48, 101)
476#ifdef WITH_VAAPI_H264_ENCODING
477 status = avcodec_send_frame(sys->codecEncoderContext,
478 sys->hwctx ? sys->hwVideoFrame : sys->videoFrame);
479#else
480 status = avcodec_send_frame(sys->codecEncoderContext, sys->videoFrame);
481#endif
482
483 if (status < 0)
484 {
485 WLog_Print(h264->log, WLOG_ERROR, "Failed to encode video frame (%s [%d])",
486 av_err2str(status), status);
487 goto fail;
488 }
489
490 status = avcodec_receive_packet(sys->codecEncoderContext, sys->packet);
491
492 if (status < 0)
493 {
494 WLog_Print(h264->log, WLOG_ERROR, "Failed to encode video frame (%s [%d])",
495 av_err2str(status), status);
496 goto fail;
497 }
498
499 gotFrame = (status == 0);
500#elif LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(54, 59, 100)
501
502 do
503 {
504 status = avcodec_encode_video2(sys->codecEncoderContext, sys->packet, sys->videoFrame,
505 &gotFrame);
506 } while ((status >= 0) && (gotFrame == 0));
507
508#else
509 sys->packet->size =
510 avpicture_get_size(sys->codecDecoderContext->pix_fmt, sys->codecDecoderContext->width,
511 sys->codecDecoderContext->height);
512 sys->packet->data = av_malloc(sys->packet->size);
513
514 if (!sys->packet->data)
515 status = -1;
516 else
517 {
518 status = avcodec_encode_video(sys->codecDecoderContext, sys->packet->data,
519 sys->packet->size, sys->videoFrame);
520 }
521
522#endif
523
524 if (status < 0)
525 {
526 WLog_Print(h264->log, WLOG_ERROR, "Failed to encode video frame (%s [%d])",
527 av_err2str(status), status);
528 goto fail;
529 }
530
531 WINPR_ASSERT(sys->packet);
532 *ppDstData = sys->packet->data;
533 *pDstSize = (UINT32)MAX(0, sys->packet->size);
534
535 if (!gotFrame)
536 {
537 WLog_Print(h264->log, WLOG_ERROR, "Did not get frame! (%s [%d])", av_err2str(status),
538 status);
539 rc = -2;
540 }
541 else
542 rc = 1;
543fail:
544 return rc;
545}
546
547static void libavcodec_uninit(H264_CONTEXT* h264)
548{
549 WINPR_ASSERT(h264);
550
551 H264_CONTEXT_LIBAVCODEC* sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
552
553 if (!sys)
554 return;
555
556 if (sys->packet)
557 {
558#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 133, 100)
559 av_packet_unref(sys->packet);
560#else
561 av_packet_free(&sys->packet);
562#endif
563 }
564
565 if (sys->videoFrame)
566 {
567#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 18, 102)
568 av_frame_free(&sys->videoFrame);
569#else
570 av_free(sys->videoFrame);
571#endif
572 }
573
574#if defined(WITH_VAAPI) || defined(WITH_VAAPI_H264_ENCODING) || defined(WITH_VIDEOTOOLBOX)
575 if (sys->hwVideoFrame)
576 {
577#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 18, 102)
578 av_frame_free(&sys->hwVideoFrame);
579#else
580 av_free(sys->hwVideoFrame);
581#endif
582 }
583
584 if (sys->hwctx)
585 av_buffer_unref(&sys->hwctx);
586
587#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(57, 80, 100) || defined(WITH_VIDEOTOOLBOX)
588
589 if (sys->hw_frames_ctx)
590 av_buffer_unref(&sys->hw_frames_ctx);
591
592#endif
593
594#endif
595
596 if (sys->codecParser)
597 av_parser_close(sys->codecParser);
598
599 if (sys->codecDecoderContext)
600 {
601#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 69, 100)
602 avcodec_free_context(&sys->codecDecoderContext);
603#else
604 avcodec_close(sys->codecDecoderContext);
605 av_free(sys->codecDecoderContext);
606#endif
607 }
608
609 libavcodec_destroy_encoder_context(h264);
610 free(sys);
611 h264->pSystemData = nullptr;
612}
613
614#if defined(WITH_VAAPI) || defined(WITH_VIDEOTOOLBOX)
615static enum AVPixelFormat libavcodec_get_format(struct AVCodecContext* ctx,
616 const enum AVPixelFormat* fmts)
617{
618 WINPR_ASSERT(ctx);
619
620 H264_CONTEXT* h264 = (H264_CONTEXT*)ctx->opaque;
621 WINPR_ASSERT(h264);
622
623 H264_CONTEXT_LIBAVCODEC* sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
624 WINPR_ASSERT(sys);
625
626 for (const enum AVPixelFormat* p = fmts; *p != AV_PIX_FMT_NONE; p++)
627 {
628 if (*p == sys->hw_pix_fmt)
629 {
630#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(57, 80, 100) || defined(WITH_VIDEOTOOLBOX)
631 if (sys->hw_frames_ctx)
632 av_buffer_unref(&sys->hw_frames_ctx);
633
634 sys->hw_frames_ctx = av_hwframe_ctx_alloc(sys->hwctx);
635
636 if (!sys->hw_frames_ctx)
637 {
638 return AV_PIX_FMT_NONE;
639 }
640
641 sys->codecDecoderContext->pix_fmt = *p;
642 AVHWFramesContext* frames = (AVHWFramesContext*)sys->hw_frames_ctx->data;
643 frames->format = *p;
644 frames->height = sys->codecDecoderContext->coded_height;
645 frames->width = sys->codecDecoderContext->coded_width;
646#ifdef WITH_VIDEOTOOLBOX
647 frames->sw_format = AV_PIX_FMT_YUV420P;
648#else
649 frames->sw_format =
650 (sys->codecDecoderContext->sw_pix_fmt == AV_PIX_FMT_YUV420P10 ? AV_PIX_FMT_P010
651 : AV_PIX_FMT_NV12);
652#endif
653 frames->initial_pool_size = 20;
654
655 if (sys->codecDecoderContext->active_thread_type & FF_THREAD_FRAME)
656 frames->initial_pool_size += sys->codecDecoderContext->thread_count;
657
658 int err = av_hwframe_ctx_init(sys->hw_frames_ctx);
659
660 if (err < 0)
661 {
662 WLog_Print(h264->log, WLOG_ERROR, "Could not init hwframes context: %s",
663 av_err2str(err));
664 return AV_PIX_FMT_NONE;
665 }
666
667 sys->codecDecoderContext->hw_frames_ctx = av_buffer_ref(sys->hw_frames_ctx);
668#endif
669 return *p;
670 }
671 }
672
673 return AV_PIX_FMT_NONE;
674}
675#endif
676
677static BOOL libavcodec_init(H264_CONTEXT* h264)
678{
679 H264_CONTEXT_LIBAVCODEC* sys = nullptr;
680
681 WINPR_ASSERT(h264);
682 sys = (H264_CONTEXT_LIBAVCODEC*)calloc(1, sizeof(H264_CONTEXT_LIBAVCODEC));
683
684 if (!sys)
685 {
686 goto EXCEPTION;
687 }
688
689 h264->pSystemData = (void*)sys;
690
691#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 10, 100)
692 avcodec_register_all();
693#endif
694
695 if (!h264->Compressor)
696 {
697 sys->codecDecoder = avcodec_find_decoder(AV_CODEC_ID_H264);
698
699 if (!sys->codecDecoder)
700 {
701 WLog_Print(h264->log, WLOG_ERROR, "Failed to find libav H.264 codec");
702 goto EXCEPTION;
703 }
704
705 sys->codecDecoderContext = avcodec_alloc_context3(sys->codecDecoder);
706
707 if (!sys->codecDecoderContext)
708 {
709 WLog_Print(h264->log, WLOG_ERROR, "Failed to allocate libav codec context");
710 goto EXCEPTION;
711 }
712
713#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(59, 18, 100)
714 if (sys->codecDecoder->capabilities & AV_CODEC_CAP_TRUNCATED)
715 {
716 sys->codecDecoderContext->flags |= AV_CODEC_FLAG_TRUNCATED;
717 }
718#endif
719
720#ifdef WITH_VAAPI
721
722 if (!sys->hwctx)
723 {
724 int ret = av_hwdevice_ctx_create(&sys->hwctx, AV_HWDEVICE_TYPE_VAAPI, VAAPI_DEVICE,
725 nullptr, 0);
726
727 if (ret < 0)
728 {
729 WLog_Print(h264->log, WLOG_ERROR,
730 "Could not initialize hardware decoder, falling back to software: %s",
731 av_err2str(ret));
732 sys->hwctx = nullptr;
733 goto fail_hwdevice_create;
734 }
735 }
736 WLog_Print(h264->log, WLOG_INFO, "Using VAAPI for accelerated H264 decoding");
737
738 sys->codecDecoderContext->get_format = libavcodec_get_format;
739 sys->hw_pix_fmt = AV_PIX_FMT_VAAPI;
740#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 80, 100)
741 sys->codecDecoderContext->hw_device_ctx = av_buffer_ref(sys->hwctx);
742#endif
743 sys->codecDecoderContext->opaque = (void*)h264;
744 fail_hwdevice_create:
745#endif
746
747#ifdef WITH_VIDEOTOOLBOX
748
749 if (!sys->hwctx)
750 {
751 int ret = av_hwdevice_ctx_create(&sys->hwctx, AV_HWDEVICE_TYPE_VIDEOTOOLBOX, nullptr,
752 nullptr, 0);
753
754 if (ret < 0)
755 {
756 WLog_Print(
757 h264->log, WLOG_ERROR,
758 "Could not initialize VideoToolbox decoder, falling back to software: %s",
759 av_err2str(ret));
760 sys->hwctx = nullptr;
761 goto fail_vt_create;
762 }
763 }
764 WLog_Print(h264->log, WLOG_INFO, "Using VideoToolbox for accelerated H264 decoding");
765
766 sys->codecDecoderContext->get_format = libavcodec_get_format;
767 sys->hw_pix_fmt = AV_PIX_FMT_VIDEOTOOLBOX;
768 sys->codecDecoderContext->hw_device_ctx = av_buffer_ref(sys->hwctx);
769 sys->codecDecoderContext->opaque = (void*)h264;
770 fail_vt_create:
771#endif
772
773 if (avcodec_open2(sys->codecDecoderContext, sys->codecDecoder, nullptr) < 0)
774 {
775 WLog_Print(h264->log, WLOG_ERROR, "Failed to open libav codec");
776 goto EXCEPTION;
777 }
778
779 sys->codecParser = av_parser_init(AV_CODEC_ID_H264);
780
781 if (!sys->codecParser)
782 {
783 WLog_Print(h264->log, WLOG_ERROR, "Failed to initialize libav parser");
784 goto EXCEPTION;
785 }
786 }
787 else
788 {
789#ifdef WITH_VAAPI_H264_ENCODING
790 if (h264->hwAccel) /* user requested hw accel */
791 {
792 sys->codecEncoder = avcodec_find_encoder_by_name("h264_vaapi");
793 if (!sys->codecEncoder)
794 {
795 WLog_Print(h264->log, WLOG_ERROR, "H264 VAAPI encoder not found");
796 }
797 else if (av_hwdevice_ctx_create(&sys->hwctx, AV_HWDEVICE_TYPE_VAAPI, VAAPI_DEVICE,
798 nullptr, 0) < 0)
799 {
800 WLog_Print(h264->log, WLOG_ERROR, "av_hwdevice_ctx_create failed");
801 sys->codecEncoder = nullptr;
802 sys->hwctx = nullptr;
803 }
804 else
805 {
806 WLog_Print(h264->log, WLOG_INFO, "Using VAAPI for accelerated H264 encoding");
807 }
808 }
809#endif
810 if (!sys->codecEncoder)
811 {
812 sys->codecEncoder = avcodec_find_encoder(AV_CODEC_ID_H264);
813 h264->hwAccel = FALSE; /* not supported */
814 }
815
816 if (!sys->codecEncoder)
817 {
818 WLog_Print(h264->log, WLOG_ERROR, "Failed to initialize H264 encoder");
819 goto EXCEPTION;
820 }
821 }
822
823#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 18, 102)
824 sys->videoFrame = av_frame_alloc();
825#if defined(WITH_VAAPI) || defined(WITH_VAAPI_H264_ENCODING) || defined(WITH_VIDEOTOOLBOX)
826 sys->hwVideoFrame = av_frame_alloc();
827#endif
828#else
829 sys->videoFrame = avcodec_alloc_frame();
830#endif
831
832 if (!sys->videoFrame)
833 {
834 WLog_Print(h264->log, WLOG_ERROR, "Failed to allocate libav frame");
835 goto EXCEPTION;
836 }
837
838#if defined(WITH_VAAPI) || defined(WITH_VAAPI_H264_ENCODING) || defined(WITH_VIDEOTOOLBOX)
839 if (!sys->hwVideoFrame)
840 {
841 WLog_Print(h264->log, WLOG_ERROR, "Failed to allocate libav hw frame");
842 goto EXCEPTION;
843 }
844
845#endif
846 sys->videoFrame->pts = 0;
847 return TRUE;
848EXCEPTION:
849 libavcodec_uninit(h264);
850 return FALSE;
851}
852
853const H264_CONTEXT_SUBSYSTEM g_Subsystem_libavcodec = { "libavcodec", libavcodec_init,
854 libavcodec_uninit, libavcodec_decompress,
855 libavcodec_compress };