27#include <winpr/assert.h>
28#include <winpr/cast.h>
29#include <winpr/cmdline.h>
30#include <winpr/string.h>
31#include <winpr/synch.h>
32#include <winpr/sysinfo.h>
33#include <winpr/thread.h>
35#include <libavcodec/avcodec.h>
36#include <libavformat/avformat.h>
37#include <libavutil/imgutils.h>
38#include <libswscale/swscale.h>
42#define TAG CHANNELS_TAG("rdpecam-movie.client")
45#define CAM_MOVIE_DEVICE_ID "RDCamera_Device_0"
47#define CAM_MOVIE_FRAMERATE_NUMERATOR_DEFAULT 30
48#define CAM_MOVIE_FRAMERATE_DENOMINATOR_DEFAULT 1
51#define CAM_MOVIE_MAX_SLEEP_MS 50
54#define CAM_MOVIE_MAX_LATE_NS (1000ull * 1000ull * 1000ull)
58 AVFormatContext* format;
59 AVCodecContext* codec;
63 struct SwsContext* sws;
65 size_t framesSinceRewind;
76 ICamHalSampleCapturedCallback sampleCallback;
80 CamMovieDecoder decoder;
95 UINT32 frameRateNumerator;
96 UINT32 frameRateDenominator;
98 CamMovieStream stream;
101static void cam_movie_decoder_close(CamMovieDecoder* decoder)
103 WINPR_ASSERT(decoder);
105 sws_freeContext(decoder->sws);
106 av_frame_free(&decoder->frame);
107 av_packet_free(&decoder->packet);
108 avcodec_free_context(&decoder->codec);
109 avformat_close_input(&decoder->format);
111 *decoder = (CamMovieDecoder){ .videoIndex = -1 };
114static const char* cam_movie_av_error(
int error,
char* buffer,
size_t size)
116 if (av_strerror(error, buffer, size) < 0)
117 (void)_snprintf(buffer, size,
"error %d", error);
122static BOOL cam_movie_decoder_open(CamMovieDecoder* decoder,
const char* file)
124 WINPR_ASSERT(decoder);
127 char buffer[128] = WINPR_C_ARRAY_INIT;
128 *decoder = (CamMovieDecoder){ .videoIndex = -1 };
130 int rc = avformat_open_input(&decoder->format, file,
nullptr,
nullptr);
133 WLog_ERR(TAG,
"Unable to open movie %s: %s", file,
134 cam_movie_av_error(rc, buffer,
sizeof(buffer)));
138 rc = avformat_find_stream_info(decoder->format,
nullptr);
141 WLog_ERR(TAG,
"Unable to read stream information of %s: %s", file,
142 cam_movie_av_error(rc, buffer,
sizeof(buffer)));
146#if LIBAVFORMAT_VERSION_MAJOR >= 59
147 const AVCodec* codec =
nullptr;
149 AVCodec* codec =
nullptr;
151 rc = av_find_best_stream(decoder->format, AVMEDIA_TYPE_VIDEO, -1, -1, &codec, 0);
154 WLog_ERR(TAG,
"No decodable video stream in %s: %s", file,
155 cam_movie_av_error(rc, buffer,
sizeof(buffer)));
158 decoder->videoIndex = rc;
160 decoder->codec = avcodec_alloc_context3(codec);
164 const AVStream* stream = decoder->format->streams[decoder->videoIndex];
165 rc = avcodec_parameters_to_context(decoder->codec, stream->codecpar);
169 rc = avcodec_open2(decoder->codec, codec,
nullptr);
172 WLog_ERR(TAG,
"Unable to open the %s decoder: %s", codec->name,
173 cam_movie_av_error(rc, buffer,
sizeof(buffer)));
177 decoder->packet = av_packet_alloc();
178 decoder->frame = av_frame_alloc();
179 if (!decoder->packet || !decoder->frame)
185 cam_movie_decoder_close(decoder);
190static BOOL cam_movie_decoder_rewind(CamMovieDecoder* decoder)
192 WINPR_ASSERT(decoder);
194 if (decoder->framesSinceRewind == 0)
196 WLog_ERR(TAG,
"The movie did not produce any frame");
200 const AVStream* stream = decoder->format->streams[decoder->videoIndex];
201 const int64_t start = (stream->start_time != AV_NOPTS_VALUE) ? stream->start_time : 0;
202 const int rc = av_seek_frame(decoder->format, decoder->videoIndex, start, AVSEEK_FLAG_BACKWARD);
205 char buffer[128] = WINPR_C_ARRAY_INIT;
206 WLog_ERR(TAG,
"Unable to rewind the movie: %s",
207 cam_movie_av_error(rc, buffer,
sizeof(buffer)));
211 avcodec_flush_buffers(decoder->codec);
212 decoder->draining = FALSE;
213 decoder->framesSinceRewind = 0;
214 WLog_DBG(TAG,
"End of movie, playing it again");
220static BOOL cam_movie_decoder_next_frame(CamMovieDecoder* decoder)
222 WINPR_ASSERT(decoder);
224 char buffer[128] = WINPR_C_ARRAY_INIT;
227 int rc = avcodec_receive_frame(decoder->codec, decoder->frame);
230 decoder->framesSinceRewind++;
235 if ((rc == AVERROR_EOF) || ((rc == AVERROR(EAGAIN)) && decoder->draining))
237 if (!cam_movie_decoder_rewind(decoder))
242 if (rc != AVERROR(EAGAIN))
244 WLog_ERR(TAG,
"Unable to decode the movie: %s",
245 cam_movie_av_error(rc, buffer,
sizeof(buffer)));
249 rc = av_read_frame(decoder->format, decoder->packet);
250 if (rc == AVERROR_EOF)
253 decoder->draining = TRUE;
254 rc = avcodec_send_packet(decoder->codec,
nullptr);
258 if (decoder->packet->stream_index == decoder->videoIndex)
259 rc = avcodec_send_packet(decoder->codec, decoder->packet);
260 av_packet_unref(decoder->packet);
263 if ((rc < 0) && (rc != AVERROR(EAGAIN)))
265 WLog_ERR(TAG,
"Unable to read the movie: %s",
266 cam_movie_av_error(rc, buffer,
sizeof(buffer)));
274static BOOL cam_movie_convert_frame(CamMovieStream* stream, UINT32 width, UINT32 height)
276 WINPR_ASSERT(stream);
278 CamMovieDecoder* decoder = &stream->decoder;
279 const AVFrame* frame = decoder->frame;
281 if ((frame->width < 0) || (frame->height < 0) || ((UINT32)frame->width < width) ||
282 ((UINT32)frame->height < height))
284 WLog_WARN(TAG,
"Skipping a %dx%d frame, smaller than the announced %" PRIu32
"x%" PRIu32,
285 frame->width, frame->height, width, height);
289 const int w = WINPR_ASSERTING_INT_CAST(
int, width);
290 const int h = WINPR_ASSERTING_INT_CAST(
int, height);
294 sws_getCachedContext(decoder->sws, w, h, (
enum AVPixelFormat)frame->format, w, h,
295 AV_PIX_FMT_YUV420P, SWS_BILINEAR,
nullptr,
nullptr,
nullptr);
298 WLog_ERR(TAG,
"Unable to convert %s frames to I420",
299 av_get_pix_fmt_name((
enum AVPixelFormat)frame->format));
303 const size_t lumaSize = 1ull * width * height;
304 BYTE* dst[4] = { stream->sample, stream->sample + lumaSize,
305 stream->sample + lumaSize + lumaSize / 4,
nullptr };
306 const int dstStride[4] = { w, w / 2, w / 2, 0 };
308 const int rc = sws_scale(decoder->sws, (
const uint8_t*
const*)frame->data, frame->linesize, 0,
315static BOOL cam_movie_wait_until(CamMovieStream* stream, UINT64 due)
317 WINPR_ASSERT(stream);
321 EnterCriticalSection(&stream->lock);
322 const BOOL streaming = stream->streaming;
323 LeaveCriticalSection(&stream->lock);
327 const UINT64 now = winpr_GetTickCount64NS();
331 UINT64 ms = (due - now) / 1000000ull;
332 if (ms > CAM_MOVIE_MAX_SLEEP_MS)
333 ms = CAM_MOVIE_MAX_SLEEP_MS;
338static DWORD WINAPI cam_movie_playback_thread(LPVOID param)
340 CamMovieHal* hal = (CamMovieHal*)param;
343 CamMovieStream* stream = &hal->stream;
344 const UINT64 frameDuration =
345 (1000000000ull * hal->frameRateDenominator) / hal->frameRateNumerator;
346 UINT64 start = winpr_GetTickCount64NS();
351 if (!cam_movie_decoder_next_frame(&stream->decoder))
354 if (!cam_movie_convert_frame(stream, hal->width, hal->height))
358 UINT64 due = start + frames * frameDuration;
359 const UINT64 now = winpr_GetTickCount64NS();
360 if (now > due + CAM_MOVIE_MAX_LATE_NS)
362 WLog_DBG(TAG,
"Playback late by %" PRIu64
" ms, resetting the clock",
363 (UINT64)((now - due) / 1000000ull));
369 if (!cam_movie_wait_until(stream, due))
372 const UINT error = stream->sampleCallback(stream->dev, stream->streamIndex, stream->sample,
374 if (error == ECAM_SAMPLE_CAPTURE_DRAINED)
376 WLog_DBG(TAG,
"No more sample requests, pausing the movie");
379 else if (error != CHANNEL_RC_OK)
380 WLog_ERR(TAG,
"Failure in sampleCallback: %" PRIu32, error);
383 EnterCriticalSection(&stream->lock);
384 stream->streaming = FALSE;
385 LeaveCriticalSection(&stream->lock);
386 return CHANNEL_RC_OK;
389static void cam_movie_stream_join(CamMovieStream* stream)
391 WINPR_ASSERT(stream);
393 EnterCriticalSection(&stream->lock);
394 stream->streaming = FALSE;
395 LeaveCriticalSection(&stream->lock);
399 (void)WaitForSingleObject(stream->thread, INFINITE);
400 (void)CloseHandle(stream->thread);
401 stream->thread =
nullptr;
405static BOOL cam_movie_activate(WINPR_ATTR_UNUSED ICamHal* ihal,
406 WINPR_ATTR_UNUSED
const char* deviceId, CAM_ERROR_CODE* errorCode)
408 *errorCode = CAM_ERROR_CODE_None;
412static BOOL cam_movie_deactivate(WINPR_ATTR_UNUSED ICamHal* ihal,
413 WINPR_ATTR_UNUSED
const char* deviceId, CAM_ERROR_CODE* errorCode)
415 *errorCode = CAM_ERROR_CODE_None;
419static UINT cam_movie_enumerate(ICamHal* ihal, ICamHalEnumCallback callback,
CameraPlugin* ecam,
422 CamMovieHal* hal = (CamMovieHal*)ihal;
425 IFCALL(callback, ecam, hchannel, CAM_MOVIE_DEVICE_ID, hal->name);
435static INT16 cam_movie_get_media_type_descriptions(
436 ICamHal* ihal, WINPR_ATTR_UNUSED
const char* deviceId, WINPR_ATTR_UNUSED
size_t streamIndex,
440 CamMovieHal* hal = (CamMovieHal*)ihal;
442 WINPR_ASSERT(nMediaTypes);
444 const size_t maxMediaTypes = *nMediaTypes;
446 if (maxMediaTypes == 0)
450 for (
size_t formatIndex = 0; formatIndex < nSupportedFormats; formatIndex++)
452 if (supportedFormats[formatIndex].inputFormat != CAM_MEDIA_FORMAT_I420)
454 if (formatIndex > INT16_MAX)
458 .Format = CAM_MEDIA_FORMAT_I420,
460 .Height = hal->height,
461 .FrameRateNumerator = hal->frameRateNumerator,
462 .FrameRateDenominator = hal->frameRateDenominator,
463 .PixelAspectRatioNumerator = 1,
464 .PixelAspectRatioDenominator = 1,
467 return (INT16)formatIndex;
470 WLog_ERR(TAG,
"I420 is not a supported input format");
474static CAM_ERROR_CODE cam_movie_stream_start(ICamHal* ihal,
CameraDevice* dev,
size_t streamIndex,
476 ICamHalSampleCapturedCallback callback)
478 CamMovieHal* hal = (CamMovieHal*)ihal;
481 WINPR_ASSERT(mediaType);
483 CamMovieStream* stream = &hal->stream;
485 if ((mediaType->Format != CAM_MEDIA_FORMAT_I420) || (mediaType->Width != hal->width) ||
486 (mediaType->Height != hal->height))
488 WLog_ERR(TAG,
"Unexpected media type %u %" PRIu32
"x%" PRIu32, mediaType->Format,
489 mediaType->Width, mediaType->Height);
490 return CAM_ERROR_CODE_InvalidMediaType;
493 EnterCriticalSection(&stream->lock);
494 const BOOL streaming = stream->streaming;
495 LeaveCriticalSection(&stream->lock);
499 WLog_ERR(TAG,
"Streaming already in progress, device %s, streamIndex %" PRIuz,
500 dev->deviceId, streamIndex);
501 return CAM_ERROR_CODE_UnexpectedError;
505 cam_movie_stream_join(stream);
507 if (!stream->decoder.format && !cam_movie_decoder_open(&stream->decoder, hal->file))
508 return CAM_ERROR_CODE_UnexpectedError;
511 stream->streamIndex = streamIndex;
512 stream->sampleCallback = callback;
514 EnterCriticalSection(&stream->lock);
515 stream->streaming = TRUE;
516 LeaveCriticalSection(&stream->lock);
518 stream->thread = CreateThread(
nullptr, 0, cam_movie_playback_thread, hal, 0,
nullptr);
521 WLog_ERR(TAG,
"CreateThread failure");
522 EnterCriticalSection(&stream->lock);
523 stream->streaming = FALSE;
524 LeaveCriticalSection(&stream->lock);
525 return CAM_ERROR_CODE_OutOfMemory;
528 WLog_INFO(TAG,
"Playing %s as camera: %" PRIu32
"x%" PRIu32
", fps: %" PRIu32
"/%" PRIu32,
529 hal->file, hal->width, hal->height, hal->frameRateNumerator,
530 hal->frameRateDenominator);
531 return CAM_ERROR_CODE_None;
534static CAM_ERROR_CODE cam_movie_stream_stop(ICamHal* ihal, WINPR_ATTR_UNUSED
const char* deviceId,
535 WINPR_ATTR_UNUSED
size_t streamIndex)
537 CamMovieHal* hal = (CamMovieHal*)ihal;
540 cam_movie_stream_join(&hal->stream);
543 cam_movie_decoder_close(&hal->stream.decoder);
544 return CAM_ERROR_CODE_None;
547static CAM_ERROR_CODE cam_movie_free(ICamHal* ihal)
549 CamMovieHal* hal = (CamMovieHal*)ihal;
552 return CAM_ERROR_CODE_NotInitialized;
554 cam_movie_stream_join(&hal->stream);
555 cam_movie_decoder_close(&hal->stream.decoder);
556 DeleteCriticalSection(&hal->stream.lock);
558 free(hal->stream.sample);
562 return CAM_ERROR_CODE_None;
567static BOOL cam_movie_probe(CamMovieHal* hal)
571 CamMovieDecoder decoder = WINPR_C_ARRAY_INIT;
572 if (!cam_movie_decoder_open(&decoder, hal->file))
576 AVStream* stream = decoder.format->streams[decoder.videoIndex];
577 const int width = decoder.codec->width;
578 const int height = decoder.codec->height;
581 if ((width < 2) || (height < 2))
583 WLog_ERR(TAG,
"Invalid movie size %dx%d", width, height);
586 hal->width = (UINT32)width & ~1u;
587 hal->height = (UINT32)height & ~1u;
589 const AVRational rate = av_guess_frame_rate(decoder.format, stream,
nullptr);
590 if ((rate.num > 0) && (rate.den > 0))
592 hal->frameRateNumerator = (UINT32)rate.num;
593 hal->frameRateDenominator = (UINT32)rate.den;
597 WLog_WARN(TAG,
"Unknown frame rate, using %d fps", CAM_MOVIE_FRAMERATE_NUMERATOR_DEFAULT);
598 hal->frameRateNumerator = CAM_MOVIE_FRAMERATE_NUMERATOR_DEFAULT;
599 hal->frameRateDenominator = CAM_MOVIE_FRAMERATE_DENOMINATOR_DEFAULT;
602 hal->stream.sampleSize = 3ull * hal->width * hal->height / 2;
603 hal->stream.sample = calloc(1, hal->stream.sampleSize);
604 if (!hal->stream.sample)
608 "Movie %s: %dx%d %s, announced as %" PRIu32
"x%" PRIu32
" at %" PRIu32
"/%" PRIu32
610 hal->file, width, height, avcodec_get_name(decoder.codec->codec_id), hal->width,
611 hal->height, hal->frameRateNumerator, hal->frameRateDenominator);
615 cam_movie_decoder_close(&decoder);
619static const char* cam_movie_basename(
const char* path)
621 const char* name = path;
622 for (
const char* cur = path; *cur; cur++)
624 if ((*cur ==
'/') || (*cur ==
'\\'))
631static BOOL cam_movie_parse_args(CamMovieHal* hal,
const ADDIN_ARGV* args)
636 nullptr,
nullptr, -1,
nullptr,
"movie to play" },
637 {
"name", COMMAND_LINE_VALUE_REQUIRED,
"<name>",
638 nullptr,
nullptr, -1,
nullptr,
"camera name" },
639 {
nullptr, 0,
nullptr,
nullptr,
nullptr, -1,
nullptr,
642 if (args && (args->argc > 1))
644 const DWORD flags = COMMAND_LINE_SIGIL_NONE | COMMAND_LINE_SEPARATOR_COLON |
645 COMMAND_LINE_IGN_UNKNOWN_KEYWORD;
646 const int status = CommandLineParseArgumentsA(args->argc, args->argv, movie_args, flags,
647 hal,
nullptr,
nullptr);
654 if (!(arg->Flags & COMMAND_LINE_VALUE_PRESENT))
657 CommandLineSwitchStart(arg) CommandLineSwitchCase(arg,
"file")
660 hal->file = _strdup(arg->Value);
664 CommandLineSwitchCase(arg,
"name")
667 hal->name = _strdup(arg->Value);
671 CommandLineSwitchDefault(arg)
674 CommandLineSwitchEnd(arg)
675 }
while ((arg = CommandLineFindNextArgumentA(arg)) !=
nullptr);
680 WLog_ERR(TAG,
"The movie backend needs a file:<path> argument");
687 winpr_asprintf(&hal->name, &len,
"Movie: %s", cam_movie_basename(hal->file));
699FREERDP_ENTRY_POINT(UINT VCAPITYPE movie_freerdp_rdpecam_client_subsystem_entry(
702 UINT ret = ERROR_INVALID_PARAMETER;
703 WINPR_ASSERT(pEntryPoints);
705 CamMovieHal* hal = (CamMovieHal*)calloc(1,
sizeof(CamMovieHal));
707 return CHANNEL_RC_NO_MEMORY;
709 hal->iHal.Enumerate = cam_movie_enumerate;
710 hal->iHal.RequestDrivenCapture = TRUE;
711 hal->iHal.GetMediaTypeDescriptions = cam_movie_get_media_type_descriptions;
712 hal->iHal.Activate = cam_movie_activate;
713 hal->iHal.Deactivate = cam_movie_deactivate;
714 hal->iHal.StartStream = cam_movie_stream_start;
715 hal->iHal.StopStream = cam_movie_stream_stop;
716 hal->iHal.Free = cam_movie_free;
717 hal->stream.decoder.videoIndex = -1;
719 if (!InitializeCriticalSectionEx(&hal->stream.lock, 0, 0))
722 return CHANNEL_RC_NO_MEMORY;
725 if (!cam_movie_parse_args(hal, pEntryPoints->args))
729 if (!cam_movie_probe(hal))
732 ret = pEntryPoints->pRegisterCameraHal(pEntryPoints->plugin, &hal->iHal);
733 if (ret != CHANNEL_RC_OK)
735 WLog_ERR(TAG,
"RegisterCameraHal failed with error %" PRIu32
"", ret);
742 cam_movie_free(&hal->iHal);