29#include <winpr/assert.h>
30#include <winpr/cast.h>
31#include <winpr/cmdline.h>
32#include <winpr/file.h>
33#include <winpr/string.h>
34#include <winpr/synch.h>
35#include <winpr/thread.h>
38#include <gst/app/gstappsink.h>
39#include <gst/video/video.h>
43#define TAG CHANNELS_TAG("rdpecam-gst.client")
46#define CAM_GST_DEVICE_ID "RDCamera_Device_0"
48#define CAM_GST_SINK_NAME "freerdpcamsink"
50#define CAM_GST_FRAMERATE_NUMERATOR_DEFAULT 30
51#define CAM_GST_FRAMERATE_DENOMINATOR_DEFAULT 1
54#define CAM_GST_START_TIMEOUT (10 * GST_SECOND)
57#define CAM_GST_PULL_TIMEOUT (100 * GST_MSECOND)
71 UINT32 frameRateNumerator;
72 UINT32 frameRateDenominator;
83 ICamHalSampleCapturedCallback sampleCallback;
90static BOOL cam_gst_log_errors(CamGstHal* hal)
95 GstBus* bus = gst_element_get_bus(hal->pipeline);
99 GstMessage* msg =
nullptr;
100 while ((msg = gst_bus_pop_filtered(bus, GST_MESSAGE_ERROR)) !=
nullptr)
102 GError* error =
nullptr;
103 gchar* debug =
nullptr;
104 gst_message_parse_error(msg, &error, &debug);
105 WLog_ERR(TAG,
"Pipeline error from %s: %s (%s)", GST_OBJECT_NAME(GST_MESSAGE_SRC(msg)),
106 error ? error->message :
"unknown", debug ? debug :
"no details");
107 g_clear_error(&error);
109 gst_message_unref(msg);
112 gst_object_unref(bus);
116static void cam_gst_pipeline_destroy(CamGstHal* hal)
122 gst_object_unref(hal->sink);
127 (void)gst_element_set_state(hal->pipeline, GST_STATE_NULL);
128 gst_object_unref(hal->pipeline);
129 hal->pipeline =
nullptr;
136static BOOL cam_gst_pipeline_create(CamGstHal* hal)
139 WINPR_ASSERT(!hal->pipeline);
141 GError* error =
nullptr;
143 gst_parse_launch_full(hal->description,
nullptr, GST_PARSE_FLAG_FATAL_ERRORS, &error);
144 if (!hal->pipeline || error)
146 WLog_ERR(TAG,
"Invalid pipeline \"%s\": %s", hal->description,
147 error ? error->message :
"unknown error");
148 g_clear_error(&error);
149 cam_gst_pipeline_destroy(hal);
153 GstElement* sink = GST_IS_BIN(hal->pipeline)
154 ? gst_bin_get_by_name(GST_BIN(hal->pipeline), CAM_GST_SINK_NAME)
158 WLog_ERR(TAG,
"The pipeline has no %s element", CAM_GST_SINK_NAME);
159 cam_gst_pipeline_destroy(hal);
162 hal->sink = GST_APP_SINK(sink);
168static BOOL cam_gst_set_state(CamGstHal* hal, GstState state, GstStateChangeReturn* pResult)
172 GstStateChangeReturn rc = gst_element_set_state(hal->pipeline, state);
173 if (rc == GST_STATE_CHANGE_ASYNC)
174 rc = gst_element_get_state(hal->pipeline,
nullptr,
nullptr, CAM_GST_START_TIMEOUT);
181 case GST_STATE_CHANGE_SUCCESS:
182 case GST_STATE_CHANGE_NO_PREROLL:
184 case GST_STATE_CHANGE_ASYNC:
185 WLog_ERR(TAG,
"Pipeline did not reach %s in time", gst_element_state_get_name(state));
187 case GST_STATE_CHANGE_FAILURE:
189 if (!cam_gst_log_errors(hal))
190 WLog_ERR(TAG,
"Pipeline failed to go to %s", gst_element_state_get_name(state));
197static BOOL cam_gst_convert_sample(CamGstHal* hal, GstSample* sample)
200 WINPR_ASSERT(sample);
203 GstCaps* caps = gst_sample_get_caps(sample);
204 GstBuffer* buffer = gst_sample_get_buffer(sample);
205 if (!caps || !buffer || !gst_video_info_from_caps(&info, caps))
207 WLog_WARN(TAG,
"Skipping a sample without video caps");
211 if ((GST_VIDEO_INFO_FORMAT(&info) != GST_VIDEO_FORMAT_I420) ||
212 (GST_VIDEO_INFO_WIDTH(&info) < 0) || (GST_VIDEO_INFO_HEIGHT(&info) < 0) ||
213 ((UINT32)GST_VIDEO_INFO_WIDTH(&info) < hal->width) ||
214 ((UINT32)GST_VIDEO_INFO_HEIGHT(&info) < hal->height))
217 TAG,
"Skipping a %s %dx%d frame, the announced format is I420 %" PRIu32
"x%" PRIu32,
218 gst_video_format_to_string(GST_VIDEO_INFO_FORMAT(&info)), GST_VIDEO_INFO_WIDTH(&info),
219 GST_VIDEO_INFO_HEIGHT(&info), hal->width, hal->height);
224 if (!gst_video_frame_map(&frame, &info, buffer, GST_MAP_READ))
226 WLog_WARN(TAG,
"Unable to map a frame");
231 BYTE* dst = hal->sample;
232 for (guint plane = 0; plane < 3; plane++)
234 const size_t rowSize = (plane == 0) ? hal->width : hal->width / 2;
235 const size_t rows = (plane == 0) ? hal->height : hal->height / 2;
236 const BYTE* src = GST_VIDEO_FRAME_PLANE_DATA(&frame, plane);
237 const size_t stride = (size_t)GST_VIDEO_FRAME_PLANE_STRIDE(&frame, plane);
239 for (
size_t y = 0; y < rows; y++)
241 memcpy(dst, &src[y * stride], rowSize);
246 gst_video_frame_unmap(&frame);
252static void cam_gst_release(CamGstHal* hal)
256 if (hal->live || !cam_gst_set_state(hal, GST_STATE_PAUSED,
nullptr))
257 cam_gst_pipeline_destroy(hal);
260static DWORD WINAPI cam_gst_capture_thread(LPVOID param)
262 CamGstHal* hal = (CamGstHal*)param;
265 BOOL endLogged = FALSE;
268 EnterCriticalSection(&hal->lock);
269 const BOOL streaming = hal->streaming;
270 LeaveCriticalSection(&hal->lock);
274 GstSample* sample = gst_app_sink_try_pull_sample(hal->sink, CAM_GST_PULL_TIMEOUT);
277 if (cam_gst_log_errors(hal))
280 if (!gst_app_sink_is_eos(hal->sink))
283 if (hal->loop && !hal->live)
285 WLog_DBG(TAG,
"End of stream, playing it again");
286 if (!gst_element_seek_simple(hal->pipeline, GST_FORMAT_TIME,
287 GST_SEEK_FLAG_FLUSH | GST_SEEK_FLAG_KEY_UNIT, 0))
289 WLog_ERR(TAG,
"Unable to rewind the pipeline");
296 WLog_INFO(TAG,
"End of stream");
302 const BOOL converted = cam_gst_convert_sample(hal, sample);
303 gst_sample_unref(sample);
308 hal->sampleCallback(hal->dev, hal->streamIndex, hal->sample, hal->sampleSize);
309 if (error == ECAM_SAMPLE_CAPTURE_DRAINED)
311 WLog_DBG(TAG,
"No more sample requests, releasing the pipeline");
312 cam_gst_release(hal);
315 else if (error != CHANNEL_RC_OK)
316 WLog_ERR(TAG,
"Failure in sampleCallback: %" PRIu32, error);
319 EnterCriticalSection(&hal->lock);
320 hal->streaming = FALSE;
321 LeaveCriticalSection(&hal->lock);
322 return CHANNEL_RC_OK;
325static void cam_gst_join(CamGstHal* hal)
329 EnterCriticalSection(&hal->lock);
330 hal->streaming = FALSE;
331 LeaveCriticalSection(&hal->lock);
335 (void)WaitForSingleObject(hal->thread, INFINITE);
336 (void)CloseHandle(hal->thread);
337 hal->thread =
nullptr;
341static BOOL cam_gst_activate(WINPR_ATTR_UNUSED ICamHal* ihal,
342 WINPR_ATTR_UNUSED
const char* deviceId, CAM_ERROR_CODE* errorCode)
344 *errorCode = CAM_ERROR_CODE_None;
348static BOOL cam_gst_deactivate(WINPR_ATTR_UNUSED ICamHal* ihal,
349 WINPR_ATTR_UNUSED
const char* deviceId, CAM_ERROR_CODE* errorCode)
351 *errorCode = CAM_ERROR_CODE_None;
355static UINT cam_gst_enumerate(ICamHal* ihal, ICamHalEnumCallback callback,
CameraPlugin* ecam,
358 CamGstHal* hal = (CamGstHal*)ihal;
361 IFCALL(callback, ecam, hchannel, CAM_GST_DEVICE_ID, hal->name);
371static INT16 cam_gst_get_media_type_descriptions(
372 ICamHal* ihal, WINPR_ATTR_UNUSED
const char* deviceId, WINPR_ATTR_UNUSED
size_t streamIndex,
376 CamGstHal* hal = (CamGstHal*)ihal;
378 WINPR_ASSERT(nMediaTypes);
380 const size_t maxMediaTypes = *nMediaTypes;
382 if (maxMediaTypes == 0)
386 for (
size_t formatIndex = 0; formatIndex < nSupportedFormats; formatIndex++)
388 if (supportedFormats[formatIndex].inputFormat != CAM_MEDIA_FORMAT_I420)
390 if (formatIndex > INT16_MAX)
394 .Format = CAM_MEDIA_FORMAT_I420,
396 .Height = hal->height,
397 .FrameRateNumerator = hal->frameRateNumerator,
398 .FrameRateDenominator = hal->frameRateDenominator,
399 .PixelAspectRatioNumerator = 1,
400 .PixelAspectRatioDenominator = 1,
403 return (INT16)formatIndex;
406 WLog_ERR(TAG,
"I420 is not a supported input format");
410static CAM_ERROR_CODE cam_gst_stream_start(ICamHal* ihal,
CameraDevice* dev,
size_t streamIndex,
412 ICamHalSampleCapturedCallback callback)
414 CamGstHal* hal = (CamGstHal*)ihal;
417 WINPR_ASSERT(mediaType);
419 if ((mediaType->Format != CAM_MEDIA_FORMAT_I420) || (mediaType->Width != hal->width) ||
420 (mediaType->Height != hal->height))
422 WLog_ERR(TAG,
"Unexpected media type %u %" PRIu32
"x%" PRIu32, mediaType->Format,
423 mediaType->Width, mediaType->Height);
424 return CAM_ERROR_CODE_InvalidMediaType;
427 EnterCriticalSection(&hal->lock);
428 const BOOL streaming = hal->streaming;
429 LeaveCriticalSection(&hal->lock);
433 WLog_ERR(TAG,
"Streaming already in progress, device %s, streamIndex %" PRIuz,
434 dev->deviceId, streamIndex);
435 return CAM_ERROR_CODE_UnexpectedError;
442 hal->streamIndex = streamIndex;
443 hal->sampleCallback = callback;
445 if (!hal->pipeline && !cam_gst_pipeline_create(hal))
446 return CAM_ERROR_CODE_UnexpectedError;
448 if (!cam_gst_set_state(hal, GST_STATE_PLAYING,
nullptr))
450 cam_gst_pipeline_destroy(hal);
451 return CAM_ERROR_CODE_UnexpectedError;
454 EnterCriticalSection(&hal->lock);
455 hal->streaming = TRUE;
456 LeaveCriticalSection(&hal->lock);
458 hal->thread = CreateThread(
nullptr, 0, cam_gst_capture_thread, hal, 0,
nullptr);
461 WLog_ERR(TAG,
"CreateThread failure");
462 EnterCriticalSection(&hal->lock);
463 hal->streaming = FALSE;
464 LeaveCriticalSection(&hal->lock);
465 cam_gst_pipeline_destroy(hal);
466 return CAM_ERROR_CODE_OutOfMemory;
469 WLog_INFO(TAG,
"Pipeline playing as camera: %" PRIu32
"x%" PRIu32
", fps: %" PRIu32
"/%" PRIu32,
470 hal->width, hal->height, hal->frameRateNumerator, hal->frameRateDenominator);
471 return CAM_ERROR_CODE_None;
474static CAM_ERROR_CODE cam_gst_stream_stop(ICamHal* ihal, WINPR_ATTR_UNUSED
const char* deviceId,
475 WINPR_ATTR_UNUSED
size_t streamIndex)
477 CamGstHal* hal = (CamGstHal*)ihal;
483 cam_gst_pipeline_destroy(hal);
484 return CAM_ERROR_CODE_None;
487static CAM_ERROR_CODE cam_gst_free(ICamHal* ihal)
489 CamGstHal* hal = (CamGstHal*)ihal;
492 return CAM_ERROR_CODE_NotInitialized;
495 cam_gst_pipeline_destroy(hal);
496 DeleteCriticalSection(&hal->lock);
499 free(hal->description);
502 return CAM_ERROR_CODE_None;
508static BOOL cam_gst_probe(CamGstHal* hal)
512 WLog_DBG(TAG,
"Pipeline: %s", hal->description);
514 if (!cam_gst_pipeline_create(hal))
519 GstSample* sample =
nullptr;
520 GstStateChangeReturn state = GST_STATE_CHANGE_FAILURE;
521 if (!cam_gst_set_state(hal, GST_STATE_PAUSED, &state))
524 hal->live = (state == GST_STATE_CHANGE_NO_PREROLL);
527 if (!cam_gst_set_state(hal, GST_STATE_PLAYING,
nullptr))
529 sample = gst_app_sink_try_pull_sample(hal->sink, CAM_GST_START_TIMEOUT);
532 sample = gst_app_sink_try_pull_preroll(hal->sink, CAM_GST_START_TIMEOUT);
536 if (!cam_gst_log_errors(hal))
537 WLog_ERR(TAG,
"The pipeline did not produce any frame");
542 GstCaps* caps = gst_sample_get_caps(sample);
543 if (!caps || !gst_video_info_from_caps(&info, caps))
545 WLog_ERR(TAG,
"The pipeline does not produce video");
550 const int width = GST_VIDEO_INFO_WIDTH(&info);
551 const int height = GST_VIDEO_INFO_HEIGHT(&info);
552 if ((width < 2) || (height < 2))
554 WLog_ERR(TAG,
"Invalid video size %dx%d", width, height);
557 hal->width = (UINT32)width & ~1u;
558 hal->height = (UINT32)height & ~1u;
561 if ((GST_VIDEO_INFO_FPS_N(&info) > 0) && (GST_VIDEO_INFO_FPS_D(&info) > 0))
563 hal->frameRateNumerator = (UINT32)GST_VIDEO_INFO_FPS_N(&info);
564 hal->frameRateDenominator = (UINT32)GST_VIDEO_INFO_FPS_D(&info);
568 WLog_WARN(TAG,
"Variable frame rate, announcing %d fps",
569 CAM_GST_FRAMERATE_NUMERATOR_DEFAULT);
570 hal->frameRateNumerator = CAM_GST_FRAMERATE_NUMERATOR_DEFAULT;
571 hal->frameRateDenominator = CAM_GST_FRAMERATE_DENOMINATOR_DEFAULT;
574 hal->sampleSize = 3ull * hal->width * hal->height / 2;
575 hal->sample = calloc(1, hal->sampleSize);
580 "%s pipeline: %dx%d, announced as %" PRIu32
"x%" PRIu32
" at %" PRIu32
"/%" PRIu32
582 hal->live ?
"Live" :
"Non live", width, height, hal->width, hal->height,
583 hal->frameRateNumerator, hal->frameRateDenominator);
588 gst_sample_unref(sample);
591 cam_gst_pipeline_destroy(hal);
595static const char* cam_gst_basename(
const char* path)
597 const char* name = path;
598 for (
const char* cur = path; *cur; cur++)
600 if ((*cur ==
'/') || (*cur ==
'\\'))
606WINPR_ATTR_MALLOC(free, 1)
607static
char* cam_gst_read_file(const
char* path)
611 char* content =
nullptr;
612 FILE* fp = winpr_fopen(path,
"rb");
615 WLog_ERR(TAG,
"Unable to open pipeline file %s", path);
619 if (_fseeki64(fp, 0, SEEK_END) != 0)
621 const INT64 size = _ftelli64(fp);
622 if ((size <= 0) || (size > 1024 * 1024) || (_fseeki64(fp, 0, SEEK_SET) != 0))
625 content = calloc(1, (
size_t)size + 1);
629 if (fread(content, 1, (
size_t)size, fp) != (
size_t)size)
637 WLog_ERR(TAG,
"Unable to read pipeline file %s", path);
643static BOOL cam_gst_parse_bool(
const char* value, BOOL* result)
646 WINPR_ASSERT(result);
648 if ((_stricmp(value,
"on") == 0) || (_stricmp(value,
"true") == 0) || (strcmp(value,
"1") == 0))
650 else if ((_stricmp(value,
"off") == 0) || (_stricmp(value,
"false") == 0) ||
651 (strcmp(value,
"0") == 0))
659static BOOL cam_gst_parse_args(CamGstHal* hal,
const ADDIN_ARGV* args)
664 {
"pipeline", COMMAND_LINE_VALUE_REQUIRED,
"<pipeline>",
nullptr,
nullptr, -1,
nullptr,
665 "pipeline producing video, in gst-launch syntax" },
666 {
"pipeline-file", COMMAND_LINE_VALUE_REQUIRED,
"<path>",
nullptr,
nullptr, -1,
nullptr,
667 "file containing the pipeline" },
668 {
"file", COMMAND_LINE_VALUE_REQUIRED,
"<path>",
nullptr,
nullptr, -1,
nullptr,
669 "media file to play" },
670 {
"name", COMMAND_LINE_VALUE_REQUIRED,
"<name>",
nullptr,
nullptr, -1,
nullptr,
672 {
"loop", COMMAND_LINE_VALUE_REQUIRED,
"<on|off>",
nullptr,
nullptr, -1,
nullptr,
673 "play the pipeline again when it ends" },
674 {
nullptr, 0,
nullptr,
nullptr,
nullptr, -1,
nullptr,
nullptr }
677 char* source =
nullptr;
678 char* defaultName =
nullptr;
683 if (args && (args->argc > 1))
685 const DWORD flags = COMMAND_LINE_SIGIL_NONE | COMMAND_LINE_SEPARATOR_COLON |
686 COMMAND_LINE_IGN_UNKNOWN_KEYWORD;
687 const int status = CommandLineParseArgumentsA(args->argc, args->argv, gst_args, flags, hal,
695 if (!(arg->Flags & COMMAND_LINE_VALUE_PRESENT))
698 CommandLineSwitchStart(arg) CommandLineSwitchCase(arg,
"pipeline")
702 source = _strdup(arg->Value);
706 CommandLineSwitchCase(arg,
"pipeline-file")
710 source = cam_gst_read_file(arg->Value);
714 CommandLineSwitchCase(arg,
"file")
720 GError* error =
nullptr;
721 gchar* uri = gst_filename_to_uri(arg->Value, &error);
724 WLog_ERR(TAG,
"Invalid file %s: %s", arg->Value,
725 error ? error->message :
"unknown error");
726 g_clear_error(&error);
732 winpr_asprintf(&source, &len,
"uridecodebin uri=\"%s\" caps=video/x-raw", uri);
738 defaultName =
nullptr;
739 winpr_asprintf(&defaultName, &len,
"Movie: %s", cam_gst_basename(arg->Value));
743 CommandLineSwitchCase(arg,
"name")
746 hal->name = _strdup(arg->Value);
750 CommandLineSwitchCase(arg,
"loop")
752 if (!cam_gst_parse_bool(arg->Value, &hal->loop))
754 WLog_ERR(TAG,
"Invalid loop value %s, expecting on or off", arg->Value);
758 CommandLineSwitchDefault(arg)
761 CommandLineSwitchEnd(arg)
762 }
while ((arg = CommandLineFindNextArgumentA(arg)) !=
nullptr);
767 WLog_ERR(TAG,
"The gst backend needs one of pipeline:<pipeline>, pipeline-file:<path> or "
772 winpr_asprintf(&hal->description, &len,
773 "%s ! videoconvert ! video/x-raw,format=I420 ! appsink name=" CAM_GST_SINK_NAME
774 " sync=true max-buffers=1 drop=true",
776 if (!hal->description)
781 hal->name = defaultName ? defaultName : _strdup(
"GStreamer camera");
782 defaultName =
nullptr;
799FREERDP_ENTRY_POINT(UINT VCAPITYPE gst_freerdp_rdpecam_client_subsystem_entry(
802 UINT ret = ERROR_INVALID_PARAMETER;
803 WINPR_ASSERT(pEntryPoints);
805 GError* error =
nullptr;
806 if (!gst_init_check(
nullptr,
nullptr, &error))
808 WLog_ERR(TAG,
"Unable to initialize GStreamer: %s",
809 error ? error->message :
"unknown error");
810 g_clear_error(&error);
811 return ERROR_INTERNAL_ERROR;
814 CamGstHal* hal = (CamGstHal*)calloc(1,
sizeof(CamGstHal));
816 return CHANNEL_RC_NO_MEMORY;
818 hal->iHal.Enumerate = cam_gst_enumerate;
819 hal->iHal.RequestDrivenCapture = TRUE;
820 hal->iHal.GetMediaTypeDescriptions = cam_gst_get_media_type_descriptions;
821 hal->iHal.Activate = cam_gst_activate;
822 hal->iHal.Deactivate = cam_gst_deactivate;
823 hal->iHal.StartStream = cam_gst_stream_start;
824 hal->iHal.StopStream = cam_gst_stream_stop;
825 hal->iHal.Free = cam_gst_free;
828 if (!InitializeCriticalSectionEx(&hal->lock, 0, 0))
831 return CHANNEL_RC_NO_MEMORY;
834 if (!cam_gst_parse_args(hal, pEntryPoints->args))
838 if (!cam_gst_probe(hal))
841 ret = pEntryPoints->pRegisterCameraHal(pEntryPoints->plugin, &hal->iHal);
842 if (ret != CHANNEL_RC_OK)
844 WLog_ERR(TAG,
"RegisterCameraHal failed with error %" PRIu32
"", ret);
851 cam_gst_free(&hal->iHal);