FreeRDP
Loading...
Searching...
No Matches
video_main.c
1
20#include <freerdp/config.h>
21
22#include <stdio.h>
23#include <stdlib.h>
24#include <string.h>
25
26#include <winpr/crt.h>
27#include <winpr/assert.h>
28#include <winpr/cast.h>
29#include <winpr/synch.h>
30#include <winpr/print.h>
31#include <winpr/stream.h>
32#include <winpr/cmdline.h>
33#include <winpr/collections.h>
34#include <winpr/interlocked.h>
35#include <winpr/sysinfo.h>
36
37#include <freerdp/addin.h>
38#include <freerdp/primitives.h>
39#include <freerdp/client/channels.h>
40#include <freerdp/client/geometry.h>
41#include <freerdp/client/video.h>
42#include <freerdp/channels/log.h>
43#include <freerdp/codec/h264.h>
44#include <freerdp/codec/yuv.h>
45#include <freerdp/timer.h>
46
47#define TAG CHANNELS_TAG("video")
48
49#include "video_main.h"
50
51typedef struct
52{
53 IWTSPlugin wtsPlugin;
54
55 IWTSListener* controlListener;
56 IWTSListener* dataListener;
57 GENERIC_LISTENER_CALLBACK* control_callback;
58 GENERIC_LISTENER_CALLBACK* data_callback;
59
60 VideoClientContext* context;
61 BOOL initialized;
62 rdpContext* rdpcontext;
63} VIDEO_PLUGIN;
64
65#define XF_VIDEO_UNLIMITED_RATE 31
66
67static const BYTE MFVideoFormat_H264[] = { 'H', '2', '6', '4', 0x00, 0x00, 0x10, 0x00,
68 0x80, 0x00, 0x00, 0xAA, 0x00, 0x38, 0x9B, 0x71 };
69
70typedef struct
71{
72 VideoClientContext* video;
73 BYTE PresentationId;
74 UINT32 ScaledWidth, ScaledHeight;
75 MAPPED_GEOMETRY* geometry;
76
77 UINT64 startTimeStamp;
78 UINT64 publishOffset;
79 H264_CONTEXT* h264;
80 wStream* currentSample;
81 UINT64 lastPublishTime, nextPublishTime;
82 volatile LONG refCounter;
83 VideoSurface* surface;
84} PresentationContext;
85
86typedef struct
87{
88 UINT64 publishTime;
89 UINT64 hnsDuration;
90 MAPPED_GEOMETRY* geometry;
91 UINT32 w, h;
92 UINT32 scanline;
93 BYTE* surfaceData;
94 PresentationContext* presentation;
95} VideoFrame;
96
98struct s_VideoClientContextPriv
99{
100 VideoClientContext* video;
101 GeometryClientContext* geometry;
102 wQueue* frames;
103 CRITICAL_SECTION framesLock;
104 wBufferPool* surfacePool;
105 UINT32 publishedFrames;
106 UINT32 droppedFrames;
107 UINT32 lastSentRate;
108 UINT64 nextFeedbackTime;
109 PresentationContext* currentPresentation;
110 FreeRDP_TimerID timerID;
111};
112
113static void PresentationContext_unref(PresentationContext** presentation);
114static void VideoClientContextPriv_free(VideoClientContextPriv* priv);
115
116WINPR_ATTR_NODISCARD
117static const char* video_command_name(BYTE cmd)
118{
119 switch (cmd)
120 {
121 case TSMM_START_PRESENTATION:
122 return "start";
123 case TSMM_STOP_PRESENTATION:
124 return "stop";
125 default:
126 return "<unknown>";
127 }
128}
129
130static void video_client_context_set_geometry(VideoClientContext* video,
131 GeometryClientContext* geometry)
132{
133 WINPR_ASSERT(video);
134 WINPR_ASSERT(video->priv);
135 video->priv->geometry = geometry;
136}
137
138WINPR_ATTR_MALLOC(VideoClientContextPriv_free, 1)
139static VideoClientContextPriv* VideoClientContextPriv_new(VideoClientContext* video)
140{
141 VideoClientContextPriv* ret = nullptr;
142
143 WINPR_ASSERT(video);
144 ret = calloc(1, sizeof(*ret));
145 if (!ret)
146 return nullptr;
147
148 ret->frames = Queue_New(TRUE, 10, 2);
149 if (!ret->frames)
150 {
151 WLog_ERR(TAG, "unable to allocate frames queue");
152 goto fail;
153 }
154
155 ret->surfacePool = BufferPool_New(FALSE, 0, 16);
156 if (!ret->surfacePool)
157 {
158 WLog_ERR(TAG, "unable to create surface pool");
159 goto fail;
160 }
161
162 if (!InitializeCriticalSectionAndSpinCount(&ret->framesLock, 4 * 1000))
163 {
164 WLog_ERR(TAG, "unable to initialize frames lock");
165 goto fail;
166 }
167
168 ret->video = video;
169
170 /* don't set to unlimited so that we have the chance to send a feedback in
171 * the first second (for servers that want feedback directly)
172 */
173 ret->lastSentRate = 30;
174 return ret;
175
176fail:
177 VideoClientContextPriv_free(ret);
178 return nullptr;
179}
180
181WINPR_ATTR_NODISCARD
182static BOOL PresentationContext_ref(PresentationContext* presentation)
183{
184 WINPR_ASSERT(presentation);
185
186 InterlockedIncrement(&presentation->refCounter);
187 return TRUE;
188}
189
190WINPR_ATTR_NODISCARD
191static PresentationContext* PresentationContext_new(VideoClientContext* video, BYTE PresentationId,
192 UINT32 x, UINT32 y, UINT32 width, UINT32 height)
193{
194 size_t s = 4ULL * width * height;
195 PresentationContext* ret = nullptr;
196
197 WINPR_ASSERT(video);
198
199 if (s > INT32_MAX)
200 return nullptr;
201
202 ret = calloc(1, sizeof(*ret));
203 if (!ret)
204 return nullptr;
205
206 ret->video = video;
207 ret->PresentationId = PresentationId;
208
209 ret->h264 = h264_context_new(FALSE);
210 if (!ret->h264)
211 {
212 WLog_ERR(TAG, "unable to create a h264 context");
213 goto fail;
214 }
215 if (!h264_context_reset(ret->h264, width, height))
216 goto fail;
217
218 ret->currentSample = Stream_New(nullptr, 4096);
219 if (!ret->currentSample)
220 {
221 WLog_ERR(TAG, "unable to create current packet stream");
222 goto fail;
223 }
224
225 ret->surface = video->createSurface(video, x, y, width, height);
226 if (!ret->surface)
227 {
228 WLog_ERR(TAG, "unable to create surface");
229 goto fail;
230 }
231
232 if (!PresentationContext_ref(ret))
233 goto fail;
234
235 return ret;
236
237fail:
238 PresentationContext_unref(&ret);
239 return nullptr;
240}
241
242static void PresentationContext_unref(PresentationContext** ppresentation)
243{
244 WINPR_ASSERT(ppresentation);
245
246 PresentationContext* presentation = *ppresentation;
247 if (!presentation)
248 return;
249
250 if (InterlockedDecrement(&presentation->refCounter) > 0)
251 return;
252
253 MAPPED_GEOMETRY* geometry = presentation->geometry;
254 if (geometry)
255 {
256 geometry->MappedGeometryUpdate = nullptr;
257 geometry->MappedGeometryClear = nullptr;
258 geometry->custom = nullptr;
259 mappedGeometryUnref(geometry);
260 }
261
262 h264_context_free(presentation->h264);
263 Stream_Free(presentation->currentSample, TRUE);
264 presentation->video->deleteSurface(presentation->video, presentation->surface);
265 free(presentation);
266 *ppresentation = nullptr;
267}
268
269static void VideoFrame_free(VideoFrame* frame)
270{
271 if (!frame)
272 return;
273
274 mappedGeometryUnref(frame->geometry);
275
276 WINPR_ASSERT(frame->presentation);
277 WINPR_ASSERT(frame->presentation->video);
278 WINPR_ASSERT(frame->presentation->video->priv);
279 BufferPool_Return(frame->presentation->video->priv->surfacePool, frame->surfaceData);
280 PresentationContext_unref(&frame->presentation);
281 free(frame);
282}
283
284WINPR_ATTR_MALLOC(VideoFrame_free, 1)
285static VideoFrame* VideoFrame_new(VideoClientContextPriv* priv, PresentationContext* presentation,
286 MAPPED_GEOMETRY* geom)
287{
288 WINPR_ASSERT(priv);
289 WINPR_ASSERT(presentation);
290 WINPR_ASSERT(geom);
291
292 const VideoSurface* surface = presentation->surface;
293 WINPR_ASSERT(surface);
294
295 VideoFrame* frame = calloc(1, sizeof(VideoFrame));
296 if (!frame)
297 goto fail;
298
299 mappedGeometryRef(geom);
300
301 frame->publishTime = presentation->lastPublishTime;
302 frame->geometry = geom;
303 frame->w = surface->alignedWidth;
304 frame->h = surface->alignedHeight;
305 frame->scanline = surface->scanline;
306
307 frame->surfaceData = BufferPool_Take(priv->surfacePool, 1ll * frame->scanline * frame->h);
308 if (!frame->surfaceData)
309 goto fail;
310
311 frame->presentation = presentation;
312 if (!PresentationContext_ref(frame->presentation))
313 goto fail;
314
315 return frame;
316
317fail:
318 VideoFrame_free(frame);
319 return nullptr;
320}
321
322void VideoClientContextPriv_free(VideoClientContextPriv* priv)
323{
324 if (!priv)
325 return;
326
327 EnterCriticalSection(&priv->framesLock);
328
329 if (priv->frames)
330 {
331 while (Queue_Count(priv->frames))
332 {
333 VideoFrame* frame = Queue_Dequeue(priv->frames);
334 if (frame)
335 VideoFrame_free(frame);
336 }
337 }
338
339 Queue_Free(priv->frames);
340 LeaveCriticalSection(&priv->framesLock);
341
342 DeleteCriticalSection(&priv->framesLock);
343
344 if (priv->currentPresentation)
345 PresentationContext_unref(&priv->currentPresentation);
346
347 BufferPool_Free(priv->surfacePool);
348 free(priv);
349}
350
351WINPR_ATTR_NODISCARD
352static UINT video_channel_write(VIDEO_PLUGIN* video, const BYTE* data, UINT32 length)
353{
354 WINPR_ASSERT(video);
355
356 if (!video->control_callback || !video->control_callback->channel_callback)
357 return ERROR_BAD_CONFIGURATION;
358 IWTSVirtualChannel* channel = video->control_callback->channel_callback->channel;
359 if (!channel || !channel->Write)
360 return ERROR_BAD_CONFIGURATION;
361 return channel->Write(channel, length, data, nullptr);
362}
363
364WINPR_ATTR_NODISCARD
365static UINT video_control_send_presentation_response(VideoClientContext* context,
366 TSMM_PRESENTATION_RESPONSE* resp)
367{
368 BYTE buf[12] = WINPR_C_ARRAY_INIT;
369
370 WINPR_ASSERT(context);
371 WINPR_ASSERT(resp);
372
373 VIDEO_PLUGIN* video = (VIDEO_PLUGIN*)context->handle;
374 WINPR_ASSERT(video);
375
376 wStream* s = Stream_New(buf, 12);
377 if (!s)
378 return CHANNEL_RC_NO_MEMORY;
379
380 Stream_Write_UINT32(s, 12); /* cbSize */
381 Stream_Write_UINT32(s, TSMM_PACKET_TYPE_PRESENTATION_RESPONSE); /* PacketType */
382 Stream_Write_UINT8(s, resp->PresentationId);
383 Stream_Zero(s, 3);
384 Stream_SealLength(s);
385 Stream_Free(s, FALSE);
386
387 return video_channel_write(video, buf, sizeof(buf));
388}
389
390WINPR_ATTR_NODISCARD
391static BOOL video_onMappedGeometryUpdate(MAPPED_GEOMETRY* geometry)
392{
393 WINPR_ASSERT(geometry);
394
395 PresentationContext* presentation = (PresentationContext*)geometry->custom;
396 WINPR_ASSERT(presentation);
397
398 RDP_RECT* r = &geometry->geometry.boundingRect;
399 WLog_DBG(TAG,
400 "geometry updated topGeom=(%" PRId32 ",%" PRId32 "-%" PRId32 "x%" PRId32
401 ") geom=(%" PRId32 ",%" PRId32 "-%" PRId32 "x%" PRId32 ") rects=(%" PRId16 ",%" PRId16
402 "-%" PRId16 "x%" PRId16 ")",
403 geometry->topLevelLeft, geometry->topLevelTop,
404 geometry->topLevelRight - geometry->topLevelLeft,
405 geometry->topLevelBottom - geometry->topLevelTop,
406
407 geometry->left, geometry->top, geometry->right - geometry->left,
408 geometry->bottom - geometry->top,
409
410 r->x, r->y, r->width, r->height);
411
412 WINPR_ASSERT(presentation->surface);
413 presentation->surface->x =
414 WINPR_ASSERTING_INT_CAST(uint32_t, geometry->topLevelLeft + geometry->left);
415 presentation->surface->y =
416 WINPR_ASSERTING_INT_CAST(uint32_t, geometry->topLevelTop + geometry->top);
417
418 return TRUE;
419}
420
421WINPR_ATTR_NODISCARD
422static BOOL video_onMappedGeometryClear(MAPPED_GEOMETRY* geometry)
423{
424 WINPR_ASSERT(geometry);
425
426 PresentationContext* presentation = (PresentationContext*)geometry->custom;
427 WINPR_ASSERT(presentation);
428
429 mappedGeometryUnref(presentation->geometry);
430 presentation->geometry = nullptr;
431 return TRUE;
432}
433
434WINPR_ATTR_NODISCARD
435static UINT video_PresentationRequest(VideoClientContext* video,
436 const TSMM_PRESENTATION_REQUEST* req)
437{
438 UINT ret = CHANNEL_RC_OK;
439
440 WINPR_ASSERT(video);
441 WINPR_ASSERT(req);
442
443 VideoClientContextPriv* priv = video->priv;
444 WINPR_ASSERT(priv);
445
446 if (req->Command == TSMM_START_PRESENTATION)
447 {
448 MAPPED_GEOMETRY* geom = nullptr;
449 TSMM_PRESENTATION_RESPONSE resp = WINPR_C_ARRAY_INIT;
450
451 if (memcmp(req->VideoSubtypeId, MFVideoFormat_H264, 16) != 0)
452 {
453 WLog_ERR(TAG, "not a H264 video, ignoring request");
454 return CHANNEL_RC_OK;
455 }
456
457 if (priv->currentPresentation)
458 {
459 if (priv->currentPresentation->PresentationId == req->PresentationId)
460 {
461 WLog_ERR(TAG, "ignoring start request for existing presentation %" PRIu8,
462 req->PresentationId);
463 return CHANNEL_RC_OK;
464 }
465
466 WLog_ERR(TAG, "releasing current presentation %" PRIu8, req->PresentationId);
467 PresentationContext_unref(&priv->currentPresentation);
468 }
469
470 if (!priv->geometry)
471 {
472 WLog_ERR(TAG, "geometry channel not ready, ignoring request");
473 return CHANNEL_RC_OK;
474 }
475
476 geom = HashTable_GetItemValue(priv->geometry->geometries, &(req->GeometryMappingId));
477 if (!geom)
478 {
479 WLog_ERR(TAG, "geometry mapping 0x%" PRIx64 " not registered", req->GeometryMappingId);
480 return CHANNEL_RC_OK;
481 }
482
483 WLog_DBG(TAG, "creating presentation 0x%x", req->PresentationId);
484 priv->currentPresentation = PresentationContext_new(
485 video, req->PresentationId,
486 WINPR_ASSERTING_INT_CAST(uint32_t, geom->topLevelLeft + geom->left),
487 WINPR_ASSERTING_INT_CAST(uint32_t, geom->topLevelTop + geom->top), req->SourceWidth,
488 req->SourceHeight);
489 if (!priv->currentPresentation)
490 {
491 WLog_ERR(TAG, "unable to create presentation video");
492 return CHANNEL_RC_NO_MEMORY;
493 }
494
495 mappedGeometryRef(geom);
496 priv->currentPresentation->geometry = geom;
497
498 priv->currentPresentation->video = video;
499 priv->currentPresentation->ScaledWidth = req->ScaledWidth;
500 priv->currentPresentation->ScaledHeight = req->ScaledHeight;
501
502 geom->custom = priv->currentPresentation;
503 geom->MappedGeometryUpdate = video_onMappedGeometryUpdate;
504 geom->MappedGeometryClear = video_onMappedGeometryClear;
505
506 /* send back response */
507 resp.PresentationId = req->PresentationId;
508 ret = video_control_send_presentation_response(video, &resp);
509 }
510 else if (req->Command == TSMM_STOP_PRESENTATION)
511 {
512 WLog_DBG(TAG, "stopping presentation 0x%x", req->PresentationId);
513 if (!priv->currentPresentation)
514 {
515 WLog_ERR(TAG, "unknown presentation to stop %" PRIu8, req->PresentationId);
516 return CHANNEL_RC_OK;
517 }
518
519 priv->droppedFrames = 0;
520 priv->publishedFrames = 0;
521 PresentationContext_unref(&priv->currentPresentation);
522 }
523
524 return ret;
525}
526
527WINPR_ATTR_NODISCARD
528static UINT video_read_tsmm_presentation_req(VideoClientContext* context, wStream* s)
529{
530 TSMM_PRESENTATION_REQUEST req = WINPR_C_ARRAY_INIT;
531
532 WINPR_ASSERT(context);
533 WINPR_ASSERT(s);
534
535 if (!Stream_CheckAndLogRequiredLength(TAG, s, 60))
536 return ERROR_INVALID_DATA;
537
538 Stream_Read_UINT8(s, req.PresentationId);
539 Stream_Read_UINT8(s, req.Version);
540 Stream_Read_UINT8(s, req.Command);
541 Stream_Read_UINT8(s, req.FrameRate); /* FrameRate - reserved and ignored */
542
543 Stream_Seek_UINT16(s); /* AverageBitrateKbps reserved and ignored */
544 Stream_Seek_UINT16(s); /* reserved */
545
546 Stream_Read_UINT32(s, req.SourceWidth);
547 Stream_Read_UINT32(s, req.SourceHeight);
548 Stream_Read_UINT32(s, req.ScaledWidth);
549 Stream_Read_UINT32(s, req.ScaledHeight);
550 Stream_Read_UINT64(s, req.hnsTimestampOffset);
551 Stream_Read_UINT64(s, req.GeometryMappingId);
552 Stream_Read(s, req.VideoSubtypeId, 16);
553
554 Stream_Read_UINT32(s, req.cbExtra);
555
556 if (!Stream_CheckAndLogRequiredLength(TAG, s, req.cbExtra))
557 return ERROR_INVALID_DATA;
558
559 req.pExtraData = Stream_Pointer(s);
560
561 WLog_DBG(TAG,
562 "presentationReq: id:%" PRIu8 " version:%" PRIu8
563 " command:%s srcWidth/srcHeight=%" PRIu32 "x%" PRIu32 " scaled Width/Height=%" PRIu32
564 "x%" PRIu32 " timestamp=%" PRIu64 " mappingId=%" PRIx64 "",
565 req.PresentationId, req.Version, video_command_name(req.Command), req.SourceWidth,
566 req.SourceHeight, req.ScaledWidth, req.ScaledHeight, req.hnsTimestampOffset,
567 req.GeometryMappingId);
568
569 return video_PresentationRequest(context, &req);
570}
571
577WINPR_ATTR_NODISCARD
578static UINT video_control_on_data_received(IWTSVirtualChannelCallback* pChannelCallback, wStream* s)
579{
580 GENERIC_CHANNEL_CALLBACK* callback = (GENERIC_CHANNEL_CALLBACK*)pChannelCallback;
581 UINT ret = CHANNEL_RC_OK;
582 UINT32 cbSize = 0;
583 UINT32 packetType = 0;
584
585 WINPR_ASSERT(callback);
586 WINPR_ASSERT(s);
587
588 VIDEO_PLUGIN* video = (VIDEO_PLUGIN*)callback->plugin;
589 WINPR_ASSERT(video);
590
591 VideoClientContext* context = (VideoClientContext*)video->wtsPlugin.pInterface;
592 WINPR_ASSERT(context);
593
594 if (!Stream_CheckAndLogRequiredLength(TAG, s, 4))
595 return ERROR_INVALID_DATA;
596
597 Stream_Read_UINT32(s, cbSize);
598 if (cbSize < 8)
599 {
600 WLog_ERR(TAG, "invalid cbSize %" PRIu32 ", expected 8", cbSize);
601 return ERROR_INVALID_DATA;
602 }
603 if (!Stream_CheckAndLogRequiredLength(TAG, s, cbSize - 4))
604 return ERROR_INVALID_DATA;
605
606 Stream_Read_UINT32(s, packetType);
607 switch (packetType)
608 {
609 case TSMM_PACKET_TYPE_PRESENTATION_REQUEST:
610 ret = video_read_tsmm_presentation_req(context, s);
611 break;
612 default:
613 WLog_ERR(TAG, "not expecting packet type %" PRIu32 "", packetType);
614 ret = ERROR_UNSUPPORTED_TYPE;
615 break;
616 }
617
618 return ret;
619}
620
621static UINT video_control_send_client_notification(VideoClientContext* context,
622 const TSMM_CLIENT_NOTIFICATION* notif)
623{
624 BYTE buf[100] = WINPR_C_ARRAY_INIT;
625
626 WINPR_ASSERT(context);
627 WINPR_ASSERT(notif);
628
629 VIDEO_PLUGIN* video = (VIDEO_PLUGIN*)context->handle;
630 WINPR_ASSERT(video);
631
632 wStream* s = Stream_New(buf, 32);
633 if (!s)
634 return CHANNEL_RC_NO_MEMORY;
635
636 UINT32 cbSize = 16;
637 Stream_Seek_UINT32(s); /* cbSize */
638 Stream_Write_UINT32(s, TSMM_PACKET_TYPE_CLIENT_NOTIFICATION); /* PacketType */
639 Stream_Write_UINT8(s, notif->PresentationId);
640 Stream_Write_UINT8(s, notif->NotificationType);
641 Stream_Zero(s, 2);
642 if (notif->NotificationType == TSMM_CLIENT_NOTIFICATION_TYPE_FRAMERATE_OVERRIDE)
643 {
644 Stream_Write_UINT32(s, 16); /* cbData */
645
646 /* TSMM_CLIENT_NOTIFICATION_FRAMERATE_OVERRIDE */
647 Stream_Write_UINT32(s, notif->FramerateOverride.Flags);
648 Stream_Write_UINT32(s, notif->FramerateOverride.DesiredFrameRate);
649 Stream_Zero(s, 4ULL * 2ULL);
650
651 cbSize += 4UL * 4UL;
652 }
653 else
654 {
655 Stream_Write_UINT32(s, 0); /* cbData */
656 }
657
658 Stream_SealLength(s);
659 Stream_ResetPosition(s);
660 Stream_Write_UINT32(s, cbSize);
661 Stream_Free(s, FALSE);
662
663 return video_channel_write(video, buf, cbSize);
664}
665
666static void video_timer(VideoClientContext* video, UINT64 now)
667{
668 VideoFrame* frame = nullptr;
669
670 WINPR_ASSERT(video);
671
672 VideoClientContextPriv* priv = video->priv;
673 WINPR_ASSERT(priv);
674
675 EnterCriticalSection(&priv->framesLock);
676 do
677 {
678 const VideoFrame* peekFrame = (VideoFrame*)Queue_Peek(priv->frames);
679 if (!peekFrame)
680 break;
681
682 if (peekFrame->publishTime > now)
683 break;
684
685 if (frame)
686 {
687 WLog_DBG(TAG, "dropping frame @%" PRIu64, frame->publishTime);
688 priv->droppedFrames++;
689 VideoFrame_free(frame);
690 }
691 frame = Queue_Dequeue(priv->frames);
692 } while (1);
693 LeaveCriticalSection(&priv->framesLock);
694
695 if (frame)
696 {
697 PresentationContext* presentation = frame->presentation;
698
699 priv->publishedFrames++;
700 memcpy(presentation->surface->data, frame->surfaceData, 1ull * frame->scanline * frame->h);
701
702 WINPR_ASSERT(video->showSurface);
703 if (!video->showSurface(video, presentation->surface, presentation->ScaledWidth,
704 presentation->ScaledHeight))
705 WLog_WARN(TAG, "showSurface failed");
706
707 VideoFrame_free(frame);
708 }
709
710 if (priv->nextFeedbackTime < now)
711 {
712 /* we can compute some feedback only if we have some published frames and
713 * a current presentation
714 */
715 if (priv->publishedFrames && priv->currentPresentation)
716 {
717 UINT32 computedRate = 0;
718
719 if (!PresentationContext_ref(priv->currentPresentation))
720 WLog_WARN(TAG, "PresentationContext_ref(priv->currentPresentation) failed");
721
722 if (priv->droppedFrames)
723 {
729 if (priv->lastSentRate == XF_VIDEO_UNLIMITED_RATE)
730 computedRate = 24;
731 else
732 {
733 computedRate = priv->lastSentRate - 2;
734 if (!computedRate)
735 computedRate = 2;
736 }
737 }
738 else
739 {
744 if (priv->lastSentRate == XF_VIDEO_UNLIMITED_RATE)
745 computedRate = XF_VIDEO_UNLIMITED_RATE; /* stay unlimited */
746 else
747 {
748 computedRate = priv->lastSentRate + 2;
749 if (computedRate > XF_VIDEO_UNLIMITED_RATE)
750 computedRate = XF_VIDEO_UNLIMITED_RATE;
751 }
752 }
753
754 if (computedRate != priv->lastSentRate)
755 {
756 TSMM_CLIENT_NOTIFICATION notif = WINPR_C_ARRAY_INIT;
757
758 WINPR_ASSERT(priv->currentPresentation);
759 notif.PresentationId = priv->currentPresentation->PresentationId;
760 notif.NotificationType = TSMM_CLIENT_NOTIFICATION_TYPE_FRAMERATE_OVERRIDE;
761 if (computedRate == XF_VIDEO_UNLIMITED_RATE)
762 {
763 notif.FramerateOverride.Flags = 0x01;
764 notif.FramerateOverride.DesiredFrameRate = 0x00;
765 }
766 else
767 {
768 notif.FramerateOverride.Flags = 0x02;
769 notif.FramerateOverride.DesiredFrameRate = computedRate;
770 }
771
772 video_control_send_client_notification(video, &notif);
773 priv->lastSentRate = computedRate;
774
775 WLog_VRB(TAG,
776 "server notified with rate %" PRIu32 " published=%" PRIu32
777 " dropped=%" PRIu32,
778 priv->lastSentRate, priv->publishedFrames, priv->droppedFrames);
779 }
780
781 PresentationContext_unref(&priv->currentPresentation);
782 }
783
784 priv->droppedFrames = 0;
785 priv->publishedFrames = 0;
786 priv->nextFeedbackTime = now + 1000;
787 }
788}
789
790WINPR_ATTR_NODISCARD
791static UINT video_VideoData(VideoClientContext* context, const TSMM_VIDEO_DATA* data)
792{
793 int status = 0;
794
795 WINPR_ASSERT(context);
796 WINPR_ASSERT(data);
797
798 VideoClientContextPriv* priv = context->priv;
799 WINPR_ASSERT(priv);
800
801 PresentationContext* presentation = priv->currentPresentation;
802 if (!presentation)
803 {
804 WLog_ERR(TAG, "no current presentation");
805 return CHANNEL_RC_OK;
806 }
807
808 if (presentation->PresentationId != data->PresentationId)
809 {
810 WLog_ERR(TAG, "current presentation id=%" PRIu8 " doesn't match data id=%" PRIu8,
811 presentation->PresentationId, data->PresentationId);
812 return CHANNEL_RC_OK;
813 }
814
815 if (!Stream_EnsureRemainingCapacity(presentation->currentSample, data->cbSample))
816 {
817 WLog_ERR(TAG, "unable to expand the current packet");
818 return CHANNEL_RC_NO_MEMORY;
819 }
820
821 Stream_Write(presentation->currentSample, data->pSample, data->cbSample);
822
823 if (data->CurrentPacketIndex == data->PacketsInSample)
824 {
825 VideoSurface* surface = presentation->surface;
826 H264_CONTEXT* h264 = presentation->h264;
827 const UINT64 startTime = winpr_GetTickCount64NS();
828 MAPPED_GEOMETRY* geom = presentation->geometry;
829
830 const RECTANGLE_16 rect = { 0, 0, WINPR_ASSERTING_INT_CAST(UINT16, surface->alignedWidth),
831 WINPR_ASSERTING_INT_CAST(UINT16, surface->alignedHeight) };
832 Stream_SealLength(presentation->currentSample);
833 Stream_ResetPosition(presentation->currentSample);
834
835 const UINT64 timeAfterH264 = winpr_GetTickCount64NS();
836 if (data->SampleNumber == 1)
837 {
838 presentation->lastPublishTime = startTime;
839 }
840
841 presentation->lastPublishTime += 100ull * data->hnsDuration;
842 if (presentation->lastPublishTime <= (10000000ull + timeAfterH264))
843 {
844 int dropped = 0;
845
846 const size_t len = Stream_Length(presentation->currentSample);
847 if (len > UINT32_MAX)
848 return CHANNEL_RC_OK;
849
850 /* if the frame is to be published in less than 10 ms, let's consider it's now */
851 status =
852 avc420_decompress(h264, Stream_Pointer(presentation->currentSample), (UINT32)len,
853 surface->data, surface->format, surface->scanline,
854 surface->alignedWidth, surface->alignedHeight, &rect, 1);
855
856 if (status < 0)
857 return CHANNEL_RC_OK;
858
859 WINPR_ASSERT(context->showSurface);
860 if (!context->showSurface(context, presentation->surface, presentation->ScaledWidth,
861 presentation->ScaledHeight))
862 return CHANNEL_RC_NOT_INITIALIZED;
863
864 priv->publishedFrames++;
865
866 /* cleanup previously scheduled frames */
867 EnterCriticalSection(&priv->framesLock);
868 while (Queue_Count(priv->frames) > 0)
869 {
870 VideoFrame* frame = Queue_Dequeue(priv->frames);
871 if (frame)
872 {
873 priv->droppedFrames++;
874 VideoFrame_free(frame);
875 dropped++;
876 }
877 }
878 LeaveCriticalSection(&priv->framesLock);
879
880 if (dropped)
881 WLog_DBG(TAG, "showing frame (%d dropped)", dropped);
882 }
883 else
884 {
885 const size_t len = Stream_Length(presentation->currentSample);
886 if (len > UINT32_MAX)
887 return CHANNEL_RC_OK;
888
889 BOOL enqueueResult = 0;
890 VideoFrame* frame = VideoFrame_new(priv, presentation, geom);
891 if (!frame)
892 {
893 WLog_ERR(TAG, "unable to create frame");
894 return CHANNEL_RC_NO_MEMORY;
895 }
896
897 status =
898 avc420_decompress(h264, Stream_Pointer(presentation->currentSample), (UINT32)len,
899 frame->surfaceData, surface->format, surface->scanline,
900 surface->alignedWidth, surface->alignedHeight, &rect, 1);
901 if (status < 0)
902 {
903 VideoFrame_free(frame);
904 return CHANNEL_RC_OK;
905 }
906
907 EnterCriticalSection(&priv->framesLock);
908 enqueueResult = Queue_Enqueue(priv->frames, frame);
909 LeaveCriticalSection(&priv->framesLock);
910
911 if (!enqueueResult)
912 {
913 WLog_ERR(TAG, "unable to enqueue frame");
914 VideoFrame_free(frame);
915 return CHANNEL_RC_NO_MEMORY;
916 }
917
918 // NOLINTNEXTLINE(clang-analyzer-unix.Malloc): Queue_Enqueue owns frame
919 WLog_DBG(TAG, "scheduling frame in %" PRIu64 " ms", (frame->publishTime - startTime));
920 }
921 }
922
923 return CHANNEL_RC_OK;
924}
925
926WINPR_ATTR_NODISCARD
927static UINT video_data_on_data_received(IWTSVirtualChannelCallback* pChannelCallback, wStream* s)
928{
929 GENERIC_CHANNEL_CALLBACK* callback = (GENERIC_CHANNEL_CALLBACK*)pChannelCallback;
930 UINT32 cbSize = 0;
931 UINT32 packetType = 0;
932 TSMM_VIDEO_DATA data;
933
934 VIDEO_PLUGIN* video = (VIDEO_PLUGIN*)callback->plugin;
935 WINPR_ASSERT(video);
936
937 VideoClientContext* context = (VideoClientContext*)video->wtsPlugin.pInterface;
938
939 if (!Stream_CheckAndLogRequiredLength(TAG, s, 4))
940 return ERROR_INVALID_DATA;
941
942 Stream_Read_UINT32(s, cbSize);
943 if (cbSize < 8)
944 {
945 WLog_ERR(TAG, "invalid cbSize %" PRIu32 ", expected >= 8", cbSize);
946 return ERROR_INVALID_DATA;
947 }
948
949 if (!Stream_CheckAndLogRequiredLength(TAG, s, cbSize - 4))
950 return ERROR_INVALID_DATA;
951
952 Stream_Read_UINT32(s, packetType);
953 if (packetType != TSMM_PACKET_TYPE_VIDEO_DATA)
954 {
955 WLog_ERR(TAG, "only expecting VIDEO_DATA on the data channel");
956 return ERROR_INVALID_DATA;
957 }
958
959 if (!Stream_CheckAndLogRequiredLength(TAG, s, 32))
960 return ERROR_INVALID_DATA;
961
962 Stream_Read_UINT8(s, data.PresentationId);
963 Stream_Read_UINT8(s, data.Version);
964 Stream_Read_UINT8(s, data.Flags);
965 Stream_Seek_UINT8(s); /* reserved */
966 Stream_Read_UINT64(s, data.hnsTimestamp);
967 Stream_Read_UINT64(s, data.hnsDuration);
968 Stream_Read_UINT16(s, data.CurrentPacketIndex);
969 Stream_Read_UINT16(s, data.PacketsInSample);
970 Stream_Read_UINT32(s, data.SampleNumber);
971 Stream_Read_UINT32(s, data.cbSample);
972 if (!Stream_CheckAndLogRequiredLength(TAG, s, data.cbSample))
973 return ERROR_INVALID_DATA;
974 data.pSample = Stream_Pointer(s);
975
976 /*
977 WLog_DBG(TAG, "videoData: id:%"PRIu8" version:%"PRIu8" flags:0x%"PRIx8" timestamp=%"PRIu64"
978 duration=%"PRIu64 " curPacketIndex:%"PRIu16" packetInSample:%"PRIu16" sampleNumber:%"PRIu32"
979 cbSample:%"PRIu32"", data.PresentationId, data.Version, data.Flags, data.hnsTimestamp,
980 data.hnsDuration, data.CurrentPacketIndex, data.PacketsInSample, data.SampleNumber,
981 data.cbSample);
982 */
983
984 return video_VideoData(context, &data);
985}
986
992WINPR_ATTR_NODISCARD
993static UINT video_control_on_close(IWTSVirtualChannelCallback* pChannelCallback)
994{
995 if (pChannelCallback)
996 {
997 GENERIC_CHANNEL_CALLBACK* listener_callback = (GENERIC_CHANNEL_CALLBACK*)pChannelCallback;
998 VIDEO_PLUGIN* video = (VIDEO_PLUGIN*)listener_callback->plugin;
999 if (video && video->control_callback)
1000 {
1001 video->control_callback->channel_callback = nullptr;
1002 }
1003 }
1004 free(pChannelCallback);
1005 return CHANNEL_RC_OK;
1006}
1007
1008WINPR_ATTR_NODISCARD
1009static UINT video_data_on_close(IWTSVirtualChannelCallback* pChannelCallback)
1010{
1011 if (pChannelCallback)
1012 {
1013 GENERIC_CHANNEL_CALLBACK* listener_callback = (GENERIC_CHANNEL_CALLBACK*)pChannelCallback;
1014 VIDEO_PLUGIN* video = (VIDEO_PLUGIN*)listener_callback->plugin;
1015 if (video && video->data_callback)
1016 {
1017 video->data_callback->channel_callback = nullptr;
1018 }
1019 }
1020 free(pChannelCallback);
1021 return CHANNEL_RC_OK;
1022}
1023
1029// NOLINTBEGIN(readability-non-const-parameter)
1030WINPR_ATTR_NODISCARD
1031static UINT video_control_on_new_channel_connection(IWTSListenerCallback* listenerCallback,
1032 IWTSVirtualChannel* channel,
1033 WINPR_ATTR_UNUSED BYTE* Data,
1034 WINPR_ATTR_UNUSED BOOL* pbAccept,
1035 IWTSVirtualChannelCallback** ppCallback)
1036// NOLINTEND(readability-non-const-parameter)
1037{
1038 GENERIC_LISTENER_CALLBACK* listener_callback = (GENERIC_LISTENER_CALLBACK*)listenerCallback;
1039
1040 GENERIC_CHANNEL_CALLBACK* callback =
1041 (GENERIC_CHANNEL_CALLBACK*)calloc(1, sizeof(GENERIC_CHANNEL_CALLBACK));
1042 if (!callback)
1043 {
1044 WLog_ERR(TAG, "calloc failed!");
1045 return CHANNEL_RC_NO_MEMORY;
1046 }
1047
1048 callback->iface.OnDataReceived = video_control_on_data_received;
1049 callback->iface.OnClose = video_control_on_close;
1050 callback->plugin = listener_callback->plugin;
1051 callback->channel_mgr = listener_callback->channel_mgr;
1052 callback->channel = channel;
1053 listener_callback->channel_callback = callback;
1054
1055 *ppCallback = &callback->iface;
1056
1057 return CHANNEL_RC_OK;
1058}
1059
1060// NOLINTBEGIN(readability-non-const-parameter)
1061WINPR_ATTR_NODISCARD
1062static UINT video_data_on_new_channel_connection(IWTSListenerCallback* pListenerCallback,
1063 IWTSVirtualChannel* pChannel,
1064 WINPR_ATTR_UNUSED BYTE* Data,
1065 WINPR_ATTR_UNUSED BOOL* pbAccept,
1066 IWTSVirtualChannelCallback** ppCallback)
1067// NOLINTEND(readability-non-const-parameter)
1068{
1069 GENERIC_LISTENER_CALLBACK* listener_callback = (GENERIC_LISTENER_CALLBACK*)pListenerCallback;
1070
1071 GENERIC_CHANNEL_CALLBACK* callback =
1072 (GENERIC_CHANNEL_CALLBACK*)calloc(1, sizeof(GENERIC_CHANNEL_CALLBACK));
1073 if (!callback)
1074 {
1075 WLog_ERR(TAG, "calloc failed!");
1076 return CHANNEL_RC_NO_MEMORY;
1077 }
1078
1079 callback->iface.OnDataReceived = video_data_on_data_received;
1080 callback->iface.OnClose = video_data_on_close;
1081 callback->plugin = listener_callback->plugin;
1082 callback->channel_mgr = listener_callback->channel_mgr;
1083 callback->channel = pChannel;
1084 listener_callback->channel_callback = callback;
1085
1086 *ppCallback = &callback->iface;
1087
1088 return CHANNEL_RC_OK;
1089}
1090
1091WINPR_ATTR_NODISCARD
1092static uint64_t timer_cb(WINPR_ATTR_UNUSED rdpContext* context, void* userdata,
1093 WINPR_ATTR_UNUSED FreeRDP_TimerID timerID, uint64_t timestamp,
1094 uint64_t interval)
1095{
1096 VideoClientContext* video = userdata;
1097 if (!video)
1098 return 0;
1099 if (!video->timer)
1100 return 0;
1101
1102 video->timer(video, timestamp);
1103
1104 return interval;
1105}
1106
1112WINPR_ATTR_NODISCARD
1113static UINT video_plugin_initialize(IWTSPlugin* plugin, IWTSVirtualChannelManager* channelMgr)
1114{
1115 UINT status = 0;
1116 VIDEO_PLUGIN* video = (VIDEO_PLUGIN*)plugin;
1117
1118 if (video->initialized)
1119 {
1120 WLog_ERR(TAG, "[%s] channel initialized twice, aborting", VIDEO_CONTROL_DVC_CHANNEL_NAME);
1121 return ERROR_INVALID_DATA;
1122 }
1123
1124 {
1125 GENERIC_LISTENER_CALLBACK* callback =
1126 (GENERIC_LISTENER_CALLBACK*)calloc(1, sizeof(GENERIC_LISTENER_CALLBACK));
1127 if (!callback)
1128 {
1129 WLog_ERR(TAG, "calloc for control callback failed!");
1130 return CHANNEL_RC_NO_MEMORY;
1131 }
1132
1133 callback->iface.OnNewChannelConnection = video_control_on_new_channel_connection;
1134 callback->plugin = plugin;
1135 callback->channel_mgr = channelMgr;
1136
1137 status = channelMgr->CreateListener(channelMgr, VIDEO_CONTROL_DVC_CHANNEL_NAME, 0,
1138 &callback->iface, &(video->controlListener));
1139 video->control_callback = callback;
1140 if (status != CHANNEL_RC_OK)
1141 return status;
1142 }
1143 video->controlListener->pInterface = video->wtsPlugin.pInterface;
1144
1145 {
1146 GENERIC_LISTENER_CALLBACK* callback =
1147 (GENERIC_LISTENER_CALLBACK*)calloc(1, sizeof(GENERIC_LISTENER_CALLBACK));
1148 if (!callback)
1149 {
1150 WLog_ERR(TAG, "calloc for data callback failed!");
1151 return CHANNEL_RC_NO_MEMORY;
1152 }
1153
1154 callback->iface.OnNewChannelConnection = video_data_on_new_channel_connection;
1155 callback->plugin = plugin;
1156 callback->channel_mgr = channelMgr;
1157
1158 status = channelMgr->CreateListener(channelMgr, VIDEO_DATA_DVC_CHANNEL_NAME, 0,
1159 &callback->iface, &(video->dataListener));
1160 video->data_callback = callback;
1161 if (status == CHANNEL_RC_OK)
1162 video->dataListener->pInterface = video->wtsPlugin.pInterface;
1163 }
1164
1165 if (status == CHANNEL_RC_OK)
1166 video->context->priv->timerID =
1167 freerdp_timer_add(video->rdpcontext, 20000000, timer_cb, video->context, true);
1168 video->initialized = video->context->priv->timerID != 0;
1169 if (!video->initialized)
1170 status = ERROR_INTERNAL_ERROR;
1171 return status;
1172}
1173
1179WINPR_ATTR_NODISCARD
1180static UINT video_plugin_terminated(IWTSPlugin* pPlugin)
1181{
1182 VIDEO_PLUGIN* video = (VIDEO_PLUGIN*)pPlugin;
1183 if (!video)
1184 return CHANNEL_RC_INVALID_INSTANCE;
1185
1186 if (video->context && video->context->priv)
1187 freerdp_timer_remove(video->rdpcontext, video->context->priv->timerID);
1188
1189 if (video->control_callback)
1190 {
1191 IWTSVirtualChannelManager* mgr = video->control_callback->channel_mgr;
1192 if (mgr)
1193 IFCALL(mgr->DestroyListener, mgr, video->controlListener);
1194 }
1195 if (video->data_callback)
1196 {
1197 IWTSVirtualChannelManager* mgr = video->data_callback->channel_mgr;
1198 if (mgr)
1199 IFCALL(mgr->DestroyListener, mgr, video->dataListener);
1200 }
1201
1202 if (video->context)
1203 VideoClientContextPriv_free(video->context->priv);
1204
1205 free(video->control_callback);
1206 free(video->data_callback);
1207 free(video->wtsPlugin.pInterface);
1208 free(pPlugin);
1209 return CHANNEL_RC_OK;
1210}
1211
1220WINPR_ATTR_NODISCARD
1221FREERDP_ENTRY_POINT(UINT VCAPITYPE video_DVCPluginEntry(IDRDYNVC_ENTRY_POINTS* pEntryPoints))
1222{
1223 UINT error = ERROR_INTERNAL_ERROR;
1224
1225 VIDEO_PLUGIN* videoPlugin = (VIDEO_PLUGIN*)pEntryPoints->GetPlugin(pEntryPoints, "video");
1226 if (!videoPlugin)
1227 {
1228 videoPlugin = (VIDEO_PLUGIN*)calloc(1, sizeof(VIDEO_PLUGIN));
1229 if (!videoPlugin)
1230 {
1231 WLog_ERR(TAG, "calloc failed!");
1232 return CHANNEL_RC_NO_MEMORY;
1233 }
1234
1235 videoPlugin->wtsPlugin.Initialize = video_plugin_initialize;
1236 videoPlugin->wtsPlugin.Connected = nullptr;
1237 videoPlugin->wtsPlugin.Disconnected = nullptr;
1238 videoPlugin->wtsPlugin.Terminated = video_plugin_terminated;
1239
1240 VideoClientContext* videoContext =
1241 (VideoClientContext*)calloc(1, sizeof(VideoClientContext));
1242 if (!videoContext)
1243 {
1244 WLog_ERR(TAG, "calloc failed!");
1245 free(videoPlugin);
1246 return CHANNEL_RC_NO_MEMORY;
1247 }
1248
1249 VideoClientContextPriv* priv = VideoClientContextPriv_new(videoContext);
1250 if (!priv)
1251 {
1252 WLog_ERR(TAG, "VideoClientContextPriv_new failed!");
1253 free(videoContext);
1254 free(videoPlugin);
1255 return CHANNEL_RC_NO_MEMORY;
1256 }
1257
1258 videoContext->handle = (void*)videoPlugin;
1259 videoContext->priv = priv;
1260 videoContext->timer = video_timer;
1261 videoContext->setGeometry = video_client_context_set_geometry;
1262
1263 videoPlugin->wtsPlugin.pInterface = (void*)videoContext;
1264 videoPlugin->context = videoContext;
1265 videoPlugin->rdpcontext = pEntryPoints->GetRdpContext(pEntryPoints);
1266 if (videoPlugin->rdpcontext)
1267 error = pEntryPoints->RegisterPlugin(pEntryPoints, "video", &videoPlugin->wtsPlugin);
1268 }
1269 else
1270 {
1271 WLog_ERR(TAG, "could not get video Plugin.");
1272 return CHANNEL_RC_BAD_CHANNEL;
1273 }
1274
1275 return error;
1276}
GENERIC_CHANNEL_CALLBACK
Definition include/freerdp/client/channels.h:35
GENERIC_LISTENER_CALLBACK
Definition include/freerdp/client/channels.h:43
RDP_RECT
Definition types.h:101
RECTANGLE_16
Definition types.h:109
RTL_CRITICAL_SECTION
Definition include/winpr/synch.h:160
TSMM_CLIENT_NOTIFICATION
a client to server notification struct
Definition channels/video.h:116
TSMM_PRESENTATION_REQUEST
presentation request struct
Definition channels/video.h:57
TSMM_PRESENTATION_RESPONSE
response to a TSMM_PRESENTATION_REQUEST
Definition channels/video.h:73
TSMM_VIDEO_DATA
a video data packet
Definition channels/video.h:87
VideoSurface
an implementation of surface used by the video channel
Definition client/video.h:36