FreeRDP
Loading...
Searching...
No Matches
video_main.c
1
20#include <freerdp/config.h>
21
22#include <stdio.h>
23#include <stdlib.h>
24#include <string.h>
25
26#include <winpr/crt.h>
27#include <winpr/assert.h>
28#include <winpr/cast.h>
29#include <winpr/synch.h>
30#include <winpr/print.h>
31#include <winpr/stream.h>
32#include <winpr/cmdline.h>
33#include <winpr/collections.h>
34#include <winpr/interlocked.h>
35#include <winpr/sysinfo.h>
36
37#include <freerdp/addin.h>
38#include <freerdp/primitives.h>
39#include <freerdp/client/channels.h>
40#include <freerdp/client/geometry.h>
41#include <freerdp/client/video.h>
42#include <freerdp/channels/log.h>
43#include <freerdp/codec/h264.h>
44#include <freerdp/codec/yuv.h>
45#include <freerdp/timer.h>
46
47#define TAG CHANNELS_TAG("video")
48
49#include "video_main.h"
50
51typedef struct
52{
53 IWTSPlugin wtsPlugin;
54
55 IWTSListener* controlListener;
56 IWTSListener* dataListener;
57 GENERIC_LISTENER_CALLBACK* control_callback;
58 GENERIC_LISTENER_CALLBACK* data_callback;
59
60 VideoClientContext* context;
61 BOOL initialized;
62 rdpContext* rdpcontext;
63} VIDEO_PLUGIN;
64
65#define XF_VIDEO_UNLIMITED_RATE 31
66
67static const BYTE MFVideoFormat_H264[] = { 'H', '2', '6', '4', 0x00, 0x00, 0x10, 0x00,
68 0x80, 0x00, 0x00, 0xAA, 0x00, 0x38, 0x9B, 0x71 };
69
70typedef struct
71{
72 VideoClientContext* video;
73 BYTE PresentationId;
74 UINT32 ScaledWidth, ScaledHeight;
75 MAPPED_GEOMETRY* geometry;
76
77 UINT64 startTimeStamp;
78 UINT64 publishOffset;
79 H264_CONTEXT* h264;
80 wStream* currentSample;
81 UINT64 lastPublishTime, nextPublishTime;
82 volatile LONG refCounter;
83 VideoSurface* surface;
84} PresentationContext;
85
86typedef struct
87{
88 UINT64 publishTime;
89 UINT64 hnsDuration;
90 MAPPED_GEOMETRY* geometry;
91 UINT32 w, h;
92 UINT32 scanline;
93 BYTE* surfaceData;
94 PresentationContext* presentation;
95} VideoFrame;
96
98struct s_VideoClientContextPriv
99{
100 VideoClientContext* video;
101 GeometryClientContext* geometry;
102 wQueue* frames;
103 CRITICAL_SECTION framesLock;
104 wBufferPool* surfacePool;
105 UINT32 publishedFrames;
106 UINT32 droppedFrames;
107 UINT32 lastSentRate;
108 UINT64 nextFeedbackTime;
109 PresentationContext* currentPresentation;
110 FreeRDP_TimerID timerID;
111};
112
113static void PresentationContext_unref(PresentationContext** presentation);
114static void VideoClientContextPriv_free(VideoClientContextPriv* priv);
115
116static const char* video_command_name(BYTE cmd)
117{
118 switch (cmd)
119 {
120 case TSMM_START_PRESENTATION:
121 return "start";
122 case TSMM_STOP_PRESENTATION:
123 return "stop";
124 default:
125 return "<unknown>";
126 }
127}
128
129static void video_client_context_set_geometry(VideoClientContext* video,
130 GeometryClientContext* geometry)
131{
132 WINPR_ASSERT(video);
133 WINPR_ASSERT(video->priv);
134 video->priv->geometry = geometry;
135}
136
137static VideoClientContextPriv* VideoClientContextPriv_new(VideoClientContext* video)
138{
139 VideoClientContextPriv* ret = nullptr;
140
141 WINPR_ASSERT(video);
142 ret = calloc(1, sizeof(*ret));
143 if (!ret)
144 return nullptr;
145
146 ret->frames = Queue_New(TRUE, 10, 2);
147 if (!ret->frames)
148 {
149 WLog_ERR(TAG, "unable to allocate frames queue");
150 goto fail;
151 }
152
153 ret->surfacePool = BufferPool_New(FALSE, 0, 16);
154 if (!ret->surfacePool)
155 {
156 WLog_ERR(TAG, "unable to create surface pool");
157 goto fail;
158 }
159
160 if (!InitializeCriticalSectionAndSpinCount(&ret->framesLock, 4 * 1000))
161 {
162 WLog_ERR(TAG, "unable to initialize frames lock");
163 goto fail;
164 }
165
166 ret->video = video;
167
168 /* don't set to unlimited so that we have the chance to send a feedback in
169 * the first second (for servers that want feedback directly)
170 */
171 ret->lastSentRate = 30;
172 return ret;
173
174fail:
175 VideoClientContextPriv_free(ret);
176 return nullptr;
177}
178
179static BOOL PresentationContext_ref(PresentationContext* presentation)
180{
181 WINPR_ASSERT(presentation);
182
183 InterlockedIncrement(&presentation->refCounter);
184 return TRUE;
185}
186
187static PresentationContext* PresentationContext_new(VideoClientContext* video, BYTE PresentationId,
188 UINT32 x, UINT32 y, UINT32 width, UINT32 height)
189{
190 size_t s = 4ULL * width * height;
191 PresentationContext* ret = nullptr;
192
193 WINPR_ASSERT(video);
194
195 if (s > INT32_MAX)
196 return nullptr;
197
198 ret = calloc(1, sizeof(*ret));
199 if (!ret)
200 return nullptr;
201
202 ret->video = video;
203 ret->PresentationId = PresentationId;
204
205 ret->h264 = h264_context_new(FALSE);
206 if (!ret->h264)
207 {
208 WLog_ERR(TAG, "unable to create a h264 context");
209 goto fail;
210 }
211 if (!h264_context_reset(ret->h264, width, height))
212 goto fail;
213
214 ret->currentSample = Stream_New(nullptr, 4096);
215 if (!ret->currentSample)
216 {
217 WLog_ERR(TAG, "unable to create current packet stream");
218 goto fail;
219 }
220
221 ret->surface = video->createSurface(video, x, y, width, height);
222 if (!ret->surface)
223 {
224 WLog_ERR(TAG, "unable to create surface");
225 goto fail;
226 }
227
228 if (!PresentationContext_ref(ret))
229 goto fail;
230
231 return ret;
232
233fail:
234 PresentationContext_unref(&ret);
235 return nullptr;
236}
237
238static void PresentationContext_unref(PresentationContext** ppresentation)
239{
240 PresentationContext* presentation = nullptr;
241 MAPPED_GEOMETRY* geometry = nullptr;
242
243 WINPR_ASSERT(ppresentation);
244
245 presentation = *ppresentation;
246 if (!presentation)
247 return;
248
249 if (InterlockedDecrement(&presentation->refCounter) > 0)
250 return;
251
252 geometry = presentation->geometry;
253 if (geometry)
254 {
255 geometry->MappedGeometryUpdate = nullptr;
256 geometry->MappedGeometryClear = nullptr;
257 geometry->custom = nullptr;
258 mappedGeometryUnref(geometry);
259 }
260
261 h264_context_free(presentation->h264);
262 Stream_Free(presentation->currentSample, TRUE);
263 presentation->video->deleteSurface(presentation->video, presentation->surface);
264 free(presentation);
265 *ppresentation = nullptr;
266}
267
268static void VideoFrame_free(VideoFrame** pframe)
269{
270 VideoFrame* frame = nullptr;
271
272 WINPR_ASSERT(pframe);
273 frame = *pframe;
274 if (!frame)
275 return;
276
277 mappedGeometryUnref(frame->geometry);
278
279 WINPR_ASSERT(frame->presentation);
280 WINPR_ASSERT(frame->presentation->video);
281 WINPR_ASSERT(frame->presentation->video->priv);
282 BufferPool_Return(frame->presentation->video->priv->surfacePool, frame->surfaceData);
283 PresentationContext_unref(&frame->presentation);
284 free(frame);
285 *pframe = nullptr;
286}
287
288static VideoFrame* VideoFrame_new(VideoClientContextPriv* priv, PresentationContext* presentation,
289 MAPPED_GEOMETRY* geom)
290{
291 VideoFrame* frame = nullptr;
292 const VideoSurface* surface = nullptr;
293
294 WINPR_ASSERT(priv);
295 WINPR_ASSERT(presentation);
296 WINPR_ASSERT(geom);
297
298 surface = presentation->surface;
299 WINPR_ASSERT(surface);
300
301 frame = calloc(1, sizeof(VideoFrame));
302 if (!frame)
303 goto fail;
304
305 mappedGeometryRef(geom);
306
307 frame->publishTime = presentation->lastPublishTime;
308 frame->geometry = geom;
309 frame->w = surface->alignedWidth;
310 frame->h = surface->alignedHeight;
311 frame->scanline = surface->scanline;
312
313 frame->surfaceData = BufferPool_Take(priv->surfacePool, 1ll * frame->scanline * frame->h);
314 if (!frame->surfaceData)
315 goto fail;
316
317 frame->presentation = presentation;
318 if (!PresentationContext_ref(frame->presentation))
319 goto fail;
320
321 return frame;
322
323fail:
324 VideoFrame_free(&frame);
325 return nullptr;
326}
327
328void VideoClientContextPriv_free(VideoClientContextPriv* priv)
329{
330 if (!priv)
331 return;
332
333 EnterCriticalSection(&priv->framesLock);
334
335 if (priv->frames)
336 {
337 while (Queue_Count(priv->frames))
338 {
339 VideoFrame* frame = Queue_Dequeue(priv->frames);
340 if (frame)
341 VideoFrame_free(&frame);
342 }
343 }
344
345 Queue_Free(priv->frames);
346 LeaveCriticalSection(&priv->framesLock);
347
348 DeleteCriticalSection(&priv->framesLock);
349
350 if (priv->currentPresentation)
351 PresentationContext_unref(&priv->currentPresentation);
352
353 BufferPool_Free(priv->surfacePool);
354 free(priv);
355}
356
357static UINT video_control_send_presentation_response(VideoClientContext* context,
358 TSMM_PRESENTATION_RESPONSE* resp)
359{
360 BYTE buf[12] = WINPR_C_ARRAY_INIT;
361 wStream* s = nullptr;
362 VIDEO_PLUGIN* video = nullptr;
363 IWTSVirtualChannel* channel = nullptr;
364 UINT ret = 0;
365
366 WINPR_ASSERT(context);
367 WINPR_ASSERT(resp);
368
369 video = (VIDEO_PLUGIN*)context->handle;
370 WINPR_ASSERT(video);
371
372 s = Stream_New(buf, 12);
373 if (!s)
374 return CHANNEL_RC_NO_MEMORY;
375
376 Stream_Write_UINT32(s, 12); /* cbSize */
377 Stream_Write_UINT32(s, TSMM_PACKET_TYPE_PRESENTATION_RESPONSE); /* PacketType */
378 Stream_Write_UINT8(s, resp->PresentationId);
379 Stream_Zero(s, 3);
380 Stream_SealLength(s);
381
382 channel = video->control_callback->channel_callback->channel;
383 ret = channel->Write(channel, 12, buf, nullptr);
384 Stream_Free(s, FALSE);
385
386 return ret;
387}
388
389static BOOL video_onMappedGeometryUpdate(MAPPED_GEOMETRY* geometry)
390{
391 PresentationContext* presentation = nullptr;
392 RDP_RECT* r = nullptr;
393
394 WINPR_ASSERT(geometry);
395
396 presentation = (PresentationContext*)geometry->custom;
397 WINPR_ASSERT(presentation);
398
399 r = &geometry->geometry.boundingRect;
400 WLog_DBG(TAG,
401 "geometry updated topGeom=(%" PRId32 ",%" PRId32 "-%" PRId32 "x%" PRId32
402 ") geom=(%" PRId32 ",%" PRId32 "-%" PRId32 "x%" PRId32 ") rects=(%" PRId16 ",%" PRId16
403 "-%" PRId16 "x%" PRId16 ")",
404 geometry->topLevelLeft, geometry->topLevelTop,
405 geometry->topLevelRight - geometry->topLevelLeft,
406 geometry->topLevelBottom - geometry->topLevelTop,
407
408 geometry->left, geometry->top, geometry->right - geometry->left,
409 geometry->bottom - geometry->top,
410
411 r->x, r->y, r->width, r->height);
412
413 presentation->surface->x =
414 WINPR_ASSERTING_INT_CAST(uint32_t, geometry->topLevelLeft + geometry->left);
415 presentation->surface->y =
416 WINPR_ASSERTING_INT_CAST(uint32_t, geometry->topLevelTop + geometry->top);
417
418 return TRUE;
419}
420
421static BOOL video_onMappedGeometryClear(MAPPED_GEOMETRY* geometry)
422{
423 PresentationContext* presentation = nullptr;
424
425 WINPR_ASSERT(geometry);
426
427 presentation = (PresentationContext*)geometry->custom;
428 WINPR_ASSERT(presentation);
429
430 mappedGeometryUnref(presentation->geometry);
431 presentation->geometry = nullptr;
432 return TRUE;
433}
434
435static UINT video_PresentationRequest(VideoClientContext* video,
436 const TSMM_PRESENTATION_REQUEST* req)
437{
438 UINT ret = CHANNEL_RC_OK;
439
440 WINPR_ASSERT(video);
441 WINPR_ASSERT(req);
442
443 VideoClientContextPriv* priv = video->priv;
444 WINPR_ASSERT(priv);
445
446 if (req->Command == TSMM_START_PRESENTATION)
447 {
448 MAPPED_GEOMETRY* geom = nullptr;
449 TSMM_PRESENTATION_RESPONSE resp;
450
451 if (memcmp(req->VideoSubtypeId, MFVideoFormat_H264, 16) != 0)
452 {
453 WLog_ERR(TAG, "not a H264 video, ignoring request");
454 return CHANNEL_RC_OK;
455 }
456
457 if (priv->currentPresentation)
458 {
459 if (priv->currentPresentation->PresentationId == req->PresentationId)
460 {
461 WLog_ERR(TAG, "ignoring start request for existing presentation %" PRIu8,
462 req->PresentationId);
463 return CHANNEL_RC_OK;
464 }
465
466 WLog_ERR(TAG, "releasing current presentation %" PRIu8, req->PresentationId);
467 PresentationContext_unref(&priv->currentPresentation);
468 }
469
470 if (!priv->geometry)
471 {
472 WLog_ERR(TAG, "geometry channel not ready, ignoring request");
473 return CHANNEL_RC_OK;
474 }
475
476 geom = HashTable_GetItemValue(priv->geometry->geometries, &(req->GeometryMappingId));
477 if (!geom)
478 {
479 WLog_ERR(TAG, "geometry mapping 0x%" PRIx64 " not registered", req->GeometryMappingId);
480 return CHANNEL_RC_OK;
481 }
482
483 WLog_DBG(TAG, "creating presentation 0x%x", req->PresentationId);
484 priv->currentPresentation = PresentationContext_new(
485 video, req->PresentationId,
486 WINPR_ASSERTING_INT_CAST(uint32_t, geom->topLevelLeft + geom->left),
487 WINPR_ASSERTING_INT_CAST(uint32_t, geom->topLevelTop + geom->top), req->SourceWidth,
488 req->SourceHeight);
489 if (!priv->currentPresentation)
490 {
491 WLog_ERR(TAG, "unable to create presentation video");
492 return CHANNEL_RC_NO_MEMORY;
493 }
494
495 mappedGeometryRef(geom);
496 priv->currentPresentation->geometry = geom;
497
498 priv->currentPresentation->video = video;
499 priv->currentPresentation->ScaledWidth = req->ScaledWidth;
500 priv->currentPresentation->ScaledHeight = req->ScaledHeight;
501
502 geom->custom = priv->currentPresentation;
503 geom->MappedGeometryUpdate = video_onMappedGeometryUpdate;
504 geom->MappedGeometryClear = video_onMappedGeometryClear;
505
506 /* send back response */
507 resp.PresentationId = req->PresentationId;
508 ret = video_control_send_presentation_response(video, &resp);
509 }
510 else if (req->Command == TSMM_STOP_PRESENTATION)
511 {
512 WLog_DBG(TAG, "stopping presentation 0x%x", req->PresentationId);
513 if (!priv->currentPresentation)
514 {
515 WLog_ERR(TAG, "unknown presentation to stop %" PRIu8, req->PresentationId);
516 return CHANNEL_RC_OK;
517 }
518
519 priv->droppedFrames = 0;
520 priv->publishedFrames = 0;
521 PresentationContext_unref(&priv->currentPresentation);
522 }
523
524 return ret;
525}
526
527static UINT video_read_tsmm_presentation_req(VideoClientContext* context, wStream* s)
528{
529 TSMM_PRESENTATION_REQUEST req = WINPR_C_ARRAY_INIT;
530
531 WINPR_ASSERT(context);
532 WINPR_ASSERT(s);
533
534 if (!Stream_CheckAndLogRequiredLength(TAG, s, 60))
535 return ERROR_INVALID_DATA;
536
537 Stream_Read_UINT8(s, req.PresentationId);
538 Stream_Read_UINT8(s, req.Version);
539 Stream_Read_UINT8(s, req.Command);
540 Stream_Read_UINT8(s, req.FrameRate); /* FrameRate - reserved and ignored */
541
542 Stream_Seek_UINT16(s); /* AverageBitrateKbps reserved and ignored */
543 Stream_Seek_UINT16(s); /* reserved */
544
545 Stream_Read_UINT32(s, req.SourceWidth);
546 Stream_Read_UINT32(s, req.SourceHeight);
547 Stream_Read_UINT32(s, req.ScaledWidth);
548 Stream_Read_UINT32(s, req.ScaledHeight);
549 Stream_Read_UINT64(s, req.hnsTimestampOffset);
550 Stream_Read_UINT64(s, req.GeometryMappingId);
551 Stream_Read(s, req.VideoSubtypeId, 16);
552
553 Stream_Read_UINT32(s, req.cbExtra);
554
555 if (!Stream_CheckAndLogRequiredLength(TAG, s, req.cbExtra))
556 return ERROR_INVALID_DATA;
557
558 req.pExtraData = Stream_Pointer(s);
559
560 WLog_DBG(TAG,
561 "presentationReq: id:%" PRIu8 " version:%" PRIu8
562 " command:%s srcWidth/srcHeight=%" PRIu32 "x%" PRIu32 " scaled Width/Height=%" PRIu32
563 "x%" PRIu32 " timestamp=%" PRIu64 " mappingId=%" PRIx64 "",
564 req.PresentationId, req.Version, video_command_name(req.Command), req.SourceWidth,
565 req.SourceHeight, req.ScaledWidth, req.ScaledHeight, req.hnsTimestampOffset,
566 req.GeometryMappingId);
567
568 return video_PresentationRequest(context, &req);
569}
570
576static UINT video_control_on_data_received(IWTSVirtualChannelCallback* pChannelCallback, wStream* s)
577{
578 GENERIC_CHANNEL_CALLBACK* callback = (GENERIC_CHANNEL_CALLBACK*)pChannelCallback;
579 VIDEO_PLUGIN* video = nullptr;
580 VideoClientContext* context = nullptr;
581 UINT ret = CHANNEL_RC_OK;
582 UINT32 cbSize = 0;
583 UINT32 packetType = 0;
584
585 WINPR_ASSERT(callback);
586 WINPR_ASSERT(s);
587
588 video = (VIDEO_PLUGIN*)callback->plugin;
589 WINPR_ASSERT(video);
590
591 context = (VideoClientContext*)video->wtsPlugin.pInterface;
592 WINPR_ASSERT(context);
593
594 if (!Stream_CheckAndLogRequiredLength(TAG, s, 4))
595 return ERROR_INVALID_DATA;
596
597 Stream_Read_UINT32(s, cbSize);
598 if (cbSize < 8)
599 {
600 WLog_ERR(TAG, "invalid cbSize %" PRIu32 ", expected 8", cbSize);
601 return ERROR_INVALID_DATA;
602 }
603 if (!Stream_CheckAndLogRequiredLength(TAG, s, cbSize - 4))
604 return ERROR_INVALID_DATA;
605
606 Stream_Read_UINT32(s, packetType);
607 switch (packetType)
608 {
609 case TSMM_PACKET_TYPE_PRESENTATION_REQUEST:
610 ret = video_read_tsmm_presentation_req(context, s);
611 break;
612 default:
613 WLog_ERR(TAG, "not expecting packet type %" PRIu32 "", packetType);
614 ret = ERROR_UNSUPPORTED_TYPE;
615 break;
616 }
617
618 return ret;
619}
620
621static UINT video_control_send_client_notification(VideoClientContext* context,
622 const TSMM_CLIENT_NOTIFICATION* notif)
623{
624 BYTE buf[100];
625 wStream* s = nullptr;
626 VIDEO_PLUGIN* video = nullptr;
627 IWTSVirtualChannel* channel = nullptr;
628 UINT ret = 0;
629 UINT32 cbSize = 0;
630
631 WINPR_ASSERT(context);
632 WINPR_ASSERT(notif);
633
634 video = (VIDEO_PLUGIN*)context->handle;
635 WINPR_ASSERT(video);
636
637 s = Stream_New(buf, 32);
638 if (!s)
639 return CHANNEL_RC_NO_MEMORY;
640
641 cbSize = 16;
642 Stream_Seek_UINT32(s); /* cbSize */
643 Stream_Write_UINT32(s, TSMM_PACKET_TYPE_CLIENT_NOTIFICATION); /* PacketType */
644 Stream_Write_UINT8(s, notif->PresentationId);
645 Stream_Write_UINT8(s, notif->NotificationType);
646 Stream_Zero(s, 2);
647 if (notif->NotificationType == TSMM_CLIENT_NOTIFICATION_TYPE_FRAMERATE_OVERRIDE)
648 {
649 Stream_Write_UINT32(s, 16); /* cbData */
650
651 /* TSMM_CLIENT_NOTIFICATION_FRAMERATE_OVERRIDE */
652 Stream_Write_UINT32(s, notif->FramerateOverride.Flags);
653 Stream_Write_UINT32(s, notif->FramerateOverride.DesiredFrameRate);
654 Stream_Zero(s, 4ULL * 2ULL);
655
656 cbSize += 4UL * 4UL;
657 }
658 else
659 {
660 Stream_Write_UINT32(s, 0); /* cbData */
661 }
662
663 Stream_SealLength(s);
664 Stream_SetPosition(s, 0);
665 Stream_Write_UINT32(s, cbSize);
666 Stream_Free(s, FALSE);
667
668 WINPR_ASSERT(video->control_callback);
669 WINPR_ASSERT(video->control_callback->channel_callback);
670
671 channel = video->control_callback->channel_callback->channel;
672 WINPR_ASSERT(channel);
673 WINPR_ASSERT(channel->Write);
674
675 ret = channel->Write(channel, cbSize, buf, nullptr);
676
677 return ret;
678}
679
680static void video_timer(VideoClientContext* video, UINT64 now)
681{
682 PresentationContext* presentation = nullptr;
683 VideoFrame* frame = nullptr;
684
685 WINPR_ASSERT(video);
686
687 VideoClientContextPriv* priv = video->priv;
688 WINPR_ASSERT(priv);
689
690 EnterCriticalSection(&priv->framesLock);
691 do
692 {
693 VideoFrame* peekFrame = (VideoFrame*)Queue_Peek(priv->frames);
694 if (!peekFrame)
695 break;
696
697 if (peekFrame->publishTime > now)
698 break;
699
700 if (frame)
701 {
702 WLog_DBG(TAG, "dropping frame @%" PRIu64, frame->publishTime);
703 priv->droppedFrames++;
704 VideoFrame_free(&frame);
705 }
706 frame = peekFrame;
707 Queue_Dequeue(priv->frames);
708 } while (1);
709 LeaveCriticalSection(&priv->framesLock);
710
711 if (frame)
712 {
713 presentation = frame->presentation;
714
715 priv->publishedFrames++;
716 memcpy(presentation->surface->data, frame->surfaceData, 1ull * frame->scanline * frame->h);
717
718 WINPR_ASSERT(video->showSurface);
719 if (!video->showSurface(video, presentation->surface, presentation->ScaledWidth,
720 presentation->ScaledHeight))
721 WLog_WARN(TAG, "showSurface failed");
722
723 VideoFrame_free(&frame);
724 }
725
726 if (priv->nextFeedbackTime < now)
727 {
728 /* we can compute some feedback only if we have some published frames and
729 * a current presentation
730 */
731 if (priv->publishedFrames && priv->currentPresentation)
732 {
733 UINT32 computedRate = 0;
734
735 PresentationContext_ref(priv->currentPresentation);
736
737 if (priv->droppedFrames)
738 {
744 if (priv->lastSentRate == XF_VIDEO_UNLIMITED_RATE)
745 computedRate = 24;
746 else
747 {
748 computedRate = priv->lastSentRate - 2;
749 if (!computedRate)
750 computedRate = 2;
751 }
752 }
753 else
754 {
759 if (priv->lastSentRate == XF_VIDEO_UNLIMITED_RATE)
760 computedRate = XF_VIDEO_UNLIMITED_RATE; /* stay unlimited */
761 else
762 {
763 computedRate = priv->lastSentRate + 2;
764 if (computedRate > XF_VIDEO_UNLIMITED_RATE)
765 computedRate = XF_VIDEO_UNLIMITED_RATE;
766 }
767 }
768
769 if (computedRate != priv->lastSentRate)
770 {
771 TSMM_CLIENT_NOTIFICATION notif;
772
773 WINPR_ASSERT(priv->currentPresentation);
774 notif.PresentationId = priv->currentPresentation->PresentationId;
775 notif.NotificationType = TSMM_CLIENT_NOTIFICATION_TYPE_FRAMERATE_OVERRIDE;
776 if (computedRate == XF_VIDEO_UNLIMITED_RATE)
777 {
778 notif.FramerateOverride.Flags = 0x01;
779 notif.FramerateOverride.DesiredFrameRate = 0x00;
780 }
781 else
782 {
783 notif.FramerateOverride.Flags = 0x02;
784 notif.FramerateOverride.DesiredFrameRate = computedRate;
785 }
786
787 video_control_send_client_notification(video, &notif);
788 priv->lastSentRate = computedRate;
789
790 WLog_VRB(TAG,
791 "server notified with rate %" PRIu32 " published=%" PRIu32
792 " dropped=%" PRIu32,
793 priv->lastSentRate, priv->publishedFrames, priv->droppedFrames);
794 }
795
796 PresentationContext_unref(&priv->currentPresentation);
797 }
798
799 priv->droppedFrames = 0;
800 priv->publishedFrames = 0;
801 priv->nextFeedbackTime = now + 1000;
802 }
803}
804
805static UINT video_VideoData(VideoClientContext* context, const TSMM_VIDEO_DATA* data)
806{
807 VideoClientContextPriv* priv = nullptr;
808 PresentationContext* presentation = nullptr;
809 int status = 0;
810
811 WINPR_ASSERT(context);
812 WINPR_ASSERT(data);
813
814 priv = context->priv;
815 WINPR_ASSERT(priv);
816
817 presentation = priv->currentPresentation;
818 if (!presentation)
819 {
820 WLog_ERR(TAG, "no current presentation");
821 return CHANNEL_RC_OK;
822 }
823
824 if (presentation->PresentationId != data->PresentationId)
825 {
826 WLog_ERR(TAG, "current presentation id=%" PRIu8 " doesn't match data id=%" PRIu8,
827 presentation->PresentationId, data->PresentationId);
828 return CHANNEL_RC_OK;
829 }
830
831 if (!Stream_EnsureRemainingCapacity(presentation->currentSample, data->cbSample))
832 {
833 WLog_ERR(TAG, "unable to expand the current packet");
834 return CHANNEL_RC_NO_MEMORY;
835 }
836
837 Stream_Write(presentation->currentSample, data->pSample, data->cbSample);
838
839 if (data->CurrentPacketIndex == data->PacketsInSample)
840 {
841 VideoSurface* surface = presentation->surface;
842 H264_CONTEXT* h264 = presentation->h264;
843 const UINT64 startTime = winpr_GetTickCount64NS();
844 MAPPED_GEOMETRY* geom = presentation->geometry;
845
846 const RECTANGLE_16 rect = { 0, 0, WINPR_ASSERTING_INT_CAST(UINT16, surface->alignedWidth),
847 WINPR_ASSERTING_INT_CAST(UINT16, surface->alignedHeight) };
848 Stream_SealLength(presentation->currentSample);
849 Stream_SetPosition(presentation->currentSample, 0);
850
851 const UINT64 timeAfterH264 = winpr_GetTickCount64NS();
852 if (data->SampleNumber == 1)
853 {
854 presentation->lastPublishTime = startTime;
855 }
856
857 presentation->lastPublishTime += 100ull * data->hnsDuration;
858 if (presentation->lastPublishTime <= (10000000ull + timeAfterH264))
859 {
860 int dropped = 0;
861
862 const size_t len = Stream_Length(presentation->currentSample);
863 if (len > UINT32_MAX)
864 return CHANNEL_RC_OK;
865
866 /* if the frame is to be published in less than 10 ms, let's consider it's now */
867 status =
868 avc420_decompress(h264, Stream_Pointer(presentation->currentSample), (UINT32)len,
869 surface->data, surface->format, surface->scanline,
870 surface->alignedWidth, surface->alignedHeight, &rect, 1);
871
872 if (status < 0)
873 return CHANNEL_RC_OK;
874
875 WINPR_ASSERT(context->showSurface);
876 if (!context->showSurface(context, presentation->surface, presentation->ScaledWidth,
877 presentation->ScaledHeight))
878 return CHANNEL_RC_NOT_INITIALIZED;
879
880 priv->publishedFrames++;
881
882 /* cleanup previously scheduled frames */
883 EnterCriticalSection(&priv->framesLock);
884 while (Queue_Count(priv->frames) > 0)
885 {
886 VideoFrame* frame = Queue_Dequeue(priv->frames);
887 if (frame)
888 {
889 priv->droppedFrames++;
890 VideoFrame_free(&frame);
891 dropped++;
892 }
893 }
894 LeaveCriticalSection(&priv->framesLock);
895
896 if (dropped)
897 WLog_DBG(TAG, "showing frame (%d dropped)", dropped);
898 }
899 else
900 {
901 const size_t len = Stream_Length(presentation->currentSample);
902 if (len > UINT32_MAX)
903 return CHANNEL_RC_OK;
904
905 BOOL enqueueResult = 0;
906 VideoFrame* frame = VideoFrame_new(priv, presentation, geom);
907 if (!frame)
908 {
909 WLog_ERR(TAG, "unable to create frame");
910 return CHANNEL_RC_NO_MEMORY;
911 }
912
913 status =
914 avc420_decompress(h264, Stream_Pointer(presentation->currentSample), (UINT32)len,
915 frame->surfaceData, surface->format, surface->scanline,
916 surface->alignedWidth, surface->alignedHeight, &rect, 1);
917 if (status < 0)
918 {
919 VideoFrame_free(&frame);
920 return CHANNEL_RC_OK;
921 }
922
923 EnterCriticalSection(&priv->framesLock);
924 enqueueResult = Queue_Enqueue(priv->frames, frame);
925 LeaveCriticalSection(&priv->framesLock);
926
927 if (!enqueueResult)
928 {
929 WLog_ERR(TAG, "unable to enqueue frame");
930 VideoFrame_free(&frame);
931 return CHANNEL_RC_NO_MEMORY;
932 }
933
934 // NOLINTNEXTLINE(clang-analyzer-unix.Malloc): Queue_Enqueue owns frame
935 WLog_DBG(TAG, "scheduling frame in %" PRIu64 " ms", (frame->publishTime - startTime));
936 }
937 }
938
939 return CHANNEL_RC_OK;
940}
941
942static UINT video_data_on_data_received(IWTSVirtualChannelCallback* pChannelCallback, wStream* s)
943{
944 GENERIC_CHANNEL_CALLBACK* callback = (GENERIC_CHANNEL_CALLBACK*)pChannelCallback;
945 VIDEO_PLUGIN* video = nullptr;
946 VideoClientContext* context = nullptr;
947 UINT32 cbSize = 0;
948 UINT32 packetType = 0;
949 TSMM_VIDEO_DATA data;
950
951 video = (VIDEO_PLUGIN*)callback->plugin;
952 context = (VideoClientContext*)video->wtsPlugin.pInterface;
953
954 if (!Stream_CheckAndLogRequiredLength(TAG, s, 4))
955 return ERROR_INVALID_DATA;
956
957 Stream_Read_UINT32(s, cbSize);
958 if (cbSize < 8)
959 {
960 WLog_ERR(TAG, "invalid cbSize %" PRIu32 ", expected >= 8", cbSize);
961 return ERROR_INVALID_DATA;
962 }
963
964 if (!Stream_CheckAndLogRequiredLength(TAG, s, cbSize - 4))
965 return ERROR_INVALID_DATA;
966
967 Stream_Read_UINT32(s, packetType);
968 if (packetType != TSMM_PACKET_TYPE_VIDEO_DATA)
969 {
970 WLog_ERR(TAG, "only expecting VIDEO_DATA on the data channel");
971 return ERROR_INVALID_DATA;
972 }
973
974 if (!Stream_CheckAndLogRequiredLength(TAG, s, 32))
975 return ERROR_INVALID_DATA;
976
977 Stream_Read_UINT8(s, data.PresentationId);
978 Stream_Read_UINT8(s, data.Version);
979 Stream_Read_UINT8(s, data.Flags);
980 Stream_Seek_UINT8(s); /* reserved */
981 Stream_Read_UINT64(s, data.hnsTimestamp);
982 Stream_Read_UINT64(s, data.hnsDuration);
983 Stream_Read_UINT16(s, data.CurrentPacketIndex);
984 Stream_Read_UINT16(s, data.PacketsInSample);
985 Stream_Read_UINT32(s, data.SampleNumber);
986 Stream_Read_UINT32(s, data.cbSample);
987 if (!Stream_CheckAndLogRequiredLength(TAG, s, data.cbSample))
988 return ERROR_INVALID_DATA;
989 data.pSample = Stream_Pointer(s);
990
991 /*
992 WLog_DBG(TAG, "videoData: id:%"PRIu8" version:%"PRIu8" flags:0x%"PRIx8" timestamp=%"PRIu64"
993 duration=%"PRIu64 " curPacketIndex:%"PRIu16" packetInSample:%"PRIu16" sampleNumber:%"PRIu32"
994 cbSample:%"PRIu32"", data.PresentationId, data.Version, data.Flags, data.hnsTimestamp,
995 data.hnsDuration, data.CurrentPacketIndex, data.PacketsInSample, data.SampleNumber,
996 data.cbSample);
997 */
998
999 return video_VideoData(context, &data);
1000}
1001
1007static UINT video_control_on_close(IWTSVirtualChannelCallback* pChannelCallback)
1008{
1009 free(pChannelCallback);
1010 return CHANNEL_RC_OK;
1011}
1012
1013static UINT video_data_on_close(IWTSVirtualChannelCallback* pChannelCallback)
1014{
1015 free(pChannelCallback);
1016 return CHANNEL_RC_OK;
1017}
1018
1024// NOLINTBEGIN(readability-non-const-parameter)
1025static UINT video_control_on_new_channel_connection(IWTSListenerCallback* listenerCallback,
1026 IWTSVirtualChannel* channel, BYTE* Data,
1027 BOOL* pbAccept,
1028 IWTSVirtualChannelCallback** ppCallback)
1029// NOLINTEND(readability-non-const-parameter)
1030{
1031 GENERIC_LISTENER_CALLBACK* listener_callback = (GENERIC_LISTENER_CALLBACK*)listenerCallback;
1032
1033 WINPR_UNUSED(Data);
1034 WINPR_UNUSED(pbAccept);
1035
1036 GENERIC_CHANNEL_CALLBACK* callback =
1037 (GENERIC_CHANNEL_CALLBACK*)calloc(1, sizeof(GENERIC_CHANNEL_CALLBACK));
1038 if (!callback)
1039 {
1040 WLog_ERR(TAG, "calloc failed!");
1041 return CHANNEL_RC_NO_MEMORY;
1042 }
1043
1044 callback->iface.OnDataReceived = video_control_on_data_received;
1045 callback->iface.OnClose = video_control_on_close;
1046 callback->plugin = listener_callback->plugin;
1047 callback->channel_mgr = listener_callback->channel_mgr;
1048 callback->channel = channel;
1049 listener_callback->channel_callback = callback;
1050
1051 *ppCallback = &callback->iface;
1052
1053 return CHANNEL_RC_OK;
1054}
1055
1056// NOLINTBEGIN(readability-non-const-parameter)
1057static UINT video_data_on_new_channel_connection(IWTSListenerCallback* pListenerCallback,
1058 IWTSVirtualChannel* pChannel, BYTE* Data,
1059 BOOL* pbAccept,
1060 IWTSVirtualChannelCallback** ppCallback)
1061// NOLINTEND(readability-non-const-parameter)
1062{
1063 GENERIC_CHANNEL_CALLBACK* callback = nullptr;
1064 GENERIC_LISTENER_CALLBACK* listener_callback = (GENERIC_LISTENER_CALLBACK*)pListenerCallback;
1065
1066 WINPR_UNUSED(Data);
1067 WINPR_UNUSED(pbAccept);
1068
1069 callback = (GENERIC_CHANNEL_CALLBACK*)calloc(1, sizeof(GENERIC_CHANNEL_CALLBACK));
1070 if (!callback)
1071 {
1072 WLog_ERR(TAG, "calloc failed!");
1073 return CHANNEL_RC_NO_MEMORY;
1074 }
1075
1076 callback->iface.OnDataReceived = video_data_on_data_received;
1077 callback->iface.OnClose = video_data_on_close;
1078 callback->plugin = listener_callback->plugin;
1079 callback->channel_mgr = listener_callback->channel_mgr;
1080 callback->channel = pChannel;
1081 listener_callback->channel_callback = callback;
1082
1083 *ppCallback = &callback->iface;
1084
1085 return CHANNEL_RC_OK;
1086}
1087
1088static uint64_t timer_cb(WINPR_ATTR_UNUSED rdpContext* context, void* userdata,
1089 WINPR_ATTR_UNUSED FreeRDP_TimerID timerID, uint64_t timestamp,
1090 uint64_t interval)
1091{
1092 VideoClientContext* video = userdata;
1093 if (!video)
1094 return 0;
1095 if (!video->timer)
1096 return 0;
1097
1098 video->timer(video, timestamp);
1099
1100 return interval;
1101}
1102
1108static UINT video_plugin_initialize(IWTSPlugin* plugin, IWTSVirtualChannelManager* channelMgr)
1109{
1110 UINT status = 0;
1111 VIDEO_PLUGIN* video = (VIDEO_PLUGIN*)plugin;
1112 GENERIC_LISTENER_CALLBACK* callback = nullptr;
1113
1114 if (video->initialized)
1115 {
1116 WLog_ERR(TAG, "[%s] channel initialized twice, aborting", VIDEO_CONTROL_DVC_CHANNEL_NAME);
1117 return ERROR_INVALID_DATA;
1118 }
1119 video->control_callback = callback =
1120 (GENERIC_LISTENER_CALLBACK*)calloc(1, sizeof(GENERIC_LISTENER_CALLBACK));
1121 if (!callback)
1122 {
1123 WLog_ERR(TAG, "calloc for control callback failed!");
1124 return CHANNEL_RC_NO_MEMORY;
1125 }
1126
1127 callback->iface.OnNewChannelConnection = video_control_on_new_channel_connection;
1128 callback->plugin = plugin;
1129 callback->channel_mgr = channelMgr;
1130
1131 status = channelMgr->CreateListener(channelMgr, VIDEO_CONTROL_DVC_CHANNEL_NAME, 0,
1132 &callback->iface, &(video->controlListener));
1133
1134 if (status != CHANNEL_RC_OK)
1135 return status;
1136 video->controlListener->pInterface = video->wtsPlugin.pInterface;
1137
1138 video->data_callback = callback =
1139 (GENERIC_LISTENER_CALLBACK*)calloc(1, sizeof(GENERIC_LISTENER_CALLBACK));
1140 if (!callback)
1141 {
1142 WLog_ERR(TAG, "calloc for data callback failed!");
1143 return CHANNEL_RC_NO_MEMORY;
1144 }
1145
1146 callback->iface.OnNewChannelConnection = video_data_on_new_channel_connection;
1147 callback->plugin = plugin;
1148 callback->channel_mgr = channelMgr;
1149
1150 status = channelMgr->CreateListener(channelMgr, VIDEO_DATA_DVC_CHANNEL_NAME, 0,
1151 &callback->iface, &(video->dataListener));
1152
1153 if (status == CHANNEL_RC_OK)
1154 video->dataListener->pInterface = video->wtsPlugin.pInterface;
1155
1156 if (status == CHANNEL_RC_OK)
1157 video->context->priv->timerID =
1158 freerdp_timer_add(video->rdpcontext, 20000000, timer_cb, video->context, true);
1159 video->initialized = video->context->priv->timerID != 0;
1160 if (!video->initialized)
1161 status = ERROR_INTERNAL_ERROR;
1162 return status;
1163}
1164
1170static UINT video_plugin_terminated(IWTSPlugin* pPlugin)
1171{
1172 VIDEO_PLUGIN* video = (VIDEO_PLUGIN*)pPlugin;
1173 if (!video)
1174 return CHANNEL_RC_INVALID_INSTANCE;
1175
1176 if (video->context && video->context->priv)
1177 freerdp_timer_remove(video->rdpcontext, video->context->priv->timerID);
1178
1179 if (video->control_callback)
1180 {
1181 IWTSVirtualChannelManager* mgr = video->control_callback->channel_mgr;
1182 if (mgr)
1183 IFCALL(mgr->DestroyListener, mgr, video->controlListener);
1184 }
1185 if (video->data_callback)
1186 {
1187 IWTSVirtualChannelManager* mgr = video->data_callback->channel_mgr;
1188 if (mgr)
1189 IFCALL(mgr->DestroyListener, mgr, video->dataListener);
1190 }
1191
1192 if (video->context)
1193 VideoClientContextPriv_free(video->context->priv);
1194
1195 free(video->control_callback);
1196 free(video->data_callback);
1197 free(video->wtsPlugin.pInterface);
1198 free(pPlugin);
1199 return CHANNEL_RC_OK;
1200}
1201
1210FREERDP_ENTRY_POINT(UINT VCAPITYPE video_DVCPluginEntry(IDRDYNVC_ENTRY_POINTS* pEntryPoints))
1211{
1212 UINT error = ERROR_INTERNAL_ERROR;
1213 VIDEO_PLUGIN* videoPlugin = nullptr;
1214 VideoClientContext* videoContext = nullptr;
1215 VideoClientContextPriv* priv = nullptr;
1216
1217 videoPlugin = (VIDEO_PLUGIN*)pEntryPoints->GetPlugin(pEntryPoints, "video");
1218 if (!videoPlugin)
1219 {
1220 videoPlugin = (VIDEO_PLUGIN*)calloc(1, sizeof(VIDEO_PLUGIN));
1221 if (!videoPlugin)
1222 {
1223 WLog_ERR(TAG, "calloc failed!");
1224 return CHANNEL_RC_NO_MEMORY;
1225 }
1226
1227 videoPlugin->wtsPlugin.Initialize = video_plugin_initialize;
1228 videoPlugin->wtsPlugin.Connected = nullptr;
1229 videoPlugin->wtsPlugin.Disconnected = nullptr;
1230 videoPlugin->wtsPlugin.Terminated = video_plugin_terminated;
1231
1232 videoContext = (VideoClientContext*)calloc(1, sizeof(VideoClientContext));
1233 if (!videoContext)
1234 {
1235 WLog_ERR(TAG, "calloc failed!");
1236 free(videoPlugin);
1237 return CHANNEL_RC_NO_MEMORY;
1238 }
1239
1240 priv = VideoClientContextPriv_new(videoContext);
1241 if (!priv)
1242 {
1243 WLog_ERR(TAG, "VideoClientContextPriv_new failed!");
1244 free(videoContext);
1245 free(videoPlugin);
1246 return CHANNEL_RC_NO_MEMORY;
1247 }
1248
1249 videoContext->handle = (void*)videoPlugin;
1250 videoContext->priv = priv;
1251 videoContext->timer = video_timer;
1252 videoContext->setGeometry = video_client_context_set_geometry;
1253
1254 videoPlugin->wtsPlugin.pInterface = (void*)videoContext;
1255 videoPlugin->context = videoContext;
1256 videoPlugin->rdpcontext = pEntryPoints->GetRdpContext(pEntryPoints);
1257 if (videoPlugin->rdpcontext)
1258 error = pEntryPoints->RegisterPlugin(pEntryPoints, "video", &videoPlugin->wtsPlugin);
1259 }
1260 else
1261 {
1262 WLog_ERR(TAG, "could not get video Plugin.");
1263 return CHANNEL_RC_BAD_CHANNEL;
1264 }
1265
1266 return error;
1267}
GENERIC_CHANNEL_CALLBACK
Definition include/freerdp/client/channels.h:35
GENERIC_LISTENER_CALLBACK
Definition include/freerdp/client/channels.h:43
RDP_RECT
Definition types.h:101
RECTANGLE_16
Definition types.h:109
RTL_CRITICAL_SECTION
Definition include/winpr/synch.h:174
TSMM_CLIENT_NOTIFICATION
a client to server notification struct
Definition channels/video.h:116
TSMM_PRESENTATION_REQUEST
presentation request struct
Definition channels/video.h:57
TSMM_PRESENTATION_RESPONSE
response to a TSMM_PRESENTATION_REQUEST
Definition channels/video.h:73
TSMM_VIDEO_DATA
a video data packet
Definition channels/video.h:87
VideoSurface
an implementation of surface used by the video channel
Definition client/video.h:36