[Bug] GStreamer sample (#1972)

* gst-sample-fix

* modify the message

* fix windows

* set to null

* fix pts and eos in sample

* unused var removed

* move null init

.github/workflows/ci.yml

@@ -556,6 +556,7 @@ jobs:
       - name: Install dependencies
         shell: powershell
         run: |
+          choco install pkgconfiglite
           choco install gstreamer --version=1.16.3
           choco install gstreamer-devel --version=1.16.3
           curl.exe -o C:\tools\pthreads-w32-2-9-1-release.zip ftp://sourceware.org/pub/pthreads-win32/pthreads-w32-2-9-1-release.zip

README.md

@@ -305,6 +305,14 @@ To run:
 ./samples/kvsWebrtcClientViewerGstSample <channelName> <mediaType>
 ```
 
+##### Known issues:
+Our GStreamer samples leverage [MatroskaMux](https://gstreamer.freedesktop.org/documentation/matroska/matroskamux.html?gi-language=c) to receive media from its peer and save it to a file. However, MatroskaMux is designed for scenarios where the media's format remains constant throughout streaming. When the media's format changes mid-streaming (referred to as "caps changes"), MatroskaMux encounters limitations, its behavior cannot be predicted and it may be unable to handle these changes, resulting in an error message like:
+
+```shell
+matroskamux matroska-mux.c:1134:gst_matroska_mux_video_pad_setcaps:<mux> error: Caps changes are not supported by Matroska
+```
+To address this issue, users need to adapt the pipeline to utilize components capable of managing dynamic changes in media formats. This might involve integrating different muxers or customizing the pipeline to handle caps changes effectively.
+
 #### Sample: Generating sample frames
 
 ##### H264
@@ -321,12 +329,12 @@ gst-launch-1.0 videotestsrc pattern=ball num-buffers=1500 ! timeoverlay ! videoc
 
 ###### ADTS LC
 ```shell
-gst-launch-1.0 audiotestsrc num-buffers=1500 ! audioconvert ! audioresample ! faac ! capsfilter caps=audio/mpeg,mpegversion=4,stream-format=adts,base-profile=lc,channels=2,rate=48000 ! multifilesink location="sample-%03d.aac" index=1
+gst-launch-1.0 audiotestsrc num-buffers=1500 ! audioconvert ! audioresample ! faac ! capsfilter caps=audio/mpeg,mpegversion=4,stream-format=adts,base-profile=lc,channels=2,rate=16000 ! multifilesink location="sample-%03d.aac" index=1
 ```
 
 ###### RAW LC
 ```shell
-gst-launch-1.0 audiotestsrc num-buffers=1500 ! audioconvert ! audioresample ! faac ! capsfilter caps=audio/mpeg,mpegversion=4,stream-format=raw,base-profile=lc,channels=2,rate=44100 ! multifilesink location="sample-%03d.aac" index=1
+gst-launch-1.0 audiotestsrc num-buffers=1500 ! audioconvert ! audioresample ! faac ! capsfilter caps=audio/mpeg,mpegversion=4,stream-format=raw,base-profile=lc,channels=2,rate=16000 ! multifilesink location="sample-%03d.aac" index=1
 ```
 
 ### Viewing Master Samples

samples/Common.c

@@ -853,6 +853,7 @@ STATUS createSampleConfiguration(PCHAR channelName, SIGNALING_CHANNEL_ROLE_TYPE
     pSampleConfiguration->trickleIce = trickleIce;
     pSampleConfiguration->useTurn = useTurn;
     pSampleConfiguration->enableSendingMetricsToViewerViaDc = FALSE;
+    pSampleConfiguration->receiveAudioVideoSource = NULL;
 
     pSampleConfiguration->channelInfo.version = CHANNEL_INFO_CURRENT_VERSION;
     pSampleConfiguration->channelInfo.pChannelName = channelName;

samples/GstAudioVideoReceiver.c

@@ -3,6 +3,9 @@
 #include <gst/app/app.h>
 #include <gst/app/gstappsink.h>
 
+static UINT64 presentationTsIncrement = 0;
+static BOOL eos = FALSE;
+
 // This function is a callback for the transceiver for every single video frame it receives
 // It writes these frames to a buffer and pushes it to the `appsrcVideo` element of the
 // GStreamer pipeline created in `receiveGstreamerAudioVideo`. Any logic to modify / discard the frames would go here
@@ -16,22 +19,28 @@ VOID onGstVideoFrameReady(UINT64 customData, PFrame pFrame)
     CHK_ERR(appsrcVideo != NULL, STATUS_NULL_ARG, "appsrcVideo is null");
     CHK_ERR(pFrame != NULL, STATUS_NULL_ARG, "Video frame is null");
 
-    buffer = gst_buffer_new_allocate(NULL, pFrame->size, NULL);
-    CHK_ERR(buffer != NULL, STATUS_NULL_ARG, "Buffer allocation failed");
+    if (!eos) {
+        buffer = gst_buffer_new_allocate(NULL, pFrame->size, NULL);
+        CHK_ERR(buffer != NULL, STATUS_NULL_ARG, "Buffer allocation failed");
+
+        DLOGV("Video frame size: %d, presentationTs: %llu", pFrame->size, presentationTsIncrement);
+
+        GST_BUFFER_DTS(buffer) = presentationTsIncrement;
+        GST_BUFFER_PTS(buffer) = presentationTsIncrement;
+        GST_BUFFER_DURATION(buffer) = gst_util_uint64_scale(1, GST_SECOND, DEFAULT_FPS_VALUE);
+        presentationTsIncrement += gst_util_uint64_scale(1, GST_SECOND, DEFAULT_FPS_VALUE);
 
-    DLOGV("Frame size: %d, presentationTs: %llu", pFrame->size, pFrame->presentationTs);
-    GST_BUFFER_PTS(buffer) = pFrame->presentationTs;
-    GST_BUFFER_DURATION(buffer) = gst_util_uint64_scale(1, GST_SECOND, DEFAULT_FPS_VALUE);
-    if (gst_buffer_fill(buffer, 0, pFrame->frameData, pFrame->size) != pFrame->size) {
-        DLOGE("Buffer fill did not complete correctly");
+        if (gst_buffer_fill(buffer, 0, pFrame->frameData, pFrame->size) != pFrame->size) {
+            DLOGE("Buffer fill did not complete correctly");
+            gst_buffer_unref(buffer);
+            return;
+        }
+        g_signal_emit_by_name(appsrcVideo, "push-buffer", buffer, &ret);
+        if (ret != GST_FLOW_OK) {
+            DLOGE("Error pushing buffer: %s", gst_flow_get_name(ret));
+        }
         gst_buffer_unref(buffer);
-        return;
-    }
-    g_signal_emit_by_name(appsrcVideo, "push-buffer", buffer, &ret);
-    if (ret != GST_FLOW_OK) {
-        DLOGE("Error pushing buffer: %s", gst_flow_get_name(ret));
     }
-    gst_buffer_unref(buffer);
 
 CleanUp:
     return;
@@ -50,25 +59,27 @@ VOID onGstAudioFrameReady(UINT64 customData, PFrame pFrame)
     CHK_ERR(appsrcAudio != NULL, STATUS_NULL_ARG, "appsrcAudio is null");
     CHK_ERR(pFrame != NULL, STATUS_NULL_ARG, "Audio frame is null");
 
-    buffer = gst_buffer_new_allocate(NULL, pFrame->size, NULL);
-    CHK_ERR(buffer != NULL, STATUS_NULL_ARG, "Buffer allocation failed");
+    if (!eos) {
+        buffer = gst_buffer_new_allocate(NULL, pFrame->size, NULL);
+        CHK_ERR(buffer != NULL, STATUS_NULL_ARG, "Buffer allocation failed");
 
-    DLOGV("Audio Frame size: %d, presentationTs: %llu", pFrame->size, pFrame->presentationTs);
-    GST_BUFFER_PTS(buffer) = pFrame->presentationTs;
+        DLOGV("Audio frame size: %d, presentationTs: %llu", pFrame->size, presentationTsIncrement);
 
-    // Recalculate the byte-rate if not using the default values
-    GST_BUFFER_DURATION(buffer) = gst_util_uint64_scale(pFrame->size, GST_SECOND, DEFAULT_AUDIO_OPUS_BYTE_RATE);
-    if (gst_buffer_fill(buffer, 0, pFrame->frameData, pFrame->size) != pFrame->size) {
-        DLOGE("Buffer fill did not complete correctly");
+        GST_BUFFER_DTS(buffer) = presentationTsIncrement;
+        GST_BUFFER_PTS(buffer) = presentationTsIncrement;
+        GST_BUFFER_DURATION(buffer) = gst_util_uint64_scale(pFrame->size, GST_SECOND, DEFAULT_AUDIO_OPUS_BYTE_RATE);
+
+        if (gst_buffer_fill(buffer, 0, pFrame->frameData, pFrame->size) != pFrame->size) {
+            DLOGE("Buffer fill did not complete correctly");
+            gst_buffer_unref(buffer);
+            return;
+        }
+        g_signal_emit_by_name(appsrcAudio, "push-buffer", buffer, &ret);
+        if (ret != GST_FLOW_OK) {
+            DLOGE("Error pushing buffer: %s", gst_flow_get_name(ret));
+        }
         gst_buffer_unref(buffer);
-        return;
     }
-    g_signal_emit_by_name(appsrcAudio, "push-buffer", buffer, &ret);
-    if (ret != GST_FLOW_OK) {
-        DLOGE("Error pushing buffer: %s", gst_flow_get_name(ret));
-    }
-    gst_buffer_unref(buffer);
-
 CleanUp:
     return;
 }
@@ -78,6 +89,7 @@ VOID onGstAudioFrameReady(UINT64 customData, PFrame pFrame)
 VOID onSampleStreamingSessionShutdown(UINT64 customData, PSampleStreamingSession pSampleStreamingSession)
 {
     (void) (pSampleStreamingSession);
+    eos = TRUE;
     GstElement* pipeline = (GstElement*) customData;
     gst_element_send_event(pipeline, gst_event_new_eos());
 }

samples/kvsWebrtcClientMasterGstSample.c

@@ -323,7 +323,6 @@ INT32 main(INT32 argc, CHAR* argv[])
 
     pSampleConfiguration->videoSource = sendGstreamerAudioVideo;
     pSampleConfiguration->mediaType = SAMPLE_STREAMING_VIDEO_ONLY;
-    pSampleConfiguration->receiveAudioVideoSource = receiveGstreamerAudioVideo;
 
 #ifdef ENABLE_DATA_CHANNEL
     pSampleConfiguration->onDataChannel = onDataChannel;