proof of concept for multi-frame streaming

- bump MLSP submodule (multi-frame transport)
- library interface for:
  - initalizing multiple harware encoders
  - encoding and sending multiple logical subframes together
- example of multi streaming

Related to #4

CMakeLists.txt

@@ -26,4 +26,6 @@ target_link_libraries(nhve-stream-h264 nhve)
 add_executable(nhve-stream-hevc10 examples/nhve_stream_hevc10.c)
 target_link_libraries(nhve-stream-hevc10 nhve)
 
+add_executable(nhve-stream-multi examples/nhve_stream_multi.c)
+target_link_libraries(nhve-stream-multi nhve)
 

README.md

@@ -121,17 +121,16 @@ See [HVE](https://github.com/bmegli/hardware-video-encoder) docs for details abo
 
 		//fill nhve_frame with increasing framenumber and
 		//pointers to your data in NV12 pixel format
-		frame.framenumber=framenumber++; //dummy framenumber
 		frame.data[0]=Y; //dummy luminance plane
 		frame.data[1]=color; //dummy UV plane
 		
 		//encode and send this frame
-		if( nhve_send_frame(streamer, &frame) != NHVE_OK)
+		if( nhve_send_frame(streamer, framenumber++, &frame) != NHVE_OK)
 			break; //break on error
 	}
 
 	//flush the streamer by sending NULL frame
-	nhve_send_frame(streamer, NULL);
+	nhve_send_frame(streamer, framenumber, NULL);
 
 	nhve_close(streamer);
 ```

examples/nhve_stream_h264.c

@@ -1,7 +1,7 @@
 /*
  * NHVE Network Hardware Video Encoder library example of streaming H.264
  *
- * Copyright 2019 (C) Bartosz Meglicki <[email protected]>
+ * Copyright 2019-2020 (C) Bartosz Meglicki <[email protected]>
  *
  * This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
@@ -91,19 +91,16 @@ int streaming_loop(struct nhve *streamer)
 		frame.data[0]=Y;
 		frame.data[1]=color;
 
-		//increase the framenumber, this is mandatory for the network protocol
-		frame.framenumber = f;
-
-		//encode and send this frame
-		if( nhve_send_frame(streamer, &frame) != NHVE_OK)
+		//encode and send this frame, the framenumber f has to increase
+		if( nhve_send_frame(streamer, f, &frame) != NHVE_OK)
 			break; //break on error
 
 		//simulate real time source (sleep according to framerate)
 		usleep(useconds_per_frame);
 	}
 
 	//flush the encoder by sending NULL frame, encode some last frames returned from hardware
-	nhve_send_frame(streamer, NULL);
+	nhve_send_frame(streamer, f, NULL);
 
 	//did we encode everything we wanted?
 	//convention 0 on success, negative on failure

examples/nhve_stream_hevc10.c

@@ -1,7 +1,7 @@
 /*
  * NHVE Network Hardware Video Encoder library example of streaming H.264
  *
- * Copyright 2019 (C) Bartosz Meglicki <[email protected]>
+ * Copyright 2019-2020 (C) Bartosz Meglicki <[email protected]>
  *
  * This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
@@ -94,19 +94,16 @@ int streaming_loop(struct nhve *streamer)
 		frame.data[0] = (uint8_t*)Y;
 		frame.data[1] = (uint8_t*)color;
 
-		//increase the framenumber, this is mandatory for the network protocol
-		frame.framenumber = f;
-
-		//encode and send this frame
-		if( nhve_send_frame(streamer, &frame) != NHVE_OK)
+		//encode and send this frame, the framenumber f has to increase
+		if( nhve_send_frame(streamer, f, &frame) != NHVE_OK)
 			break; //break on error
 
 		//simulate real time source (sleep according to framerate)
 		usleep(useconds_per_frame);
 	}
 
 	//flush the encoder by sending NULL frame, encode some last frames returned from hardware
-	nhve_send_frame(streamer, NULL);
+	nhve_send_frame(streamer, f, NULL);
 
 	//did we encode everything we wanted?
 	//convention 0 on success, negative on failure

examples/nhve_stream_multi.c

@@ -0,0 +1,154 @@
+/*
+ * NHVE Network Hardware Video Encoder library example of
+ * streaming with multiple simultanous hardware encoders
+ *
+ * Copyright 2019-2020 (C) Bartosz Meglicki <[email protected]>
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ */
+
+#include <stdio.h> //printf, fprintf
+#include <inttypes.h> //uint8_t
+#include <unistd.h> //usleep
+
+#include "../nhve.h"
+
+const char *IP; //e.g "127.0.0.1"
+unsigned short PORT; //e.g. 9667 
+
+const int WIDTH=640;
+const int HEIGHT=360;
+const int FRAMERATE=30;
+int SECONDS=10;
+const char *DEVICE; //NULL for default or device e.g. "/dev/dri/renderD128"
+const char *ENCODER=NULL;//NULL for default (h264_vaapi) or FFmpeg encoder e.g. "hevc_vaapi", ...
+const char *PIXEL_FORMAT="nv12"; //NULL / "" for default (NV12) or pixel format e.g. "rgb0"
+const int PROFILE=FF_PROFILE_H264_HIGH; //or FF_PROFILE_H264_MAIN, FF_PROFILE_H264_CONSTRAINED_BASELINE, ...
+const int BFRAMES=0; //max_b_frames, set to 0 to minimize latency, non-zero to minimize size
+const int BITRATE1=500000; //average bitrate in VBR mode (bit_rate != 0 and qp == 0)
+const int BITRATE2=2000000; //average bitrate in VBR mode (bit_rate != 0 and qp == 0)
+const int QP=0; //quantization parameter in CQP mode (qp != 0 and bit_rate == 0)
+const int GOP_SIZE=0; //group of pictures size, 0 for default (determines keyframe period)
+const int COMPRESSION_LEVEL=0; //speed-quality tradeoff, 0 for default, 1 for the highest quality, 7 for the fastest
+
+//IP, PORT, SECONDS and DEVICE are read from user input
+
+int streaming_loop(struct nhve *streamer);
+int process_user_input(int argc, char* argv[]);
+int hint_user_on_failure(char *argv[]);
+void hint_user_on_success();
+
+int main(int argc, char* argv[])
+{
+	//get SECONDS and DEVICE from the command line
+	if( process_user_input(argc, argv) < 0 )
+		return -1;
+
+	//prepare library data
+	struct nhve_net_config net_config = {IP, PORT};
+
+	struct nhve_hw_config hw_config[2] = 
+	{  //those could be completely different encoders using different hardware, here just different bitrate
+		{WIDTH, HEIGHT, FRAMERATE, DEVICE, ENCODER, PIXEL_FORMAT, PROFILE, BFRAMES, BITRATE1, QP, GOP_SIZE, COMPRESSION_LEVEL},
+		{WIDTH, HEIGHT, FRAMERATE, DEVICE, ENCODER, PIXEL_FORMAT, PROFILE, BFRAMES, BITRATE2, QP, GOP_SIZE, COMPRESSION_LEVEL}
+	};
+
+	struct nhve *streamer;
+
+	//initialize library with nhve_multi_init
+	if( (streamer = nhve_multi_init(&net_config, hw_config, 2)) == NULL )
+		return hint_user_on_failure(argv);
+
+	//do the actual encoding
+	int status = streaming_loop(streamer);
+
+	nhve_close(streamer);
+
+	if(status == 0)
+		hint_user_on_success();
+
+	return status;
+}
+
+int streaming_loop(struct nhve *streamer)
+{
+	const int TOTAL_FRAMES = SECONDS*FRAMERATE;
+	const useconds_t useconds_per_frame = 1000000/FRAMERATE;
+	int f;
+	struct nhve_frame frames[2] = { 0 };
+
+
+	//we are working with NV12 because we specified nv12 pixel formats
+	//when calling nhve_multi_init, in principle we could use other format
+	//if hardware supported it (e.g. RGB0 is supported on my Intel)
+	uint8_t Y1[WIDTH*HEIGHT]; //dummy NV12 luminance data for encoder 1
+	uint8_t Y2[WIDTH*HEIGHT]; //                          for encoder 2
+
+	uint8_t color[WIDTH*HEIGHT/2]; //same dummy NV12 color data for both encoders
+
+	//fill with your stride (width including padding if any)
+	frames[0].linesize[0] = frames[1].linesize[0] = WIDTH; //Y planes stride
+	frames[0].linesize[1] = frames[1].linesize[1] = WIDTH; //UV planes stride 
+
+	for(f=0;f<TOTAL_FRAMES;++f)
+	{
+		//prepare dummy images data, normally you would take it from cameras or other source
+		memset(Y1, f % 255, WIDTH*HEIGHT); //NV12 luminance (ride through greyscale)
+		memset(Y2, 255 - (f % 255), WIDTH*HEIGHT); //NV12 luminance (reverse ride through greyscale)
+		memset(color, 128, WIDTH*HEIGHT/2); //NV12 UV (no color really)
+
+		//fill nhve_frame with pointers to your data in NV12 pixel format
+		frames[0].data[0]=Y1;
+		frames[0].data[1]=color;
+
+		frames[1].data[0]=Y2;
+		frames[1].data[1]=color;
+
+		//encode and send this frame, the framenumber f has to increase
+		if( nhve_send_frames(streamer, f, frames, 2) != NHVE_OK)
+			break; //break on error
+
+		//simulate real time source (sleep according to framerate)
+		usleep(useconds_per_frame);
+	}
+
+	//flush the encoder by sending NULL frame, encode some last frames returned from hardware
+	nhve_send_frame(streamer, f, NULL);
+
+	//did we encode everything we wanted?
+	//convention 0 on success, negative on failure
+	return f == TOTAL_FRAMES ? 0 : -1;
+}
+
+int process_user_input(int argc, char* argv[])
+{
+	if(argc < 4)
+	{
+		fprintf(stderr, "Usage: %s <ip> <port> <seconds> [device]\n", argv[0]);
+		fprintf(stderr, "\nexamples:\n");
+		fprintf(stderr, "%s 127.0.0.1 9766 10\n", argv[0]);
+		fprintf(stderr, "%s 127.0.0.1 9766 10 /dev/dri/renderD128\n", argv[0]);
+		return -1;
+	}
+
+	IP = argv[1];
+	PORT = atoi(argv[2]);
+	SECONDS = atoi(argv[3]);
+	DEVICE=argv[4]; //NULL as last argv argument, or device path
+
+	return 0;
+}
+
+int hint_user_on_failure(char *argv[])
+{
+	fprintf(stderr, "unable to initalize, try to specify device e.g:\n\n");
+	fprintf(stderr, "%s 127.0.0.1 9766 10 /dev/dri/renderD128\n", argv[0]);
+	return -1;
+}
+void hint_user_on_success()
+{
+	printf("finished successfully\n");
+}

nhve.c

@@ -92,52 +92,89 @@ void nhve_close(struct nhve *n)
 	mlsp_close(n->network_streamer);
 	for(int i=0;i<n->hardware_encoders_size;++i)
 		hve_close(n->hardware_encoder[i]);
-	free(n->hardware_encoder);
 	free(n);
 }
 
-// NULL frame or NULL frame.data[0] to flush
-int nhve_send_frame(struct nhve *n,struct nhve_frame *frame)
+// NULL frame to flush
+int nhve_send_frame(struct nhve *n, uint16_t framenumber, struct nhve_frame *frame)
 {
-	struct hve_frame *final_video_frame = NULL, video_frame;
-
-	// NULL is also valid input meaning - flush the encoder
-	if(frame != NULL)
-	{	//copy pointers to data and linesizes (just a few bytes)
-		memcpy(video_frame.data, frame->data, sizeof(frame->data));
-		memcpy(video_frame.linesize, frame->linesize, sizeof(frame->linesize));
-		final_video_frame = &video_frame;
-	}
+	return nhve_send_frames(n, framenumber, frame, 1);
+}
 
-	// send video frame data to hardware for encoding
-	// in case frame argument was NULL pass NULL here for flushing encoder
-	if( hve_send_frame(n->hardware_encoder[0], final_video_frame) != HVE_OK)
+//NULL frames to flush all encoders
+//NULL frames[i].data[0] is legal and silently skipped
+//this is necessary to support multiple encoders with e.g. different B frames
+int nhve_send_frames(struct nhve *n, uint16_t framenumber, struct nhve_frame *frames, int frames_size)
+{
+	struct hve_frame video_frames[NHVE_MAX_ENCODERS] = {0};
+
+	if(frames_size > n->hardware_encoders_size)
 	{
-		fprintf(stderr, "nhve: failed to send frame to hardware\n");
+		fprintf(stderr, "nhve: requested to send more frames than initialized encoders\n");
 		return NHVE_ERROR;
 	}
 
-	AVPacket *encoded_frame;
-	int failed;
+	if(frames) //NULL frames is valid input - flush the encoders
+		for(int i=0;i<frames_size;++i)
+		{	//copy pointers to data and linesizes (just a few bytes)
+			memcpy(video_frames[i].data, frames[i].data, sizeof(frames[i].data));
+			memcpy(video_frames[i].linesize, frames[i].linesize, sizeof(frames[i].linesize));
+		}
 
-	//get the encoded frame data from hardware
-	while( (encoded_frame=hve_receive_packet(n->hardware_encoder[0], &failed)) )
+	for(int i=0;i<frames_size;++i)
 	{
-		struct mlsp_frame network_frame = {frame->framenumber, encoded_frame->data, encoded_frame->size};
-		//send encoded frame data over network
-		if ( mlsp_send(n->network_streamer, &network_frame) != MLSP_OK )
+		if(!frames) //flush all encoders
+			if( hve_send_frame(n->hardware_encoder[i], NULL) != HVE_OK)
+			{
+				fprintf(stderr, "nhve: failed to send flush frame to hardware\n");
+				return NHVE_ERROR;
+			}
+
+		//sending data only to selected encoders is legal and will not result in flushing
+		if( video_frames[i].data[0] && (hve_send_frame(n->hardware_encoder[i], &video_frames[i]) != HVE_OK) )
 		{
-			fprintf(stderr, "nhve: failed to send frame\n");
+			fprintf(stderr, "nhve: failed to send frame to hardware\n");
 			return NHVE_ERROR;
 		}
 	}
 
-	//NULL packet and non-zero failed indicates failure during encoding
-	if(failed  != HVE_OK)
+	AVPacket *encoded_frames[NHVE_MAX_ENCODERS];
+	int failed[NHVE_MAX_ENCODERS] = {0};
+	int keep_working;
+
+	//while any of encoders still produces data we should send frames
+	do
 	{
-		fprintf(stderr, "nhve: failed to encode frame\n");
-		return NHVE_ERROR;
-	}
+		struct mlsp_frame network_frame = {0};
+
+		keep_working = 0;
+
+		for(int i=0;i<frames_size;++i)
+			if( (encoded_frames[i] = hve_receive_packet(n->hardware_encoder[i], &failed[i])) )
+			{
+				network_frame.data[i] = encoded_frames[i]->data;
+				network_frame.size[i] = encoded_frames[i]->size;
+				//this should be increased with every frame, otherwise the world will explode
+				//this means that it no good for flushing now
+				network_frame.framenumber = framenumber;
+				keep_working = 1; //some encoder still produces output
+			}
+
+		if(keep_working && mlsp_send(n->network_streamer, &network_frame) != MLSP_OK )
+		{
+			fprintf(stderr, "nhve: failed to send frame\n");
+			return NHVE_ERROR;
+		}
+
+	} while(keep_working);
+
+	//NULL packet and non-zero failed indicates failure during encoding
+	for(int i=0;i<frames_size;++i)
+		if(failed[i] != HVE_OK)
+		{
+			fprintf(stderr, "nhve: failed to encode frame\n");
+			return NHVE_ERROR;
+		}
 
 	return NHVE_OK;
 }

nhve.h

@@ -46,7 +46,6 @@ struct nhve_net_config
 
 struct nhve_frame
 {
-	uint16_t framenumber;
 	uint8_t *data[AV_NUM_DATA_POINTERS]; //!< array of pointers to frame planes (e.g. Y plane and UV plane)
 	int linesize[AV_NUM_DATA_POINTERS]; //!< array of strides (width + padding) for planar frame formats
 };
@@ -68,7 +67,10 @@ void nhve_close(struct nhve *n);
 
 //NHVE_OK on success, NHVE_ERROR on error
 //pass NULL frame to flush encoder
-int nhve_send_frame(struct nhve *n,struct nhve_frame *frame);
+int nhve_send_frame(struct nhve *n, uint16_t framenumber, struct nhve_frame *frame);
+
+int nhve_send_frames(struct nhve *n, uint16_t framenumber, struct nhve_frame *frames, int frames_size);
+
 
 #ifdef __cplusplus
 }