diff --git a/README.md b/README.md index 8259123..3e17d20 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Realsense hardware video/depth encoding and streaming over custom [MLSP](https:/ This includes streaming: - color (H.264, HEVC Main) -- infrared (H.264, HEVC Main) +- infrared/infrared-rgb (H.264, HEVC Main) - depth (HEVC Main10) - textured depth (HEVC Main10 + HEVC Main) @@ -35,7 +35,7 @@ Requires Intel VAAPI compatible hardware encoder (QuickSync Video). For depth en [Other technologies](https://github.com/bmegli/realsense-network-hardware-video-encoder/wiki/Hardware) may also work but were not tested. -Infrared textured depth encoding is implemented for D435, D455 and L515. +Infrared textured depth encoding is implemented for D415, D435, D455 and L515. Color textured depth encoding is implemented for D415, D435, D455, L515. ## Dependencies @@ -81,58 +81,64 @@ make ## Running -Stream H.264 Realsense color/infrared video over UDP. +Stream H.264 Realsense color/infrared/infrared-rgb video over UDP. ```bash -Usage: -./realsense-nhve-h264 [device] [bitrate] +Usage: ./realsense-nhve-h264 [device] [bitrate] +examples: ./realsense-nhve-h264 127.0.0.1 9766 color 640 360 30 5 -./realsense-nhve-h264 127.0.0.1 9766 infrared 640 360 30 5 +./realsense-nhve-h264 127.0.0.1 9766 ir 640 360 30 5 +./realsense-nhve-h264 127.0.0.1 9766 ir-rgb 640 360 30 5 ./realsense-nhve-h264 127.0.0.1 9766 color 640 360 30 5 /dev/dri/renderD128 -./realsense-nhve-h264 127.0.0.1 9766 infrared 640 360 30 5 /dev/dri/renderD128 -./realsense-nhve-h264 192.168.0.125 9766 color 640 360 30 50 /dev/dri/renderD128 500000 -``` +./realsense-nhve-h264 127.0.0.1 9766 ir 640 360 30 5 /dev/dri/renderD128 +./realsense-nhve-h264 127.0.0.1 9766 ir-rgb 640 360 30 5 /dev/dri/renderD128 +./realsense-nhve-h264 192.168.0.125 9766 color 640 360 30 50 /dev/dri/renderD128 500000``` Stream Realsense: -- color/infrared with HEVC Main +- color/infrared/infrared-rgb with HEVC Main - depth with HEVC Main10 ```bash -Usage: -./realsense-nhve-hevc [device] [bitrate] [depth units] [json] +Usage: ./realsense-nhve-hevc [device] [bitrate] [depth units] [json] +examples: ./realsense-nhve-hevc 127.0.0.1 9766 color 640 360 30 5 -./realsense-nhve-hevc 127.0.0.1 9766 infrared 640 360 30 5 +./realsense-nhve-hevc 127.0.0.1 9766 ir 640 360 30 5 +./realsense-nhve-hevc 127.0.0.1 9766 ir-rgb 640 360 30 5 ./realsense-nhve-hevc 127.0.0.1 9766 depth 640 360 30 5 ./realsense-nhve-hevc 127.0.0.1 9766 color 640 360 30 5 /dev/dri/renderD128 -./realsense-nhve-hevc 127.0.0.1 9766 infrared 640 360 30 5 /dev/dri/renderD128 +./realsense-nhve-hevc 127.0.0.1 9766 ir 640 360 30 5 /dev/dri/renderD128 +./realsense-nhve-hevc 127.0.0.1 9766 ir-rgb 640 360 30 5 /dev/dri/renderD128 ./realsense-nhve-hevc 127.0.0.1 9766 depth 640 360 30 5 /dev/dri/renderD128 ./realsense-nhve-hevc 192.168.0.125 9766 color 640 360 30 50 /dev/dri/renderD128 500000 -./realsense-nhve-hevc 192.168.0.125 9768 depth 848 480 30 50 /dev/dri/renderD128 2000000 -./realsense-nhve-hevc 192.168.0.125 9768 depth 848 480 30 50 /dev/dri/renderD128 8000000 0.0001 +./realsense-nhve-hevc 127.0.0.1 9768 depth 848 480 30 50 /dev/dri/renderD128 2000000 +./realsense-nhve-hevc 192.168.0.100 9768 depth 848 480 30 500 /dev/dri/renderD128 2000000 0.0001 +./realsense-nhve-hevc 192.168.0.100 9768 depth 848 480 30 500 /dev/dri/renderD128 2000000 0.00005 +./realsense-nhve-hevc 192.168.0.100 9768 depth 848 480 30 500 /dev/dri/renderD128 2000000 0.000025 +./realsense-nhve-hevc 192.168.0.100 9768 depth 848 480 30 500 /dev/dri/renderD128 2000000 0.0000125 ./realsense-nhve-hevc 192.168.0.100 9768 depth 640 480 30 500 /dev/dri/renderD128 8000000 0.0000390625 my_config.json ``` -Stream Realsense D435/D455/L515: -- depth with HEVC Main10, infrared with HEVC +Stream Realsense D415/D435/D455/L515: +- depth with HEVC Main10, infrared or infrared rgb with HEVC ```bash -Usage: -./realsense-nhve-depth-ir [device] [bitrate_depth] [bitrate_ir] [depth units] [json] +Usage: ./realsense-nhve-depth-ir [device] [bitrate_depth] [bitrate_ir] [depth units] [json] examples: -./realsense-nhve-depth-ir 127.0.0.1 9766 640 360 30 5 -./realsense-nhve-depth-ir 127.0.0.1 9766 640 360 30 5 /dev/dri/renderD128 -./realsense-nhve-depth-ir 192.168.0.125 9766 640 360 30 50 /dev/dri/renderD128 4000000 1000000 -./realsense-nhve-depth-ir 192.168.0.100 9768 848 480 30 500 /dev/dri/renderD128 8000000 1000000 0.0001 -./realsense-nhve-depth-ir 192.168.0.100 9768 848 480 30 500 /dev/dri/renderD128 8000000 1000000 0.00005 -./realsense-nhve-depth-ir 192.168.0.100 9768 848 480 30 500 /dev/dri/renderD128 8000000 1000000 0.000025 -./realsense-nhve-depth-ir 192.168.0.100 9768 848 480 30 500 /dev/dri/renderD128 8000000 1000000 0.0000125 -./realsense-nhve-depth-ir 192.168.0.100 9768 848 480 30 500 /dev/dri/renderD128 8000000 1000000 0.0000125 -./realsense-nhve-depth-ir 192.168.0.100 9768 640 480 30 500 /dev/dri/renderD128 8000000 1000000 0.0000390625 my_config.json -``` +./realsense-nhve-depth-ir 127.0.0.1 9766 ir 640 360 30 5 +./realsense-nhve-depth-ir 127.0.0.1 9766 ir-rgb 640 360 30 5 +./realsense-nhve-depth-ir 127.0.0.1 9766 ir 640 360 30 5 /dev/dri/renderD128 +./realsense-nhve-depth-ir 192.168.0.125 9766 ir-rgb 640 360 30 50 /dev/dri/renderD128 4000000 1000000 +./realsense-nhve-depth-ir 192.168.0.100 9768 ir 848 480 30 500 /dev/dri/renderD128 8000000 1000000 0.0001 +./realsense-nhve-depth-ir 192.168.0.100 9768 ir-rgb 848 480 30 500 /dev/dri/renderD128 8000000 1000000 0.00005 +./realsense-nhve-depth-ir 192.168.0.100 9768 ir 848 480 30 500 /dev/dri/renderD128 8000000 1000000 0.000025 +./realsense-nhve-depth-ir 192.168.0.100 9768 ir-rgb 848 480 30 500 /dev/dri/renderD128 8000000 1000000 0.0000125 +./realsense-nhve-depth-ir 192.168.0.100 9768 ir 848 480 30 500 /dev/dri/renderD128 8000000 1000000 0.0000125 +./realsense-nhve-depth-ir 192.168.0.100 9768 ir-rgb 848 480 30 500 /dev/dri/renderD128 8000000 1000000 0.00003125 +./realsense-nhve-depth-ir 192.168.0.100 9768 ir 640 480 30 500 /dev/dri/renderD128 8000000 1000000 0.0000390625 my_config.json``` Stream Realsense D415/D435/D455/L515: - depth with HEVC Main10, color with HEVC (aligned) diff --git a/rnhve_depth_ir.cpp b/rnhve_depth_ir.cpp index 4048f7a..d5d2071 100644 --- a/rnhve_depth_ir.cpp +++ b/rnhve_depth_ir.cpp @@ -3,6 +3,7 @@ * * Realsense hardware encoded UDP HEVC multi-streaming * - depth (Main10) + infrared (Main) + * - depth (Main10) + infrared rgb (Main) * * Copyright 2020 (C) Bartosz Meglicki * @@ -28,6 +29,8 @@ using namespace std; int hint_user_on_failure(char *argv[]); +enum StreamType {INFRARED, INFRARED_RGB}; + //user supplied input struct input_args { @@ -36,6 +39,7 @@ struct input_args int framerate; int seconds; float depth_units; + StreamType stream; std::string json; bool needs_postprocessing; }; @@ -92,7 +96,7 @@ bool main_loop(const input_args& input, rs2::pipeline& realsense, nhve *streamer nhve_frame frame[2] = { {0}, {0} }; uint16_t *depth_uv = NULL; //data of dummy color plane for P010LE - uint8_t *ir_uv = NULL; //data of dummy color plane for NV12 + uint8_t *ir_uv = NULL; //data of dummy color plane for NV12 for Realsense infrared for(f = 0; f < frames; ++f) { @@ -109,7 +113,7 @@ bool main_loop(const input_args& input, rs2::pipeline& realsense, nhve *streamer if(input.needs_postprocessing) process_depth_data(input, depth); - if(!depth_uv || !ir_uv) + if(!depth_uv) { //prepare dummy color plane for P010LE format, half the size of Y //we can't alloc it in advance, this is the first time we know realsense stride //the stride will be at least width * 2 (Realsense Z16, VAAPI P010LE) @@ -119,8 +123,11 @@ bool main_loop(const input_args& input, rs2::pipeline& realsense, nhve *streamer depth_uv[i] = UINT16_MAX / 2; //dummy middle value for U/V, equals 128 << 8, equals 32768 //prepare dummy color plane for NV12 format, half the size of Y - ir_uv = new uint8_t[ir_stride * h /2]; - memset(ir_uv, 128, ir_stride * h /2); + if(input.stream == INFRARED) + { + ir_uv = new uint8_t[ir_stride * h /2]; + memset(ir_uv, 128, ir_stride * h /2); + } } //supply realsense depth frame data as ffmpeg frame data @@ -135,9 +142,11 @@ bool main_loop(const input_args& input, rs2::pipeline& realsense, nhve *streamer } //supply realsense infrared frame data as ffmpeg frame data - frame[1].linesize[0] = frame[1].linesize[1] = ir_stride; //the strides of Y and UV are equal + frame[1].linesize[0] = ir_stride; frame[1].data[0] = (uint8_t*) ir.get_data(); - frame[1].data[1] = ir_uv; + + frame[1].linesize[1] = (input.stream == INFRARED) ? ir_stride : 0; //NV12 strides of Y and UV are equal, UYVY is single plane + frame[1].data[1] = ir_uv; //data for NV12 or NULL for single plane UYVY if(nhve_send(streamer, &frame[1], 1) != NHVE_OK) { @@ -180,7 +189,10 @@ void init_realsense(rs2::pipeline& pipe, input_args& input) rs2::config cfg; cfg.enable_stream(RS2_STREAM_DEPTH, input.width, input.height, RS2_FORMAT_Z16, input.framerate); - cfg.enable_stream(RS2_STREAM_INFRARED, input.width, input.height, RS2_FORMAT_Y8, input.framerate); + if(input.stream == INFRARED) + cfg.enable_stream(RS2_STREAM_INFRARED, input.width, input.height, RS2_FORMAT_Y8, input.framerate); + else //INFRARED_RGB + cfg.enable_stream(RS2_STREAM_INFRARED, input.width, input.height, RS2_FORMAT_UYVY, input.framerate); rs2::pipeline_profile profile = pipe.start(cfg); @@ -274,20 +286,21 @@ void print_intrinsics(const rs2::pipeline_profile& profile, rs2_stream stream) int process_user_input(int argc, char* argv[], input_args* input, nhve_net_config *net_config, nhve_hw_config *hw_config) { - if(argc < 7) + if(argc < 8) { - cerr << "Usage: " << argv[0] << " [device] [bitrate_depth] [bitrate_ir] [depth units] [json]" << endl; + cerr << "Usage: " << argv[0] << " [device] [bitrate_depth] [bitrate_ir] [depth units] [json]" << endl; cerr << endl << "examples: " << endl; - cerr << argv[0] << " 127.0.0.1 9766 640 360 30 5" << endl; - cerr << argv[0] << " 127.0.0.1 9766 640 360 30 5 /dev/dri/renderD128" << endl; - cerr << argv[0] << " 192.168.0.125 9766 640 360 30 50 /dev/dri/renderD128 4000000 1000000" << endl; - cerr << argv[0] << " 192.168.0.100 9768 848 480 30 500 /dev/dri/renderD128 8000000 1000000 0.0001" << endl; - cerr << argv[0] << " 192.168.0.100 9768 848 480 30 500 /dev/dri/renderD128 8000000 1000000 0.00005" << endl; - cerr << argv[0] << " 192.168.0.100 9768 848 480 30 500 /dev/dri/renderD128 8000000 1000000 0.000025" << endl; - cerr << argv[0] << " 192.168.0.100 9768 848 480 30 500 /dev/dri/renderD128 8000000 1000000 0.0000125" << endl; - cerr << argv[0] << " 192.168.0.100 9768 848 480 30 500 /dev/dri/renderD128 8000000 1000000 0.0000125" << endl; - cerr << argv[0] << " 192.168.0.100 9768 848 480 30 500 /dev/dri/renderD128 8000000 1000000 0.00003125" << endl; - cerr << argv[0] << " 192.168.0.100 9768 640 480 30 500 /dev/dri/renderD128 8000000 1000000 0.0000390625 my_config.json" << endl; + cerr << argv[0] << " 127.0.0.1 9766 ir 640 360 30 5" << endl; + cerr << argv[0] << " 127.0.0.1 9766 ir-rgb 640 360 30 5" << endl; + cerr << argv[0] << " 127.0.0.1 9766 ir 640 360 30 5 /dev/dri/renderD128" << endl; + cerr << argv[0] << " 192.168.0.125 9766 ir-rgb 640 360 30 50 /dev/dri/renderD128 4000000 1000000" << endl; + cerr << argv[0] << " 192.168.0.100 9768 ir 848 480 30 500 /dev/dri/renderD128 8000000 1000000 0.0001" << endl; + cerr << argv[0] << " 192.168.0.100 9768 ir-rgb 848 480 30 500 /dev/dri/renderD128 8000000 1000000 0.00005" << endl; + cerr << argv[0] << " 192.168.0.100 9768 ir 848 480 30 500 /dev/dri/renderD128 8000000 1000000 0.000025" << endl; + cerr << argv[0] << " 192.168.0.100 9768 ir-rgb 848 480 30 500 /dev/dri/renderD128 8000000 1000000 0.0000125" << endl; + cerr << argv[0] << " 192.168.0.100 9768 ir 848 480 30 500 /dev/dri/renderD128 8000000 1000000 0.0000125" << endl; + cerr << argv[0] << " 192.168.0.100 9768 ir-rgb 848 480 30 500 /dev/dri/renderD128 8000000 1000000 0.00003125" << endl; + cerr << argv[0] << " 192.168.0.100 9768 ir 640 480 30 500 /dev/dri/renderD128 8000000 1000000 0.0000390625 my_config.json" << endl; return -1; } @@ -295,6 +308,10 @@ int process_user_input(int argc, char* argv[], input_args* input, nhve_net_confi net_config->ip = argv[1]; net_config->port = atoi(argv[2]); + input->stream = INFRARED; + if(strlen(argv[3]) > 3 && argv[3][2] == '-') + input->stream = INFRARED_RGB; + //for depth encoding we use 10 bit P010LE pixel format //that can be directly matched with Realsense output as P016LE Y plane //with precision/range trade-off controlled by Realsense Depth Units @@ -307,33 +324,36 @@ int process_user_input(int argc, char* argv[], input_args* input, nhve_net_confi //for both cases (depth and infrared) we use native hardware formats //and there is no need for data processing on the host CPU + //for infrared rgb encoding we use uyvy_422 VAAPI format + //matched with Realsense UYVY infrared rgb data + //DEPTH hardware encoding configuration hw_config[DEPTH].profile = FF_PROFILE_HEVC_MAIN_10; hw_config[DEPTH].pixel_format = "p010le"; hw_config[DEPTH].encoder = "hevc_vaapi"; - hw_config[DEPTH].width = input->width = atoi(argv[3]); - hw_config[DEPTH].height = input->height = atoi(argv[4]); - hw_config[DEPTH].framerate = input->framerate = atoi(argv[5]); + hw_config[DEPTH].width = input->width = atoi(argv[4]); + hw_config[DEPTH].height = input->height = atoi(argv[5]); + hw_config[DEPTH].framerate = input->framerate = atoi(argv[6]); - input->seconds = atoi(argv[6]); + input->seconds = atoi(argv[7]); - hw_config[DEPTH].device = argv[7]; //NULL as last argv argument, or device path + hw_config[DEPTH].device = argv[8]; //NULL as last argv argument, or device path - if(argc > 8) - hw_config[DEPTH].bit_rate = atoi(argv[8]); + if(argc > 9) + hw_config[DEPTH].bit_rate = atoi(argv[9]); //INFRARED hardware encoding configuration hw_config[IR].profile = FF_PROFILE_HEVC_MAIN; - hw_config[IR].pixel_format = "nv12"; + hw_config[IR].pixel_format = (input->stream == INFRARED) ? "nv12" : "uyvy422"; hw_config[IR].encoder = "hevc_vaapi"; - hw_config[IR].width = input->width = atoi(argv[3]); - hw_config[IR].height = input->height = atoi(argv[4]); - hw_config[IR].framerate = input->framerate = atoi(argv[5]); + hw_config[IR].width = input->width = atoi(argv[4]); + hw_config[IR].height = input->height = atoi(argv[5]); + hw_config[IR].framerate = input->framerate = atoi(argv[6]); - hw_config[IR].device = argv[7]; //NULL as last argv argument, or device path + hw_config[IR].device = argv[8]; //NULL as last argv argument, or device path - if(argc > 9) - hw_config[IR].bit_rate = atoi(argv[9]); + if(argc > 10) + hw_config[IR].bit_rate = atoi(argv[10]); //set highest quality and slowest encoding //this adds around 3 ms and 10% GPU usage on my 2017 KabyLake @@ -347,15 +367,15 @@ int process_user_input(int argc, char* argv[], input_args* input, nhve_net_confi //optionally set gop_size (determines keyframes period) //hw_config[].gop_size = ...; - if(argc > 10) - input->depth_units = strtof(argv[10], NULL); - if(argc > 11) + input->depth_units = strtof(argv[11], NULL); + + if(argc > 12) { - ifstream file(argv[11]); + ifstream file(argv[12]); if(!file) { - cerr << "unable to open file " << argv[11] << endl; + cerr << "unable to open file " << argv[12] << endl; return -1; } @@ -370,7 +390,7 @@ int process_user_input(int argc, char* argv[], input_args* input, nhve_net_confi int hint_user_on_failure(char *argv[]) { cerr << "unable to initalize, try to specify device e.g:" << endl << endl; - cerr << argv[0] << " 127.0.0.1 9766 640 360 30 5 /dev/dri/renderD128" << endl; - cerr << argv[0] << " 127.0.0.1 9766 640 360 30 5 /dev/dri/renderD129" << endl; + cerr << argv[0] << " 127.0.0.1 9766 ir 640 360 30 5 /dev/dri/renderD128" << endl; + cerr << argv[0] << " 127.0.0.1 9766 ir 640 360 30 5 /dev/dri/renderD129" << endl; return -1; } diff --git a/rnhve_h264.cpp b/rnhve_h264.cpp index cf5a278..7ca4ecb 100644 --- a/rnhve_h264.cpp +++ b/rnhve_h264.cpp @@ -22,6 +22,8 @@ using namespace std; int hint_user_on_failure(char *argv[]); +enum StreamType {COLOR, INFRARED, INFRARED_RGB}; + //user supplied input struct input_args { @@ -29,7 +31,7 @@ struct input_args int height; int framerate; int seconds; - bool stream_color; //true for color, infrared otherwise + StreamType stream; }; bool main_loop(const input_args& input, rs2::pipeline& realsense, nhve *streamer); @@ -73,17 +75,17 @@ bool main_loop(const input_args& input, rs2::pipeline& realsense, nhve *streamer const int frames = input.seconds * input.framerate; int f; nhve_frame frame = {0}; - uint8_t *color_data = NULL; //data of dummy color plane for NV12 + uint8_t *color_data = NULL; //data of dummy color plane for NV12 with Realsense infrared for(f = 0; f < frames; ++f) { rs2::frameset frameset = realsense.wait_for_frames(); - rs2::video_frame video_frame = input.stream_color ? frameset.get_color_frame() : frameset.get_infrared_frame(1); + rs2::video_frame video_frame = (input.stream == COLOR) ? frameset.get_color_frame() : frameset.get_infrared_frame(0); - if(!input.stream_color && !color_data) - { //prepare dummy color plane for NV12 format, half the size of Y - //we can't alloc it in advance, this is the first time we know realsense stride + if(input.stream == INFRARED && !color_data) + { //prepare dummy color plane for NV12 format, half the size of Y + //we can't alloc it in advance, this is the first time we know realsense stride int size = video_frame.get_stride_in_bytes()*video_frame.get_height()/2; color_data = new uint8_t[size]; memset(color_data, 128, size); @@ -93,7 +95,7 @@ bool main_loop(const input_args& input, rs2::pipeline& realsense, nhve *streamer frame.data[0] = (uint8_t*) video_frame.get_data(); //if we are streaming infrared we have 2 planes (luminance and color) - frame.linesize[1] = input.stream_color ? 0 : frame.linesize[0]; + frame.linesize[1] = (input.stream == INFRARED) ? frame.linesize[0] : 0; frame.data[1] = color_data; //dummy color plane for infrared if(nhve_send(streamer, &frame, 0) != NHVE_OK) @@ -116,13 +118,12 @@ void init_realsense(rs2::pipeline& pipe, const input_args& input) { rs2::config cfg; - if(input.stream_color) + if(input.stream == COLOR) cfg.enable_stream(RS2_STREAM_COLOR, input.width, input.height, RS2_FORMAT_YUYV, input.framerate); - else - {// depth stream seems to be required for infrared to work - cfg.enable_stream(RS2_STREAM_DEPTH, input.width, input.height, RS2_FORMAT_Z16, input.framerate); + else if(input.stream == INFRARED) cfg.enable_stream(RS2_STREAM_INFRARED, input.width, input.height, RS2_FORMAT_Y8, input.framerate); - } + else //INFRARED_RGB + cfg.enable_stream(RS2_STREAM_INFRARED, input.width, input.height, RS2_FORMAT_UYVY, input.framerate); rs2::pipeline_profile profile = pipe.start(cfg); } @@ -131,12 +132,14 @@ int process_user_input(int argc, char* argv[], input_args* input, nhve_net_confi { if(argc < 8) { - cerr << "Usage: " << argv[0] << " [device] [bitrate]" << endl; + cerr << "Usage: " << argv[0] << " [device] [bitrate]" << endl; cerr << endl << "examples: " << endl; cerr << argv[0] << " 127.0.0.1 9766 color 640 360 30 5" << endl; - cerr << argv[0] << " 127.0.0.1 9766 infrared 640 360 30 5" << endl; + cerr << argv[0] << " 127.0.0.1 9766 ir 640 360 30 5" << endl; + cerr << argv[0] << " 127.0.0.1 9766 ir-rgb 640 360 30 5" << endl; cerr << argv[0] << " 127.0.0.1 9766 color 640 360 30 5 /dev/dri/renderD128" << endl; - cerr << argv[0] << " 127.0.0.1 9766 infrared 640 360 30 5 /dev/dri/renderD128" << endl; + cerr << argv[0] << " 127.0.0.1 9766 ir 640 360 30 5 /dev/dri/renderD128" << endl; + cerr << argv[0] << " 127.0.0.1 9766 ir-rgb 640 360 30 5 /dev/dri/renderD128" << endl; cerr << argv[0] << " 192.168.0.125 9766 color 640 360 30 50 /dev/dri/renderD128 500000" << endl; return -1; @@ -145,17 +148,30 @@ int process_user_input(int argc, char* argv[], input_args* input, nhve_net_confi net_config->ip = argv[1]; net_config->port = atoi(argv[2]); - input->stream_color = argv[3][0] == 'c'; + input->stream = COLOR; + if(argv[3][0] == 'i') + input->stream = INFRARED; + if(strlen(argv[3]) > 3 && argv[3][2] == '-') + input->stream = INFRARED_RGB; //native format of Realsense RGB sensor is YUYV (YUY2, YUYV422) //see https://github.com/IntelRealSense/librealsense/issues/3042 + //Realsense datasheet mentions uyvy format for IR rgb data + //see https://dev.intelrealsense.com/docs/intel-realsense-d400-series-product-family-datasheet //on the other hand native format for VAAPI is nv12 //we will match: //- Realsense RGB sensor YUYV with VAAPI YUYV422 (same format) //- Realsense IR sensor Y8 with VAAPI NV12 (luminance plane with dummy color plane) + //- Realsense IR sensor rgb data UYVY with VAAPI uyvy422 //this way we always have optimal format at least on one side and hardware conversion on other - hw_config->pixel_format = input->stream_color ? "yuyv422" : "nv12"; + hw_config->pixel_format = "yuyv422"; + + if(input->stream == INFRARED) + hw_config->pixel_format = "nv12"; + else if(input->stream == INFRARED_RGB) + hw_config->pixel_format = "uyvy422"; + hw_config->encoder = "h264_vaapi"; hw_config->width = input->width = atoi(argv[4]); hw_config->height = input->height = atoi(argv[5]); @@ -186,6 +202,7 @@ int hint_user_on_failure(char *argv[]) { cerr << "unable to initalize, try to specify device e.g:" << endl << endl; cerr << argv[0] << " 127.0.0.1 9766 color 640 360 30 5 /dev/dri/renderD128" << endl; - cerr << argv[0] << " 127.0.0.1 9766 infrared 640 360 30 5 /dev/dri/renderD128" << endl; + cerr << argv[0] << " 127.0.0.1 9766 ir 640 360 30 5 /dev/dri/renderD128" << endl; + cerr << argv[0] << " 127.0.0.1 9766 ir-rgb 640 360 30 5 /dev/dri/renderD128" << endl; return -1; } diff --git a/rnhve_hevc.cpp b/rnhve_hevc.cpp index bfbcc04..785a9cd 100644 --- a/rnhve_hevc.cpp +++ b/rnhve_hevc.cpp @@ -2,7 +2,7 @@ * Realsense Network Hardware Video Encoder * * Realsense hardware encoded UDP HEVC streaming - * - color/infrared (Main) + * - color/infrared/infrared rgb (Main) * - depth (Main10) * * Copyright 2020 (C) Bartosz Meglicki @@ -28,7 +28,7 @@ using namespace std; int hint_user_on_failure(char *argv[]); -enum StreamType {COLOR, INFRARED, DEPTH}; +enum StreamType {COLOR, INFRARED, INFRARED_RGB, DEPTH}; //user supplied input struct input_args @@ -78,10 +78,10 @@ int main(int argc, char* argv[]) bool status = false; - if(user_input.stream == COLOR || user_input.stream == INFRARED) - status = main_loop_color_infrared(user_input, realsense, streamer); if(user_input.stream == DEPTH) status = main_loop_depth(user_input, realsense, streamer); + else //color, infrared, infrared rgb + status = main_loop_color_infrared(user_input, realsense, streamer); nhve_close(streamer); @@ -97,17 +97,17 @@ bool main_loop_color_infrared(const input_args& input, rs2::pipeline& realsense, const int frames = input.seconds * input.framerate; int f; nhve_frame frame = {0}; - uint8_t *color_data = NULL; //data of dummy color plane for NV12 + uint8_t *color_data = NULL; //data of dummy color plane for NV12 with Realsense infrared for(f = 0; f < frames; ++f) { rs2::frameset frameset = realsense.wait_for_frames(); - rs2::video_frame video_frame = input.stream == COLOR ? frameset.get_color_frame() : frameset.get_infrared_frame(1); + rs2::video_frame video_frame = (input.stream == COLOR) ? frameset.get_color_frame() : frameset.get_infrared_frame(0); - if(!(input.stream == COLOR) && !color_data) - { //prepare dummy color plane for NV12 format, half the size of Y - //we can't alloc it in advance, this is the first time we know realsense stride + if(input.stream == INFRARED && !color_data) + { //prepare dummy color plane for NV12 format, half the size of Y + //we can't alloc it in advance, this is the first time we know realsense stride int size = video_frame.get_stride_in_bytes()*video_frame.get_height()/2; color_data = new uint8_t[size]; memset(color_data, 128, size); @@ -116,8 +116,8 @@ bool main_loop_color_infrared(const input_args& input, rs2::pipeline& realsense, frame.linesize[0] = video_frame.get_stride_in_bytes(); frame.data[0] = (uint8_t*) video_frame.get_data(); - //if we are streaming infrared we have 2 planes (luminance and color) - frame.linesize[1] = (input.stream == COLOR) ? 0 : frame.linesize[0]; + //if we are streaming infrared we have 2 planes (luminance and dummy color) + frame.linesize[1] = (input.stream == INFRARED) ? frame.linesize[0] : 0; frame.data[1] = color_data; //dummy color plane for infrared if(nhve_send(streamer, &frame, 0) != NHVE_OK) @@ -212,10 +212,9 @@ void init_realsense(rs2::pipeline& pipe, input_args& input) if(input.stream == COLOR) cfg.enable_stream(RS2_STREAM_COLOR, input.width, input.height, RS2_FORMAT_YUYV, input.framerate); else if(input.stream == INFRARED) - {// depth stream seems to be required for infrared to work - cfg.enable_stream(RS2_STREAM_DEPTH, input.width, input.height, RS2_FORMAT_Z16, input.framerate); cfg.enable_stream(RS2_STREAM_INFRARED, input.width, input.height, RS2_FORMAT_Y8, input.framerate); - } + else if(input.stream == INFRARED_RGB) + cfg.enable_stream(RS2_STREAM_INFRARED, input.width, input.height, RS2_FORMAT_UYVY, input.framerate); else if(input.stream == DEPTH) cfg.enable_stream(RS2_STREAM_DEPTH, input.width, input.height, RS2_FORMAT_Z16, input.framerate); @@ -315,13 +314,15 @@ int process_user_input(int argc, char* argv[], input_args* input, nhve_net_confi { if(argc < 8) { - cerr << "Usage: " << argv[0] << " [device] [bitrate] [depth units] [json]" << endl; + cerr << "Usage: " << argv[0] << " [device] [bitrate] [depth units] [json]" << endl; cerr << endl << "examples: " << endl; cerr << argv[0] << " 127.0.0.1 9766 color 640 360 30 5" << endl; - cerr << argv[0] << " 127.0.0.1 9766 infrared 640 360 30 5" << endl; + cerr << argv[0] << " 127.0.0.1 9766 ir 640 360 30 5" << endl; + cerr << argv[0] << " 127.0.0.1 9766 ir-rgb 640 360 30 5" << endl; cerr << argv[0] << " 127.0.0.1 9766 depth 640 360 30 5" << endl; cerr << argv[0] << " 127.0.0.1 9766 color 640 360 30 5 /dev/dri/renderD128" << endl; - cerr << argv[0] << " 127.0.0.1 9766 infrared 640 360 30 5 /dev/dri/renderD128" << endl; + cerr << argv[0] << " 127.0.0.1 9766 ir 640 360 30 5 /dev/dri/renderD128" << endl; + cerr << argv[0] << " 127.0.0.1 9766 ir-rgb 640 360 30 5 /dev/dri/renderD128" << endl; cerr << argv[0] << " 127.0.0.1 9766 depth 640 360 30 5 /dev/dri/renderD128" << endl; cerr << argv[0] << " 192.168.0.125 9766 color 640 360 30 50 /dev/dri/renderD128 500000" << endl; cerr << argv[0] << " 127.0.0.1 9768 depth 848 480 30 50 /dev/dri/renderD128 2000000" << endl; @@ -339,8 +340,13 @@ int process_user_input(int argc, char* argv[], input_args* input, nhve_net_confi char c = argv[3][0]; //color, infrared, depth if(c == 'c') input->stream = COLOR; - else if(c == 'i') input->stream = INFRARED; else if(c == 'd') input->stream = DEPTH; + else if(c == 'i') + { + input->stream = INFRARED; + if(strlen(argv[3]) > 3 && argv[3][2] == '-') + input->stream = INFRARED_RGB; + } else { cerr << "unknown stream: " << argv[3]; @@ -349,11 +355,14 @@ int process_user_input(int argc, char* argv[], input_args* input, nhve_net_confi //native format of Realsense RGB sensor is YUYV (YUY2, YUYV422) //see https://github.com/IntelRealSense/librealsense/issues/3042 + //Realsense datasheet mentions uyvy format for IR rgb data + //see https://dev.intelrealsense.com/docs/intel-realsense-d400-series-product-family-datasheet //on the other hand native format for VAAPI is nv12 //we will match: //- Realsense RGB sensor YUYV with VAAPI YUYV422 (same format) //- Realsense IR sensor Y8 with VAAPI NV12 (luminance plane with dummy color plane) + //- Realsense IR sensor rgb data UYVY with VAAPI uyvy422 //this way we always have optimal format at least on one side and hardware conversion on other //for depth encoding we use 10 bit P010LE pixel format @@ -368,6 +377,8 @@ int process_user_input(int argc, char* argv[], input_args* input, nhve_net_confi hw_config->pixel_format = "yuyv422"; else if(input->stream == INFRARED) hw_config->pixel_format = "nv12"; + else if(input->stream == INFRARED_RGB) + hw_config->pixel_format = "uyvy422"; else //DEPTH { hw_config->pixel_format = "p010le"; @@ -421,7 +432,8 @@ int hint_user_on_failure(char *argv[]) { cerr << "unable to initalize, try to specify device e.g:" << endl << endl; cerr << argv[0] << " 127.0.0.1 9766 color 640 360 30 5 /dev/dri/renderD128" << endl; - cerr << argv[0] << " 127.0.0.1 9766 infrared 640 360 30 5 /dev/dri/renderD128" << endl; + cerr << argv[0] << " 127.0.0.1 9766 ir 640 360 30 5 /dev/dri/renderD128" << endl; + cerr << argv[0] << " 127.0.0.1 9766 ir-rgb 640 360 30 5 /dev/dri/renderD128" << endl; cerr << argv[0] << " 127.0.0.1 9766 depth 640 360 30 5 /dev/dri/renderD128" << endl; return -1; }