From 40571ff8e15b30ecee01f02a1391effa0346a2b4 Mon Sep 17 00:00:00 2001 From: Bartosz Meglicki Date: Fri, 28 Feb 2020 09:30:16 +0100 Subject: [PATCH] textured depth streaming (#5) Implementation with: - depth encoded in luminance plane - infrared encoded in chroma U/V plane Requires at least KabyLake architecture. Textured depth is implemented only for D435. It is possible to also make it working for D415 (greyscale infrared, not implemented now). Closes #2 implements encoding side of [hardware-video-streaming#2](https://github.com/bmegli/hardware-video-streaming/issues/2) --- README.md | 13 +++++++-- rnhve_hevc.cpp | 73 ++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 76 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 409feda..a3dbb1b 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ This includes streaming: - color (H.264, HEVC Main) - infrared (H.264, HEVC Main) - depth (HEVC Main10) +- textured depth (HEVC Main10) See [unity-network-hardware-video-decoder](https://github.com/bmegli/unity-network-hardware-video-decoder) as example network decoder & renderer (color, infrared and depth). @@ -33,7 +34,11 @@ ATI/AMD may also work through VAAPI (libva-mesa-driver, not tested however). The dependency is through [HVE](https://github.com/bmegli/hardware-video-encoder) implementation (see [HVE issues](https://github.com/bmegli/hardware-video-encoder/issues/5)). -Tested on LattePanda Alpha and i7-7820HK laptop. +Depth encoding (HEVC Main10) requires at least Intel KabyLake. + +Textured depth encoding is implemented only for D435. + +Tested on LattePanda Alpha and i7-7820HK laptop with Realsense D435 camera. ## Dependencies @@ -92,18 +97,22 @@ Stream H.264 Realsense color/infrared video over UDP. Stream Realsense: - color/infrared with HEVC Main - depth with HEVC Main10 +- textured depth with HEVC Main10 ```bash -# Usage: ./realsense-nhve-hevc [device] [bitrate] [depth units] +# Usage: ./realsense-nhve-hevc [device] [bitrate] [depth units] ./realsense-nhve-hevc 127.0.0.1 9766 color 640 360 30 5 #./realsense-nhve-hevc 127.0.0.1 9766 infrared 640 360 30 5 #./realsense-nhve-hevc 127.0.0.1 9766 depth 640 360 30 5 +#./realsense-nhve-hevc 127.0.0.1 9766 depth+ir 848 480 30 5 #./realsense-nhve-hevc 127.0.0.1 9766 color 640 360 30 5 /dev/dri/renderD128 #./realsense-nhve-hevc 127.0.0.1 9766 infrared 640 360 30 5 /dev/dri/renderD128 #./realsense-nhve-hevc 127.0.0.1 9766 depth 640 360 30 5 /dev/dri/renderD128 +#./realsense-nhve-hevc 127.0.0.1 9766 depth+ir 848 480 30 5 /dev/dri/renderD128 #./realsense-nhve-hevc 192.168.0.125 9766 color 640 360 30 50 /dev/dri/renderD128 500000 #./realsense-nhve-hevc 192.168.0.125 9768 depth 848 480 30 50 /dev/dri/renderD128 2000000 #./realsense-nhve-hevc 192.168.0.125 9768 depth 848 480 30 50 /dev/dri/renderD128 8000000 0.0001 +#./realsense-nhve-hevc 192.168.0.100 9768 depth+ir 848 480 30 50 /dev/dri/renderD128 8000000 0.0001 ``` You may need to specify VAAPI device if you have more than one (e.g. NVIDIA GPU + Intel CPU). diff --git a/rnhve_hevc.cpp b/rnhve_hevc.cpp index 2950875..54c9a90 100644 --- a/rnhve_hevc.cpp +++ b/rnhve_hevc.cpp @@ -3,7 +3,8 @@ * * Realsense hardware encoded UDP HEVC streaming * - color/infrared (Main) - * - depth streaming (Main10) + * - depth (Main10) + * - depth with infrared (Main10) * * Copyright 2020 (C) Bartosz Meglicki * @@ -22,11 +23,12 @@ #include #include +#include using namespace std; int hint_user_on_failure(char *argv[]); -enum StreamType {COLOR, INFRARED, DEPTH}; +enum StreamType {COLOR, INFRARED, DEPTH, DEPTH_WITH_INFRARED}; //user supplied input struct input_args @@ -41,6 +43,7 @@ struct input_args bool main_loop_color_infrared(const input_args& input, rs2::pipeline& realsense, nhve *streamer); bool main_loop_depth(const input_args& input, rs2::pipeline& realsense, nhve *streamer); +bool main_loop_depth_with_infrared(const input_args& input, rs2::pipeline& realsense, nhve *streamer); void init_realsense(rs2::pipeline& pipe, const input_args& input); int process_user_input(int argc, char* argv[], input_args* input, nhve_net_config *net_config, nhve_hw_config *hw_config); @@ -48,7 +51,6 @@ const uint16_t P010LE_MAX = 0xFFC0; //in binary 10 ones followed by 6 zeroes int main(int argc, char* argv[]) { - //struct nhve_hw_config hw_config = {WIDTH, HEIGHT, FRAMERATE, DEVICE, ENCODER, PIXEL_FORMAT, PROFILE, BFRAMES, BITRATE}; //prepare NHVE Network Hardware Video Encoder struct nhve_net_config net_config = {0}; struct nhve_hw_config hw_config = {0}; @@ -73,6 +75,8 @@ int main(int argc, char* argv[]) status = main_loop_color_infrared(user_input, realsense, streamer); if(user_input.stream == DEPTH) status = main_loop_depth(user_input, realsense, streamer); + if(user_input.stream == DEPTH_WITH_INFRARED) + status = main_loop_depth_with_infrared(user_input, realsense, streamer); nhve_close(streamer); @@ -178,13 +182,55 @@ bool main_loop_depth(const input_args& input, rs2::pipeline& realsense, nhve *st return f==frames; } +//true on success, false on failure +bool main_loop_depth_with_infrared(const input_args& input, rs2::pipeline& realsense, nhve *streamer) +{ + const int frames = input.seconds * input.framerate; + int f; + nhve_frame frame = {0}; + + for(f = 0; f < frames; ++f) + { + rs2::frameset frameset = realsense.wait_for_frames(); + rs2::depth_frame depth = frameset.get_depth_frame(); + rs2::video_frame video = frameset.get_infrared_frame(1); + + const int w = depth.get_width(); + const int h = depth.get_height(); + const int stride=depth.get_stride_in_bytes(); + const int ir_stride=video.get_stride_in_bytes(); + + assert(video.get_width() == ir_stride && 2*ir_stride == stride); + + //supply realsense frame data as ffmpeg frame data + frame.linesize[0] = frame.linesize[1] = stride; //the stride of Y and interleaved UV is equal + frame.data[0] = (uint8_t*) depth.get_data(); + frame.data[1] = (uint8_t*) video.get_data(); + + frame.framenumber = f; + + if(nhve_send_frame(streamer, &frame) != NHVE_OK) + { + cerr << "failed to send" << endl; + break; + } + } + + //flush the streamer by sending NULL frame + nhve_send_frame(streamer, NULL); + + //all the requested frames processed? + return f==frames; +} + + void init_realsense(rs2::pipeline& pipe, const input_args& input) { rs2::config cfg; if(input.stream == COLOR) cfg.enable_stream(RS2_STREAM_COLOR, input.width, input.height, RS2_FORMAT_YUYV, input.framerate); - else if(input.stream == INFRARED) + else if(input.stream == INFRARED || input.stream == DEPTH_WITH_INFRARED) {// depth stream seems to be required for infrared to work cfg.enable_stream(RS2_STREAM_DEPTH, input.width, input.height, RS2_FORMAT_Z16, input.framerate); cfg.enable_stream(RS2_STREAM_INFRARED, 1, input.width, input.height, RS2_FORMAT_Y8, input.framerate); @@ -194,7 +240,7 @@ void init_realsense(rs2::pipeline& pipe, const input_args& input) rs2::pipeline_profile profile = pipe.start(cfg); - if(input.stream != DEPTH) + if(input.stream != DEPTH && input.stream != DEPTH_WITH_INFRARED) return; rs2::depth_sensor depth_sensor = profile.get_device().first(); @@ -245,7 +291,7 @@ int process_user_input(int argc, char* argv[], input_args* input, nhve_net_confi { if(argc < 8) { - cerr << "Usage: " << argv[0] << " [device] [bitrate] [depth units]" << endl; + cerr << "Usage: " << argv[0] << " [device] [bitrate] [depth units]" << endl; cerr << endl << "examples: " << endl; cerr << argv[0] << " 127.0.0.1 9766 color 640 360 30 5" << endl; cerr << argv[0] << " 127.0.0.1 9766 infrared 640 360 30 5" << endl; @@ -253,12 +299,15 @@ int process_user_input(int argc, char* argv[], input_args* input, nhve_net_confi cerr << argv[0] << " 127.0.0.1 9766 color 640 360 30 5 /dev/dri/renderD128" << endl; cerr << argv[0] << " 127.0.0.1 9766 infrared 640 360 30 5 /dev/dri/renderD128" << endl; cerr << argv[0] << " 127.0.0.1 9766 depth 640 360 30 5 /dev/dri/renderD128" << endl; + cerr << argv[0] << " 127.0.0.1 9766 depth+ir 640 360 30 5 /dev/dri/renderD128" << endl; cerr << argv[0] << " 192.168.0.125 9766 color 640 360 30 50 /dev/dri/renderD128 500000" << endl; cerr << argv[0] << " 127.0.0.1 9768 depth 848 480 30 50 /dev/dri/renderD128 2000000" << endl; cerr << argv[0] << " 192.168.0.100 9768 depth 848 480 30 500 /dev/dri/renderD128 2000000 0.0001" << endl; cerr << argv[0] << " 192.168.0.100 9768 depth 848 480 30 500 /dev/dri/renderD128 2000000 0.00005" << endl; cerr << argv[0] << " 192.168.0.100 9768 depth 848 480 30 500 /dev/dri/renderD128 2000000 0.000025" << endl; cerr << argv[0] << " 192.168.0.100 9768 depth 848 480 30 500 /dev/dri/renderD128 2000000 0.0000125" << endl; + cerr << argv[0] << " 192.168.0.100 9768 depth+ir 848 480 30 500 /dev/dri/renderD128 2000000 0.0000125" << endl; + cerr << argv[0] << " 192.168.0.100 9768 depth+ir 848 480 30 500 /dev/dri/renderD128 8000000 0.00003125f" << endl; return -1; } @@ -269,7 +318,8 @@ int process_user_input(int argc, char* argv[], input_args* input, nhve_net_confi char c = argv[3][0]; //color, infrared, depth if(c == 'c') input->stream = COLOR; else if(c == 'i') input->stream = INFRARED; - else if(c == 'd') input->stream = DEPTH; + else if(c == 'd') + input->stream = (strlen(argv[3]) <= 5) ? DEPTH : DEPTH_WITH_INFRARED; else { cerr << "unknown stream: " << argv[3]; @@ -291,13 +341,19 @@ int process_user_input(int argc, char* argv[], input_args* input, nhve_net_confi //for explanation see: //https://github.com/bmegli/realsense-depth-to-vaapi-hevc10/wiki/How-it-works + //for depth with infrared encoding we use 10 bit P010LE pixel format + //with depth encoded as above + //the infrared plane is encoded in chroma U/V plane + //for explanation see: + //https://github.com/bmegli/hardware-video-streaming/issues/2 + hw_config->profile = FF_PROFILE_HEVC_MAIN; if(input->stream == COLOR) hw_config->pixel_format = "yuyv422"; else if(input->stream == INFRARED) hw_config->pixel_format = "nv12"; - else //DEPTH + else //DEPTH, DEPTH_WITH_INFRARED { hw_config->pixel_format = "p010le"; hw_config->profile = FF_PROFILE_HEVC_MAIN_10; @@ -327,5 +383,6 @@ int hint_user_on_failure(char *argv[]) cerr << argv[0] << " 127.0.0.1 9766 color 640 360 30 5 /dev/dri/renderD128" << endl; cerr << argv[0] << " 127.0.0.1 9766 infrared 640 360 30 5 /dev/dri/renderD128" << endl; cerr << argv[0] << " 127.0.0.1 9766 depth 640 360 30 5 /dev/dri/renderD128" << endl; + cerr << argv[0] << " 127.0.0.1 9766 depth+infrared 640 360 30 5 /dev/dri/renderD128" << endl; return -1; }