Skip to content

Commit

Permalink
textured depth streaming (#5)
Browse files Browse the repository at this point in the history 
Implementation with:
- depth encoded in luminance plane
- infrared encoded in chroma U/V plane

Requires at least KabyLake architecture.
Textured depth is implemented only for D435.
It is possible to also make it working for D415 (greyscale infrared, not implemented now).

Closes #2
implements encoding side of [hardware-video-streaming#2](bmegli/hardware-video-streaming#2)
  • Loading branch information
@bmegli
bmegli authored Feb 28, 2020
1 parent 150be7e commit 40571ff
Show file tree
Hide file tree
Showing 2 changed files with 76 additions and 10 deletions.
13 changes: 11 additions & 2 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ This includes streaming:
- color (H.264, HEVC Main)
- infrared (H.264, HEVC Main)
- depth (HEVC Main10)
- textured depth (HEVC Main10)

See [unity-network-hardware-video-decoder](https://github.com/bmegli/unity-network-hardware-video-decoder) as example network decoder & renderer (color, infrared and depth).

Expand Down Expand Up @@ -33,7 +34,11 @@ ATI/AMD may also work through VAAPI (libva-mesa-driver, not tested however).

The dependency is through [HVE](https://github.com/bmegli/hardware-video-encoder) implementation (see [HVE issues](https://github.com/bmegli/hardware-video-encoder/issues/5)).

Tested on LattePanda Alpha and i7-7820HK laptop.
Depth encoding (HEVC Main10) requires at least Intel KabyLake.

Textured depth encoding is implemented only for D435.

Tested on LattePanda Alpha and i7-7820HK laptop with Realsense D435 camera.

## Dependencies

Expand Down Expand Up @@ -92,18 +97,22 @@ Stream H.264 Realsense color/infrared video over UDP.
Stream Realsense:
- color/infrared with HEVC Main
- depth with HEVC Main10
- textured depth with HEVC Main10

```bash
# Usage: ./realsense-nhve-hevc <host> <port> <color/ir/depth> <width> <height> <framerate> <seconds> [device] [bitrate] [depth units]
# Usage: ./realsense-nhve-hevc <host> <port> <color/ir/depth/depth+ir> <width> <height> <framerate> <seconds> [device] [bitrate] [depth units]
./realsense-nhve-hevc 127.0.0.1 9766 color 640 360 30 5
#./realsense-nhve-hevc 127.0.0.1 9766 infrared 640 360 30 5
#./realsense-nhve-hevc 127.0.0.1 9766 depth 640 360 30 5
#./realsense-nhve-hevc 127.0.0.1 9766 depth+ir 848 480 30 5
#./realsense-nhve-hevc 127.0.0.1 9766 color 640 360 30 5 /dev/dri/renderD128
#./realsense-nhve-hevc 127.0.0.1 9766 infrared 640 360 30 5 /dev/dri/renderD128
#./realsense-nhve-hevc 127.0.0.1 9766 depth 640 360 30 5 /dev/dri/renderD128
#./realsense-nhve-hevc 127.0.0.1 9766 depth+ir 848 480 30 5 /dev/dri/renderD128
#./realsense-nhve-hevc 192.168.0.125 9766 color 640 360 30 50 /dev/dri/renderD128 500000
#./realsense-nhve-hevc 192.168.0.125 9768 depth 848 480 30 50 /dev/dri/renderD128 2000000
#./realsense-nhve-hevc 192.168.0.125 9768 depth 848 480 30 50 /dev/dri/renderD128 8000000 0.0001
#./realsense-nhve-hevc 192.168.0.100 9768 depth+ir 848 480 30 50 /dev/dri/renderD128 8000000 0.0001
```

You may need to specify VAAPI device if you have more than one (e.g. NVIDIA GPU + Intel CPU).
Expand Down
73 changes: 65 additions & 8 deletions rnhve_hevc.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@
*
* Realsense hardware encoded UDP HEVC streaming
* - color/infrared (Main)
* - depth streaming (Main10)
* - depth (Main10)
* - depth with infrared (Main10)
*
* Copyright 2020 (C) Bartosz Meglicki <[email protected]>
*
Expand All @@ -22,11 +23,12 @@

#include <fstream>
#include <iostream>
#include <cassert>
using namespace std;

int hint_user_on_failure(char *argv[]);

enum StreamType {COLOR, INFRARED, DEPTH};
enum StreamType {COLOR, INFRARED, DEPTH, DEPTH_WITH_INFRARED};

//user supplied input
struct input_args
Expand All @@ -41,14 +43,14 @@ struct input_args

bool main_loop_color_infrared(const input_args& input, rs2::pipeline& realsense, nhve *streamer);
bool main_loop_depth(const input_args& input, rs2::pipeline& realsense, nhve *streamer);
bool main_loop_depth_with_infrared(const input_args& input, rs2::pipeline& realsense, nhve *streamer);
void init_realsense(rs2::pipeline& pipe, const input_args& input);
int process_user_input(int argc, char* argv[], input_args* input, nhve_net_config *net_config, nhve_hw_config *hw_config);

const uint16_t P010LE_MAX = 0xFFC0; //in binary 10 ones followed by 6 zeroes

int main(int argc, char* argv[])
{
//struct nhve_hw_config hw_config = {WIDTH, HEIGHT, FRAMERATE, DEVICE, ENCODER, PIXEL_FORMAT, PROFILE, BFRAMES, BITRATE};
//prepare NHVE Network Hardware Video Encoder
struct nhve_net_config net_config = {0};
struct nhve_hw_config hw_config = {0};
Expand All @@ -73,6 +75,8 @@ int main(int argc, char* argv[])
status = main_loop_color_infrared(user_input, realsense, streamer);
if(user_input.stream == DEPTH)
status = main_loop_depth(user_input, realsense, streamer);
if(user_input.stream == DEPTH_WITH_INFRARED)
status = main_loop_depth_with_infrared(user_input, realsense, streamer);

nhve_close(streamer);

Expand Down Expand Up @@ -178,13 +182,55 @@ bool main_loop_depth(const input_args& input, rs2::pipeline& realsense, nhve *st
return f==frames;
}

//true on success, false on failure
bool main_loop_depth_with_infrared(const input_args& input, rs2::pipeline& realsense, nhve *streamer)
{
const int frames = input.seconds * input.framerate;
int f;
nhve_frame frame = {0};

for(f = 0; f < frames; ++f)
{
rs2::frameset frameset = realsense.wait_for_frames();
rs2::depth_frame depth = frameset.get_depth_frame();
rs2::video_frame video = frameset.get_infrared_frame(1);

const int w = depth.get_width();
const int h = depth.get_height();
const int stride=depth.get_stride_in_bytes();
const int ir_stride=video.get_stride_in_bytes();

assert(video.get_width() == ir_stride && 2*ir_stride == stride);

//supply realsense frame data as ffmpeg frame data
frame.linesize[0] = frame.linesize[1] = stride; //the stride of Y and interleaved UV is equal
frame.data[0] = (uint8_t*) depth.get_data();
frame.data[1] = (uint8_t*) video.get_data();

frame.framenumber = f;

if(nhve_send_frame(streamer, &frame) != NHVE_OK)
{
cerr << "failed to send" << endl;
break;
}
}

//flush the streamer by sending NULL frame
nhve_send_frame(streamer, NULL);

//all the requested frames processed?
return f==frames;
}


void init_realsense(rs2::pipeline& pipe, const input_args& input)
{
rs2::config cfg;

if(input.stream == COLOR)
cfg.enable_stream(RS2_STREAM_COLOR, input.width, input.height, RS2_FORMAT_YUYV, input.framerate);
else if(input.stream == INFRARED)
else if(input.stream == INFRARED || input.stream == DEPTH_WITH_INFRARED)
{// depth stream seems to be required for infrared to work
cfg.enable_stream(RS2_STREAM_DEPTH, input.width, input.height, RS2_FORMAT_Z16, input.framerate);
cfg.enable_stream(RS2_STREAM_INFRARED, 1, input.width, input.height, RS2_FORMAT_Y8, input.framerate);
Expand All @@ -194,7 +240,7 @@ void init_realsense(rs2::pipeline& pipe, const input_args& input)

rs2::pipeline_profile profile = pipe.start(cfg);

if(input.stream != DEPTH)
if(input.stream != DEPTH && input.stream != DEPTH_WITH_INFRARED)
return;

rs2::depth_sensor depth_sensor = profile.get_device().first<rs2::depth_sensor>();
Expand Down Expand Up @@ -245,20 +291,23 @@ int process_user_input(int argc, char* argv[], input_args* input, nhve_net_confi
{
if(argc < 8)
{
cerr << "Usage: " << argv[0] << " <host> <port> <color/ir/depth> <width> <height> <framerate> <seconds> [device] [bitrate] [depth units]" << endl;
cerr << "Usage: " << argv[0] << " <host> <port> <color/ir/depth/depth+ir> <width> <height> <framerate> <seconds> [device] [bitrate] [depth units]" << endl;
cerr << endl << "examples: " << endl;
cerr << argv[0] << " 127.0.0.1 9766 color 640 360 30 5" << endl;
cerr << argv[0] << " 127.0.0.1 9766 infrared 640 360 30 5" << endl;
cerr << argv[0] << " 127.0.0.1 9766 depth 640 360 30 5" << endl;
cerr << argv[0] << " 127.0.0.1 9766 color 640 360 30 5 /dev/dri/renderD128" << endl;
cerr << argv[0] << " 127.0.0.1 9766 infrared 640 360 30 5 /dev/dri/renderD128" << endl;
cerr << argv[0] << " 127.0.0.1 9766 depth 640 360 30 5 /dev/dri/renderD128" << endl;
cerr << argv[0] << " 127.0.0.1 9766 depth+ir 640 360 30 5 /dev/dri/renderD128" << endl;
cerr << argv[0] << " 192.168.0.125 9766 color 640 360 30 50 /dev/dri/renderD128 500000" << endl;
cerr << argv[0] << " 127.0.0.1 9768 depth 848 480 30 50 /dev/dri/renderD128 2000000" << endl;
cerr << argv[0] << " 192.168.0.100 9768 depth 848 480 30 500 /dev/dri/renderD128 2000000 0.0001" << endl;
cerr << argv[0] << " 192.168.0.100 9768 depth 848 480 30 500 /dev/dri/renderD128 2000000 0.00005" << endl;
cerr << argv[0] << " 192.168.0.100 9768 depth 848 480 30 500 /dev/dri/renderD128 2000000 0.000025" << endl;
cerr << argv[0] << " 192.168.0.100 9768 depth 848 480 30 500 /dev/dri/renderD128 2000000 0.0000125" << endl;
cerr << argv[0] << " 192.168.0.100 9768 depth+ir 848 480 30 500 /dev/dri/renderD128 2000000 0.0000125" << endl;
cerr << argv[0] << " 192.168.0.100 9768 depth+ir 848 480 30 500 /dev/dri/renderD128 8000000 0.00003125f" << endl;

return -1;
}
Expand All @@ -269,7 +318,8 @@ int process_user_input(int argc, char* argv[], input_args* input, nhve_net_confi
char c = argv[3][0]; //color, infrared, depth
if(c == 'c') input->stream = COLOR;
else if(c == 'i') input->stream = INFRARED;
else if(c == 'd') input->stream = DEPTH;
else if(c == 'd')
input->stream = (strlen(argv[3]) <= 5) ? DEPTH : DEPTH_WITH_INFRARED;
else
{
cerr << "unknown stream: " << argv[3];
Expand All @@ -291,13 +341,19 @@ int process_user_input(int argc, char* argv[], input_args* input, nhve_net_confi
//for explanation see:
//https://github.com/bmegli/realsense-depth-to-vaapi-hevc10/wiki/How-it-works

//for depth with infrared encoding we use 10 bit P010LE pixel format
//with depth encoded as above
//the infrared plane is encoded in chroma U/V plane
//for explanation see:
//https://github.com/bmegli/hardware-video-streaming/issues/2

hw_config->profile = FF_PROFILE_HEVC_MAIN;

if(input->stream == COLOR)
hw_config->pixel_format = "yuyv422";
else if(input->stream == INFRARED)
hw_config->pixel_format = "nv12";
else //DEPTH
else //DEPTH, DEPTH_WITH_INFRARED
{
hw_config->pixel_format = "p010le";
hw_config->profile = FF_PROFILE_HEVC_MAIN_10;
Expand Down Expand Up @@ -327,5 +383,6 @@ int hint_user_on_failure(char *argv[])
cerr << argv[0] << " 127.0.0.1 9766 color 640 360 30 5 /dev/dri/renderD128" << endl;
cerr << argv[0] << " 127.0.0.1 9766 infrared 640 360 30 5 /dev/dri/renderD128" << endl;
cerr << argv[0] << " 127.0.0.1 9766 depth 640 360 30 5 /dev/dri/renderD128" << endl;
cerr << argv[0] << " 127.0.0.1 9766 depth+infrared 640 360 30 5 /dev/dri/renderD128" << endl;
return -1;
}

0 comments on commit 40571ff

Please sign in to comment.