From a781c17dd8cb5c66f716ddc1e00476769095e4fe Mon Sep 17 00:00:00 2001 From: Bartosz Meglicki Date: Tue, 31 Dec 2019 13:09:45 +0100 Subject: [PATCH] depth encoding to HEVC Main10 proof-of-concept - working proof of concept for encoding depth stream - into HEVC Main10, with p010le pixel format Looks great at first sight but this needs more tests. relevant to #9 --- hardware-video-encoder | 2 +- high_accuracy.json | 94 ++++++++++++++++++++++++++++++++++++++++++ main.cpp | 67 +++++++++++++++++++++++------- 3 files changed, 148 insertions(+), 15 deletions(-) create mode 100644 high_accuracy.json diff --git a/hardware-video-encoder b/hardware-video-encoder index 82a7010..ac3a4c1 160000 --- a/hardware-video-encoder +++ b/hardware-video-encoder @@ -1 +1 @@ -Subproject commit 82a7010a58c8204508b50e20ccde2ddf033e02cb +Subproject commit ac3a4c1eadc621b26794a02536681c2a13027606 diff --git a/high_accuracy.json b/high_accuracy.json new file mode 100644 index 0000000..74ee83d --- /dev/null +++ b/high_accuracy.json @@ -0,0 +1,94 @@ +{ + "aux-param-autoexposure-setpoint": "1536", + "aux-param-colorcorrection1": "0.298828", + "aux-param-colorcorrection10": "0", + "aux-param-colorcorrection11": "0", + "aux-param-colorcorrection12": "0", + "aux-param-colorcorrection2": "0.293945", + "aux-param-colorcorrection3": "0.293945", + "aux-param-colorcorrection4": "0.114258", + "aux-param-colorcorrection5": "0", + "aux-param-colorcorrection6": "0", + "aux-param-colorcorrection7": "0", + "aux-param-colorcorrection8": "0", + "aux-param-colorcorrection9": "0", + "aux-param-depthclampmax": "65536", + "aux-param-depthclampmin": "0", + "aux-param-disparityshift": "0", + "controls-autoexposure-auto": "True", + "controls-autoexposure-manual": "8500", + "controls-color-autoexposure-auto": "True", + "controls-color-autoexposure-manual": "166", + "controls-color-backlight-compensation": "0", + "controls-color-brightness": "0", + "controls-color-contrast": "50", + "controls-color-gain": "64", + "controls-color-gamma": "300", + "controls-color-hue": "0", + "controls-color-power-line-frequency": "3", + "controls-color-saturation": "64", + "controls-color-sharpness": "50", + "controls-color-white-balance-auto": "True", + "controls-color-white-balance-manual": "4600", + "controls-depth-gain": "16", + "controls-laserpower": "150", + "controls-laserstate": "on", + "ignoreSAD": "0", + "param-autoexposure-setpoint": "1536", + "param-censusenablereg-udiameter": "9", + "param-censusenablereg-vdiameter": "9", + "param-censususize": "9", + "param-censusvsize": "9", + "param-depthclampmax": "65536", + "param-depthclampmin": "0", + "param-depthunits": "100", + "param-disableraucolor": "0", + "param-disablesadcolor": "0", + "param-disablesadnormalize": "0", + "param-disablesloleftcolor": "0", + "param-disableslorightcolor": "1", + "param-disparitymode": "0", + "param-disparityshift": "0", + "param-lambdaad": "751", + "param-lambdacensus": "6", + "param-leftrightthreshold": "10", + "param-maxscorethreshb": "2893", + "param-medianthreshold": "796", + "param-minscorethresha": "4", + "param-neighborthresh": "108", + "param-raumine": "6", + "param-rauminn": "3", + "param-rauminnssum": "7", + "param-raumins": "2", + "param-rauminw": "2", + "param-rauminwesum": "12", + "param-regioncolorthresholdb": "0.785714", + "param-regioncolorthresholdg": "0.565558", + "param-regioncolorthresholdr": "0.985323", + "param-regionshrinku": "3", + "param-regionshrinkv": "0", + "param-robbinsmonrodecrement": "25", + "param-robbinsmonroincrement": "2", + "param-rsmdiffthreshold": "1.65625", + "param-rsmrauslodiffthreshold": "0.71875", + "param-rsmremovethreshold": "0.809524", + "param-scanlineedgetaub": "13", + "param-scanlineedgetaug": "15", + "param-scanlineedgetaur": "30", + "param-scanlinep1": "155", + "param-scanlinep1onediscon": "160", + "param-scanlinep1twodiscon": "59", + "param-scanlinep2": "190", + "param-scanlinep2onediscon": "507", + "param-scanlinep2twodiscon": "493", + "param-secondpeakdelta": "647", + "param-texturecountthresh": "0", + "param-texturedifferencethresh": "1722", + "param-usersm": "1", + "param-zunits": "100", + "stream-depth-format": "Z16", + "stream-fps": "30", + "stream-height": "480", + "stream-ir-format": "Y8", + "stream-width": "848" +} \ No newline at end of file diff --git a/main.cpp b/main.cpp index 650fc6b..e37c165 100644 --- a/main.cpp +++ b/main.cpp @@ -25,10 +25,21 @@ // Realsense API #include +//loading json settings +#include +//#include +//#include +//#include +//end of json settings + #include #include using namespace std; +const char *ENCODER="hevc_vaapi";//NULL for default (h264_vaapi) or FFmpeg encoder e.g. "hevc_vaapi", ... +const char *PIXEL_FORMAT="p010le"; //NULL for default (nv12) or pixel format e.g. "rgb0", ... +const int PROFILE=FF_PROFILE_HEVC_MAIN_10; //or FF_PROFILE_HEVC_MAIN, ... + //user supplied input struct input_args { @@ -39,8 +50,9 @@ struct input_args }; bool main_loop(const input_args& input, rs2::pipeline& realsense, hve *avctx, ofstream& out_file); -void dump_frame_info(rs2::video_frame &frame); +void dump_frame_info(const rs2::depth_frame &frame); void init_realsense(rs2::pipeline& pipe, const input_args& input); +void load_camera_json(const rs2::pipeline_profile &profile, const char *filename); int process_user_input(int argc, char* argv[], input_args* input, hve_config *config); int main(int argc, char* argv[]) @@ -49,12 +61,16 @@ int main(int argc, char* argv[]) struct hve_config hardware_config = {0}; struct input_args user_input = {0}; - ofstream out_file("output.h264", ofstream::binary); + ofstream out_file("output.hevc", ofstream::binary); rs2::pipeline realsense; if(process_user_input(argc, argv, &user_input, &hardware_config) < 0) return 1; + hardware_config.encoder=ENCODER; + hardware_config.pixel_format=PIXEL_FORMAT; + hardware_config.profile=PROFILE; + if(!out_file) return 2; @@ -72,7 +88,7 @@ int main(int argc, char* argv[]) if(status) { cout << "Finished successfully." << endl; - cout << "Test with: " << endl << endl << "ffplay output.h264" << endl; + cout << "Test with: " << endl << endl << "ffplay output.hevc" << endl; } return 0; @@ -84,28 +100,33 @@ bool main_loop(const input_args& input, rs2::pipeline& realsense, hve *he, ofstr const int frames = input.seconds * input.framerate; int f, failed; hve_frame frame = {0}; - uint8_t *color_data = NULL; //data of dummy color plane for NV12 + uint16_t *color_data = NULL; //data of dummy color plane for NV12 AVPacket *packet; for(f = 0; f < frames; ++f) { rs2::frameset frameset = realsense.wait_for_frames(); - rs2::video_frame ir_frame = frameset.get_infrared_frame(1); + //rs2::video_frame ir_frame = frameset.get_infrared_frame(1); + rs2::frame depth = frameset.get_depth_frame(); + + const int w = depth.as().get_width(); + const int h = depth.as().get_height(); + const int stride=depth.as().get_stride_in_bytes(); if(!color_data) - { //prepare dummy color plane for NV12 format, half the size of Y + { //prepare dummy color plane for P010LE format, half the size of Y //we can't alloc it in advance, this is the first time we know realsense stride - int size = ir_frame.get_stride_in_bytes()*ir_frame.get_height()/2; - color_data = new uint8_t[size]; - memset(color_data, 128, size); + color_data = new uint16_t[stride/2*h/2]; + for(int i=0;i(t)), std::istreambuf_iterator()); + rs400::advanced_mode dev = profile.get_device(); + dev.load_json(str); +} + + + + int process_user_input(int argc, char* argv[], input_args* input, hve_config *config) { if(argc < 5)