diff --git a/CMakeLists.txt b/CMakeLists.txt index 54c255b..7f4c2cd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,7 +5,7 @@ project( ) add_library(hve hve.c) -target_link_libraries(hve avcodec avutil) +target_link_libraries(hve avcodec avutil avfilter) install(TARGETS hve DESTINATION lib) install(FILES hve.h DESTINATION include) diff --git a/README.md b/README.md index c592443..3ec85c3 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # HVE - Hardware Video Encoder C library -This library wraps hardware video encoding in a simple interface. +This library wraps hardware video encoding and scaling in a simple interface. There are no performance loses (at the cost of library flexibility). Currently it supports VAAPI and various codecs (H.264, HEVC, ...).\ @@ -20,7 +20,7 @@ Raw encoding (H264, HEVC, ...): - raw dumping (H264, HEVC, ...) - ... -Complex pipelines (muxing, scaling, color conversions, filtering) are beyond the scope of this library. +Complex pipelines (muxing, filtering) are beyond the scope of this library. ## Platforms @@ -34,7 +34,7 @@ Intel VAAPI compatible hardware encoders ([Quick Sync Video](https://ark.intel.c ## Dependencies Library depends on: -- FFmpeg `avcodec` and `avutil` (at least 3.4 version) +- FFmpeg `avcodec`, `avutil`, `avfilter` (at least 3.4 version) Works with system FFmpeg on Ubuntu 18.04 and doesn't on 16.04 (outdated FFmpeg and VAAPI ecosystem). @@ -46,7 +46,7 @@ Tested on Ubuntu 18.04. # update package repositories sudo apt-get update # get avcodec and avutil (and ffmpeg for testing) -sudo apt-get install ffmpeg libavcodec-dev libavutil-dev +sudo apt-get install ffmpeg libavcodec-dev libavutil-dev libavfilter-dev # get compilers and make and cmake sudo apt-get install build-essential # get cmake - we need to specify libcurl4 for Ubuntu 18.04 dependencies problem @@ -108,7 +108,7 @@ You should see procedurally generated video (moving through greyscale). ## Using -See examples directory for a more complete and commented examples with error handling. +See examples directory for more complete and commented examples with error handling. There are just 4 functions and 3 user-visible data types: - `hve_init` @@ -117,15 +117,15 @@ There are just 4 functions and 3 user-visible data types: - `hve_close` ```C - struct hve_config hardware_config = {WIDTH, HEIGHT, FRAMERATE, DEVICE, ENCODER, - PIXEL_FORMAT, PROFILE, BFRAMES, BITRATE, QP, GOP_SIZE, COMPRESSION_LEVEL}; + struct hve_config hardware_config = {WIDTH, HEIGHT, INPUT_WIDTH, INPUT_HEIGHT, FRAMERATE, + DEVICE, ENCODER, PIXEL_FORMAT, PROFILE, BFRAMES, BITRATE, QP, GOP_SIZE, COMPRESSION_LEVEL}; struct hve *hardware_encoder=hve_init(&hardware_config); struct hve_frame frame = { 0 }; //later assuming PIXEL_FORMAT is "nv12" (you may use something else) //fill with your stride (width including padding if any) - frame.linesize[0] = frame.linesize[1] = WIDTH; + frame.linesize[0] = frame.linesize[1] = INPUT_WIDTH; AVPacket *packet; //encoded data is returned in FFmpeg packet int failed; //error indicator while encoding @@ -170,13 +170,13 @@ You have several options. For static linking of HVE and dynamic linking of FFmpeg libraries (easiest): - copy `hve.h` and `hve.c` to your project and add them in your favourite IDE -- add `avcodec` and `avutil` to linked libraries in IDE project configuration +- add `avcodec`, `avutil`, `avfilter` to linked libraries in IDE project configuration For dynamic linking of HVE and FFmpeg libraries: - place `hve.h` where compiler can find it (e.g. `make install` for `/usr/local/include/hve.h`) - place `libhve.so` where linker can find it (e.g. `make install` for `/usr/local/lib/libhve.so`) - make sure `/usr/local/...` is considered for libraries -- add `hve`, `avcodec` and `avutil` to linked libraries in IDE project configuration +- add `hve`, `avcodec`, `avutil`, `avfilter` to linked libraries in IDE project configuration - make sure `libhve.so` is reachable to you program at runtime (e.g. set `LD_LIBRARIES_PATH`) ### CMake @@ -208,7 +208,7 @@ add_library(hve SHARED hardware-video-encoder/hve.c) add_executable(your-project main.cpp) target_include_directories(your-project PRIVATE hardware-video-encoder) -target_link_libraries(your-project hve avcodec avutil) +target_link_libraries(your-project hve avcodec avutil avfilter) ``` For example see [realsense-ir-to-vaapi-h264](https://github.com/bmegli/realsense-ir-to-vaapi-h264) @@ -219,14 +219,14 @@ Assuming your `main.c`/`main.cpp` and `hve.h`, `hve.c` are all in the same direc C ```bash -gcc main.c hve.c -lavcodec -lavutil -o your-program +gcc main.c hve.c -lavcodec -lavutil -lavfilter -o your-program ``` C++ ```bash gcc -c hve.c g++ -c main.cpp -g++ hve.o main.o -lavcodec -lavutil -o your program +g++ hve.o main.o -lavcodec -lavutil -lavfilter -o your program ``` ## License @@ -240,7 +240,7 @@ This is similiar to LGPL but more permissive: Like in LGPL, if you modify this library, you have to make your changes available. Making a github fork of the library with your changes satisfies those requirements perfectly. -Since you are linking to FFmpeg libraries. Consider also `avcodec` and `avutil` licensing. +Since you are linking to FFmpeg libraries consider also `avcodec`, `avutil` and `avfilter` licensing. ## Additional information diff --git a/examples/hve_encode_raw_h264.c b/examples/hve_encode_raw_h264.c index 820e619..d4b743f 100644 --- a/examples/hve_encode_raw_h264.c +++ b/examples/hve_encode_raw_h264.c @@ -16,6 +16,8 @@ const int WIDTH=1280; const int HEIGHT=720; +const int INPUT_WIDTH=1280; //optional hardware scaling if different from width +const int INPUT_HEIGHT=720; //optional hardware scaling if different from height const int FRAMERATE=30; int SECONDS=10; const char *DEVICE=NULL; //NULL for default or device e.g. "/dev/dri/renderD128" @@ -40,8 +42,8 @@ int main(int argc, char* argv[]) return -1; //prepare library data - struct hve_config hardware_config = {WIDTH, HEIGHT, FRAMERATE, DEVICE, ENCODER, - PIXEL_FORMAT, PROFILE, BFRAMES, BITRATE, QP, GOP_SIZE, COMPRESSION_LEVEL}; + struct hve_config hardware_config = {WIDTH, HEIGHT, INPUT_WIDTH, INPUT_HEIGHT, FRAMERATE, + DEVICE, ENCODER, PIXEL_FORMAT, PROFILE, BFRAMES, BITRATE, QP, GOP_SIZE, COMPRESSION_LEVEL}; struct hve *hardware_encoder; //prepare file for raw H.264 output @@ -76,11 +78,11 @@ int encoding_loop(struct hve *hardware_encoder, FILE *output_file) //we are working with NV12 because we specified nv12 pixel format //when calling hve_init, in principle we could use other format //if hardware supported it (e.g. RGB0 is supported on my Intel) - uint8_t Y[WIDTH*HEIGHT]; //dummy NV12 luminance data - uint8_t color[WIDTH*HEIGHT/2]; //dummy NV12 color data + uint8_t Y[INPUT_WIDTH*INPUT_HEIGHT]; //dummy NV12 luminance data + uint8_t color[INPUT_WIDTH*INPUT_HEIGHT/2]; //dummy NV12 color data //fill with your stride (width including padding if any) - frame.linesize[0] = frame.linesize[1] = WIDTH; + frame.linesize[0] = frame.linesize[1] = INPUT_WIDTH; //encoded data is returned in FFmpeg packet AVPacket *packet; @@ -88,8 +90,8 @@ int encoding_loop(struct hve *hardware_encoder, FILE *output_file) for(f=0;f * @@ -15,6 +15,8 @@ #include #include #include +#include +#include #include //fprintf #include //malloc @@ -22,17 +24,32 @@ // internal library data passed around by the user struct hve { - AVBufferRef* hw_device_ctx; enum AVPixelFormat sw_pix_fmt; + AVBufferRef* hw_device_ctx; AVCodecContext* avctx; - AVFrame *sw_frame; - AVFrame *hw_frame; + + //accelerated scaling related + AVFilterContext *buffersrc_ctx; + AVFilterContext *buffersink_ctx; + AVFilterGraph *filter_graph; + + AVFrame *sw_frame; //software + AVFrame *hw_frame; //hardware + AVFrame *fr_frame; //filter AVPacket enc_pkt; }; -static int init_hwframes_context(struct hve* h, const struct hve_config *config); static struct hve *hve_close_and_return_null(struct hve *h, const char *msg); + +static int init_hwframes_context(struct hve* h, const struct hve_config *config); +static int init_hardware_scaling(struct hve *h, const struct hve_config *config); + static int HVE_ERROR_MSG(const char *msg); +static int HVE_ERROR_MSG_FILTER(AVFilterInOut *ins, AVFilterInOut *outs, const char *msg); + +static int hw_upload(struct hve *h); +static int scale_encode(struct hve *h); +static int encode(struct hve *h); // NULL on error struct hve *hve_init(const struct hve_config *config) @@ -47,7 +64,8 @@ struct hve *hve_init(const struct hve_config *config) *h = zero_hve; //set all members of dynamically allocated struct to 0 in a portable way avcodec_register_all(); - av_log_set_level(AV_LOG_VERBOSE); + avfilter_register_all(); + av_log_set_level(AV_LOG_DEBUG); //specified device or NULL / empty string for default const char *device = (config->device != NULL && config->device[0] != '\0') ? config->device : NULL; @@ -110,11 +128,20 @@ struct hve *hve_init(const struct hve_config *config) av_dict_free(&opts); + if(config->input_width && config->input_width != config->width || + config->input_height && config->input_height != config->height) + if(init_hardware_scaling(h, config) < 0) + return hve_close_and_return_null(h, "failed to init filter"); + //from now on h->filter_graph may be used to check if scaling was requested + if(h->filter_graph) + if(!(h->fr_frame = av_frame_alloc())) + return hve_close_and_return_null(h, "av_frame_alloc not enough memory (filter frame)"); + if(!(h->sw_frame = av_frame_alloc())) - return hve_close_and_return_null(h, "av_frame_alloc not enough memory"); + return hve_close_and_return_null(h, "av_frame_alloc not enough memory (software frame"); - h->sw_frame->width = config->width; - h->sw_frame->height = config->height; + h->sw_frame->width = config->input_width ? config->input_width : config->width; + h->sw_frame->height = config->input_height ? config->input_height : config->height; h->sw_frame->format = h->sw_pix_fmt; av_init_packet(&h->enc_pkt); @@ -131,7 +158,11 @@ void hve_close(struct hve* h) av_packet_unref(&h->enc_pkt); av_frame_free(&h->sw_frame); + av_frame_free(&h->fr_frame); av_frame_free(&h->hw_frame); + + avfilter_graph_free(&h->filter_graph); + avcodec_free_context(&h->avctx); av_buffer_unref(&h->hw_device_ctx); @@ -148,12 +179,6 @@ static struct hve *hve_close_and_return_null(struct hve *h, const char *msg) return NULL; } -static int HVE_ERROR_MSG(const char *msg) -{ - fprintf(stderr, "hve: %s\n", msg); - return HVE_ERROR; -} - static int init_hwframes_context(struct hve* h, const struct hve_config *config) { AVBufferRef* hw_frames_ref; @@ -166,8 +191,10 @@ static int init_hwframes_context(struct hve* h, const struct hve_config *config) frames_ctx = (AVHWFramesContext*)(hw_frames_ref->data); frames_ctx->format = AV_PIX_FMT_VAAPI; frames_ctx->sw_format = h->sw_pix_fmt; - frames_ctx->width = config->width; - frames_ctx->height = config->height; + + frames_ctx->width = config->input_width ? config->input_width : config->width; + frames_ctx->height = config->input_height ? config->input_height : config->height; + frames_ctx->initial_pool_size = 20; if((err = av_hwframe_ctx_init(hw_frames_ref)) < 0) @@ -185,12 +212,97 @@ static int init_hwframes_context(struct hve* h, const struct hve_config *config) return err == 0 ? HVE_OK : HVE_ERROR; } -int hve_send_frame(struct hve *h,struct hve_frame *frame) +static int init_hardware_scaling(struct hve *h, const struct hve_config *config) { - AVCodecContext* avctx=h->avctx; - AVFrame *sw_frame=h->sw_frame; + const AVFilter *buffersrc, *buffersink; + AVFilterInOut *ins, *outs; + char temp_str[128]; int err = 0; + if( !(buffersrc = avfilter_get_by_name("buffer")) ) + return HVE_ERROR_MSG("unable to find filter 'buffer'"); + + if( !(buffersink = avfilter_get_by_name("buffersink")) ) + return HVE_ERROR_MSG("unable to find filter 'buffersink'"); + + //allocate memory + ins = avfilter_inout_alloc(); + outs = avfilter_inout_alloc(); + h->filter_graph = avfilter_graph_alloc(); //has to be fred with HVE cleanup + + if (!ins || !outs || !h->filter_graph) + return HVE_ERROR_MSG_FILTER(ins, outs, "unable to allocate memory for the filter"); + + //prepare filter source + snprintf(temp_str, sizeof(temp_str), "video_size=%dx%d:pix_fmt=%d:time_base=1/%d:pixel_aspect=1/1", + config->input_width, config->input_height, AV_PIX_FMT_VAAPI, config->framerate); + + if(avfilter_graph_create_filter(&h->buffersrc_ctx, buffersrc, "in", temp_str, NULL, h->filter_graph) < 0) + return HVE_ERROR_MSG_FILTER(ins, outs, "cannot create buffer source"); + + outs->name = av_strdup("in"); + outs->filter_ctx = h->buffersrc_ctx; + outs->pad_idx = 0; + outs->next = NULL; + + //initialize buffersrc with hw frames context + AVBufferSrcParameters *par; + + if (!(par = av_buffersrc_parameters_alloc()) ) + return HVE_ERROR_MSG_FILTER(ins, outs, "unable to allocate memory for the filter (params)"); + + par->hw_frames_ctx = h->avctx->hw_frames_ctx; + + err = av_buffersrc_parameters_set(h->buffersrc_ctx, par); + av_free(par); + if(err < 0) + return HVE_ERROR_MSG_FILTER(ins, outs, "unable to initialize buffersrc with hw frames context"); + + //prepare filter sink + if(avfilter_graph_create_filter(&h->buffersink_ctx, buffersink, "out", NULL, NULL, h->filter_graph) < 0) + return HVE_ERROR_MSG_FILTER(ins, outs, "cannot create buffer sink"); + + ins->name = av_strdup("out"); + ins->filter_ctx = h->buffersink_ctx; + ins->pad_idx = 0; + ins->next = NULL; + + //the actual description of the graph + snprintf(temp_str, sizeof(temp_str), "format=vaapi,scale_vaapi=w=%d:h=%d", config->width, config->height); + + if(avfilter_graph_parse_ptr(h->filter_graph, temp_str, &ins, &outs, NULL) < 0) + return HVE_ERROR_MSG_FILTER(ins, outs, "failed to parse filter graph description"); + + for (int i = 0; i < h->filter_graph->nb_filters; i++) + if( !(h->filter_graph->filters[i]->hw_device_ctx = av_buffer_ref(h->hw_device_ctx)) ) + return HVE_ERROR_MSG_FILTER(ins, outs, "not enough memory to reference hw device ctx by filters"); + + if(avfilter_graph_config(h->filter_graph, NULL) < 0) + return HVE_ERROR_MSG_FILTER(ins, outs, "failed to configure filter graph"); + + avfilter_inout_free(&ins); + avfilter_inout_free(&outs); + + return HVE_OK; +} + +static int HVE_ERROR_MSG(const char *msg) +{ + fprintf(stderr, "hve: %s\n", msg); + return HVE_ERROR; +} + +static int HVE_ERROR_MSG_FILTER(AVFilterInOut *ins, AVFilterInOut *outs, const char *msg) +{ + avfilter_inout_free(&ins); + avfilter_inout_free(&outs); + //h->filter_graph is fred in reaction to init_hardware_scaling HVE_ERROR return + + return HVE_ERROR_MSG(msg); +} + +int hve_send_frame(struct hve *h,struct hve_frame *frame) +{ //note - in case hardware frame preparation fails, the frame is fred: // - here (this is next user try) // - or in av_close (this is user decision to terminate) @@ -199,29 +311,75 @@ int hve_send_frame(struct hve *h,struct hve_frame *frame) // NULL frame is used for flushing the encoder if(frame == NULL) { - if ( (err = avcodec_send_frame(avctx, NULL) ) < 0) + if(h->filter_graph) + if(av_buffersrc_add_frame_flags(h->buffersrc_ctx, NULL, AV_BUFFERSRC_FLAG_KEEP_REF | AV_BUFFERSRC_FLAG_PUSH)) + fprintf(stderr, "hve: error while marking filter EOF\n"); + + if (avcodec_send_frame(h->avctx, NULL) < 0) return HVE_ERROR_MSG("error while flushing encoder"); return HVE_OK; } //this just copies a few ints and pointers, not the actual frame data - memcpy(sw_frame->linesize, frame->linesize, sizeof(frame->linesize)); - memcpy(sw_frame->data, frame->data, sizeof(frame->data)); + memcpy(h->sw_frame->linesize, frame->linesize, sizeof(frame->linesize)); + memcpy(h->sw_frame->data, frame->data, sizeof(frame->data)); + + if(hw_upload(h) < 0) + return HVE_ERROR_MSG("failed to upload frame data to hardware"); + + if(h->filter_graph) + return scale_encode(h); + + return encode(h); +} +static int hw_upload(struct hve *h) +{ if(!(h->hw_frame = av_frame_alloc())) - return HVE_ERROR_MSG("av_frame_alloc not enough memory"); + return HVE_ERROR_MSG("av_frame_alloc not enough memory for hw_frame"); - if((err = av_hwframe_get_buffer(avctx->hw_frames_ctx, h->hw_frame, 0)) < 0) + if(av_hwframe_get_buffer(h->avctx->hw_frames_ctx, h->hw_frame, 0) < 0) return HVE_ERROR_MSG("av_hwframe_get_buffer error"); if(!h->hw_frame->hw_frames_ctx) return HVE_ERROR_MSG("hw_frame->hw_frames_ctx not enough memory"); - if((err = av_hwframe_transfer_data(h->hw_frame, sw_frame, 0)) < 0) + if(av_hwframe_transfer_data(h->hw_frame, h->sw_frame, 0) < 0) return HVE_ERROR_MSG("error while transferring frame data to surface"); - if((err = avcodec_send_frame(avctx, h->hw_frame)) < 0) + return HVE_OK; +} + +static int scale_encode(struct hve *h) +{ + int err; + + if (av_buffersrc_add_frame_flags(h->buffersrc_ctx, h->hw_frame, AV_BUFFERSRC_FLAG_KEEP_REF | AV_BUFFERSRC_FLAG_PUSH) < 0) + return HVE_ERROR_MSG("failed to push frame to filtergraph"); + + while((err = av_buffersink_get_frame(h->buffersink_ctx, h->fr_frame)) >= 0) + { + if(avcodec_send_frame(h->avctx, h->fr_frame) < 0) + { + av_frame_unref(h->fr_frame); + return HVE_ERROR_MSG("send_frame error (after scaling)"); + } + av_frame_unref(h->fr_frame); + } + + if(err == AVERROR(EAGAIN) || err == AVERROR_EOF) + return HVE_OK; + + if(err < 0) + return HVE_ERROR_MSG("failed to get frame from filtergraph"); + + return HVE_OK; +} + +static int encode(struct hve *h) +{ + if(avcodec_send_frame(h->avctx, h->hw_frame) < 0) return HVE_ERROR_MSG("send_frame error"); return HVE_OK; diff --git a/hve.h b/hve.h index a853e0c..7d21ef3 100644 --- a/hve.h +++ b/hve.h @@ -1,5 +1,5 @@ /* - * HVE Hardware Video Encoding C library header + * HVE Hardware Video Encoder C library header * * Copyright 2019-2020 (C) Bartosz Meglicki * @@ -46,6 +46,11 @@ struct hve; * @struct hve_config * @brief Encoder configuration * + * The width and height are dimmensions of the encoded data. + * + * To enable hardware accelerated scaling specify non-zero + * input_width and input_height different from width and height. + * * The device can be: * - NULL or empty string (select automatically) * - point to valid device e.g. "/dev/dri/renderD128" for vaapi @@ -134,6 +139,8 @@ struct hve_config { int width; //!< width of the encoded frames int height; //!< height of the encoded frames + int input_width; //!< optional scaling if non-zero and different from width + int input_height; //!< optional scaling if non-zero and different from height int framerate; //!< framerate of the encoded video const char *device; //!< NULL / "" or device, e.g. "/dev/dri/renderD128" const char *encoder; //!< NULL / "" or encoder, e.g. "h264_vaapi" @@ -206,6 +213,10 @@ void hve_close(struct hve* h); * * The pixel format of the frame should match the one specified in hve_init. * + * Hardware accelerated scaling is performed before encoding if non-zero + * input_width and input_height different from width and height were specified in hve_init. + * + * * Perfomance hints: * - don't copy data from your source, just pass the pointers to data planes *