Ffmpeg compositor native code

.vscode/settings.json

@@ -0,0 +1,8 @@
+{
+    "C_Cpp.errorSquiggles": "Disabled",
+    "files.associations": {
+        "__locale": "c",
+        "filter.h": "c",
+        "raw_video.h": "c"
+    }
+}

bundlex.exs

@@ -0,0 +1,21 @@
+defmodule Membrane.VideoCompositor.BundlexProject do
+  use Bundlex.Project
+
+  def project do
+    [
+      natives: natives()
+    ]
+  end
+
+  defp natives() do
+    [
+      video_compositor: [
+        interface: :nif,
+        sources: ["video_compositor.c", "filter.c", "raw_video.c"],
+        pkg_configs: ["libavutil", "libavfilter"],
+        preprocessor: Unifex,
+        src_base: "ffmpeg_video_compositor"
+      ]
+    ]
+  end
+end

c_src/ffmpeg_video_compositor/_generated/.gitignore

@@ -0,0 +1,3 @@
+**/*.h
+**/*.c
+**/*.cpp

c_src/ffmpeg_video_compositor/filter.c

@@ -0,0 +1,120 @@
+#include "filter.h"
+/**
+ * @brief Append a header of the filter description to the string (buffer).
+ * Creates two input pads with output pads named [in_1] and [in_2]
+ *
+ * @param filters_str Description destination string (buffer)
+ * @param filters_size Remaining size of the buffer
+ * @param width Width of the videos
+ * @param height Height of the videos
+ * @param pixel_format Pixel format code of the videos
+ * @return Number of characters written to the buffer
+ */
+static int init_filters_string(char *filters_str, int filters_size, int width,
+                               int height, int pixel_format);
+
+/**
+ * @brief Append a main filter description (transformation graph) to the string
+ * (buffer)
+ *
+ * @param filters_str Description destination string (buffer)
+ * @param filters_size Remaining size of the buffer
+ * @return Number of characters written to the buffer
+ */
+static int apply_filters_options_string(char *filters_str, int filters_size);
+
+/**
+ * @brief Append a footer filter description to the string
+ * (buffer). Assumes that previous filter description provides one output pad
+ * named [out]
+ *
+ * @param filters_str Description destination string (buffer)
+ * @param filters_size Remaining size of the buffer
+ * @return Number of characters written to the buffer
+ */
+static int finish_filters_string(char *filter_str, int filters_size);
+
+static void cs_printAVError(const char *msg, int returnCode) {
+    fprintf(stderr, "%s: %s\n", msg, av_err2str(returnCode));
+}
+
+int create_filter_description(char *filter_str, int filters_size, int width,
+                              int height, int pixel_format) {
+    int filter_end = 0;
+    filter_end +=
+        init_filters_string(filter_str + filter_end, filters_size - filter_end,
+                            width, height, pixel_format);
+    filter_end += apply_filters_options_string(filter_str + filter_end,
+                                               filters_size - filter_end);
+    filter_end += finish_filters_string(filter_str + filter_end,
+                                        filters_size - filter_end);
+    return filter_end;
+}
+
+static int init_filters_string(char *filters_str, int filters_size, int width,
+                               int height, int pixel_format) {
+    const int n_videos = 2;
+    int filter_end = 0;
+    for (int i = 0; i < n_videos; ++i) {
+        filter_end +=
+            snprintf(filters_str + filter_end, filters_size - filter_end,
+                     "buffer=video_size=%dx%d:pix_fmt=%d:time_base=%d/"
+                     "%d [in_%d];\n",
+                     width, height, pixel_format, 1, 1, i + 1);
+    }
+    return filter_end;
+}
+
+static int apply_filters_options_string(char *filters_str, int filters_size) {
+    int filter_end = 0;
+    // Temporary main filter description. It creates space for the second video
+    // (pad) and then overlay them on top of each other (overlay)
+    const char *filter_descr =
+        "[in_1]pad=iw:ih*2[src]; "
+        "[src][in_2]overlay=0:h[out];\n";
+    filter_end += snprintf(filters_str + filter_end, filters_size - filter_end,
+                           "%s", filter_descr);
+    return filter_end;
+}
+
+static int finish_filters_string(char *filters_str, int filters_size) {
+    int filter_end = 0;
+
+    filter_end += snprintf(filters_str + filter_end, filters_size - filter_end,
+                           "%s", "[out] buffersink");
+    return filter_end;
+}
+
+int init_filters(const char *filters_str, FilterState *filter) {
+    filter->graph = avfilter_graph_alloc();
+    AVFilterGraph *graph = filter->graph;
+
+    if (graph == NULL) {
+        fprintf(stderr, "Cannot allocate filter graph.");
+        return -1;
+    }
+
+    AVFilterInOut *gis = NULL;
+    AVFilterInOut *gos = NULL;
+
+    int ret = avfilter_graph_parse2(graph, filters_str, &gis, &gos);
+    if (ret < 0) {
+        cs_printAVError("Cannot parse graph.", ret);
+        goto end;
+    }
+
+    ret = avfilter_graph_config(graph, NULL);
+    if (ret < 0) {
+        cs_printAVError("Cannot configure graph.", ret);
+        goto end;
+    }
+
+    filter->inputs[0] = graph->filters[0];
+    filter->inputs[1] = graph->filters[1];
+    filter->output = graph->filters[graph->nb_filters - 1 - 1];
+
+end:
+    avfilter_inout_free(&gis);
+    avfilter_inout_free(&gos);
+    return ret;
+}

c_src/ffmpeg_video_compositor/filter.h

@@ -0,0 +1,46 @@
+#include <libavfilter/buffersink.h>
+#include <libavfilter/buffersrc.h>
+#include <libavutil/imgutils.h>
+#include <libavutil/opt.h>
+#include <libavutil/parseutils.h>
+#include <stdio.h>
+
+#include "raw_video.h"
+
+#define SIZE(x) ((int)(sizeof(x) / sizeof(x[0])))
+
+typedef struct FilterState {
+    AVFilterContext *inputs[2];
+    AVFilterContext *output;
+    AVFilterGraph *graph;
+} FilterState;
+
+typedef struct VState {
+    FilterState filter;
+    RawVideo videos[2];
+} VState;
+
+/**
+ * @brief Create a filter graph from string description and stores it in the
+ * given filter.
+ *
+ * @param filters_descr String description of the filter graph. This should
+ * follow ffmpeg filter documentation.
+ * @param filter  Pointer to the filter graph.
+ * @return int Return code. Return 0 on success, negative value on failure.
+ */
+int init_filters(const char *filters_descr, FilterState *filter);
+
+/**
+ * @brief Create a filter description string in FFmpeg format and store it in
+ * the given string. It assumes two input videos.
+ *
+ * @param filter_str Description destination (buffer)
+ * @param filter_size Maximum size of the filter description (buffer size)
+ * @param width Width of the videos
+ * @param height Height of the videos
+ * @param pixel_format Pixel format code of the videos
+ * @return Number of characters written to the buffer
+ */
+int create_filter_description(char *filter_str, int filter_size, int width,
+                              int height, int pixel_format);

c_src/ffmpeg_video_compositor/raw_video.c

@@ -0,0 +1,15 @@
+#include "raw_video.h"
+
+#include <string.h>
+
+enum AVPixelFormat get_pixel_format(const char *fmt_name) {
+    enum AVPixelFormat pix_fmt = AV_PIX_FMT_NONE;
+    if (strcmp(fmt_name, "I420") == 0) {
+        pix_fmt = AV_PIX_FMT_YUV420P;
+    } else if (strcmp(fmt_name, "I422") == 0) {
+        pix_fmt = AV_PIX_FMT_YUV422P;
+    } else if (strcmp(fmt_name, "I444") == 0) {
+        pix_fmt = AV_PIX_FMT_YUV444P;
+    }
+    return pix_fmt;
+}

c_src/ffmpeg_video_compositor/raw_video.h

@@ -0,0 +1,12 @@
+#include <libavutil/pixfmt.h>
+
+typedef struct RawVideo {
+    int width;
+    int height;
+    enum AVPixelFormat pixel_format;
+} RawVideo;
+
+enum AVPixelFormat get_pixel_format(const char *fmt_name);
+/*
+Returns the specified pixel code.
+*/

c_src/ffmpeg_video_compositor/video_compositor.c

@@ -0,0 +1,160 @@
+#include "video_compositor.h"
+
+static UNIFEX_TERM create_unifex_filter(UnifexEnv *env,
+                                        const char *filter_description,
+                                        int pixel_format, int width,
+                                        int height);
+
+/**
+ * @brief Initialize the state of the video compositor, create filter graph
+ *
+ * @param env Unifex environment
+ * @param width Width of the videos
+ * @param height Height of the videos
+ * @param pixel_format_name Pixel format of the videos, given in a string
+ * @return UNIFEX_TERM
+ */
+UNIFEX_TERM create(UnifexEnv *env, int width, int height,
+                   char *pixel_format_name) {
+    UNIFEX_TERM result;
+    char filter_str[512];
+    int pixel_format = get_pixel_format(pixel_format_name);
+    if (pixel_format < 0) {
+        result = create_result_error(env, "unsupported_pixel_format");
+        goto end;
+    }
+    create_filter_description(filter_str, sizeof filter_str, width, height,
+                              pixel_format);
+    result = create_unifex_filter(env, filter_str, pixel_format, width, height);
+end:
+    return result;
+}
+
+/**
+ * @brief Create a unifex filter object
+ *
+ * @param env Unifex environment
+ * @param filter_description Description of the FFmpeg filter (transformation
+ * graph), given in a string
+ * @param pixel_format Pixel format code of the videos
+ * @param width Width of the videos
+ * @param height Height of the videos
+ * @return UNIFEX_TERM
+ */
+static UNIFEX_TERM create_unifex_filter(UnifexEnv *env,
+                                        const char *filter_description,
+                                        int pixel_format, int width,
+                                        int height) {
+    UNIFEX_TERM result;
+
+    State *state = unifex_alloc_state(env);
+    for (int i = 0; i < SIZE(state->vstate.videos); i++) {
+        state->vstate.videos[i].height = height;
+        state->vstate.videos[i].width = width;
+        state->vstate.videos[i].pixel_format = pixel_format;
+    }
+
+    if (init_filters(filter_description, &state->vstate.filter) < 0) {
+        result = create_result_error(env, "error_creating_filters");
+        goto exit_create;
+    }
+    result = create_result_ok(env, state);
+
+exit_create:
+    unifex_release_state(env, state);
+    return result;
+}
+
+/**
+ * @brief Apply a filter on the given frames (compose them) and stores the
+ * result in the environment.
+ *
+ * @param env Unifex environment
+ * @param left_payload First frame
+ * @param right_payload Second frame
+ * @param state State with the initialised filter
+ * @return UNIFEX_TERM
+ */
+UNIFEX_TERM apply_filter(UnifexEnv *env, UnifexPayload *left_payload,
+                         UnifexPayload *right_payload, State *state) {
+    UNIFEX_TERM res;
+    int ret = 0;
+    UnifexPayload *payloads[] = {left_payload, right_payload};
+    AVFrame *frames[] = {av_frame_alloc(), av_frame_alloc()};
+    AVFrame *filtered_frame = av_frame_alloc();
+
+    if (!frames[0] || !frames[1] || !filtered_frame) {
+        res = apply_filter_result_error(env, "error_allocating_frame");
+        goto exit_filter;
+    }
+
+    for (int i = 0; i < SIZE(frames); i++) {
+        AVFrame *frame = frames[i];
+        RawVideo *video = &state->vstate.videos[i];
+        UnifexPayload *payload = payloads[i];
+        frame->format = video->pixel_format;
+        frame->width = video->width;
+        frame->height = video->height;
+        av_image_fill_arrays(frame->data, frame->linesize, payload->data,
+                             frame->format, frame->width, frame->height, 1);
+    }
+
+    /* feed the filtergraph */
+    FilterState *filter = &state->vstate.filter;
+    for (int i = 0; i < SIZE(filter->inputs); ++i) {
+        AVFilterContext *input = filter->inputs[i];
+        AVFrame *frame = frames[i];
+        if (av_buffersrc_add_frame_flags(input, frame,
+                                         AV_BUFFERSRC_FLAG_KEEP_REF) < 0) {
+            res = apply_filter_result_error(env, "error_feeding_filtergraph");
+            goto exit_filter;
+        }
+    }
+
+    /* pull filtered frame from the filtergraph - in drawtext filter there
+     * should always be 1 frame on output for each frame on input*/
+    ret = av_buffersink_get_frame(filter->output, filtered_frame);
+    if (ret < 0) {
+        res = apply_filter_result_error(env, "error_pulling_from_filtergraph");
+        goto exit_filter;
+    }
+
+    UnifexPayload payload_frame;
+    size_t payload_size =
+        av_image_get_buffer_size(filtered_frame->format, filtered_frame->width,
+                                 filtered_frame->height, 1);
+    unifex_payload_alloc(env, UNIFEX_PAYLOAD_BINARY, payload_size,
+                         &payload_frame);
+
+    if (av_image_copy_to_buffer(payload_frame.data, payload_size,
+                                (const uint8_t *const *)filtered_frame->data,
+                                filtered_frame->linesize,
+                                filtered_frame->format, filtered_frame->width,
+                                filtered_frame->height, 1) < 0) {
+        res = apply_filter_result_error(env, "copy_to_payload");
+        goto exit_filter;
+    }
+    res = apply_filter_result_ok(env, &payload_frame);
+exit_filter:
+    if (frames[0] != NULL) av_frame_free(&frames[0]);
+    if (frames[1] != NULL) av_frame_free(&frames[1]);
+    if (filtered_frame != NULL) av_frame_free(&filtered_frame);
+    return res;
+}
+
+/**
+ * @brief Clean up the state
+ *
+ * @param env Unifex environment
+ * @param state State
+ */
+void handle_destroy_state(UnifexEnv *env, State *state) {
+    UNIFEX_UNUSED(env);
+    FilterState *filter = &state->vstate.filter;
+    if (filter->graph != NULL) {
+        avfilter_graph_free(&filter->graph);
+    }
+    filter->inputs[0] = NULL;
+    filter->inputs[1] = NULL;
+    filter->output = NULL;
+}