Merge pull request #2 from membraneframework-labs/ffmpeg-compositor

Ffmpeg compositor native code

.formatter.exs

@@ -1,8 +1,9 @@
 [
   inputs: [
     "{lib,test,config}/**/*.{ex,exs}",
+    "c_src/**/*.spec.exs",
     ".formatter.exs",
     "*.exs"
   ],
-  import_deps: [:membrane_core]
+  import_deps: [:membrane_core, :bundlex, :unifex]
 ]

.vscode/settings.json

@@ -0,0 +1,18 @@
+{
+    "C_Cpp.errorSquiggles": "Disabled",
+    "files.associations": {
+        "__locale": "c",
+        "filter.h": "c",
+        "raw_video.h": "c",
+        "functional": "c",
+        "locale": "c",
+        "concepts": "c",
+        "ios": "c",
+        "istream": "c",
+        "numeric": "c",
+        "ostream": "c",
+        "random": "c",
+        "*.inc": "c",
+        "video_compositor.h": "c"
+    }
+}

bundlex.exs

@@ -0,0 +1,22 @@
+defmodule Membrane.VideoCompositor.BundlexProject do
+  use Bundlex.Project
+
+  def project do
+    [
+      natives: natives()
+    ]
+  end
+
+  defp natives() do
+    [
+      video_compositor: [
+        interface: :nif,
+        sources: ["video_compositor.c", "filter.c", "raw_video.c"],
+        includes: ["filter.h", "raw_video.h"],
+        pkg_configs: ["libavutil", "libavfilter"],
+        preprocessor: Unifex,
+        src_base: "ffmpeg_video_compositor"
+      ]
+    ]
+  end
+end

c_src/ffmpeg_video_compositor/_generated/.gitignore

@@ -0,0 +1,3 @@
+**/*.h
+**/*.c
+**/*.cpp

c_src/ffmpeg_video_compositor/filter.c

@@ -0,0 +1,154 @@
+#include "filter.h"
+/**
+ * @brief Append a header of the filter description to the string (buffer).
+ * Creates \p n_videos input nodes with output pads named [in_1], [in_2], ..,
+ * [in_ \p n_videos].
+ *
+ * @param filters_str Description destination string (buffer)
+ * @param filters_size Remaining size of the buffer
+ * @param videos Array of input videos.
+ * @param n_videos Size of the videos array
+ * @return Number of characters written to the buffer
+ */
+static int append_input_nodes_filters_string(char *filters_str,
+                                             int filters_size,
+                                             RawVideo videos[], int n_videos);
+
+/**
+ * @brief Append a main filter description (transformation graph) to the string
+ * (buffer)
+ *
+ * @param filters_str Description destination string (buffer)
+ * @param filters_size Remaining size of the buffer
+ * @return Number of characters written to the buffer
+ */
+static int apply_filters_options_string(char *filters_str, int filters_size);
+
+/**
+ * @brief Append a footer filter description to the string
+ * (buffer). Assumes that the previous filter description provides one output
+ * pad named [out]
+ *
+ * @param filters_str Description destination string (buffer)
+ * @param filters_size Remaining size of the buffer
+ * @return Number of characters written to the buffer
+ */
+static int finish_filters_string(char *filter_str, int filters_size);
+
+/**
+ * @brief Print error message to the stderr with formatted error code.
+ *
+ * @param msg
+ * @param error_code
+ */
+static void print_av_error(const char *msg, int error_code) {
+  fprintf(stderr, "%s: %s\n", msg, av_err2str(error_code));
+}
+
+/**
+ * @brief Creates a filter description string in an FFmpeg format and stores it
+ * in the given string.
+ *
+ * @param filter_str Description destination (buffer)
+ * @param filter_size Maximum size of the filter description (buffer size)
+ * @param videos Array of input videos.
+ * @param n_videos Size of the videos array
+ * @return Number of characters written to the buffer
+ */
+int get_filter_description(char *filter_str, int filter_size, RawVideo videos[],
+                           int n_videos) {
+  int filter_end = 0;
+  filter_end += append_input_nodes_filters_string(
+      filter_str + filter_end, filter_size - filter_end, videos, n_videos);
+  filter_end += apply_filters_options_string(filter_str + filter_end,
+                                             filter_size - filter_end);
+  filter_end +=
+      finish_filters_string(filter_str + filter_end, filter_size - filter_end);
+  return filter_end;
+}
+
+static int append_input_nodes_filters_string(char *filters_str,
+                                             int filters_size,
+                                             RawVideo videos[], int n_videos) {
+  int filter_end = 0;
+  for (int i = 0; i < n_videos; ++i) {
+    RawVideo *video = &videos[i];
+    const char *video_description_format =
+        "buffer="
+        "video_size=%dx%d"
+        ":pix_fmt=%d"
+        ":time_base=%d/%d"
+        "[in_%d];\n";
+    const int time_base_num = 1;
+    const int time_base_den = 1;
+    const int input_pad_idx = i + 1;
+    filter_end += snprintf(filters_str + filter_end, filters_size - filter_end,
+                           video_description_format, video->width,
+                           video->height, video->pixel_format, time_base_num,
+                           time_base_den, input_pad_idx);
+  }
+  return filter_end;
+}
+
+static int apply_filters_options_string(char *filters_str, int filters_size) {
+  int filter_end = 0;
+  // Temporary main filter description. It creates space for the second video
+  // (pad) and then overlay them on top of each other (overlay)
+  const char *filter_descr =
+      "[in_1]pad=iw:ih*2[src]; "
+      "[src][in_2]overlay=0:h[out];\n";
+  filter_end += snprintf(filters_str + filter_end, filters_size - filter_end,
+                         "%s", filter_descr);
+  return filter_end;
+}
+
+static int finish_filters_string(char *filters_str, int filters_size) {
+  int filter_end = 0;
+
+  filter_end += snprintf(filters_str + filter_end, filters_size - filter_end,
+                         "%s", "[out] buffersink");
+  return filter_end;
+}
+
+/**
+ * @brief Creates a filter graph from the string description and stores it in
+ * the given filter.
+ *
+ * @param filters_descr String description of the filter graph. This should
+ * follow FFmpeg filter documentation.
+ * @param filter  Pointer to the filter graph.
+ * @return Return code. Return 0 on success, a negative value on failure.
+ */
+int init_filters_graph(const char *filters_str, FilterState *filter) {
+  filter->graph = avfilter_graph_alloc();
+  AVFilterGraph *graph = filter->graph;
+
+  if (graph == NULL) {
+    fprintf(stderr, "Cannot allocate filter graph.");
+    return -1;
+  }
+
+  AVFilterInOut *gis = NULL;
+  AVFilterInOut *gos = NULL;
+
+  int ret = avfilter_graph_parse2(graph, filters_str, &gis, &gos);
+  if (ret < 0) {
+    print_av_error("Cannot parse graph.", ret);
+    goto end;
+  }
+
+  ret = avfilter_graph_config(graph, NULL);
+  if (ret < 0) {
+    print_av_error("Cannot configure graph.", ret);
+    goto end;
+  }
+
+  filter->inputs[0] = graph->filters[0];
+  filter->inputs[1] = graph->filters[1];
+  filter->output = graph->filters[graph->nb_filters - 1 - 1];
+
+end:
+  avfilter_inout_free(&gis);
+  avfilter_inout_free(&gos);
+  return ret;
+}

c_src/ffmpeg_video_compositor/filter.h

@@ -0,0 +1,26 @@
+#include <libavfilter/buffersink.h>
+#include <libavfilter/buffersrc.h>
+#include <libavutil/imgutils.h>
+#include <libavutil/opt.h>
+#include <libavutil/parseutils.h>
+#include <stdio.h>
+
+#include "raw_video.h"
+
+#define SIZE(x) ((int)(sizeof(x) / sizeof(x[0])))
+
+typedef struct FilterState {
+  AVFilterContext *inputs[2];
+  AVFilterContext *output;
+  AVFilterGraph *graph;
+} FilterState;
+
+typedef struct VState {
+  FilterState filter;
+  RawVideo videos[2];
+} VState;
+
+int init_filters_graph(const char *filters_descr, FilterState *filter);
+
+int get_filter_description(char *filter_str, int filter_size, RawVideo videos[],
+                           int n_videos);

c_src/ffmpeg_video_compositor/raw_video.c

@@ -0,0 +1,43 @@
+#include "raw_video.h"
+
+#include <string.h>
+
+/**
+ * @brief Returns the specified pixel code.
+ *
+ * @param fmt_name Pixel format string
+ * @return Pixel format code
+ */
+enum AVPixelFormat get_pixel_format(const char *fmt_name) {
+  enum AVPixelFormat pix_fmt = AV_PIX_FMT_NONE;
+  if (strcmp(fmt_name, "I420") == 0) {
+    pix_fmt = AV_PIX_FMT_YUV420P;
+  } else if (strcmp(fmt_name, "I422") == 0) {
+    pix_fmt = AV_PIX_FMT_YUV422P;
+  } else if (strcmp(fmt_name, "I444") == 0) {
+    pix_fmt = AV_PIX_FMT_YUV444P;
+  }
+  return pix_fmt;
+}
+
+/**
+ * @brief Init the raw video with the given parameters
+ *
+ * @param raw_video Destination video
+ * @param width Video width
+ * @param height Video height
+ * @param pixel_format_name Pixel format name given in a string. It will be
+ * converted into the corresponding enum code
+ * @return Return code. Return 0 on success, negative value otherwise
+ */
+int init_raw_video(RawVideo *raw_video, int width, int height,
+                   const char *pixel_format_name) {
+  int pixel_format = get_pixel_format(pixel_format_name);
+  if (pixel_format < 0) {
+    return -1;
+  }
+  raw_video->width = width;
+  raw_video->height = height;
+  raw_video->pixel_format = pixel_format;
+  return 0;
+}

c_src/ffmpeg_video_compositor/raw_video.h

@@ -0,0 +1,12 @@
+#include <libavutil/pixfmt.h>
+
+typedef struct RawVideo {
+  int width;
+  int height;
+  enum AVPixelFormat pixel_format;
+} RawVideo;
+
+enum AVPixelFormat get_pixel_format(const char *fmt_name);
+
+int init_raw_video(RawVideo *raw_video, int width, int height,
+                   const char *pixel_format_name);