Add support for hvc1 and hev1

lib/membrane_mp4/container/schema.ex

@@ -14,23 +14,24 @@ defmodule Membrane.MP4.Container.Schema do
     flags: {:uint24, store: :fo_flags}
   ]
 
+  @visual_sample_entry @full_box ++
+                         [
+                           num_of_entries: :uint32,
+                           reserved: <<0::128>>,
+                           width: :uint16,
+                           height: :uint16,
+                           horizresolution: :fp16d16,
+                           vertresolution: :fp16d16,
+                           reserved: <<0::32>>,
+                           frame_count: :uint16,
+                           compressor_name: :str256,
+                           depth: :uint16,
+                           reserved: <<-1::16-integer>>
+                         ]
+
   @avc_schema [
     version: 0,
-    fields:
-      @full_box ++
-        [
-          num_of_entries: :uint32,
-          reserved: <<0::128>>,
-          width: :uint16,
-          height: :uint16,
-          horizresolution: :fp16d16,
-          vertresolution: :fp16d16,
-          reserved: <<0::32>>,
-          frame_count: :uint16,
-          compressor_name: :str256,
-          depth: :uint16,
-          reserved: <<-1::16-integer>>
-        ],
+    fields: @visual_sample_entry,
     avcC: [
       black_box?: true
     ],
@@ -42,6 +43,20 @@ defmodule Membrane.MP4.Container.Schema do
     ]
   ]
 
+  @hevc_schema [
+    version: 0,
+    fields: @visual_sample_entry,
+    hvcC: [
+      black_box?: true
+    ],
+    pasp: [
+      fields: [
+        h_spacing: :uint32,
+        v_spacing: :uint32
+      ]
+    ]
+  ]
+
   @schema_def ftyp: [
                 fields: [
                   major_brand: :str32,
@@ -178,6 +193,8 @@ defmodule Membrane.MP4.Container.Schema do
                               ],
                           avc1: @avc_schema,
                           avc3: @avc_schema,
+                          hvc1: @hevc_schema,
+                          hev1: @hevc_schema,
                           mp4a: [
                             fields: [
                               reserved: <<0::6*8>>,

lib/membrane_mp4/demuxer/isom.ex

@@ -28,6 +28,10 @@ defmodule Membrane.MP4.Demuxer.ISOM do
           stream_structure: {:avc1, _dcr},
           alignment: :au
         },
+        %Membrane.H265{
+          stream_structure: {_hevc, _dcr},
+          alignment: :au
+        },
         %Membrane.Opus{self_delimiting?: false}
       ),
     availability: :on_request

lib/membrane_mp4/movie_box/sample_table_box.ex

@@ -3,7 +3,7 @@ defmodule Membrane.MP4.MovieBox.SampleTableBox do
 
   require Membrane.H264
   alias Membrane.MP4.{Container, Helper, Track.SampleTable}
-  alias Membrane.{AAC, H264, Opus}
+  alias Membrane.{AAC, H264, H265, Opus}
 
   @spec assemble(SampleTable.t()) :: Container.t()
   def assemble(table) do
@@ -102,6 +102,39 @@ defmodule Membrane.MP4.MovieBox.SampleTableBox do
     ]
   end
 
+  defp assemble_sample_description(%H265{
+         stream_structure: {hvc, dcr},
+         width: width,
+         height: height
+       }) do
+    [
+      {hvc,
+       %{
+         children: [
+           hvcC: %{
+             content: dcr
+           },
+           pasp: %{
+             children: [],
+             fields: %{h_spacing: 1, v_spacing: 1}
+           }
+         ],
+         fields: %{
+           compressor_name: <<0::size(32)-unit(8)>>,
+           depth: 24,
+           flags: 0,
+           frame_count: 1,
+           height: height,
+           horizresolution: {0, 0},
+           num_of_entries: 1,
+           version: 0,
+           vertresolution: {0, 0},
+           width: width
+         }
+       }}
+    ]
+  end
+
   defp assemble_sample_description(%AAC{
          config: {:esds, esds},
          channels: channels,
@@ -244,6 +277,15 @@ defmodule Membrane.MP4.MovieBox.SampleTableBox do
     }
   end
 
+  defp unpack_sample_description(%{children: [{hevc, %{children: boxes, fields: fields}}]})
+       when hevc in [:hvc1, :hev1] do
+    %H265{
+      width: fields.width,
+      height: fields.height,
+      stream_structure: {hevc, boxes[:hvcC].content}
+    }
+  end
+
   defp unpack_sample_description(%{children: [{:mp4a, %{children: boxes, fields: fields}}]}) do
     {sample_rate, 0} = fields.sample_rate
 

lib/membrane_mp4/movie_box/track_box.ex

@@ -11,7 +11,9 @@ defmodule Membrane.MP4.MovieBox.TrackBox do
   """
   alias Membrane.MP4.{Container, MovieBox.SampleTableBox, Track}
 
-  defguardp is_audio(track) when not is_struct(track.stream_format, Membrane.H264)
+  defguardp is_audio(track)
+            when not is_struct(track.stream_format, Membrane.H264) and
+                   not is_struct(track.stream_format, Membrane.H265)
 
   @spec assemble(Track.t()) :: Container.t()
   def assemble(track) do

lib/membrane_mp4/muxer/isom.ex

@@ -20,6 +20,10 @@ defmodule Membrane.MP4.Muxer.ISOM do
           stream_structure: {:avc1, _dcr},
           alignment: :au
         },
+        %Membrane.H265{
+          stream_structure: {_hevc, _dcr},
+          alignment: :au
+        },
         %Membrane.Opus{self_delimiting?: false}
       ),
     availability: :on_request

lib/membrane_mp4/track.ex

@@ -8,7 +8,7 @@ defmodule Membrane.MP4.Track do
   """
   require Membrane.H264
   alias __MODULE__.SampleTable
-  alias Membrane.{AAC, H264}
+  alias Membrane.{AAC, H264, H265}
   alias Membrane.MP4.Helper
 
   @type t :: %__MODULE__{
@@ -125,11 +125,20 @@ defmodule Membrane.MP4.Track do
 
   defp get_timescale(stream_format) do
     case stream_format do
-      %Membrane.Opus{} -> 48_000
-      %Membrane.AAC{sample_rate: sample_rate} -> sample_rate
-      %Membrane.H264{framerate: nil} -> 30 * 1024
-      %Membrane.H264{framerate: {0, _denominator}} -> 30 * 1024
-      %Membrane.H264{framerate: {nominator, _denominator}} -> nominator * 1024
+      %Membrane.Opus{} ->
+        48_000
+
+      %Membrane.AAC{sample_rate: sample_rate} ->
+        sample_rate
+
+      %module{framerate: nil} when module in [H264, H265] ->
+        30 * 1024
+
+      %module{framerate: {0, _denominator}} when module in [H264, H265] ->
+        30 * 1024
+
+      %module{framerate: {nominator, _denominator}} when module in [H264, H265] ->
+        nominator * 1024
     end
   end
 

lib/membrane_mp4/track/sample_table.ex

@@ -148,9 +148,13 @@ defmodule Membrane.MP4.Track.SampleTable do
     end)
   end
 
-  defp maybe_store_sync_sample(sample_table, %Buffer{
-         metadata: %{h264: %{key_frame?: true}}
-       }) do
+  defp maybe_store_sync_sample(sample_table, %Buffer{metadata: %{h264: h264}})
+       when h264.key_frame? do
+    Map.update!(sample_table, :sync_samples, &[sample_table.sample_count | &1])
+  end
+
+  defp maybe_store_sync_sample(sample_table, %Buffer{metadata: %{h265: h265}})
+       when h265.key_frame? do
     Map.update!(sample_table, :sync_samples, &[sample_table.sample_count | &1])
   end
 

mix.exs

@@ -42,10 +42,12 @@ defmodule Membrane.MP4.Plugin.MixProject do
       {:membrane_cmaf_format, "~> 0.7.0"},
       {:membrane_aac_format, "~> 0.8.0"},
       {:membrane_h264_format, "~> 0.6.1"},
+      {:membrane_h265_format, "~> 0.2.0"},
       {:membrane_opus_format, "~> 0.3.0"},
       {:membrane_file_plugin, "~> 0.15.0"},
       {:membrane_h264_plugin, "~> 0.7.0"},
       {:bunch, "~> 1.5"},
+      {:membrane_h265_plugin, "~> 0.3.1", only: :test},
       {:membrane_aac_plugin, "~> 0.16.0", only: :test},
       {:membrane_opus_plugin, "~> 0.17.0", only: :test},
       {:membrane_stream_plugin, "~> 0.3.0", only: :test},

mix.lock

@@ -26,6 +26,8 @@
   "membrane_file_plugin": {:hex, :membrane_file_plugin, "0.15.0", "ddf9535fda82aae5b0688a98de1d02268287ffc8bcc6dba1a85e057d71c522af", [:mix], [{:membrane_core, "~> 0.12.0", [hex: :membrane_core, repo: "hexpm", optional: false]}], "hexpm", "fa2f7219f96c9e815475dc0d8c238c0a5648012917584756eb3eee476f737ce2"},
   "membrane_h264_format": {:hex, :membrane_h264_format, "0.6.1", "44836cd9de0abe989b146df1e114507787efc0cf0da2368f17a10c47b4e0738c", [:mix], [], "hexpm", "4b79be56465a876d2eac2c3af99e115374bbdc03eb1dea4f696ee9a8033cd4b0"},
   "membrane_h264_plugin": {:hex, :membrane_h264_plugin, "0.7.3", "c558137cdfe72ca68a1bcf05d30cfd1bdc2264a3fbea0b002e9682a6832efc7d", [:mix], [{:bunch, "~> 1.4", [hex: :bunch, repo: "hexpm", optional: false]}, {:membrane_core, "~> 0.12.7", [hex: :membrane_core, repo: "hexpm", optional: false]}, {:membrane_h264_format, "~> 0.6.0", [hex: :membrane_h264_format, repo: "hexpm", optional: false]}], "hexpm", "291b0b5ed07e0e39ec0369f6a3d47a189544ddb62ce2b7a1443742beb6618992"},
+  "membrane_h265_format": {:hex, :membrane_h265_format, "0.2.0", "1903c072cf7b0980c4d0c117ab61a2cd33e88782b696290de29570a7fab34819", [:mix], [], "hexpm", "6df418bdf242c0d9f7dbf2e5aea4c2d182e34ac9ad5a8b8cef2610c290002e83"},
+  "membrane_h265_plugin": {:hex, :membrane_h265_plugin, "0.3.1", "486479f67ec90cd84fd78e3de45693305fad67b41ef39d624188a96fdb6c9669", [:mix], [{:bunch, "~> 1.4", [hex: :bunch, repo: "hexpm", optional: false]}, {:membrane_core, "~> 0.12.0", [hex: :membrane_core, repo: "hexpm", optional: false]}, {:membrane_h265_format, "~> 0.2.0", [hex: :membrane_h265_format, repo: "hexpm", optional: false]}], "hexpm", "63b748ac7bb63b2e5145c4f79e061faacfcc4f218b96908e5a9648d6a4f8aca2"},
   "membrane_mp4_format": {:hex, :membrane_mp4_format, "0.8.0", "8c6e7d68829228117d333b4fbb030e7be829aab49dd8cb047fdc664db1812e6a", [:mix], [], "hexpm", "148dea678a1f82ccfd44dbde6f936d2f21255f496cb45a22cc6eec427f025522"},
   "membrane_opus_format": {:hex, :membrane_opus_format, "0.3.0", "3804d9916058b7cfa2baa0131a644d8186198d64f52d592ae09e0942513cb4c2", [:mix], [], "hexpm", "8fc89c97be50de23ded15f2050fe603dcce732566fe6fdd15a2de01cb6b81afe"},
   "membrane_opus_plugin": {:hex, :membrane_opus_plugin, "0.17.1", "533d694d8c6f38fbf7f09a514e16037e8980d167899d4c35cb5017b088122ce9", [:mix], [{:bunch, "~> 1.3", [hex: :bunch, repo: "hexpm", optional: false]}, {:membrane_common_c, "~> 0.15.0", [hex: :membrane_common_c, repo: "hexpm", optional: false]}, {:membrane_core, "~> 0.12.3", [hex: :membrane_core, repo: "hexpm", optional: false]}, {:membrane_opus_format, "~> 0.3.0", [hex: :membrane_opus_format, repo: "hexpm", optional: false]}, {:membrane_raw_audio_format, "~> 0.11.0", [hex: :membrane_raw_audio_format, repo: "hexpm", optional: false]}, {:unifex, "~> 1.0", [hex: :unifex, repo: "hexpm", optional: false]}], "hexpm", "8d2bf3d97651f136fd393ce80c099a4f282bbe1dfafde887f8ab18668fd349aa"},

test/membrane_mp4/demuxer/isom/demuxer_test.exs

@@ -64,6 +64,20 @@ defmodule Membrane.MP4.Demuxer.ISOM.DemuxerTest do
       perform_test(pipeline, "video", out_path)
     end
 
+    @tag :tmp_dir
+    test "a single non-fast-start H265 track", %{tmp_dir: dir} do
+      in_path = "test/fixtures/isom/ref_video_hevc.mp4"
+      out_path = Path.join(dir, "out")
+
+      pipeline =
+        start_testing_pipeline!(
+          input_file: in_path,
+          output_file: out_path
+        )
+
+      perform_test(pipeline, "video_hevc", out_path)
+    end
+
     @tag :tmp_dir
     test "a single non-fast-start AAC track", %{tmp_dir: dir} do
       in_path = "test/fixtures/isom/ref_aac.mp4"

test/membrane_mp4/muxer/isom/integration_test.exs

@@ -62,6 +62,24 @@ defmodule Membrane.MP4.Muxer.ISOM.IntegrationTest do
       perform_test(pid, "video")
     end
 
+    test "single H265 track" do
+      prepare_test("video_hevc")
+
+      structure = [
+        child(:file, %Membrane.File.Source{location: "test/fixtures/in_video_hevc.h265"})
+        |> child(:parser, %Membrane.H265.Parser{
+          generate_best_effort_timestamps: %{framerate: {30, 1}},
+          output_stream_structure: :hvc1
+        })
+        |> child(:muxer, %Membrane.MP4.Muxer.ISOM{chunk_duration: Time.seconds(1)})
+        |> child(:sink, %Membrane.File.Sink{location: out_path_for("video_hevc")})
+      ]
+
+      pid = Pipeline.start_link_supervised!(structure: structure)
+
+      perform_test(pid, "video_hevc")
+    end
+
     test "single AAC track" do
       prepare_test("aac")