Add H265 support to Sink and Source

Author: georgeday-firefly

lib/membrane_webrtc/ex_webrtc/sink.ex

@@ -161,7 +161,7 @@ defmodule Membrane.WebRTC.ExWebRTCSink do
     {[], state}
   end
 
-  defp set_pc_sender_video_codec(state, pad, codec) when codec in [:vp8, :h264, :av1] do
+  defp set_pc_sender_video_codec(state, pad, codec) when codec in [:vp8, :h264, :h265, :av1] do
     mime_type = mime_type_from_codec(codec)
     transceiver = get_transceiver(state, pad)
 
@@ -383,6 +383,7 @@ defmodule Membrane.WebRTC.ExWebRTCSink do
   end
 
   defp mime_type_from_codec(:h264), do: "video/H264"
+  defp mime_type_from_codec(:h265), do: "video/H265"
   defp mime_type_from_codec(:vp8), do: "video/VP8"
   defp mime_type_from_codec(:av1), do: "video/AV1"
 end

lib/membrane_webrtc/ex_webrtc/source.ex

@@ -45,8 +45,8 @@ defmodule Membrane.WebRTC.ExWebRTCSource do
             status: :init | :connecting | :connected | :closed,
             audio_params: [ExWebRTC.RTPCodecParameters.t()],
             video_params: [ExWebRTC.RTPCodecParameters.t()],
-            allowed_video_codecs: [:h264 | :vp8],
-            preferred_video_codec: :h264 | :vp8,
+            allowed_video_codecs: [:h264 | :h265 | :vp8],
+            preferred_video_codec: :h264 | :h265 | :vp8,
             ice_servers: [ExWebRTC.PeerConnection.Configuration.ice_server()],
             ice_port_range: Enumerable.t(non_neg_integer()),
             ice_ip_filter: (:inet.ip_address() -> boolean()),

lib/membrane_webrtc/ex_webrtc/utils.ex

@@ -3,7 +3,7 @@ defmodule Membrane.WebRTC.ExWebRTCUtils do
 
   alias ExWebRTC.RTPCodecParameters
 
-  @type codec :: :opus | :h264 | :vp8 | :av1
+  @type codec :: :opus | :h264 | :h265 | :vp8 | :av1
   @type codec_or_codecs :: codec() | [codec()]
 
   @spec codec_params(codec_or_codecs()) :: [RTPCodecParameters.t()]
@@ -33,6 +33,22 @@ defmodule Membrane.WebRTC.ExWebRTCUtils do
     ]
   end
 
+  def codec_params(:h265) do
+    [
+      %RTPCodecParameters{
+        payload_type: 98,
+        mime_type: "video/H265",
+        clock_rate: codec_clock_rate(:h265),
+        sdp_fmtp_line: %ExSDP.Attribute.FMTP{
+          pt: 98,
+          profile_id: 1,
+          tier_flag: false,
+          level_id: 153
+        }
+      }
+    ]
+  end
+
   def codec_params(:vp8) do
     [
       %RTPCodecParameters{
@@ -67,19 +83,22 @@ defmodule Membrane.WebRTC.ExWebRTCUtils do
   def codec_clock_rate(:opus), do: 48_000
   def codec_clock_rate(:vp8), do: 90_000
   def codec_clock_rate(:h264), do: 90_000
+  def codec_clock_rate(:h265), do: 90_000
   def codec_clock_rate(:av1), do: 90_000
 
   def codec_clock_rate(codecs) when is_list(codecs) do
     cond do
       codecs == [:opus] ->
         48_000
 
-      codecs != [] and Enum.all?(codecs, &(&1 in [:vp8, :h264])) ->
+      codecs != [] and Enum.all?(codecs, &(&1 in [:vp8, :h264, :h265])) ->
         90_000
     end
   end
 
-  @spec get_video_codecs_from_sdp(ExWebRTC.SessionDescription.t()) :: [:h264 | :vp8 | :av1]
+  @spec get_video_codecs_from_sdp(ExWebRTC.SessionDescription.t()) :: [
+          :h264 | :h265 | :vp8 | :av1
+        ]
   def get_video_codecs_from_sdp(%ExWebRTC.SessionDescription{sdp: sdp}) do
     ex_sdp = ExSDP.parse!(sdp)
 
@@ -90,6 +109,7 @@ defmodule Membrane.WebRTC.ExWebRTCUtils do
     end)
     |> Enum.flat_map(fn
       %ExSDP.Attribute.RTPMapping{encoding: "H264"} -> [:h264]
+      %ExSDP.Attribute.RTPMapping{encoding: "H265"} -> [:h265]
       %ExSDP.Attribute.RTPMapping{encoding: "VP8"} -> [:vp8]
       %ExSDP.Attribute.RTPMapping{encoding: "AV1"} -> [:av1]
       _attribute -> []

lib/membrane_webrtc/sink.ex

@@ -18,6 +18,7 @@ defmodule Membrane.WebRTC.Sink do
   use Membrane.Bin
 
   alias Membrane.H264
+  alias Membrane.H265
   alias Membrane.RemoteStream
   alias Membrane.VP8
   alias Membrane.WebRTC.{ExWebRTCSink, Signaling, SimpleWebSocketServer}
@@ -68,7 +69,7 @@ defmodule Membrane.WebRTC.Sink do
                 """
               ],
               video_codec: [
-                spec: :vp8 | :h264 | :av1 | [:vp8 | :h264 | :av1],
+                spec: :vp8 | :h264 | :h265 | :av1 | [:vp8 | :h264 | :h265 | :av1],
                 default: [:vp8, :h264],
                 description: """
                 Video codecs, that #{inspect(__MODULE__)} will try to negotiatie in SDP
@@ -102,6 +103,7 @@ defmodule Membrane.WebRTC.Sink do
     accepted_format:
       any_of(
         %Membrane.H264{alignment: :nalu},
+        %Membrane.H265{alignment: :nalu},
         %Membrane.RemoteStream{content_format: Membrane.VP8},
         %Membrane.RemoteStream{content_format: Membrane.RTP},
         Membrane.VP8,
@@ -189,6 +191,9 @@ defmodule Membrane.WebRTC.Sink do
         %H264{} ->
           :h264
 
+        %H265{} ->
+          :h265
+
         %VP8{} ->
           :vp8
 
@@ -206,6 +211,7 @@ defmodule Membrane.WebRTC.Sink do
     payloader =
       case codec do
         :h264 -> %Membrane.RTP.H264.Payloader{max_payload_size: 1000}
+        :h265 -> %Membrane.RTP.H265.Payloader{max_payload_size: 1000}
         :vp8 -> Membrane.RTP.VP8.Payloader
       end
 

lib/membrane_webrtc/source.ex

@@ -64,13 +64,13 @@ defmodule Membrane.WebRTC.Source do
                 """
               ],
               allowed_video_codecs: [
-                spec: :vp8 | :h264 | :av1 | [:vp8 | :h264 | :av1],
+                spec: :vp8 | :h264 | :h265 | :av1 | [:vp8 | :h264 | :h265 | :av1],
                 default: :vp8,
                 description: """
                 Specifies, which video codecs can be accepted by the source during the SDP
                 negotiaion.
 
-                Either `:vp8`, `:h264` or a list containing both options.
+                Either `:vp8`, `:h264`, `:h265` or a list containing these options.
 
                 Event if it is set to `[:h264, :vp8]`, the source will negotiate at most
                 one video codec. Negotiated codec can be deduced from
@@ -83,7 +83,7 @@ defmodule Membrane.WebRTC.Source do
                 """
               ],
               preferred_video_codec: [
-                spec: :vp8 | :h264,
+                spec: :vp8 | :h264 | :h265,
                 default: :vp8,
                 description: """
                 Specyfies, which video codec will be preferred by the source, if both of
@@ -129,6 +129,7 @@ defmodule Membrane.WebRTC.Source do
     accepted_format:
       any_of(
         Membrane.H264,
+        Membrane.H265,
         %Membrane.RemoteStream{content_format: Membrane.VP8},
         %Membrane.RemoteStream{content_format: Membrane.Opus},
         Membrane.RTP
@@ -252,12 +253,18 @@ defmodule Membrane.WebRTC.Source do
       state.allowed_video_codecs == [:h264] ->
         get_h264_depayloader(builder)
 
+      state.allowed_video_codecs == [:h265] ->
+        get_h265_depayloader(builder)
+
       state.negotiated_video_codecs == [:vp8] ->
         get_vp8_depayloader(builder)
 
       state.negotiated_video_codecs == [:h264] ->
         get_h264_depayloader(builder)
 
+      state.negotiated_video_codecs == [:h265] ->
+        get_h265_depayloader(builder)
+
       state.negotiated_video_codecs == nil ->
         raise "Cannot select depayloader before end of SDP messages exchange"
 
@@ -285,5 +292,12 @@ defmodule Membrane.WebRTC.Source do
     })
   end
 
+  defp get_h265_depayloader(builder) do
+    child(builder, {:depayloader_bin, make_ref()}, %Membrane.RTP.DepayloaderBin{
+      depayloader: Membrane.RTP.H265.Depayloader,
+      clock_rate: ExWebRTCUtils.codec_clock_rate(:h265)
+    })
+  end
+
   def default_ice_ip_filter(_ip), do: true
 end

mix.exs

@@ -46,6 +46,7 @@ defmodule Membrane.WebRTC.Plugin.Mixfile do
       {:membrane_core, "~> 1.2 and >= 1.2.2"},
       {:membrane_rtp_plugin, "~> 0.31.1"},
       {:membrane_rtp_h264_plugin, "~> 0.20.1"},
+      {:membrane_rtp_h265_plugin, "~> 0.5.3"},
       {:membrane_rtp_vp8_plugin, "~> 0.9.4"},
       {:membrane_rtp_opus_plugin, "~> 0.10.0"},
 

mix.lock

@@ -53,6 +53,7 @@
   "membrane_realtimer_plugin": {:hex, :membrane_realtimer_plugin, "0.10.1", "6f00f499570609f0fd717edb3c069fdcd8ece893ae683867c9ea3f7aa3de9dd0", [:mix], [{:membrane_core, "~> 1.0", [hex: :membrane_core, repo: "hexpm", optional: false]}], "hexpm", "e961cf8aab3857f686eba9a61aaa91a818fe70b1e1282d09f4d6db06acb9dd67"},
   "membrane_rtp_format": {:hex, :membrane_rtp_format, "0.11.0", "0091465530ab946bd4d8affe849e9a01a79e46b750f6ec420521bcb806d5be49", [:mix], [{:membrane_core, "~> 1.0", [hex: :membrane_core, repo: "hexpm", optional: false]}], "hexpm", "bea52c2f286d5edb90d647bf3916f4e9e0270d5c6eddb708b65880570c1e42d2"},
   "membrane_rtp_h264_plugin": {:hex, :membrane_rtp_h264_plugin, "0.20.5", "31220642282806f87226d8e472be09873d48ed5a519feeb0ccdb884be8410fd0", [:mix], [{:bunch, "~> 1.5", [hex: :bunch, repo: "hexpm", optional: false]}, {:membrane_core, "~> 1.0", [hex: :membrane_core, repo: "hexpm", optional: false]}, {:membrane_h264_format, "~> 0.6.0", [hex: :membrane_h264_format, repo: "hexpm", optional: false]}, {:membrane_rtp_format, "~> 0.11.0", [hex: :membrane_rtp_format, repo: "hexpm", optional: false]}], "hexpm", "ea1cd860874527d7c63afdc68ec4d90aa46c3bb25ea320019af2a89ba7dc0937"},
+  "membrane_rtp_h265_plugin": {:hex, :membrane_rtp_h265_plugin, "0.5.4", "578dc8e0afc62396e9fd37ab83b32106265e3188ac53a83c57bd96cb592c6cc7", [:mix], [{:membrane_core, "~> 1.0", [hex: :membrane_core, repo: "hexpm", optional: false]}, {:membrane_h265_format, "~> 0.2.0", [hex: :membrane_h265_format, repo: "hexpm", optional: false]}, {:membrane_rtp_format, "~> 0.11.0", [hex: :membrane_rtp_format, repo: "hexpm", optional: false]}], "hexpm", "1f643c8a25a5e721ea4eeacbf7b1ae5e5a5423e1f17400414aefa85a54c2cd47"},
   "membrane_rtp_opus_plugin": {:hex, :membrane_rtp_opus_plugin, "0.10.2", "cf614e5f62e4896e48030016f4ac04d690351185aad83ca874c09383fe376698", [:mix], [{:membrane_core, "~> 1.0", [hex: :membrane_core, repo: "hexpm", optional: false]}, {:membrane_opus_format, "~> 0.3.0", [hex: :membrane_opus_format, repo: "hexpm", optional: false]}, {:membrane_rtp_format, "~> 0.11.0", [hex: :membrane_rtp_format, repo: "hexpm", optional: false]}], "hexpm", "1f02501a017e55f74a47fa6b724ae73d5821887fa4a207a2ecf7cb7d7f7539c1"},
   "membrane_rtp_plugin": {:hex, :membrane_rtp_plugin, "0.31.4", "671144c967f9ec1935d8c08fe7c7ebdfca081a69ef588e0806e2bc0081482766", [:mix], [{:bimap, "~> 1.2", [hex: :bimap, repo: "hexpm", optional: false]}, {:bunch, "~> 1.5", [hex: :bunch, repo: "hexpm", optional: false]}, {:ex_libsrtp, "~> 0.6.0 or ~> 0.7.0", [hex: :ex_libsrtp, repo: "hexpm", optional: true]}, {:ex_rtcp, "~> 0.4.0", [hex: :ex_rtcp, repo: "hexpm", optional: false]}, {:ex_rtp, "~> 0.4.0", [hex: :ex_rtp, repo: "hexpm", optional: false]}, {:heap, "~> 2.0.2", [hex: :heap, repo: "hexpm", optional: false]}, {:membrane_core, "~> 1.0", [hex: :membrane_core, repo: "hexpm", optional: false]}, {:membrane_funnel_plugin, "~> 0.9.0", [hex: :membrane_funnel_plugin, repo: "hexpm", optional: false]}, {:membrane_rtp_format, "~> 0.11.0", [hex: :membrane_rtp_format, repo: "hexpm", optional: false]}, {:membrane_telemetry_metrics, "~> 0.1.0", [hex: :membrane_telemetry_metrics, repo: "hexpm", optional: false]}, {:qex, "~> 0.5.1", [hex: :qex, repo: "hexpm", optional: false]}], "hexpm", "5a1e6b2512628619a597ef609d2583e8891411446122da55f248c745a2f1d348"},
   "membrane_rtp_vp8_plugin": {:hex, :membrane_rtp_vp8_plugin, "0.9.6", "49b93f09cd23f8d5a69b622466df02c6db1b8467cfbabf4715421a51f2eea3f9", [:mix], [{:membrane_core, "~> 1.0", [hex: :membrane_core, repo: "hexpm", optional: false]}, {:membrane_rtp_format, "~> 0.11.0", [hex: :membrane_rtp_format, repo: "hexpm", optional: false]}, {:membrane_vp8_format, "~> 0.4.0 or ~> 0.5.0", [hex: :membrane_vp8_format, repo: "hexpm", optional: false]}], "hexpm", "ad74006ad4b22218676558c44e15057c389651887dea82c1e581139c16176630"},

test/fixtures/input_bbb.h265

@@ -226,6 +226,70 @@ defmodule Membrane.WebRTC.IntegrationTest do
     end
   end
 
+  defmodule H265SendRecv do
+    use ExUnit.Case, async: true
+
+    import Membrane.Testing.Assertions
+    import Utils
+
+    test "send and receive an H265 stream" do
+      signaling = Signaling.new()
+
+      send_pipeline =
+        Testing.Pipeline.start_link_supervised!(
+          spec:
+            child(%Membrane.File.Source{location: "test/fixtures/input_bbb.h265"})
+            |> child(%Membrane.H265.Parser{
+              output_alignment: :nalu,
+              output_stream_structure: :annexb,
+              generate_best_effort_timestamps: %{framerate: {24, 1}}
+            })
+            |> child(Membrane.Realtimer)
+            |> via_in(Pad.ref(:input, :video), options: [kind: :video])
+            |> child(:webrtc, %WebRTC.Sink{
+              signaling: signaling,
+              tracks: [:video],
+              video_codec: [:h265]
+            })
+        )
+
+      receive_pipeline =
+        Testing.Pipeline.start_link_supervised!(
+          spec:
+            child(:webrtc, %WebRTC.Source{
+              signaling: signaling,
+              allowed_video_codecs: [:h265],
+              preferred_video_codec: :h265
+            })
+            |> via_out(Pad.ref(:output, :video), options: [kind: :video])
+            |> child(:video_sink, Membrane.Testing.Sink)
+        )
+
+      [send_pipeline, receive_pipeline]
+      |> Enum.each(fn pipeline ->
+        assert_pipeline_notified(
+          pipeline,
+          :webrtc,
+          {:negotiated_video_codecs, [:h265]},
+          fixture_processing_timeout()
+        )
+      end)
+
+      assert_sink_stream_format(receive_pipeline, :video_sink, %Membrane.H265{}, 10_000)
+      assert_sink_buffer(receive_pipeline, :video_sink, %Membrane.Buffer{}, 10_000)
+
+      assert_pipeline_notified(
+        send_pipeline,
+        :webrtc,
+        {:end_of_stream, :video},
+        fixture_processing_timeout()
+      )
+
+      Testing.Pipeline.terminate(send_pipeline)
+      Testing.Pipeline.terminate(receive_pipeline)
+    end
+  end
+
   defmodule DynamicTracks do
     use ExUnit.Case, async: true