webrtc: support publishing H265 tracks (#3435) (#3492)

IMPORTANT NOTE: this doesn't allow to read H265 tracks with WebRTC, just to publish them. The inability to read H265 tracks with WebRTC is not in any way related to the server but depends on browsers and on the fact that they are not legally entitled to embed a H265 decoder inside them.
bluenviron · Jun 19, 2024 · a1dc9f4 · a1dc9f4
1 parent 65d90f7
commit a1dc9f4
Show file tree

Hide file tree

Showing 5 changed files with 56 additions and 26 deletions.
diff --git a/README.md b/README.md
@@ -22,8 +22,8 @@ Live streams can be published to the server with:
 |--------|--------|------------|------------|
 |[SRT clients](#srt-clients)||H265, H264, MPEG-4 Video (H263, Xvid), MPEG-1/2 Video|Opus, MPEG-4 Audio (AAC), MPEG-1/2 Audio (MP3), AC-3|
 |[SRT cameras and servers](#srt-cameras-and-servers)||H265, H264, MPEG-4 Video (H263, Xvid), MPEG-1/2 Video|Opus, MPEG-4 Audio (AAC), MPEG-1/2 Audio (MP3), AC-3|
-|[WebRTC clients](#webrtc-clients)|Browser-based, WHIP|AV1, VP9, VP8, H264|Opus, G722, G711 (PCMA, PCMU)|
-|[WebRTC servers](#webrtc-servers)|WHEP|AV1, VP9, VP8, H264|Opus, G722, G711 (PCMA, PCMU)|
+|[WebRTC clients](#webrtc-clients)|Browser-based, WHIP|AV1, VP9, VP8, H265, H264|Opus, G722, G711 (PCMA, PCMU)|
+|[WebRTC servers](#webrtc-servers)|WHEP|AV1, VP9, VP8, H265, H264|Opus, G722, G711 (PCMA, PCMU)|
 |[RTSP clients](#rtsp-clients)|UDP, TCP, RTSPS|AV1, VP9, VP8, H265, H264, MPEG-4 Video (H263, Xvid), MPEG-1/2 Video, M-JPEG and any RTP-compatible codec|Opus, MPEG-4 Audio (AAC), MPEG-1/2 Audio (MP3), AC-3, G726, G722, G711 (PCMA, PCMU), LPCM and any RTP-compatible codec|
 |[RTSP cameras and servers](#rtsp-cameras-and-servers)|UDP, UDP-Multicast, TCP, RTSPS|AV1, VP9, VP8, H265, H264, MPEG-4 Video (H263, Xvid), MPEG-1/2 Video, M-JPEG and any RTP-compatible codec|Opus, MPEG-4 Audio (AAC), MPEG-1/2 Audio (MP3), AC-3, G726, G722, G711 (PCMA, PCMU), LPCM and any RTP-compatible codec|
 |[RTMP clients](#rtmp-clients)|RTMP, RTMPS, Enhanced RTMP|AV1, VP9, H265, H264|MPEG-4 Audio (AAC), MPEG-1/2 Audio (MP3), G711 (PCMA, PCMU), LPCM|

diff --git a/internal/protocols/webrtc/incoming_track.go b/internal/protocols/webrtc/incoming_track.go
@@ -5,6 +5,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/bluenviron/gortsplib/v4/pkg/description"
 	"github.com/bluenviron/gortsplib/v4/pkg/format"
 	"github.com/bluenviron/gortsplib/v4/pkg/liberrors"
 	"github.com/bluenviron/gortsplib/v4/pkg/rtpreorderer"
@@ -79,21 +80,28 @@ var incomingVideoCodecs = []webrtc.RTPCodecParameters{
 		},
 		PayloadType: 102,
 	},
+	{
+		RTPCodecCapability: webrtc.RTPCodecCapability{
+			MimeType:  webrtc.MimeTypeH265,
+			ClockRate: 90000,
+		},
+		PayloadType: 103,
+	},
 	{
 		RTPCodecCapability: webrtc.RTPCodecCapability{
 			MimeType:    webrtc.MimeTypeH264,
 			ClockRate:   90000,
 			SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f",
 		},
-		PayloadType: 103,
+		PayloadType: 104,
 	},
 	{
 		RTPCodecCapability: webrtc.RTPCodecCapability{
 			MimeType:    webrtc.MimeTypeH264,
 			ClockRate:   90000,
 			SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f",
 		},
-		PayloadType: 104,
+		PayloadType: 105,
 	},
 }
 
@@ -229,6 +237,7 @@ type IncomingTrack struct {
 	track *webrtc.TrackRemote
 	log   logger.Writer
 
+	typ       description.MediaType
 	format    format.Format
 	reorderer *rtpreorderer.Reorderer
 	pkts      []*rtp.Packet
@@ -246,41 +255,47 @@ func newIncomingTrack(
 		reorderer: rtpreorderer.New(),
 	}
 
-	isVideo := false
-
 	switch strings.ToLower(track.Codec().MimeType) {
 	case strings.ToLower(webrtc.MimeTypeAV1):
-		isVideo = true
+		t.typ = description.MediaTypeVideo
 		t.format = &format.AV1{
 			PayloadTyp: uint8(track.PayloadType()),
 		}
 
 	case strings.ToLower(webrtc.MimeTypeVP9):
-		isVideo = true
+		t.typ = description.MediaTypeVideo
 		t.format = &format.VP9{
 			PayloadTyp: uint8(track.PayloadType()),
 		}
 
 	case strings.ToLower(webrtc.MimeTypeVP8):
-		isVideo = true
+		t.typ = description.MediaTypeVideo
 		t.format = &format.VP8{
 			PayloadTyp: uint8(track.PayloadType()),
 		}
 
+	case strings.ToLower(webrtc.MimeTypeH265):
+		t.typ = description.MediaTypeVideo
+		t.format = &format.H265{
+			PayloadTyp: uint8(track.PayloadType()),
+		}
+
 	case strings.ToLower(webrtc.MimeTypeH264):
-		isVideo = true
+		t.typ = description.MediaTypeVideo
 		t.format = &format.H264{
 			PayloadTyp:        uint8(track.PayloadType()),
 			PacketizationMode: 1,
 		}
 
 	case strings.ToLower(mimeTypeMultiopus):
+		t.typ = description.MediaTypeAudio
 		t.format = &format.Opus{
 			PayloadTyp:   uint8(track.PayloadType()),
 			ChannelCount: int(track.Codec().Channels),
 		}
 
 	case strings.ToLower(webrtc.MimeTypeOpus):
+		t.typ = description.MediaTypeAudio
 		t.format = &format.Opus{
 			PayloadTyp: uint8(track.PayloadType()),
 			ChannelCount: func() int {
@@ -292,9 +307,12 @@ func newIncomingTrack(
 		}
 
 	case strings.ToLower(webrtc.MimeTypeG722):
+		t.typ = description.MediaTypeAudio
 		t.format = &format.G722{}
 
 	case strings.ToLower(webrtc.MimeTypePCMU):
+		t.typ = description.MediaTypeAudio
+
 		channels := track.Codec().Channels
 		if channels == 0 {
 			channels = 1
@@ -313,6 +331,8 @@ func newIncomingTrack(
 		}
 
 	case strings.ToLower(webrtc.MimeTypePCMA):
+		t.typ = description.MediaTypeAudio
+
 		channels := track.Codec().Channels
 		if channels == 0 {
 			channels = 1
@@ -331,6 +351,7 @@ func newIncomingTrack(
 		}
 
 	case strings.ToLower(mimeTypeL16):
+		t.typ = description.MediaTypeAudio
 		t.format = &format.LPCM{
 			PayloadTyp:   uint8(track.PayloadType()),
 			BitDepth:     16,
@@ -339,7 +360,7 @@ func newIncomingTrack(
 		}
 
 	default:
-		return nil, fmt.Errorf("unsupported codec: %+v", track.Codec())
+		return nil, fmt.Errorf("unsupported codec: %+v", track.Codec().RTPCodecCapability)
 	}
 
 	// read incoming RTCP packets to make interceptors work
@@ -354,7 +375,7 @@ func newIncomingTrack(
 	}()
 
 	// send period key frame requests
-	if isVideo {
+	if t.typ == description.MediaTypeVideo {
 		go func() {
 			keyframeTicker := time.NewTicker(keyFrameInterval)
 			defer keyframeTicker.Stop()

diff --git a/internal/protocols/webrtc/outgoing_track.go b/internal/protocols/webrtc/outgoing_track.go
@@ -54,6 +54,15 @@ func (t *OutgoingTrack) codecParameters() (webrtc.RTPCodecParameters, error) {
 			PayloadType: 96,
 		}, nil
 
+	case *format.H265:
+		return webrtc.RTPCodecParameters{
+			RTPCodecCapability: webrtc.RTPCodecCapability{
+				MimeType:  webrtc.MimeTypeH265,
+				ClockRate: 90000,
+			},
+			PayloadType: 96,
+		}, nil
+
 	case *format.H264:
 		return webrtc.RTPCodecParameters{
 			RTPCodecCapability: webrtc.RTPCodecCapability{
@@ -205,6 +214,7 @@ func (t *OutgoingTrack) isVideo() bool {
 	case *format.AV1,
 		*format.VP9,
 		*format.VP8,
+		*format.H265,
 		*format.H264:
 		return true
 	}

diff --git a/internal/protocols/webrtc/peer_connection_test.go b/internal/protocols/webrtc/peer_connection_test.go
@@ -83,6 +83,17 @@ func TestPeerConnectionPublishRead(t *testing.T) {
 				PayloadTyp: 96,
 			},
 		},
+		{
+			"h265",
+			test.FormatH265,
+			webrtc.RTPCodecCapability{
+				MimeType:  "video/H265",
+				ClockRate: 90000,
+			},
+			&format.H265{
+				PayloadTyp: 96,
+			},
+		},
 		{
 			"h264",
 			test.FormatH264,

diff --git a/internal/protocols/webrtc/tracks_to_medias.go b/internal/protocols/webrtc/tracks_to_medias.go
@@ -10,21 +10,9 @@ func TracksToMedias(tracks []*IncomingTrack) []*description.Media {
 	ret := make([]*description.Media, len(tracks))
 
 	for i, track := range tracks {
-		forma := track.Format()
-
-		var mediaType description.MediaType
-
-		switch forma.(type) {
-		case *format.AV1, *format.VP9, *format.VP8, *format.H264:
-			mediaType = description.MediaTypeVideo
-
-		default:
-			mediaType = description.MediaTypeAudio
-		}
-
 		ret[i] = &description.Media{
-			Type:    mediaType,
-			Formats: []format.Format{forma},
+			Type:    track.typ,
+			Formats: []format.Format{track.format},
 		}
 	}