From e3cfb58b3fc2b3f23b955b0cf20d240c3e9eed53 Mon Sep 17 00:00:00 2001 From: aler9 <46489434+aler9@users.noreply.github.com> Date: Wed, 19 Jun 2024 20:54:46 +0200 Subject: [PATCH] webrtc: support publishing H265 tracks (#3435) IMPORTANT NOTE: this doesn't allow to read H265 tracks with WebRTC, just to publish them. The inability to read H265 tracks with WebRTC is not in any way related to the server but depends on browsers and on the fact that they are not legally entitled to embed a H265 decoder inside them. --- README.md | 4 +- internal/protocols/webrtc/incoming_track.go | 41 ++++++++++++++----- internal/protocols/webrtc/outgoing_track.go | 10 +++++ .../protocols/webrtc/peer_connection_test.go | 11 +++++ internal/protocols/webrtc/tracks_to_medias.go | 16 +------- 5 files changed, 56 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 25ec0c46688..3e09e429893 100644 --- a/README.md +++ b/README.md @@ -22,8 +22,8 @@ Live streams can be published to the server with: |--------|--------|------------|------------| |[SRT clients](#srt-clients)||H265, H264, MPEG-4 Video (H263, Xvid), MPEG-1/2 Video|Opus, MPEG-4 Audio (AAC), MPEG-1/2 Audio (MP3), AC-3| |[SRT cameras and servers](#srt-cameras-and-servers)||H265, H264, MPEG-4 Video (H263, Xvid), MPEG-1/2 Video|Opus, MPEG-4 Audio (AAC), MPEG-1/2 Audio (MP3), AC-3| -|[WebRTC clients](#webrtc-clients)|Browser-based, WHIP|AV1, VP9, VP8, H264|Opus, G722, G711 (PCMA, PCMU)| -|[WebRTC servers](#webrtc-servers)|WHEP|AV1, VP9, VP8, H264|Opus, G722, G711 (PCMA, PCMU)| +|[WebRTC clients](#webrtc-clients)|Browser-based, WHIP|AV1, VP9, VP8, H265, H264|Opus, G722, G711 (PCMA, PCMU)| +|[WebRTC servers](#webrtc-servers)|WHEP|AV1, VP9, VP8, H265, H264|Opus, G722, G711 (PCMA, PCMU)| |[RTSP clients](#rtsp-clients)|UDP, TCP, RTSPS|AV1, VP9, VP8, H265, H264, MPEG-4 Video (H263, Xvid), MPEG-1/2 Video, M-JPEG and any RTP-compatible codec|Opus, MPEG-4 Audio (AAC), MPEG-1/2 Audio (MP3), AC-3, G726, G722, G711 (PCMA, PCMU), LPCM and any RTP-compatible codec| |[RTSP cameras and servers](#rtsp-cameras-and-servers)|UDP, UDP-Multicast, TCP, RTSPS|AV1, VP9, VP8, H265, H264, MPEG-4 Video (H263, Xvid), MPEG-1/2 Video, M-JPEG and any RTP-compatible codec|Opus, MPEG-4 Audio (AAC), MPEG-1/2 Audio (MP3), AC-3, G726, G722, G711 (PCMA, PCMU), LPCM and any RTP-compatible codec| |[RTMP clients](#rtmp-clients)|RTMP, RTMPS, Enhanced RTMP|AV1, VP9, H265, H264|MPEG-4 Audio (AAC), MPEG-1/2 Audio (MP3), G711 (PCMA, PCMU), LPCM| diff --git a/internal/protocols/webrtc/incoming_track.go b/internal/protocols/webrtc/incoming_track.go index 9694403137b..71016e96314 100644 --- a/internal/protocols/webrtc/incoming_track.go +++ b/internal/protocols/webrtc/incoming_track.go @@ -5,6 +5,7 @@ import ( "strings" "time" + "github.com/bluenviron/gortsplib/v4/pkg/description" "github.com/bluenviron/gortsplib/v4/pkg/format" "github.com/bluenviron/gortsplib/v4/pkg/liberrors" "github.com/bluenviron/gortsplib/v4/pkg/rtpreorderer" @@ -79,13 +80,20 @@ var incomingVideoCodecs = []webrtc.RTPCodecParameters{ }, PayloadType: 102, }, + { + RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: webrtc.MimeTypeH265, + ClockRate: 90000, + }, + PayloadType: 103, + }, { RTPCodecCapability: webrtc.RTPCodecCapability{ MimeType: webrtc.MimeTypeH264, ClockRate: 90000, SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f", }, - PayloadType: 103, + PayloadType: 104, }, { RTPCodecCapability: webrtc.RTPCodecCapability{ @@ -93,7 +101,7 @@ var incomingVideoCodecs = []webrtc.RTPCodecParameters{ ClockRate: 90000, SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f", }, - PayloadType: 104, + PayloadType: 105, }, } @@ -229,6 +237,7 @@ type IncomingTrack struct { track *webrtc.TrackRemote log logger.Writer + typ description.MediaType format format.Format reorderer *rtpreorderer.Reorderer pkts []*rtp.Packet @@ -246,41 +255,47 @@ func newIncomingTrack( reorderer: rtpreorderer.New(), } - isVideo := false - switch strings.ToLower(track.Codec().MimeType) { case strings.ToLower(webrtc.MimeTypeAV1): - isVideo = true + t.typ = description.MediaTypeVideo t.format = &format.AV1{ PayloadTyp: uint8(track.PayloadType()), } case strings.ToLower(webrtc.MimeTypeVP9): - isVideo = true + t.typ = description.MediaTypeVideo t.format = &format.VP9{ PayloadTyp: uint8(track.PayloadType()), } case strings.ToLower(webrtc.MimeTypeVP8): - isVideo = true + t.typ = description.MediaTypeVideo t.format = &format.VP8{ PayloadTyp: uint8(track.PayloadType()), } + case strings.ToLower(webrtc.MimeTypeH265): + t.typ = description.MediaTypeVideo + t.format = &format.H265{ + PayloadTyp: uint8(track.PayloadType()), + } + case strings.ToLower(webrtc.MimeTypeH264): - isVideo = true + t.typ = description.MediaTypeVideo t.format = &format.H264{ PayloadTyp: uint8(track.PayloadType()), PacketizationMode: 1, } case strings.ToLower(mimeTypeMultiopus): + t.typ = description.MediaTypeAudio t.format = &format.Opus{ PayloadTyp: uint8(track.PayloadType()), ChannelCount: int(track.Codec().Channels), } case strings.ToLower(webrtc.MimeTypeOpus): + t.typ = description.MediaTypeAudio t.format = &format.Opus{ PayloadTyp: uint8(track.PayloadType()), ChannelCount: func() int { @@ -292,9 +307,12 @@ func newIncomingTrack( } case strings.ToLower(webrtc.MimeTypeG722): + t.typ = description.MediaTypeAudio t.format = &format.G722{} case strings.ToLower(webrtc.MimeTypePCMU): + t.typ = description.MediaTypeAudio + channels := track.Codec().Channels if channels == 0 { channels = 1 @@ -313,6 +331,8 @@ func newIncomingTrack( } case strings.ToLower(webrtc.MimeTypePCMA): + t.typ = description.MediaTypeAudio + channels := track.Codec().Channels if channels == 0 { channels = 1 @@ -331,6 +351,7 @@ func newIncomingTrack( } case strings.ToLower(mimeTypeL16): + t.typ = description.MediaTypeAudio t.format = &format.LPCM{ PayloadTyp: uint8(track.PayloadType()), BitDepth: 16, @@ -339,7 +360,7 @@ func newIncomingTrack( } default: - return nil, fmt.Errorf("unsupported codec: %+v", track.Codec()) + return nil, fmt.Errorf("unsupported codec: %+v", track.Codec().RTPCodecCapability) } // read incoming RTCP packets to make interceptors work @@ -354,7 +375,7 @@ func newIncomingTrack( }() // send period key frame requests - if isVideo { + if t.typ == description.MediaTypeVideo { go func() { keyframeTicker := time.NewTicker(keyFrameInterval) defer keyframeTicker.Stop() diff --git a/internal/protocols/webrtc/outgoing_track.go b/internal/protocols/webrtc/outgoing_track.go index 3f72c1cc650..adc60e51478 100644 --- a/internal/protocols/webrtc/outgoing_track.go +++ b/internal/protocols/webrtc/outgoing_track.go @@ -54,6 +54,15 @@ func (t *OutgoingTrack) codecParameters() (webrtc.RTPCodecParameters, error) { PayloadType: 96, }, nil + case *format.H265: + return webrtc.RTPCodecParameters{ + RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: webrtc.MimeTypeH265, + ClockRate: 90000, + }, + PayloadType: 96, + }, nil + case *format.H264: return webrtc.RTPCodecParameters{ RTPCodecCapability: webrtc.RTPCodecCapability{ @@ -205,6 +214,7 @@ func (t *OutgoingTrack) isVideo() bool { case *format.AV1, *format.VP9, *format.VP8, + *format.H265, *format.H264: return true } diff --git a/internal/protocols/webrtc/peer_connection_test.go b/internal/protocols/webrtc/peer_connection_test.go index 1927786fafe..60454133afd 100644 --- a/internal/protocols/webrtc/peer_connection_test.go +++ b/internal/protocols/webrtc/peer_connection_test.go @@ -83,6 +83,17 @@ func TestPeerConnectionPublishRead(t *testing.T) { PayloadTyp: 96, }, }, + { + "h265", + test.FormatH265, + webrtc.RTPCodecCapability{ + MimeType: "video/H265", + ClockRate: 90000, + }, + &format.H265{ + PayloadTyp: 96, + }, + }, { "h264", test.FormatH264, diff --git a/internal/protocols/webrtc/tracks_to_medias.go b/internal/protocols/webrtc/tracks_to_medias.go index 809eeec4d53..81ea085061a 100644 --- a/internal/protocols/webrtc/tracks_to_medias.go +++ b/internal/protocols/webrtc/tracks_to_medias.go @@ -10,21 +10,9 @@ func TracksToMedias(tracks []*IncomingTrack) []*description.Media { ret := make([]*description.Media, len(tracks)) for i, track := range tracks { - forma := track.Format() - - var mediaType description.MediaType - - switch forma.(type) { - case *format.AV1, *format.VP9, *format.VP8, *format.H264: - mediaType = description.MediaTypeVideo - - default: - mediaType = description.MediaTypeAudio - } - ret[i] = &description.Media{ - Type: mediaType, - Formats: []format.Format{forma}, + Type: track.typ, + Formats: []format.Format{track.format}, } }