Skip to content

Commit

Permalink
Add Retransmission and FEC to TrackLocal
Browse files Browse the repository at this point in the history
If the MediaEngine contains support for them a SSRC will be generated
appropriately

Co-authored-by: aggresss <[email protected]>
Co-authored-by: Kevin Wang <[email protected]>

Resolves #1989
Resolves #1675
  • Loading branch information
Sean-Der committed Oct 4, 2024
1 parent bd2309f commit d2c9d16
Show file tree
Hide file tree
Showing 21 changed files with 455 additions and 109 deletions.
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ require (
github.com/pion/datachannel v1.5.9
github.com/pion/dtls/v3 v3.0.2
github.com/pion/ice/v4 v4.0.1
github.com/pion/interceptor v0.1.31
github.com/pion/interceptor v0.1.32
github.com/pion/logging v0.2.2
github.com/pion/randutil v0.1.0
github.com/pion/rtcp v1.2.14
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,8 @@ github.com/pion/dtls/v3 v3.0.2 h1:425DEeJ/jfuTTghhUDW0GtYZYIwwMtnKKJNMcWccTX0=
github.com/pion/dtls/v3 v3.0.2/go.mod h1:dfIXcFkKoujDQ+jtd8M6RgqKK3DuaUilm3YatAbGp5k=
github.com/pion/ice/v4 v4.0.1 h1:2d3tPoTR90F3TcGYeXUwucGlXI3hds96cwv4kjZmb9s=
github.com/pion/ice/v4 v4.0.1/go.mod h1:2dpakjpd7+74L5j3TAe6gvkbI5UIzOgAnkimm9SuHvA=
github.com/pion/interceptor v0.1.31 h1:9enhHjP1fDfMI8sqvpO5c/9QuTQnCf2dzPHwwIH4x5w=
github.com/pion/interceptor v0.1.31/go.mod h1:JzxbJ4umVTlZAf+/utHzNesY8tmRkM2lVmkS82TTj8Y=
github.com/pion/interceptor v0.1.32 h1:DYbusOBhWKjPMiA5ifyczW03Tnh12gCaYn4VOvLMGk4=
github.com/pion/interceptor v0.1.32/go.mod h1:JzxbJ4umVTlZAf+/utHzNesY8tmRkM2lVmkS82TTj8Y=
github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY=
github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms=
github.com/pion/mdns/v2 v2.0.7 h1:c9kM8ewCgjslaAmicYMFQIde2H9/lrZpjBkN8VwoVtM=
Expand Down
2 changes: 1 addition & 1 deletion interceptor.go
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@ func (i *interceptorToTrackLocalWriter) Write(b []byte) (int, error) {
return i.WriteRTP(&packet.Header, packet.Payload)
}

func createStreamInfo(id string, ssrc SSRC, payloadType PayloadType, codec RTPCodecCapability, webrtcHeaderExtensions []RTPHeaderExtensionParameter) *interceptor.StreamInfo {
func createStreamInfo(id string, ssrc, ssrcFEC, ssrcRTX SSRC, payloadType PayloadType, codec RTPCodecCapability, webrtcHeaderExtensions []RTPHeaderExtensionParameter) *interceptor.StreamInfo {

Check failure on line 162 in interceptor.go

View workflow job for this annotation

GitHub Actions / lint / Go

unused-parameter: parameter 'ssrcFEC' seems to be unused, consider removing or renaming it as _ (revive)
headerExtensions := make([]interceptor.RTPHeaderExtension, 0, len(webrtcHeaderExtensions))
for _, h := range webrtcHeaderExtensions {
headerExtensions = append(headerExtensions, interceptor.RTPHeaderExtension{ID: h.ID, URI: h.URI})
Expand Down
6 changes: 3 additions & 3 deletions interceptor_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -196,18 +196,18 @@ func Test_Interceptor_BindUnbind(t *testing.T) {
if cnt := atomic.LoadUint32(&cntUnbindLocalStream); cnt != 1 {
t.Errorf("UnbindLocalStreamFn is expected to be called once, but called %d times", cnt)
}
if cnt := atomic.LoadUint32(&cntBindRemoteStream); cnt != 1 {
if cnt := atomic.LoadUint32(&cntBindRemoteStream); cnt != 2 {
t.Errorf("BindRemoteStreamFn is expected to be called once, but called %d times", cnt)
}
if cnt := atomic.LoadUint32(&cntUnbindRemoteStream); cnt != 1 {
if cnt := atomic.LoadUint32(&cntUnbindRemoteStream); cnt != 2 {
t.Errorf("UnbindRemoteStreamFn is expected to be called once, but called %d times", cnt)
}

// BindRTCPWriter/Reader and Close should be called from both side.
if cnt := atomic.LoadUint32(&cntBindRTCPWriter); cnt != 2 {
t.Errorf("BindRTCPWriterFn is expected to be called twice, but called %d times", cnt)
}
if cnt := atomic.LoadUint32(&cntBindRTCPReader); cnt != 2 {
if cnt := atomic.LoadUint32(&cntBindRTCPReader); cnt != 3 {
t.Errorf("BindRTCPReaderFn is expected to be called twice, but called %d times", cnt)
}
if cnt := atomic.LoadUint32(&cntClose); cnt != 2 {
Expand Down
48 changes: 37 additions & 11 deletions mediaengine.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,12 @@ const (
// MimeTypePCMA PCMA MIME type
// Note: Matching should be case insensitive.
MimeTypePCMA = "audio/PCMA"
// MimeTypeRTX RTX MIME type
// Note: Matching should be case insensitive.
MimeTypeRTX = "video/rtx"
// MimeTypeFlexFEC FEC MIME Type
// Note: Matching should be case insensitive.
MimeTypeFlexFEC = "video/flexfec"
)

type mediaEngineHeaderExtension struct {
Expand Down Expand Up @@ -106,7 +112,7 @@ func (m *MediaEngine) RegisterDefaultCodecs() error {
PayloadType: 96,
},
{
RTPCodecCapability: RTPCodecCapability{"video/rtx", 90000, 0, "apt=96", nil},
RTPCodecCapability: RTPCodecCapability{MimeTypeRTX, 90000, 0, "apt=96", nil},
PayloadType: 97,
},

Expand All @@ -115,7 +121,7 @@ func (m *MediaEngine) RegisterDefaultCodecs() error {
PayloadType: 102,
},
{
RTPCodecCapability: RTPCodecCapability{"video/rtx", 90000, 0, "apt=102", nil},
RTPCodecCapability: RTPCodecCapability{MimeTypeRTX, 90000, 0, "apt=102", nil},
PayloadType: 103,
},

Expand All @@ -124,7 +130,7 @@ func (m *MediaEngine) RegisterDefaultCodecs() error {
PayloadType: 104,
},
{
RTPCodecCapability: RTPCodecCapability{"video/rtx", 90000, 0, "apt=104", nil},
RTPCodecCapability: RTPCodecCapability{MimeTypeRTX, 90000, 0, "apt=104", nil},
PayloadType: 105,
},

Expand All @@ -133,7 +139,7 @@ func (m *MediaEngine) RegisterDefaultCodecs() error {
PayloadType: 106,
},
{
RTPCodecCapability: RTPCodecCapability{"video/rtx", 90000, 0, "apt=106", nil},
RTPCodecCapability: RTPCodecCapability{MimeTypeRTX, 90000, 0, "apt=106", nil},
PayloadType: 107,
},

Expand All @@ -142,7 +148,7 @@ func (m *MediaEngine) RegisterDefaultCodecs() error {
PayloadType: 108,
},
{
RTPCodecCapability: RTPCodecCapability{"video/rtx", 90000, 0, "apt=108", nil},
RTPCodecCapability: RTPCodecCapability{MimeTypeRTX, 90000, 0, "apt=108", nil},
PayloadType: 109,
},

Expand All @@ -151,7 +157,7 @@ func (m *MediaEngine) RegisterDefaultCodecs() error {
PayloadType: 127,
},
{
RTPCodecCapability: RTPCodecCapability{"video/rtx", 90000, 0, "apt=127", nil},
RTPCodecCapability: RTPCodecCapability{MimeTypeRTX, 90000, 0, "apt=127", nil},
PayloadType: 125,
},

Expand All @@ -160,7 +166,7 @@ func (m *MediaEngine) RegisterDefaultCodecs() error {
PayloadType: 39,
},
{
RTPCodecCapability: RTPCodecCapability{"video/rtx", 90000, 0, "apt=39", nil},
RTPCodecCapability: RTPCodecCapability{MimeTypeRTX, 90000, 0, "apt=39", nil},
PayloadType: 40,
},

Expand All @@ -169,7 +175,7 @@ func (m *MediaEngine) RegisterDefaultCodecs() error {
PayloadType: 45,
},
{
RTPCodecCapability: RTPCodecCapability{"video/rtx", 90000, 0, "apt=45", nil},
RTPCodecCapability: RTPCodecCapability{MimeTypeRTX, 90000, 0, "apt=45", nil},
PayloadType: 46,
},

Expand All @@ -178,7 +184,7 @@ func (m *MediaEngine) RegisterDefaultCodecs() error {
PayloadType: 98,
},
{
RTPCodecCapability: RTPCodecCapability{"video/rtx", 90000, 0, "apt=98", nil},
RTPCodecCapability: RTPCodecCapability{MimeTypeRTX, 90000, 0, "apt=98", nil},
PayloadType: 99,
},

Expand All @@ -187,7 +193,7 @@ func (m *MediaEngine) RegisterDefaultCodecs() error {
PayloadType: 100,
},
{
RTPCodecCapability: RTPCodecCapability{"video/rtx", 90000, 0, "apt=100", nil},
RTPCodecCapability: RTPCodecCapability{MimeTypeRTX, 90000, 0, "apt=100", nil},
PayloadType: 101,
},

Expand All @@ -196,7 +202,7 @@ func (m *MediaEngine) RegisterDefaultCodecs() error {
PayloadType: 112,
},
{
RTPCodecCapability: RTPCodecCapability{"video/rtx", 90000, 0, "apt=112", nil},
RTPCodecCapability: RTPCodecCapability{MimeTypeRTX, 90000, 0, "apt=112", nil},
PayloadType: 113,
},
} {
Expand Down Expand Up @@ -702,3 +708,23 @@ func payloaderForCodec(codec RTPCodecCapability) (rtp.Payloader, error) {
return nil, ErrNoPayloaderForCodec
}
}

func (m *MediaEngine) isRTXEnabled(typ RTPCodecType, directions []RTPTransceiverDirection) bool {
for _, p := range m.getRTPParametersByKind(typ, directions).Codecs {
if p.MimeType == MimeTypeRTX {
return true
}
}

return false
}

func (m *MediaEngine) isFECEnabled(typ RTPCodecType, directions []RTPTransceiverDirection) bool {
for _, p := range m.getRTPParametersByKind(typ, directions).Codecs {
if strings.Contains(p.MimeType, MimeTypeFlexFEC) {
return true
}
}

return false
}
16 changes: 8 additions & 8 deletions mediaengine_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -364,31 +364,31 @@ a=fmtp:97 apt=96
PayloadType: 96,
}, RTPCodecTypeVideo))
assert.NoError(t, m.RegisterCodec(RTPCodecParameters{
RTPCodecCapability: RTPCodecCapability{"video/rtx", 90000, 0, "apt=96", nil},
RTPCodecCapability: RTPCodecCapability{MimeTypeRTX, 90000, 0, "apt=96", nil},
PayloadType: 97,
}, RTPCodecTypeVideo))
assert.NoError(t, m.RegisterCodec(RTPCodecParameters{
RTPCodecCapability: RTPCodecCapability{MimeTypeH264, 90000, 0, "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f", nil},
PayloadType: 102,
}, RTPCodecTypeVideo))
assert.NoError(t, m.RegisterCodec(RTPCodecParameters{
RTPCodecCapability: RTPCodecCapability{"video/rtx", 90000, 0, "apt=102", nil},
RTPCodecCapability: RTPCodecCapability{MimeTypeRTX, 90000, 0, "apt=102", nil},
PayloadType: 103,
}, RTPCodecTypeVideo))
assert.NoError(t, m.RegisterCodec(RTPCodecParameters{
RTPCodecCapability: RTPCodecCapability{MimeTypeH264, 90000, 0, "level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42001f", nil},
PayloadType: 104,
}, RTPCodecTypeVideo))
assert.NoError(t, m.RegisterCodec(RTPCodecParameters{
RTPCodecCapability: RTPCodecCapability{"video/rtx", 90000, 0, "apt=104", nil},
RTPCodecCapability: RTPCodecCapability{MimeTypeRTX, 90000, 0, "apt=104", nil},
PayloadType: 105,
}, RTPCodecTypeVideo))
assert.NoError(t, m.RegisterCodec(RTPCodecParameters{
RTPCodecCapability: RTPCodecCapability{MimeTypeVP9, 90000, 0, "profile-id=2", nil},
PayloadType: 98,
}, RTPCodecTypeVideo))
assert.NoError(t, m.RegisterCodec(RTPCodecParameters{
RTPCodecCapability: RTPCodecCapability{"video/rtx", 90000, 0, "apt=98", nil},
RTPCodecCapability: RTPCodecCapability{MimeTypeRTX, 90000, 0, "apt=98", nil},
PayloadType: 99,
}, RTPCodecTypeVideo))
assert.NoError(t, m.updateFromRemoteDescription(mustParse(profileLevels)))
Expand All @@ -400,15 +400,15 @@ a=fmtp:97 apt=96
assert.Equal(t, vp9Codec.MimeType, MimeTypeVP9)
vp9RTX, _, err := m.getCodecByPayload(97)
assert.NoError(t, err)
assert.Equal(t, vp9RTX.MimeType, "video/rtx")
assert.Equal(t, vp9RTX.MimeType, MimeTypeRTX)

h264P1Codec, _, err := m.getCodecByPayload(106)
assert.NoError(t, err)
assert.Equal(t, h264P1Codec.MimeType, MimeTypeH264)
assert.Equal(t, h264P1Codec.SDPFmtpLine, "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f")
h264P1RTX, _, err := m.getCodecByPayload(107)
assert.NoError(t, err)
assert.Equal(t, h264P1RTX.MimeType, "video/rtx")
assert.Equal(t, h264P1RTX.MimeType, MimeTypeRTX)
assert.Equal(t, h264P1RTX.SDPFmtpLine, "apt=106")

h264P0Codec, _, err := m.getCodecByPayload(108)
Expand All @@ -417,7 +417,7 @@ a=fmtp:97 apt=96
assert.Equal(t, h264P0Codec.SDPFmtpLine, "level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42001f")
h264P0RTX, _, err := m.getCodecByPayload(109)
assert.NoError(t, err)
assert.Equal(t, h264P0RTX.MimeType, "video/rtx")
assert.Equal(t, h264P0RTX.MimeType, MimeTypeRTX)
assert.Equal(t, h264P0RTX.SDPFmtpLine, "apt=108")
})

Expand All @@ -443,7 +443,7 @@ a=fmtp:97 apt=96
PayloadType: 96,
}, RTPCodecTypeVideo))
assert.NoError(t, m.RegisterCodec(RTPCodecParameters{
RTPCodecCapability: RTPCodecCapability{"video/rtx", 90000, 0, "apt=96", nil},
RTPCodecCapability: RTPCodecCapability{MimeTypeRTX, 90000, 0, "apt=96", nil},
PayloadType: 97,
}, RTPCodecTypeVideo))
assert.NoError(t, m.updateFromRemoteDescription(mustParse(profileLevels)))
Expand Down
7 changes: 6 additions & 1 deletion peerconnection.go
Original file line number Diff line number Diff line change
Expand Up @@ -1048,6 +1048,11 @@ func (pc *PeerConnection) SetRemoteDescription(desc SessionDescription) error {
return err
}

// Disable RTX/FEC on RTPSenders if the remote didn't support it
for _, sender := range pc.GetSenders() {
sender.configureRTXAndFEC()
}

var t *RTPTransceiver
localTransceivers := append([]*RTPTransceiver{}, pc.GetTransceivers()...)
detectedPlanB := descriptionIsPlanB(pc.RemoteDescription(), pc.log)
Expand Down Expand Up @@ -1616,7 +1621,7 @@ func (pc *PeerConnection) handleIncomingSSRC(rtpStream io.Reader, ssrc SSRC) err
return err
}

streamInfo := createStreamInfo("", ssrc, params.Codecs[0].PayloadType, params.Codecs[0].RTPCodecCapability, params.HeaderExtensions)
streamInfo := createStreamInfo("", ssrc, 0, 0, params.Codecs[0].PayloadType, params.Codecs[0].RTPCodecCapability, params.HeaderExtensions)
readStream, interceptor, rtcpReadStream, rtcpInterceptor, err := pc.dtlsTransport.streamsForSSRC(ssrc, *streamInfo)
if err != nil {
return err
Expand Down
5 changes: 2 additions & 3 deletions peerconnection_media_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ import (
"time"

"github.com/pion/logging"
"github.com/pion/randutil"
"github.com/pion/rtcp"
"github.com/pion/rtp"
"github.com/pion/sdp/v3"
Expand Down Expand Up @@ -778,7 +777,7 @@ func TestAddTransceiverFromTrackFailsRecvOnly(t *testing.T) {
func TestPlanBMediaExchange(t *testing.T) {
runTest := func(trackCount int, t *testing.T) {
addSingleTrack := func(p *PeerConnection) *TrackLocalStaticSample {
track, err := NewTrackLocalStaticSample(RTPCodecCapability{MimeType: MimeTypeVP8}, fmt.Sprintf("video-%d", randutil.NewMathRandomGenerator().Uint32()), fmt.Sprintf("video-%d", randutil.NewMathRandomGenerator().Uint32()))
track, err := NewTrackLocalStaticSample(RTPCodecCapability{MimeType: MimeTypeVP8}, fmt.Sprintf("video-%d", util.RandUint32()), fmt.Sprintf("video-%d", util.RandUint32()))
assert.NoError(t, err)

_, err = p.AddTrack(track)
Expand Down Expand Up @@ -1020,7 +1019,7 @@ func TestPeerConnection_Simulcast_Probe(t *testing.T) {
if len(track.bindings) == 1 {
_, err = track.bindings[0].writeStream.WriteRTP(&rtp.Header{
Version: 2,
SSRC: randutil.NewMathRandomGenerator().Uint32(),
SSRC: util.RandUint32(),
}, []byte{0, 1, 2, 3, 4, 5})
assert.NoError(t, err)
}
Expand Down
4 changes: 2 additions & 2 deletions pkg/media/oggwriter/oggwriter.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,9 @@ import (
"io"
"os"

"github.com/pion/randutil"
"github.com/pion/rtp"
"github.com/pion/rtp/codecs"
"github.com/pion/webrtc/v4/internal/util"
)

const (
Expand Down Expand Up @@ -68,7 +68,7 @@ func NewWith(out io.Writer, sampleRate uint32, channelCount uint16) (*OggWriter,
stream: out,
sampleRate: sampleRate,
channelCount: channelCount,
serial: randutil.NewMathRandomGenerator().Uint32(),
serial: util.RandUint32(),
checksumTable: generateChecksumTable(),

// Timestamp and Granule MUST start from 1
Expand Down
13 changes: 13 additions & 0 deletions rtpcodec.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
package webrtc

import (
"fmt"
"strings"

"github.com/pion/webrtc/v4/internal/fmtp"
Expand Down Expand Up @@ -123,3 +124,15 @@ func codecParametersFuzzySearch(needle RTPCodecParameters, haystack []RTPCodecPa

return RTPCodecParameters{}, codecMatchNone
}

// Given a CodecParameters find the RTX CodecParameters if one exists
func findRTXCodecParameters(needle PayloadType, haystack []RTPCodecParameters) (RTPCodecParameters, bool) {
aptStr := fmt.Sprintf("apt=%d", needle)
for _, c := range haystack {
if aptStr == c.SDPFmtpLine {
return c, true
}
}

return RTPCodecParameters{}, false
}
7 changes: 7 additions & 0 deletions rtpcodingparameters.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,12 @@ type RTPRtxParameters struct {
SSRC SSRC `json:"ssrc"`
}

// RTPFecParameters dictionary contains information relating to forward error correction (FEC) settings.
// https://draft.ortc.org/#dom-rtcrtpfecparameters
type RTPFecParameters struct {
SSRC SSRC `json:"ssrc"`
}

// RTPCodingParameters provides information relating to both encoding and decoding.
// This is a subset of the RFC since Pion WebRTC doesn't implement encoding/decoding itself
// http://draft.ortc.org/#dom-rtcrtpcodingparameters
Expand All @@ -17,4 +23,5 @@ type RTPCodingParameters struct {
SSRC SSRC `json:"ssrc"`
PayloadType PayloadType `json:"payloadType"`
RTX RTPRtxParameters `json:"rtx"`
FEC RTPFecParameters `json:"fec"`
}
4 changes: 2 additions & 2 deletions rtpreceiver.go
Original file line number Diff line number Diff line change
Expand Up @@ -210,14 +210,14 @@ func (r *RTPReceiver) startReceive(parameters RTPReceiveParameters) error {
return fmt.Errorf("%w: %d", errRTPReceiverWithSSRCTrackStreamNotFound, parameters.Encodings[i].SSRC)
}

t.streamInfo = createStreamInfo("", parameters.Encodings[i].SSRC, 0, codec, globalParams.HeaderExtensions)
t.streamInfo = createStreamInfo("", parameters.Encodings[i].SSRC, 0, 0, 0, codec, globalParams.HeaderExtensions)
var err error
if t.rtpReadStream, t.rtpInterceptor, t.rtcpReadStream, t.rtcpInterceptor, err = r.transport.streamsForSSRC(parameters.Encodings[i].SSRC, *t.streamInfo); err != nil {
return err
}

if rtxSsrc := parameters.Encodings[i].RTX.SSRC; rtxSsrc != 0 {
streamInfo := createStreamInfo("", rtxSsrc, 0, codec, globalParams.HeaderExtensions)
streamInfo := createStreamInfo("", rtxSsrc, 0, 0, 0, codec, globalParams.HeaderExtensions)
rtpReadStream, rtpInterceptor, rtcpReadStream, rtcpInterceptor, err := r.transport.streamsForSSRC(rtxSsrc, *streamInfo)
if err != nil {
return err
Expand Down
Loading

0 comments on commit d2c9d16

Please sign in to comment.