From dc2ef97b5a4a6f98bc91d47aff152634f08754d0 Mon Sep 17 00:00:00 2001
From: aler9 <46489434+aler9@users.noreply.github.com>
Date: Thu, 29 Dec 2022 19:02:48 +0100
Subject: [PATCH 1/2] support reading H265 tracks with HLS
---
README.md | 2 +-
go.mod | 8 +-
go.sum | 8 +-
internal/core/formatprocessor_h264.go | 41 +++--
internal/core/formatprocessor_h265.go | 93 +++++++++--
internal/core/hls_muxer.go | 102 ++++++++----
internal/core/hls_source.go | 8 +-
internal/core/rpicamera_source.go | 8 +-
internal/core/rtmp_conn.go | 34 ++--
internal/core/rtmp_source.go | 8 +-
internal/core/webrtc_conn.go | 6 +-
internal/hls/fmp4/init_track.go | 155 +++++++++++++++---
internal/hls/mpegts/writer.go | 2 +-
internal/hls/muxer.go | 16 +-
internal/hls/muxer_primary_playlist.go | 38 ++++-
internal/hls/muxer_test.go | 42 ++---
internal/hls/muxer_variant.go | 4 +-
internal/hls/muxer_variant_fmp4.go | 79 ++++++---
internal/hls/muxer_variant_fmp4_part.go | 4 +-
internal/hls/muxer_variant_fmp4_playlist.go | 4 +-
internal/hls/muxer_variant_fmp4_segment.go | 4 +-
internal/hls/muxer_variant_fmp4_segmenter.go | 134 +++++++++------
internal/hls/muxer_variant_mpegts.go | 21 ++-
.../hls/muxer_variant_mpegts_segmenter.go | 5 +-
24 files changed, 577 insertions(+), 249 deletions(-)
diff --git a/README.md b/README.md
index 77ae556ef70..5cea6a77de7 100644
--- a/README.md
+++ b/README.md
@@ -22,7 +22,7 @@ And can be read from the server with:
|--------|--------|------|
|RTSP|UDP, UDP-Multicast, TCP, RTSPS|H264, H265, VP8, VP9, AV1, MPEG2, M-JPEG, MP3, MPEG4 Audio (AAC), Opus, G711, G722, LPCM and any RTP-compatible codec|
|RTMP|RTMP, RTMPS|H264, MPEG4 Audio (AAC)|
-|HLS|Low-Latency HLS, MP4-based HLS, legacy HLS|H264, MPEG4 Audio (AAC)|
+|HLS|Low-Latency HLS, MP4-based HLS, legacy HLS|H264, H265, MPEG4 Audio (AAC)|
|WebRTC||H264, VP8, VP9, Opus, G711, G722|
Features:
diff --git a/go.mod b/go.mod
index 0c8b26e5999..3b63b7ad0fc 100644
--- a/go.mod
+++ b/go.mod
@@ -3,9 +3,9 @@ module github.com/aler9/rtsp-simple-server
go 1.18
require (
- code.cloudfoundry.org/bytefmt v0.0.0-20211005130812-5bb3c17173e5
- github.com/abema/go-mp4 v0.8.0
- github.com/aler9/gortsplib/v2 v2.0.0-20221228192116-da21f946e562
+ code.cloudfoundry.org/bytefmt v0.0.0
+ github.com/abema/go-mp4 v0.0.0
+ github.com/aler9/gortsplib/v2 v2.0.0-20221229123705-ce25207cb823
github.com/asticode/go-astits v1.10.1-0.20220319093903-4abe66a9b757
github.com/fsnotify/fsnotify v1.4.9
github.com/gin-gonic/gin v1.8.1
@@ -68,3 +68,5 @@ require (
replace github.com/orcaman/writerseeker => github.com/aler9/writerseeker v0.0.0-20220601075008-6f0e685b9c82
replace code.cloudfoundry.org/bytefmt => github.com/cloudfoundry/bytefmt v0.0.0-20211005130812-5bb3c17173e5
+
+replace github.com/abema/go-mp4 => github.com/aler9/go-mp4 v0.0.0-20221229152535-34c82c552218
diff --git a/go.sum b/go.sum
index 8fbbc4b3794..c7d8f796364 100644
--- a/go.sum
+++ b/go.sum
@@ -1,11 +1,11 @@
-github.com/abema/go-mp4 v0.8.0 h1:JHYkOvTfBpTnqJHiFFOXe8d6wiFy5MtDnA10fgccNqY=
-github.com/abema/go-mp4 v0.8.0/go.mod h1:vPl9t5ZK7K0x68jh12/+ECWBCXoWuIDtNgPtU2f04ws=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E=
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
-github.com/aler9/gortsplib/v2 v2.0.0-20221228192116-da21f946e562 h1://BJIsHw2vYKdPL6sKbxZEnlGPpj2PTznNzRpou87ds=
-github.com/aler9/gortsplib/v2 v2.0.0-20221228192116-da21f946e562/go.mod h1:lMdAxc6daduSzVwh75yQkvH9UHCYHpng/vJ8uXKFzdA=
+github.com/aler9/go-mp4 v0.0.0-20221229152535-34c82c552218 h1:Zak89uY+y0q/gL7jaKbl2XeyMOLT/5qVuW6TIJphEJY=
+github.com/aler9/go-mp4 v0.0.0-20221229152535-34c82c552218/go.mod h1:vPl9t5ZK7K0x68jh12/+ECWBCXoWuIDtNgPtU2f04ws=
+github.com/aler9/gortsplib/v2 v2.0.0-20221229123705-ce25207cb823 h1:EFq9LqgA15drNgXj3hNlmAouxjMYb9jyyBq6hmjDO8U=
+github.com/aler9/gortsplib/v2 v2.0.0-20221229123705-ce25207cb823/go.mod h1:lMdAxc6daduSzVwh75yQkvH9UHCYHpng/vJ8uXKFzdA=
github.com/aler9/writerseeker v0.0.0-20220601075008-6f0e685b9c82 h1:9WgSzBLo3a9ToSVV7sRTBYZ1GGOZUpq4+5H3SN0UZq4=
github.com/aler9/writerseeker v0.0.0-20220601075008-6f0e685b9c82/go.mod h1:qsMrZCbeBf/mCLOeF16KDkPu4gktn/pOWyaq1aYQE7U=
github.com/asticode/go-astikit v0.20.0 h1:+7N+J4E4lWx2QOkRdOf6DafWJMv6O4RRfgClwQokrH8=
diff --git a/internal/core/formatprocessor_h264.go b/internal/core/formatprocessor_h264.go
index 9d390fa706c..ec1f230c2df 100644
--- a/internal/core/formatprocessor_h264.go
+++ b/internal/core/formatprocessor_h264.go
@@ -71,7 +71,7 @@ type dataH264 struct {
rtpPackets []*rtp.Packet
ntp time.Time
pts time.Duration
- nalus [][]byte
+ au [][]byte
}
func (d *dataH264) getRTPPackets() []*rtp.Packet {
@@ -134,22 +134,23 @@ func (t *formatProcessorH264) updateTrackParametersFromNALUs(nalus [][]byte) {
}
}
-// remux is needed to fix corrupted streams and make streams
-// compatible with all protocols.
-func (t *formatProcessorH264) remuxNALUs(nalus [][]byte) [][]byte {
- addSPSPPS := false
+func (t *formatProcessorH264) remuxAccessUnit(nalus [][]byte) [][]byte {
+ addParameters := false
n := 0
+
for _, nalu := range nalus {
typ := h264.NALUType(nalu[0] & 0x1F)
+
switch typ {
- case h264.NALUTypeSPS, h264.NALUTypePPS:
+ case h264.NALUTypeSPS, h264.NALUTypePPS: // remove parameters
continue
- case h264.NALUTypeAccessUnitDelimiter:
+
+ case h264.NALUTypeAccessUnitDelimiter: // remove AUDs
continue
- case h264.NALUTypeIDR:
- // prepend SPS and PPS to the group if there's at least an IDR
- if !addSPSPPS {
- addSPSPPS = true
+
+ case h264.NALUTypeIDR: // prepend parameters if there's at least an IDR
+ if !addParameters {
+ addParameters = true
n += 2
}
}
@@ -163,7 +164,7 @@ func (t *formatProcessorH264) remuxNALUs(nalus [][]byte) [][]byte {
filteredNALUs := make([][]byte, n)
i := 0
- if addSPSPPS {
+ if addParameters {
filteredNALUs[0] = t.format.SafeSPS()
filteredNALUs[1] = t.format.SafePPS()
i = 2
@@ -171,13 +172,12 @@ func (t *formatProcessorH264) remuxNALUs(nalus [][]byte) [][]byte {
for _, nalu := range nalus {
typ := h264.NALUType(nalu[0] & 0x1F)
+
switch typ {
case h264.NALUTypeSPS, h264.NALUTypePPS:
- // remove since they're automatically added
continue
case h264.NALUTypeAccessUnitDelimiter:
- // remove since it is not needed
continue
}
@@ -227,7 +227,7 @@ func (t *formatProcessorH264) process(dat data, hasNonRTSPReaders bool) error {
}
// DecodeUntilMarker() is necessary, otherwise Encode() generates partial groups
- nalus, pts, err := t.decoder.DecodeUntilMarker(pkt)
+ au, pts, err := t.decoder.DecodeUntilMarker(pkt)
if err != nil {
if err == rtph264.ErrNonStartingPacketAndNoPrevious || err == rtph264.ErrMorePacketsNeeded {
return nil
@@ -235,10 +235,9 @@ func (t *formatProcessorH264) process(dat data, hasNonRTSPReaders bool) error {
return err
}
- tdata.nalus = nalus
+ tdata.au = au
tdata.pts = pts
-
- tdata.nalus = t.remuxNALUs(tdata.nalus)
+ tdata.au = t.remuxAccessUnit(tdata.au)
}
// route packet as is
@@ -246,11 +245,11 @@ func (t *formatProcessorH264) process(dat data, hasNonRTSPReaders bool) error {
return nil
}
} else {
- t.updateTrackParametersFromNALUs(tdata.nalus)
- tdata.nalus = t.remuxNALUs(tdata.nalus)
+ t.updateTrackParametersFromNALUs(tdata.au)
+ tdata.au = t.remuxAccessUnit(tdata.au)
}
- pkts, err := t.encoder.Encode(tdata.nalus, tdata.pts)
+ pkts, err := t.encoder.Encode(tdata.au, tdata.pts)
if err != nil {
return err
}
diff --git a/internal/core/formatprocessor_h265.go b/internal/core/formatprocessor_h265.go
index 70af98f96e2..1d5cdf81dd6 100644
--- a/internal/core/formatprocessor_h265.go
+++ b/internal/core/formatprocessor_h265.go
@@ -78,7 +78,7 @@ type dataH265 struct {
rtpPackets []*rtp.Packet
ntp time.Time
pts time.Duration
- nalus [][]byte
+ au [][]byte
}
func (d *dataH265) getRTPPackets() []*rtp.Packet {
@@ -128,12 +128,82 @@ func (t *formatProcessorH265) updateTrackParametersFromRTPPacket(pkt *rtp.Packet
}
func (t *formatProcessorH265) updateTrackParametersFromNALUs(nalus [][]byte) {
- // TODO: extract VPS, SPS, PPS and set them into the track
+ for _, nalu := range nalus {
+ typ := h265.NALUType((nalu[0] >> 1) & 0b111111)
+
+ switch typ {
+ case h265.NALUType_VPS_NUT:
+ if !bytes.Equal(nalu, t.format.SafeVPS()) {
+ t.format.SafeSetVPS(nalu)
+ }
+
+ case h265.NALUType_SPS_NUT:
+ if !bytes.Equal(nalu, t.format.SafePPS()) {
+ t.format.SafeSetSPS(nalu)
+ }
+
+ case h265.NALUType_PPS_NUT:
+ if !bytes.Equal(nalu, t.format.SafePPS()) {
+ t.format.SafeSetPPS(nalu)
+ }
+ }
+ }
}
-func (t *formatProcessorH265) remuxNALUs(nalus [][]byte) [][]byte {
- // TODO: add VPS, SPS, PPS before IDRs
- return nalus
+func (t *formatProcessorH265) remuxAccessUnit(nalus [][]byte) [][]byte {
+ addParameters := false
+ n := 0
+
+ for _, nalu := range nalus {
+ typ := h265.NALUType((nalu[0] >> 1) & 0b111111)
+
+ switch typ {
+ case h265.NALUType_VPS_NUT, h265.NALUType_SPS_NUT, h265.NALUType_PPS_NUT: // remove parameters
+ continue
+
+ case h265.NALUType_AUD_NUT: // remove AUDs
+ continue
+
+ // prepend parameters if there's at least a random access unit
+ case h265.NALUType_IDR_W_RADL, h265.NALUType_IDR_N_LP, h265.NALUType_CRA_NUT:
+ if !addParameters {
+ addParameters = true
+ n += 3
+ }
+ }
+ n++
+ }
+
+ if n == 0 {
+ return nil
+ }
+
+ filteredNALUs := make([][]byte, n)
+ i := 0
+
+ if addParameters {
+ filteredNALUs[0] = t.format.SafeVPS()
+ filteredNALUs[1] = t.format.SafeSPS()
+ filteredNALUs[2] = t.format.SafePPS()
+ i = 3
+ }
+
+ for _, nalu := range nalus {
+ typ := h265.NALUType((nalu[0] >> 1) & 0b111111)
+
+ switch typ {
+ case h265.NALUType_VPS_NUT, h265.NALUType_SPS_NUT, h265.NALUType_PPS_NUT:
+ continue
+
+ case h265.NALUType_AUD_NUT:
+ continue
+ }
+
+ filteredNALUs[i] = nalu
+ i++
+ }
+
+ return filteredNALUs
}
func (t *formatProcessorH265) process(dat data, hasNonRTSPReaders bool) error { //nolint:dupl
@@ -175,7 +245,7 @@ func (t *formatProcessorH265) process(dat data, hasNonRTSPReaders bool) error {
}
// DecodeUntilMarker() is necessary, otherwise Encode() generates partial groups
- nalus, pts, err := t.decoder.DecodeUntilMarker(pkt)
+ au, pts, err := t.decoder.DecodeUntilMarker(pkt)
if err != nil {
if err == rtph265.ErrNonStartingPacketAndNoPrevious || err == rtph265.ErrMorePacketsNeeded {
return nil
@@ -183,10 +253,9 @@ func (t *formatProcessorH265) process(dat data, hasNonRTSPReaders bool) error {
return err
}
- tdata.nalus = nalus
+ tdata.au = au
tdata.pts = pts
-
- tdata.nalus = t.remuxNALUs(tdata.nalus)
+ tdata.au = t.remuxAccessUnit(tdata.au)
}
// route packet as is
@@ -194,11 +263,11 @@ func (t *formatProcessorH265) process(dat data, hasNonRTSPReaders bool) error {
return nil
}
} else {
- t.updateTrackParametersFromNALUs(tdata.nalus)
- tdata.nalus = t.remuxNALUs(tdata.nalus)
+ t.updateTrackParametersFromNALUs(tdata.au)
+ tdata.au = t.remuxAccessUnit(tdata.au)
}
- pkts, err := t.encoder.Encode(tdata.nalus, tdata.pts)
+ pkts, err := t.encoder.Encode(tdata.au, tdata.pts)
if err != nil {
return err
}
diff --git a/internal/core/hls_muxer.go b/internal/core/hls_muxer.go
index 8889b9aca26..3dce010baed 100644
--- a/internal/core/hls_muxer.go
+++ b/internal/core/hls_muxer.go
@@ -243,38 +243,18 @@ func (m *hlsMuxer) runInner(innerCtx context.Context, innerReady chan struct{})
m.path.readerRemove(pathReaderRemoveReq{author: m})
}()
- var videoFormat *format.H264
- videoMedia := res.stream.medias().FindFormat(&videoFormat)
-
- var audioFormat *format.MPEG4Audio
- audioMedia := res.stream.medias().FindFormat(&audioFormat)
-
- if videoFormat == nil && audioFormat == nil {
- return fmt.Errorf("the stream doesn't contain an H264 track or an AAC track")
- }
-
- var err error
- m.muxer, err = hls.NewMuxer(
- hls.MuxerVariant(m.hlsVariant),
- m.hlsSegmentCount,
- time.Duration(m.hlsSegmentDuration),
- time.Duration(m.hlsPartDuration),
- uint64(m.hlsSegmentMaxSize),
- videoFormat,
- audioFormat,
- )
- if err != nil {
- return fmt.Errorf("muxer error: %v", err)
- }
- defer m.muxer.Close()
-
- innerReady <- struct{}{}
-
m.ringBuffer, _ = ringbuffer.New(uint64(m.readBufferCount))
var medias media.Medias
- if videoMedia != nil {
+ var videoFormat format.Format
+ var videoMedia *media.Media
+
+ var videoFormatH265 *format.H265
+ videoMedia = res.stream.medias().FindFormat(&videoFormatH265)
+
+ if videoFormatH265 != nil {
+ videoFormat = videoFormatH265
medias = append(medias, videoMedia)
videoStartPTSFilled := false
@@ -282,9 +262,9 @@ func (m *hlsMuxer) runInner(innerCtx context.Context, innerReady chan struct{})
res.stream.readerAdd(m, videoMedia, videoFormat, func(dat data) {
m.ringBuffer.Push(func() error {
- tdata := dat.(*dataH264)
+ tdata := dat.(*dataH265)
- if tdata.nalus == nil {
+ if tdata.au == nil {
return nil
}
@@ -294,7 +274,7 @@ func (m *hlsMuxer) runInner(innerCtx context.Context, innerReady chan struct{})
}
pts := tdata.pts - videoStartPTS
- err := m.muxer.WriteH264(tdata.ntp, pts, tdata.nalus)
+ err := m.muxer.WriteH26x(tdata.ntp, pts, tdata.au)
if err != nil {
return fmt.Errorf("muxer error: %v", err)
}
@@ -302,9 +282,46 @@ func (m *hlsMuxer) runInner(innerCtx context.Context, innerReady chan struct{})
return nil
})
})
+ } else {
+ var videoFormatH264 *format.H264
+ videoMedia = res.stream.medias().FindFormat(&videoFormatH264)
+
+ if videoFormatH264 != nil {
+ videoFormat = videoFormatH264
+ medias = append(medias, videoMedia)
+
+ videoStartPTSFilled := false
+ var videoStartPTS time.Duration
+
+ res.stream.readerAdd(m, videoMedia, videoFormat, func(dat data) {
+ m.ringBuffer.Push(func() error {
+ tdata := dat.(*dataH264)
+
+ if tdata.au == nil {
+ return nil
+ }
+
+ if !videoStartPTSFilled {
+ videoStartPTSFilled = true
+ videoStartPTS = tdata.pts
+ }
+ pts := tdata.pts - videoStartPTS
+
+ err := m.muxer.WriteH26x(tdata.ntp, pts, tdata.au)
+ if err != nil {
+ return fmt.Errorf("muxer error: %v", err)
+ }
+
+ return nil
+ })
+ })
+ }
}
- if audioMedia != nil {
+ var audioFormat *format.MPEG4Audio
+ audioMedia := res.stream.medias().FindFormat(&audioFormat)
+
+ if audioFormat != nil {
medias = append(medias, audioMedia)
audioStartPTSFilled := false
@@ -342,6 +359,27 @@ func (m *hlsMuxer) runInner(innerCtx context.Context, innerReady chan struct{})
defer res.stream.readerRemove(m)
+ if medias == nil {
+ return fmt.Errorf("the stream doesn't contain a supported video or audio track")
+ }
+
+ var err error
+ m.muxer, err = hls.NewMuxer(
+ hls.MuxerVariant(m.hlsVariant),
+ m.hlsSegmentCount,
+ time.Duration(m.hlsSegmentDuration),
+ time.Duration(m.hlsPartDuration),
+ uint64(m.hlsSegmentMaxSize),
+ videoFormat,
+ audioFormat,
+ )
+ if err != nil {
+ return fmt.Errorf("muxer error: %v", err)
+ }
+ defer m.muxer.Close()
+
+ innerReady <- struct{}{}
+
m.log(logger.Info, "is converting into HLS, %s",
sourceMediaInfo(medias))
diff --git a/internal/core/hls_source.go b/internal/core/hls_source.go
index 1b46ef70625..742bd76afb9 100644
--- a/internal/core/hls_source.go
+++ b/internal/core/hls_source.go
@@ -84,11 +84,11 @@ func (s *hlsSource) run(ctx context.Context) error {
return nil
}
- onVideoData := func(pts time.Duration, nalus [][]byte) {
+ onVideoData := func(pts time.Duration, au [][]byte) {
err := stream.writeData(videoMedia, videoMedia.Formats[0], &dataH264{
- pts: pts,
- nalus: nalus,
- ntp: time.Now(),
+ pts: pts,
+ au: au,
+ ntp: time.Now(),
})
if err != nil {
s.Log(logger.Warn, "%v", err)
diff --git a/internal/core/rpicamera_source.go b/internal/core/rpicamera_source.go
index b293b995b34..6a40d314ee2 100644
--- a/internal/core/rpicamera_source.go
+++ b/internal/core/rpicamera_source.go
@@ -48,7 +48,7 @@ func (s *rpiCameraSource) run(ctx context.Context) error {
medias := media.Medias{medi}
var stream *stream
- onData := func(dts time.Duration, nalus [][]byte) {
+ onData := func(dts time.Duration, au [][]byte) {
if stream == nil {
res := s.parent.sourceStaticImplSetReady(pathSourceStaticSetReadyReq{
medias: medias,
@@ -63,9 +63,9 @@ func (s *rpiCameraSource) run(ctx context.Context) error {
}
err := stream.writeData(medi, medi.Formats[0], &dataH264{
- pts: dts,
- nalus: nalus,
- ntp: time.Now(),
+ pts: dts,
+ au: au,
+ ntp: time.Now(),
})
if err != nil {
s.Log(logger.Warn, "%v", err)
diff --git a/internal/core/rtmp_conn.go b/internal/core/rtmp_conn.go
index f6767f926a7..419428aeccd 100644
--- a/internal/core/rtmp_conn.go
+++ b/internal/core/rtmp_conn.go
@@ -281,7 +281,7 @@ func (c *rtmpConn) runRead(ctx context.Context, u *url.URL) error {
ringBuffer.Push(func() error {
tdata := dat.(*dataH264)
- if tdata.nalus == nil {
+ if tdata.au == nil {
return nil
}
@@ -294,7 +294,7 @@ func (c *rtmpConn) runRead(ctx context.Context, u *url.URL) error {
idrPresent := false
nonIDRPresent := false
- for _, nalu := range tdata.nalus {
+ for _, nalu := range tdata.au {
typ := h264.NALUType(nalu[0] & 0x1F)
switch typ {
case h264.NALUTypeIDR:
@@ -317,7 +317,7 @@ func (c *rtmpConn) runRead(ctx context.Context, u *url.URL) error {
videoDTSExtractor = h264.NewDTSExtractor()
var err error
- dts, err = videoDTSExtractor.Extract(tdata.nalus, pts)
+ dts, err = videoDTSExtractor.Extract(tdata.au, pts)
if err != nil {
return err
}
@@ -331,7 +331,7 @@ func (c *rtmpConn) runRead(ctx context.Context, u *url.URL) error {
}
var err error
- dts, err = videoDTSExtractor.Extract(tdata.nalus, pts)
+ dts, err = videoDTSExtractor.Extract(tdata.au, pts)
if err != nil {
return err
}
@@ -340,7 +340,7 @@ func (c *rtmpConn) runRead(ctx context.Context, u *url.URL) error {
pts -= videoStartDTS
}
- avcc, err := h264.AVCCMarshal(tdata.nalus)
+ avcc, err := h264.AVCCMarshal(tdata.au)
if err != nil {
return err
}
@@ -538,22 +538,22 @@ func (c *rtmpConn) runPublish(ctx context.Context, u *url.URL) error {
var onVideoData func(time.Duration, [][]byte)
if _, ok := videoFormat.(*format.H264); ok {
- onVideoData = func(pts time.Duration, nalus [][]byte) {
+ onVideoData = func(pts time.Duration, au [][]byte) {
err = rres.stream.writeData(videoMedia, videoFormat, &dataH264{
- pts: pts,
- nalus: nalus,
- ntp: time.Now(),
+ pts: pts,
+ au: au,
+ ntp: time.Now(),
})
if err != nil {
c.log(logger.Warn, "%v", err)
}
}
} else {
- onVideoData = func(pts time.Duration, nalus [][]byte) {
+ onVideoData = func(pts time.Duration, au [][]byte) {
err = rres.stream.writeData(videoMedia, videoFormat, &dataH265{
- pts: pts,
- nalus: nalus,
- ntp: time.Now(),
+ pts: pts,
+ au: au,
+ ntp: time.Now(),
})
if err != nil {
c.log(logger.Warn, "%v", err)
@@ -577,15 +577,15 @@ func (c *rtmpConn) runPublish(ctx context.Context, u *url.URL) error {
return fmt.Errorf("unable to parse H264 config: %v", err)
}
- nalus := [][]byte{
+ au := [][]byte{
conf.SPS,
conf.PPS,
}
err := rres.stream.writeData(videoMedia, videoFormat, &dataH264{
- pts: tmsg.DTS + tmsg.PTSDelta,
- nalus: nalus,
- ntp: time.Now(),
+ pts: tmsg.DTS + tmsg.PTSDelta,
+ au: au,
+ ntp: time.Now(),
})
if err != nil {
c.log(logger.Warn, "%v", err)
diff --git a/internal/core/rtmp_source.go b/internal/core/rtmp_source.go
index d0b44dae9b5..49051fedb61 100644
--- a/internal/core/rtmp_source.go
+++ b/internal/core/rtmp_source.go
@@ -176,15 +176,15 @@ func (s *rtmpSource) run(ctx context.Context) error {
return fmt.Errorf("received an H264 packet, but track is not set up")
}
- nalus, err := h264.AVCCUnmarshal(tmsg.Payload)
+ au, err := h264.AVCCUnmarshal(tmsg.Payload)
if err != nil {
return fmt.Errorf("unable to decode AVCC: %v", err)
}
err = res.stream.writeData(videoMedia, videoFormat, &dataH264{
- pts: tmsg.DTS + tmsg.PTSDelta,
- nalus: nalus,
- ntp: time.Now(),
+ pts: tmsg.DTS + tmsg.PTSDelta,
+ au: au,
+ ntp: time.Now(),
})
if err != nil {
s.Log(logger.Warn, "%v", err)
diff --git a/internal/core/webrtc_conn.go b/internal/core/webrtc_conn.go
index 1743b01d17f..8b3a33e1302 100644
--- a/internal/core/webrtc_conn.go
+++ b/internal/core/webrtc_conn.go
@@ -519,12 +519,12 @@ func (c *webRTCConn) allocateTracks(medias media.Medias) ([]*webRTCTrack, error)
cb: func(dat data, ctx context.Context, writeError chan error) {
tdata := dat.(*dataH264)
- if tdata.nalus == nil {
+ if tdata.au == nil {
return
}
if !firstNALUReceived {
- if !h264.IDRPresent(tdata.nalus) {
+ if !h264.IDRPresent(tdata.au) {
return
}
@@ -541,7 +541,7 @@ func (c *webRTCConn) allocateTracks(medias media.Medias) ([]*webRTCTrack, error)
lastPTS = tdata.pts
}
- packets, err := encoder.Encode(tdata.nalus, tdata.pts)
+ packets, err := encoder.Encode(tdata.au, tdata.pts)
if err != nil {
return
}
diff --git a/internal/hls/fmp4/init_track.go b/internal/hls/fmp4/init_track.go
index 737876f3dda..bd7f7aba8c1 100644
--- a/internal/hls/fmp4/init_track.go
+++ b/internal/hls/fmp4/init_track.go
@@ -3,6 +3,7 @@ package fmp4
import (
gomp4 "github.com/abema/go-mp4"
"github.com/aler9/gortsplib/v2/pkg/codecs/h264"
+ "github.com/aler9/gortsplib/v2/pkg/codecs/h265"
"github.com/aler9/gortsplib/v2/pkg/format"
)
@@ -46,24 +47,56 @@ func (track *InitTrack) marshal(w *mp4Writer) error {
return err
}
- var sps []byte
- var pps []byte
- var spsp h264.SPS
+ var h264SPS []byte
+ var h264PPS []byte
+ var h264SPSP h264.SPS
+
+ var h265VPS []byte
+ var h265SPS []byte
+ var h265PPS []byte
+ var h265SPSP h265.SPS
+
var width int
var height int
switch ttrack := track.Format.(type) {
case *format.H264:
- sps = ttrack.SafeSPS()
- pps = ttrack.SafePPS()
+ h264SPS = ttrack.SafeSPS()
+ h264PPS = ttrack.SafePPS()
+
+ err = h264SPSP.Unmarshal(h264SPS)
+ if err != nil {
+ return err
+ }
+
+ width = h264SPSP.Width()
+ height = h264SPSP.Height()
+
+ _, err = w.WriteBox(&gomp4.Tkhd{ //
+ FullBox: gomp4.FullBox{
+ Flags: [3]byte{0, 0, 3},
+ },
+ TrackID: uint32(track.ID),
+ Width: uint32(width * 65536),
+ Height: uint32(height * 65536),
+ Matrix: [9]int32{0x10000, 0, 0, 0, 0x10000, 0, 0, 0, 0x40000000},
+ })
+ if err != nil {
+ return err
+ }
+
+ case *format.H265:
+ h265VPS = ttrack.SafeVPS()
+ h265SPS = ttrack.SafeSPS()
+ h265PPS = ttrack.SafePPS()
- err = spsp.Unmarshal(sps)
+ err = h265SPSP.Unmarshal(h265SPS)
if err != nil {
return err
}
- width = spsp.Width()
- height = spsp.Height()
+ width = h265SPSP.Width()
+ height = h265SPSP.Height()
_, err = w.WriteBox(&gomp4.Tkhd{ //
FullBox: gomp4.FullBox{
@@ -72,7 +105,7 @@ func (track *InitTrack) marshal(w *mp4Writer) error {
TrackID: uint32(track.ID),
Width: uint32(width * 65536),
Height: uint32(height * 65536),
- Matrix: [9]int32{0x00010000, 0, 0, 0, 0x00010000, 0, 0, 0, 0x40000000},
+ Matrix: [9]int32{0x10000, 0, 0, 0, 0x10000, 0, 0, 0, 0x40000000},
})
if err != nil {
return err
@@ -86,7 +119,7 @@ func (track *InitTrack) marshal(w *mp4Writer) error {
TrackID: uint32(track.ID),
AlternateGroup: 1,
Volume: 256,
- Matrix: [9]int32{0x00010000, 0, 0, 0, 0x00010000, 0, 0, 0, 0x40000000},
+ Matrix: [9]int32{0x10000, 0, 0, 0, 0x10000, 0, 0, 0, 0x40000000},
})
if err != nil {
return err
@@ -107,7 +140,7 @@ func (track *InitTrack) marshal(w *mp4Writer) error {
}
switch track.Format.(type) {
- case *format.H264:
+ case *format.H264, *format.H265:
_, err = w.WriteBox(&gomp4.Hdlr{ //
HandlerType: [4]byte{'v', 'i', 'd', 'e'},
Name: "VideoHandler",
@@ -132,7 +165,7 @@ func (track *InitTrack) marshal(w *mp4Writer) error {
}
switch track.Format.(type) {
- case *format.H264:
+ case *format.H264, *format.H265:
_, err = w.WriteBox(&gomp4.Vmhd{ //
FullBox: gomp4.FullBox{
Flags: [3]byte{0, 0, 1},
@@ -219,22 +252,22 @@ func (track *InitTrack) marshal(w *mp4Writer) error {
Type: gomp4.BoxTypeAvcC(),
},
ConfigurationVersion: 1,
- Profile: spsp.ProfileIdc,
- ProfileCompatibility: sps[2],
- Level: spsp.LevelIdc,
+ Profile: h264SPSP.ProfileIdc,
+ ProfileCompatibility: h264SPS[2],
+ Level: h264SPSP.LevelIdc,
LengthSizeMinusOne: 3,
NumOfSequenceParameterSets: 1,
SequenceParameterSets: []gomp4.AVCParameterSet{
{
- Length: uint16(len(sps)),
- NALUnit: sps,
+ Length: uint16(len(h264SPS)),
+ NALUnit: h264SPS,
},
},
NumOfPictureParameterSets: 1,
PictureParameterSets: []gomp4.AVCParameterSet{
{
- Length: uint16(len(pps)),
- NALUnit: pps,
+ Length: uint16(len(h264PPS)),
+ NALUnit: h264PPS,
},
},
})
@@ -255,6 +288,90 @@ func (track *InitTrack) marshal(w *mp4Writer) error {
return err
}
+ case *format.H265:
+ _, err = w.writeBoxStart(&gomp4.VisualSampleEntry{ //
+ SampleEntry: gomp4.SampleEntry{
+ AnyTypeBox: gomp4.AnyTypeBox{
+ Type: gomp4.BoxTypeHev1(),
+ },
+ DataReferenceIndex: 1,
+ },
+ Width: uint16(width),
+ Height: uint16(height),
+ Horizresolution: 4718592,
+ Vertresolution: 4718592,
+ FrameCount: 1,
+ Depth: 24,
+ PreDefined3: -1,
+ })
+ if err != nil {
+ return err
+ }
+
+ _, err = w.WriteBox(&gomp4.HvcC{ //
+ ConfigurationVersion: 1,
+ GeneralProfileIdc: h265SPSP.ProfileTierLevel.GeneralProfileIdc,
+ GeneralProfileCompatibility: h265SPSP.ProfileTierLevel.GeneralProfileCompatibilityFlag,
+ GeneralConstraintIndicator: [6]uint8{
+ h265SPS[7], h265SPS[8], h265SPS[9],
+ h265SPS[10], h265SPS[11], h265SPS[12],
+ },
+ GeneralLevelIdc: h265SPSP.ProfileTierLevel.GeneralLevelIdc,
+ // MinSpatialSegmentationIdc
+ // ParallelismType
+ ChromaFormatIdc: uint8(h265SPSP.ChromaFormatIdc),
+ BitDepthLumaMinus8: uint8(h265SPSP.BitDepthLumaMinus8),
+ BitDepthChromaMinus8: uint8(h265SPSP.BitDepthChromaMinus8),
+ // AvgFrameRate
+ // ConstantFrameRate
+ NumTemporalLayers: 1,
+ // TemporalIdNested
+ LengthSizeMinusOne: 3,
+ NumOfNaluArrays: 3,
+ NaluArrays: []gomp4.HEVCNaluArray{
+ {
+ NaluType: byte(h265.NALUType_VPS_NUT),
+ NumNalus: 1,
+ Nalus: []gomp4.HEVCNalu{{
+ Length: uint16(len(h265VPS)),
+ NALUnit: h265VPS,
+ }},
+ },
+ {
+ NaluType: byte(h265.NALUType_SPS_NUT),
+ NumNalus: 1,
+ Nalus: []gomp4.HEVCNalu{{
+ Length: uint16(len(h265SPS)),
+ NALUnit: h265SPS,
+ }},
+ },
+ {
+ NaluType: byte(h265.NALUType_PPS_NUT),
+ NumNalus: 1,
+ Nalus: []gomp4.HEVCNalu{{
+ Length: uint16(len(h265PPS)),
+ NALUnit: h265PPS,
+ }},
+ },
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ _, err = w.WriteBox(&gomp4.Btrt{ //
+ MaxBitrate: 1000000,
+ AvgBitrate: 1000000,
+ })
+ if err != nil {
+ return err
+ }
+
+ err = w.writeBoxEnd() //
+ if err != nil {
+ return err
+ }
+
case *format.MPEG4Audio:
_, err = w.writeBoxStart(&gomp4.AudioSampleEntry{ //
SampleEntry: gomp4.SampleEntry{
diff --git a/internal/hls/mpegts/writer.go b/internal/hls/mpegts/writer.go
index 7b59a12c512..4f112612eb6 100644
--- a/internal/hls/mpegts/writer.go
+++ b/internal/hls/mpegts/writer.go
@@ -86,7 +86,7 @@ func (w *Writer) GenerateSegment() []byte {
return ret
}
-// WriteH264 writes a group of H264 NALUs.
+// WriteH264 writes a H264 access unit.
func (w *Writer) WriteH264(
pcr time.Duration,
dts time.Duration,
diff --git a/internal/hls/muxer.go b/internal/hls/muxer.go
index b731434ff8b..17a6686866f 100644
--- a/internal/hls/muxer.go
+++ b/internal/hls/muxer.go
@@ -28,20 +28,24 @@ func NewMuxer(
segmentDuration time.Duration,
partDuration time.Duration,
segmentMaxSize uint64,
- videoTrack *format.H264,
+ videoTrack format.Format,
audioTrack *format.MPEG4Audio,
) (*Muxer, error) {
m := &Muxer{}
switch variant {
case MuxerVariantMPEGTS:
- m.variant = newMuxerVariantMPEGTS(
+ var err error
+ m.variant, err = newMuxerVariantMPEGTS(
segmentCount,
segmentDuration,
segmentMaxSize,
videoTrack,
audioTrack,
)
+ if err != nil {
+ return nil, err
+ }
case MuxerVariantFMP4:
m.variant = newMuxerVariantFMP4(
@@ -76,12 +80,12 @@ func (m *Muxer) Close() {
m.variant.close()
}
-// WriteH264 writes H264 NALUs, grouped by timestamp.
-func (m *Muxer) WriteH264(ntp time.Time, pts time.Duration, nalus [][]byte) error {
- return m.variant.writeH264(ntp, pts, nalus)
+// WriteH26x writes an H264 or an H265 access unit.
+func (m *Muxer) WriteH26x(ntp time.Time, pts time.Duration, au [][]byte) error {
+ return m.variant.writeH26x(ntp, pts, au)
}
-// WriteAAC writes AAC AUs, grouped by timestamp.
+// WriteAAC writes an AAC access unit.
func (m *Muxer) WriteAAC(ntp time.Time, pts time.Duration, au []byte) error {
return m.variant.writeAAC(ntp, pts, au)
}
diff --git a/internal/hls/muxer_primary_playlist.go b/internal/hls/muxer_primary_playlist.go
index d145459577f..33111929a35 100644
--- a/internal/hls/muxer_primary_playlist.go
+++ b/internal/hls/muxer_primary_playlist.go
@@ -8,18 +8,43 @@ import (
"strconv"
"strings"
+ "github.com/aler9/gortsplib/v2/pkg/codecs/h265"
"github.com/aler9/gortsplib/v2/pkg/format"
)
+func codecParameters(track format.Format) string {
+ switch ttrack := track.(type) {
+ case *format.H264:
+ sps := ttrack.SafeSPS()
+ if len(sps) >= 4 {
+ return "avc1." + hex.EncodeToString(sps[1:4])
+ }
+
+ case *format.H265:
+ var sps h265.SPS
+ err := sps.Unmarshal(ttrack.SafeSPS())
+ if err == nil {
+ return "hvc1." + strconv.FormatInt(int64(sps.ProfileTierLevel.GeneralProfileIdc), 10) +
+ ".4.L" + strconv.FormatInt(int64(sps.ProfileTierLevel.GeneralLevelIdc), 10) + ".B0"
+ }
+
+ case *format.MPEG4Audio:
+ // https://developer.mozilla.org/en-US/docs/Web/Media/Formats/codecs_parameter
+ return "mp4a.40." + strconv.FormatInt(int64(ttrack.Config.Type), 10)
+ }
+
+ return ""
+}
+
type muxerPrimaryPlaylist struct {
fmp4 bool
- videoTrack *format.H264
+ videoTrack format.Format
audioTrack *format.MPEG4Audio
}
func newMuxerPrimaryPlaylist(
fmp4 bool,
- videoTrack *format.H264,
+ videoTrack format.Format,
audioTrack *format.MPEG4Audio,
) *muxerPrimaryPlaylist {
return &muxerPrimaryPlaylist{
@@ -39,15 +64,10 @@ func (p *muxerPrimaryPlaylist) file() *MuxerFileResponse {
var codecs []string
if p.videoTrack != nil {
- sps := p.videoTrack.SafeSPS()
- if len(sps) >= 4 {
- codecs = append(codecs, "avc1."+hex.EncodeToString(sps[1:4]))
- }
+ codecs = append(codecs, codecParameters(p.videoTrack))
}
-
- // https://developer.mozilla.org/en-US/docs/Web/Media/Formats/codecs_parameter
if p.audioTrack != nil {
- codecs = append(codecs, "mp4a.40."+strconv.FormatInt(int64(p.audioTrack.Config.Type), 10))
+ codecs = append(codecs, codecParameters(p.audioTrack))
}
var version int
diff --git a/internal/hls/muxer_test.go b/internal/hls/muxer_test.go
index 0acb153ff16..76eb597a276 100644
--- a/internal/hls/muxer_test.go
+++ b/internal/hls/muxer_test.go
@@ -57,17 +57,17 @@ func TestMuxerVideoAudio(t *testing.T) {
require.NoError(t, err)
defer m.Close()
- // group without IDR
+ // access unit without IDR
d := 1 * time.Second
- err = m.WriteH264(testTime.Add(d-1*time.Second), d, [][]byte{
+ err = m.WriteH26x(testTime.Add(d-1*time.Second), d, [][]byte{
{0x06},
{0x07},
})
require.NoError(t, err)
- // group with IDR
+ // access unit with IDR
d = 2 * time.Second
- err = m.WriteH264(testTime.Add(d-1*time.Second), d, [][]byte{
+ err = m.WriteH26x(testTime.Add(d-1*time.Second), d, [][]byte{
testSPS, // SPS
{8}, // PPS
{5}, // IDR
@@ -86,9 +86,9 @@ func TestMuxerVideoAudio(t *testing.T) {
})
require.NoError(t, err)
- // group without IDR
+ // access unit without IDR
d = 4 * time.Second
- err = m.WriteH264(testTime.Add(d-1*time.Second), d, [][]byte{
+ err = m.WriteH26x(testTime.Add(d-1*time.Second), d, [][]byte{
{1}, // non-IDR
})
require.NoError(t, err)
@@ -99,16 +99,16 @@ func TestMuxerVideoAudio(t *testing.T) {
})
require.NoError(t, err)
- // group with IDR
+ // access unit with IDR
d = 6 * time.Second
- err = m.WriteH264(testTime.Add(d-1*time.Second), d, [][]byte{
+ err = m.WriteH26x(testTime.Add(d-1*time.Second), d, [][]byte{
{5}, // IDR
})
require.NoError(t, err)
- // group with IDR
+ // access unit with IDR
d = 7 * time.Second
- err = m.WriteH264(testTime.Add(d-1*time.Second), d, [][]byte{
+ err = m.WriteH26x(testTime.Add(d-1*time.Second), d, [][]byte{
{5}, // IDR
})
require.NoError(t, err)
@@ -203,25 +203,25 @@ func TestMuxerVideoOnly(t *testing.T) {
require.NoError(t, err)
defer m.Close()
- // group with IDR
+ // access unit with IDR
d := 2 * time.Second
- err = m.WriteH264(testTime.Add(d-2*time.Second), d, [][]byte{
+ err = m.WriteH26x(testTime.Add(d-2*time.Second), d, [][]byte{
testSPS, // SPS
{8}, // PPS
{5}, // IDR
})
require.NoError(t, err)
- // group with IDR
+ // access unit with IDR
d = 6 * time.Second
- err = m.WriteH264(testTime.Add(d-2*time.Second), d, [][]byte{
+ err = m.WriteH26x(testTime.Add(d-2*time.Second), d, [][]byte{
{5}, // IDR
})
require.NoError(t, err)
- // group with IDR
+ // access unit with IDR
d = 7 * time.Second
- err = m.WriteH264(testTime.Add(d-2*time.Second), d, [][]byte{
+ err = m.WriteH26x(testTime.Add(d-2*time.Second), d, [][]byte{
{5}, // IDR
})
require.NoError(t, err)
@@ -415,8 +415,8 @@ func TestMuxerCloseBeforeFirstSegmentReader(t *testing.T) {
m, err := NewMuxer(MuxerVariantMPEGTS, 3, 1*time.Second, 0, 50*1024*1024, videoTrack, nil)
require.NoError(t, err)
- // group with IDR
- err = m.WriteH264(testTime, 2*time.Second, [][]byte{
+ // access unit with IDR
+ err = m.WriteH26x(testTime, 2*time.Second, [][]byte{
testSPS, // SPS
{8}, // PPS
{5}, // IDR
@@ -441,7 +441,7 @@ func TestMuxerMaxSegmentSize(t *testing.T) {
require.NoError(t, err)
defer m.Close()
- err = m.WriteH264(testTime, 2*time.Second, [][]byte{
+ err = m.WriteH26x(testTime, 2*time.Second, [][]byte{
testSPS,
{5}, // IDR
})
@@ -460,14 +460,14 @@ func TestMuxerDoubleRead(t *testing.T) {
require.NoError(t, err)
defer m.Close()
- err = m.WriteH264(testTime, 0, [][]byte{
+ err = m.WriteH26x(testTime, 0, [][]byte{
testSPS,
{5}, // IDR
{1},
})
require.NoError(t, err)
- err = m.WriteH264(testTime, 2*time.Second, [][]byte{
+ err = m.WriteH26x(testTime, 2*time.Second, [][]byte{
{5}, // IDR
{2},
})
diff --git a/internal/hls/muxer_variant.go b/internal/hls/muxer_variant.go
index 104562748a9..1b7af3340a9 100644
--- a/internal/hls/muxer_variant.go
+++ b/internal/hls/muxer_variant.go
@@ -16,7 +16,7 @@ const (
type muxerVariant interface {
close()
- writeH264(ntp time.Time, pts time.Duration, nalus [][]byte) error
- writeAAC(ntp time.Time, pts time.Duration, au []byte) error
+ writeH26x(time.Time, time.Duration, [][]byte) error
+ writeAAC(time.Time, time.Duration, []byte) error
file(name string, msn string, part string, skip string) *MuxerFileResponse
}
diff --git a/internal/hls/muxer_variant_fmp4.go b/internal/hls/muxer_variant_fmp4.go
index 4d837ce53c4..5017d4334d7 100644
--- a/internal/hls/muxer_variant_fmp4.go
+++ b/internal/hls/muxer_variant_fmp4.go
@@ -11,16 +11,48 @@ import (
"github.com/aler9/rtsp-simple-server/internal/hls/fmp4"
)
+func extractVideoParams(track format.Format) [][]byte {
+ switch ttrack := track.(type) {
+ case *format.H264:
+ params := make([][]byte, 2)
+ params[0] = ttrack.SafeSPS()
+ params[1] = ttrack.SafePPS()
+ return params
+
+ case *format.H265:
+ params := make([][]byte, 3)
+ params[0] = ttrack.SafeVPS()
+ params[1] = ttrack.SafeSPS()
+ params[2] = ttrack.SafePPS()
+ return params
+
+ default:
+ return nil
+ }
+}
+
+func videoParamsEqual(p1 [][]byte, p2 [][]byte) bool {
+ if len(p1) != len(p2) {
+ return true
+ }
+
+ for i, p := range p1 {
+ if !bytes.Equal(p2[i], p) {
+ return false
+ }
+ }
+ return true
+}
+
type muxerVariantFMP4 struct {
playlist *muxerVariantFMP4Playlist
segmenter *muxerVariantFMP4Segmenter
- videoTrack *format.H264
+ videoTrack format.Format
audioTrack *format.MPEG4Audio
- mutex sync.Mutex
- videoLastSPS []byte
- videoLastPPS []byte
- initContent []byte
+ mutex sync.Mutex
+ lastVideoParams [][]byte
+ initContent []byte
}
func newMuxerVariantFMP4(
@@ -29,7 +61,7 @@ func newMuxerVariantFMP4(
segmentDuration time.Duration,
partDuration time.Duration,
segmentMaxSize uint64,
- videoTrack *format.H264,
+ videoTrack format.Format,
audioTrack *format.MPEG4Audio,
) *muxerVariantFMP4 {
v := &muxerVariantFMP4{
@@ -63,28 +95,34 @@ func (v *muxerVariantFMP4) close() {
v.playlist.close()
}
-func (v *muxerVariantFMP4) writeH264(ntp time.Time, pts time.Duration, nalus [][]byte) error {
- return v.segmenter.writeH264(ntp, pts, nalus)
+func (v *muxerVariantFMP4) writeH26x(ntp time.Time, pts time.Duration, au [][]byte) error {
+ return v.segmenter.writeH26x(ntp, pts, au)
}
func (v *muxerVariantFMP4) writeAAC(ntp time.Time, pts time.Duration, au []byte) error {
return v.segmenter.writeAAC(ntp, pts, au)
}
+func (v *muxerVariantFMP4) mustRegenerateInit() bool {
+ if v.videoTrack == nil {
+ return false
+ }
+
+ videoParams := extractVideoParams(v.videoTrack)
+ if !videoParamsEqual(videoParams, v.lastVideoParams) {
+ v.lastVideoParams = videoParams
+ return true
+ }
+
+ return false
+}
+
func (v *muxerVariantFMP4) file(name string, msn string, part string, skip string) *MuxerFileResponse {
if name == "init.mp4" {
v.mutex.Lock()
defer v.mutex.Unlock()
- var sps []byte
- var pps []byte
- if v.videoTrack != nil {
- sps = v.videoTrack.SafeSPS()
- pps = v.videoTrack.SafePPS()
- }
-
- if v.initContent == nil ||
- (v.videoTrack != nil && (!bytes.Equal(v.videoLastSPS, sps) || !bytes.Equal(v.videoLastPPS, pps))) {
+ if v.initContent == nil || v.mustRegenerateInit() {
init := fmp4.Init{}
trackID := 1
@@ -105,14 +143,11 @@ func (v *muxerVariantFMP4) file(name string, msn string, part string, skip strin
})
}
- initContent, err := init.Marshal()
+ var err error
+ v.initContent, err = init.Marshal()
if err != nil {
return &MuxerFileResponse{Status: http.StatusInternalServerError}
}
-
- v.videoLastSPS = sps
- v.videoLastPPS = pps
- v.initContent = initContent
}
return &MuxerFileResponse{
diff --git a/internal/hls/muxer_variant_fmp4_part.go b/internal/hls/muxer_variant_fmp4_part.go
index 5ad4a66b97f..9483d6a47ee 100644
--- a/internal/hls/muxer_variant_fmp4_part.go
+++ b/internal/hls/muxer_variant_fmp4_part.go
@@ -17,7 +17,7 @@ func fmp4PartName(id uint64) string {
}
type muxerVariantFMP4Part struct {
- videoTrack *format.H264
+ videoTrack format.Format
audioTrack *format.MPEG4Audio
id uint64
@@ -33,7 +33,7 @@ type muxerVariantFMP4Part struct {
}
func newMuxerVariantFMP4Part(
- videoTrack *format.H264,
+ videoTrack format.Format,
audioTrack *format.MPEG4Audio,
id uint64,
) *muxerVariantFMP4Part {
diff --git a/internal/hls/muxer_variant_fmp4_playlist.go b/internal/hls/muxer_variant_fmp4_playlist.go
index 3dc9765757a..dbcb3f3b68e 100644
--- a/internal/hls/muxer_variant_fmp4_playlist.go
+++ b/internal/hls/muxer_variant_fmp4_playlist.go
@@ -70,7 +70,7 @@ func partTargetDuration(
type muxerVariantFMP4Playlist struct {
lowLatency bool
segmentCount int
- videoTrack *format.H264
+ videoTrack format.Format
audioTrack *format.MPEG4Audio
mutex sync.Mutex
@@ -89,7 +89,7 @@ type muxerVariantFMP4Playlist struct {
func newMuxerVariantFMP4Playlist(
lowLatency bool,
segmentCount int,
- videoTrack *format.H264,
+ videoTrack format.Format,
audioTrack *format.MPEG4Audio,
) *muxerVariantFMP4Playlist {
p := &muxerVariantFMP4Playlist{
diff --git a/internal/hls/muxer_variant_fmp4_segment.go b/internal/hls/muxer_variant_fmp4_segment.go
index d5122b16066..017bced7d02 100644
--- a/internal/hls/muxer_variant_fmp4_segment.go
+++ b/internal/hls/muxer_variant_fmp4_segment.go
@@ -45,7 +45,7 @@ type muxerVariantFMP4Segment struct {
startTime time.Time
startDTS time.Duration
segmentMaxSize uint64
- videoTrack *format.H264
+ videoTrack format.Format
audioTrack *format.MPEG4Audio
genPartID func() uint64
onPartFinalized func(*muxerVariantFMP4Part)
@@ -63,7 +63,7 @@ func newMuxerVariantFMP4Segment(
startTime time.Time,
startDTS time.Duration,
segmentMaxSize uint64,
- videoTrack *format.H264,
+ videoTrack format.Format,
audioTrack *format.MPEG4Audio,
genPartID func() uint64,
onPartFinalized func(*muxerVariantFMP4Part),
diff --git a/internal/hls/muxer_variant_fmp4_segmenter.go b/internal/hls/muxer_variant_fmp4_segmenter.go
index 72fbf84edf0..37d21da5ada 100644
--- a/internal/hls/muxer_variant_fmp4_segmenter.go
+++ b/internal/hls/muxer_variant_fmp4_segmenter.go
@@ -1,10 +1,11 @@
package hls
import (
- "bytes"
+ "fmt"
"time"
"github.com/aler9/gortsplib/v2/pkg/codecs/h264"
+ "github.com/aler9/gortsplib/v2/pkg/codecs/h265"
"github.com/aler9/gortsplib/v2/pkg/format"
"github.com/aler9/rtsp-simple-server/internal/hls/fmp4"
@@ -45,6 +46,21 @@ func findCompatiblePartDuration(
return i
}
+type dtsExtractor interface {
+ Extract([][]byte, time.Duration) (time.Duration, error)
+}
+
+func allocateDTSExtractor(track format.Format) dtsExtractor {
+ switch track.(type) {
+ case *format.H264:
+ return h264.NewDTSExtractor()
+
+ case *format.H265:
+ return h265.NewDTSExtractor()
+ }
+ return nil
+}
+
type augmentedVideoSample struct {
fmp4.PartSample
dts time.Duration
@@ -62,23 +78,23 @@ type muxerVariantFMP4Segmenter struct {
segmentDuration time.Duration
partDuration time.Duration
segmentMaxSize uint64
- videoTrack *format.H264
+ videoTrack format.Format
audioTrack *format.MPEG4Audio
onSegmentFinalized func(*muxerVariantFMP4Segment)
onPartFinalized func(*muxerVariantFMP4Part)
- startDTS time.Duration
- videoFirstIDRReceived bool
- videoDTSExtractor *h264.DTSExtractor
- videoSPS []byte
- currentSegment *muxerVariantFMP4Segment
- nextSegmentID uint64
- nextPartID uint64
- nextVideoSample *augmentedVideoSample
- nextAudioSample *augmentedAudioSample
- firstSegmentFinalized bool
- sampleDurations map[time.Duration]struct{}
- adjustedPartDuration time.Duration
+ startDTS time.Duration
+ videoFirstRandomAccessReceived bool
+ videoDTSExtractor dtsExtractor
+ lastVideoParams [][]byte
+ currentSegment *muxerVariantFMP4Segment
+ nextSegmentID uint64
+ nextPartID uint64
+ nextVideoSample *augmentedVideoSample
+ nextAudioSample *augmentedAudioSample
+ firstSegmentFinalized bool
+ sampleDurations map[time.Duration]struct{}
+ adjustedPartDuration time.Duration
}
func newMuxerVariantFMP4Segmenter(
@@ -87,7 +103,7 @@ func newMuxerVariantFMP4Segmenter(
segmentDuration time.Duration,
partDuration time.Duration,
segmentMaxSize uint64,
- videoTrack *format.H264,
+ videoTrack format.Format,
audioTrack *format.MPEG4Audio,
onSegmentFinalized func(*muxerVariantFMP4Segment),
onPartFinalized func(*muxerVariantFMP4Part),
@@ -140,50 +156,65 @@ func (m *muxerVariantFMP4Segmenter) adjustPartDuration(du time.Duration) {
}
}
-func (m *muxerVariantFMP4Segmenter) writeH264(ntp time.Time, pts time.Duration, nalus [][]byte) error {
- idrPresent := false
- nonIDRPresent := false
+func (m *muxerVariantFMP4Segmenter) writeH26x(ntp time.Time, pts time.Duration, au [][]byte) error {
+ randomAccessPresent := false
+
+ switch m.videoTrack.(type) {
+ case *format.H264:
+ nonIDRPresent := false
+
+ for _, nalu := range au {
+ typ := h264.NALUType(nalu[0] & 0x1F)
- for _, nalu := range nalus {
- typ := h264.NALUType(nalu[0] & 0x1F)
- switch typ {
- case h264.NALUTypeIDR:
- idrPresent = true
+ switch typ {
+ case h264.NALUTypeIDR:
+ randomAccessPresent = true
- case h264.NALUTypeNonIDR:
- nonIDRPresent = true
+ case h264.NALUTypeNonIDR:
+ nonIDRPresent = true
+ }
}
- }
- if !idrPresent && !nonIDRPresent {
- return nil
+ if !randomAccessPresent && !nonIDRPresent {
+ return nil
+ }
+
+ case *format.H265:
+ for _, nalu := range au {
+ typ := h265.NALUType((nalu[0] >> 1) & 0b111111)
+
+ switch typ {
+ case h265.NALUType_IDR_W_RADL, h265.NALUType_IDR_N_LP, h265.NALUType_CRA_NUT:
+ randomAccessPresent = true
+ }
+ }
}
- return m.writeH264Entry(ntp, pts, nalus, idrPresent)
+ return m.writeH26xEntry(ntp, pts, au, randomAccessPresent)
}
-func (m *muxerVariantFMP4Segmenter) writeH264Entry(
+func (m *muxerVariantFMP4Segmenter) writeH26xEntry(
ntp time.Time,
pts time.Duration,
- nalus [][]byte,
- idrPresent bool,
+ au [][]byte,
+ randomAccessPresent bool,
) error {
var dts time.Duration
- if !m.videoFirstIDRReceived {
+ if !m.videoFirstRandomAccessReceived {
// skip sample silently until we find one with an IDR
- if !idrPresent {
+ if !randomAccessPresent {
return nil
}
- m.videoFirstIDRReceived = true
- m.videoDTSExtractor = h264.NewDTSExtractor()
- m.videoSPS = m.videoTrack.SafeSPS()
+ m.videoFirstRandomAccessReceived = true
+ m.videoDTSExtractor = allocateDTSExtractor(m.videoTrack)
+ m.lastVideoParams = extractVideoParams(m.videoTrack)
var err error
- dts, err = m.videoDTSExtractor.Extract(nalus, pts)
+ dts, err = m.videoDTSExtractor.Extract(au, pts)
if err != nil {
- return err
+ return fmt.Errorf("unable to extract DTS: %v", err)
}
m.startDTS = dts
@@ -191,16 +222,16 @@ func (m *muxerVariantFMP4Segmenter) writeH264Entry(
pts -= m.startDTS
} else {
var err error
- dts, err = m.videoDTSExtractor.Extract(nalus, pts)
+ dts, err = m.videoDTSExtractor.Extract(au, pts)
if err != nil {
- return err
+ return fmt.Errorf("unable to extract DTS: %v", err)
}
dts -= m.startDTS
pts -= m.startDTS
}
- avcc, err := h264.AVCCMarshal(nalus)
+ avcc, err := h264.AVCCMarshal(au)
if err != nil {
return err
}
@@ -208,7 +239,7 @@ func (m *muxerVariantFMP4Segmenter) writeH264Entry(
sample := &augmentedVideoSample{
PartSample: fmp4.PartSample{
PTSOffset: int32(durationGoToMp4(pts-dts, 90000)),
- IsNonSyncSample: !idrPresent,
+ IsNonSyncSample: !randomAccessPresent,
Payload: avcc,
},
dts: dts,
@@ -247,12 +278,12 @@ func (m *muxerVariantFMP4Segmenter) writeH264Entry(
}
// switch segment
- if idrPresent {
- sps := m.videoTrack.SafeSPS()
- spsChanged := !bytes.Equal(m.videoSPS, sps)
+ if randomAccessPresent {
+ videoParams := extractVideoParams(m.videoTrack)
+ paramsChanged := !videoParamsEqual(m.lastVideoParams, videoParams)
if (m.nextVideoSample.dts-m.currentSegment.startDTS) >= m.segmentDuration ||
- spsChanged {
+ paramsChanged {
err := m.currentSegment.finalize(m.nextVideoSample.dts)
if err != nil {
return err
@@ -273,10 +304,11 @@ func (m *muxerVariantFMP4Segmenter) writeH264Entry(
m.onPartFinalized,
)
- // if SPS changed, reset adjusted part duration
- if spsChanged {
- m.videoSPS = sps
+ if paramsChanged {
+ m.lastVideoParams = videoParams
m.firstSegmentFinalized = false
+
+ // reset adjusted part duration
m.sampleDurations = make(map[time.Duration]struct{})
}
}
@@ -288,7 +320,7 @@ func (m *muxerVariantFMP4Segmenter) writeH264Entry(
func (m *muxerVariantFMP4Segmenter) writeAAC(ntp time.Time, dts time.Duration, au []byte) error {
if m.videoTrack != nil {
// wait for the video track
- if !m.videoFirstIDRReceived {
+ if !m.videoFirstRandomAccessReceived {
return nil
}
diff --git a/internal/hls/muxer_variant_mpegts.go b/internal/hls/muxer_variant_mpegts.go
index 790f2e65bd2..04312588f81 100644
--- a/internal/hls/muxer_variant_mpegts.go
+++ b/internal/hls/muxer_variant_mpegts.go
@@ -1,6 +1,7 @@
package hls
import (
+ "fmt"
"time"
"github.com/aler9/gortsplib/v2/pkg/format"
@@ -15,9 +16,19 @@ func newMuxerVariantMPEGTS(
segmentCount int,
segmentDuration time.Duration,
segmentMaxSize uint64,
- videoTrack *format.H264,
+ videoTrack format.Format,
audioTrack *format.MPEG4Audio,
-) *muxerVariantMPEGTS {
+) (*muxerVariantMPEGTS, error) {
+ var videoTrackH264 *format.H264
+ if videoTrack != nil {
+ var ok bool
+ videoTrackH264, ok = videoTrack.(*format.H264)
+ if !ok {
+ return nil, fmt.Errorf(
+ "the MPEG-TS variant of HLS doesn't support H265. Use the fMP4 or Low-Latency variants instead")
+ }
+ }
+
v := &muxerVariantMPEGTS{}
v.playlist = newMuxerVariantMPEGTSPlaylist(segmentCount)
@@ -25,21 +36,21 @@ func newMuxerVariantMPEGTS(
v.segmenter = newMuxerVariantMPEGTSSegmenter(
segmentDuration,
segmentMaxSize,
- videoTrack,
+ videoTrackH264,
audioTrack,
func(seg *muxerVariantMPEGTSSegment) {
v.playlist.pushSegment(seg)
},
)
- return v
+ return v, nil
}
func (v *muxerVariantMPEGTS) close() {
v.playlist.close()
}
-func (v *muxerVariantMPEGTS) writeH264(ntp time.Time, pts time.Duration, nalus [][]byte) error {
+func (v *muxerVariantMPEGTS) writeH26x(ntp time.Time, pts time.Duration, nalus [][]byte) error {
return v.segmenter.writeH264(ntp, pts, nalus)
}
diff --git a/internal/hls/muxer_variant_mpegts_segmenter.go b/internal/hls/muxer_variant_mpegts_segmenter.go
index c9cccf0c926..a954249909c 100644
--- a/internal/hls/muxer_variant_mpegts_segmenter.go
+++ b/internal/hls/muxer_variant_mpegts_segmenter.go
@@ -1,6 +1,7 @@
package hls
import (
+ "fmt"
"time"
"github.com/aler9/gortsplib/v2/pkg/codecs/h264"
@@ -84,7 +85,7 @@ func (m *muxerVariantMPEGTSSegmenter) writeH264(ntp time.Time, pts time.Duration
var err error
dts, err = m.videoDTSExtractor.Extract(nalus, pts)
if err != nil {
- return err
+ return fmt.Errorf("unable to extract DTS: %v", err)
}
m.startPCR = ntp
@@ -108,7 +109,7 @@ func (m *muxerVariantMPEGTSSegmenter) writeH264(ntp time.Time, pts time.Duration
var err error
dts, err = m.videoDTSExtractor.Extract(nalus, pts)
if err != nil {
- return err
+ return fmt.Errorf("unable to extract DTS: %v", err)
}
dts -= m.startDTS
From 2d107445187c8fb367a99b75b47a8a6a0df059cb Mon Sep 17 00:00:00 2001
From: aler9 <46489434+aler9@users.noreply.github.com>
Date: Thu, 29 Dec 2022 19:47:11 +0100
Subject: [PATCH 2/2] update README
---
README.md | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
diff --git a/README.md b/README.md
index 5cea6a77de7..b930a15d108 100644
--- a/README.md
+++ b/README.md
@@ -89,6 +89,7 @@ Features:
* [Encryption](#encryption-1)
* [HLS protocol](#hls-protocol)
* [General usage](#general-usage-2)
+ * [Browser support](#browser-support)
* [Embedding](#embedding)
* [Low-Latency variant](#low-latency-variant)
* [Decreasing latency](#decreasing-latency)
@@ -903,7 +904,13 @@ http://localhost:8888/mystream
where `mystream` is the name of a stream that is being published.
-Please be aware that HLS only supports a single H264 video track and a single AAC audio track due to limitations of most browsers. If you want to use HLS with streams that use other codecs, you have to re-encode them, for instance by using _FFmpeg_:
+### Browser support
+
+Although the server can produce HLS with a variety of video and audio codecs (that are listed at the beginningo of the README), not all browsers can read all codecs. You can check what codecs your browser can read by visiting this page:
+
+https://jsfiddle.net/7nwxmLto
+
+If you want to increase the compatibility of the stream in order to support most browsers, you have to re-encode it by using the H264 and AAC codecs, for instance by using _FFmpeg_:
```
ffmpeg -i rtsp://original-source -pix_fmt yuv420p -c:v libx264 -preset ultrafast -b:v 600k -c:a aac -b:a 160k -f rtsp rtsp://localhost:8554/mystream