From a0a3d539beccbcb33c87f6c4c06a8beb589362ec Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 23 Oct 2024 16:11:50 -0400 Subject: [PATCH] Add ivfwriter support for VP9 Adds the necessary wiring to get VP9 to work with `ivfwriter` as well as a save-to-disk example. It currently assumes 30 fps but it seems that the other codecs also assume 30 fps so that is not a net-new assumption. --- examples/save-to-disk-vp9/README.md | 36 +++++ examples/save-to-disk-vp9/main.go | 215 ++++++++++++++++++++++++++++ pkg/media/ivfwriter/ivfwriter.go | 53 +++++-- 3 files changed, 295 insertions(+), 9 deletions(-) create mode 100644 examples/save-to-disk-vp9/README.md create mode 100644 examples/save-to-disk-vp9/main.go diff --git a/examples/save-to-disk-vp9/README.md b/examples/save-to-disk-vp9/README.md new file mode 100644 index 00000000000..2fd882185fe --- /dev/null +++ b/examples/save-to-disk-vp9/README.md @@ -0,0 +1,36 @@ +# save-to-disk-vp9 +save-to-disk-vp9 is a simple application that shows how to save a video to disk using VP9. + +If you wish to save VP8 and Opus instead of VP9 see [save-to-disk](https://github.com/pion/webrtc/tree/master/examples/save-to-disk) + +If you wish to save VP8/Opus inside the same file see [save-to-webm](https://github.com/pion/example-webrtc-applications/tree/master/save-to-webm) + +You can then send this video back to your browser using [play-from-disk](https://github.com/pion/example-webrtc-applications/tree/master/play-from-disk) + +## Instructions +### Download save-to-disk-vp9 +``` +go install github.com/pion/webrtc/v4/examples/save-to-disk-vp9@latest +``` + +### Open save-to-disk-vp9 example page +[jsfiddle.net](https://jsfiddle.net/xjcve6d3/) you should see your Webcam, two text-areas and two buttons: `Copy browser SDP to clipboard`, `Start Session`. + +### Run save-to-disk-vp9, with your browsers SessionDescription as stdin +In the jsfiddle the top textarea is your browser's Session Description. Press `Copy browser SDP to clipboard` or copy the base64 string manually. +We will use this value in the next step. + +#### Linux/macOS +Run `echo $BROWSER_SDP | save-to-disk-vp9` +#### Windows +1. Paste the SessionDescription into a file. +1. Run `save-to-disk-vp9 < my_file` + +### Input save-to-disk-vp9's SessionDescription into your browser +Copy the text that `save-to-disk-vp9` just emitted and copy into second text area + +### Hit 'Start Session' in jsfiddle, wait, close jsfiddle, enjoy your video! +In the folder you ran `save-to-disk-vp9` you should now have a file `output.ivf` play with your video player of choice! +> Note: In order to correctly create the files, the remote client (JSFiddle) should be closed. The Go example will automatically close itself. + +Congrats, you have used Pion WebRTC! Now start building something cool diff --git a/examples/save-to-disk-vp9/main.go b/examples/save-to-disk-vp9/main.go new file mode 100644 index 00000000000..f9687f3b7e2 --- /dev/null +++ b/examples/save-to-disk-vp9/main.go @@ -0,0 +1,215 @@ +// SPDX-FileCopyrightText: 2023 The Pion community +// SPDX-License-Identifier: MIT + +//go:build !js +// +build !js + +// save-to-disk-vp9 is a simple application that shows how to save a video to disk using VP9. +package main + +import ( + "bufio" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "strings" + + "github.com/pion/interceptor" + "github.com/pion/interceptor/pkg/intervalpli" + "github.com/pion/webrtc/v4" + "github.com/pion/webrtc/v4/pkg/media" + "github.com/pion/webrtc/v4/pkg/media/ivfwriter" +) + +func saveToDisk(i media.Writer, track *webrtc.TrackRemote) { + defer func() { + if err := i.Close(); err != nil { + panic(err) + } + }() + + for { + rtpPacket, _, err := track.ReadRTP() + if err != nil { + fmt.Println(err) + return + } + if err := i.WriteRTP(rtpPacket); err != nil { + fmt.Println(err) + return + } + } +} + +func main() { + // Everything below is the Pion WebRTC API! Thanks for using it ❤️. + + // Create a MediaEngine object to configure the supported codec + m := &webrtc.MediaEngine{} + + // Setup the codecs you want to use. + // We'll use a VP9 and Opus but you can also define your own + if err := m.RegisterCodec(webrtc.RTPCodecParameters{ + RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP9, ClockRate: 90000, Channels: 0, SDPFmtpLine: "", RTCPFeedback: nil}, + PayloadType: 96, + }, webrtc.RTPCodecTypeVideo); err != nil { + panic(err) + } + + // Create a InterceptorRegistry. This is the user configurable RTP/RTCP Pipeline. + // This provides NACKs, RTCP Reports and other features. If you use `webrtc.NewPeerConnection` + // this is enabled by default. If you are manually managing You MUST create a InterceptorRegistry + // for each PeerConnection. + i := &interceptor.Registry{} + + // Register a intervalpli factory + // This interceptor sends a PLI every 3 seconds. A PLI causes a video keyframe to be generated by the sender. + // This makes our video seekable and more error resilent, but at a cost of lower picture quality and higher bitrates + // A real world application should process incoming RTCP packets from viewers and forward them to senders + intervalPliFactory, err := intervalpli.NewReceiverInterceptor() + if err != nil { + panic(err) + } + i.Add(intervalPliFactory) + + // Use the default set of Interceptors + if err = webrtc.RegisterDefaultInterceptors(m, i); err != nil { + panic(err) + } + + // Create the API object with the MediaEngine + api := webrtc.NewAPI(webrtc.WithMediaEngine(m), webrtc.WithInterceptorRegistry(i)) + + // Prepare the configuration + config := webrtc.Configuration{} + + // Create a new RTCPeerConnection + peerConnection, err := api.NewPeerConnection(config) + if err != nil { + panic(err) + } + + // Allow us to receive 1 video track + if _, err = peerConnection.AddTransceiverFromKind(webrtc.RTPCodecTypeVideo); err != nil { + panic(err) + } + + ivfFile, err := ivfwriter.New("output.ivf", ivfwriter.WithCodec(webrtc.MimeTypeVP9)) + if err != nil { + panic(err) + } + + // Set a handler for when a new remote track starts, this handler saves buffers to disk as + // an ivf file, since we could have multiple video tracks we provide a counter. + // In your application this is where you would handle/process video + peerConnection.OnTrack(func(track *webrtc.TrackRemote, receiver *webrtc.RTPReceiver) { //nolint: revive + if strings.EqualFold(track.Codec().MimeType, webrtc.MimeTypeVP9) { + fmt.Println("Got VP9 track, saving to disk as output.ivf") + saveToDisk(ivfFile, track) + } + }) + + // Set the handler for ICE connection state + // This will notify you when the peer has connected/disconnected + peerConnection.OnICEConnectionStateChange(func(connectionState webrtc.ICEConnectionState) { + fmt.Printf("Connection State has changed %s \n", connectionState.String()) + + if connectionState == webrtc.ICEConnectionStateConnected { + fmt.Println("Ctrl+C the remote client to stop the demo") + } else if connectionState == webrtc.ICEConnectionStateFailed || connectionState == webrtc.ICEConnectionStateClosed { + if closeErr := ivfFile.Close(); closeErr != nil { + panic(closeErr) + } + + fmt.Println("Done writing media files") + + // Gracefully shutdown the peer connection + if closeErr := peerConnection.Close(); closeErr != nil { + panic(closeErr) + } + + os.Exit(0) + } + }) + + // Wait for the offer to be pasted + offer := webrtc.SessionDescription{} + decode(readUntilNewline(), &offer) + + // Set the remote SessionDescription + err = peerConnection.SetRemoteDescription(offer) + if err != nil { + panic(err) + } + + // Create answer + answer, err := peerConnection.CreateAnswer(nil) + if err != nil { + panic(err) + } + + // Create channel that is blocked until ICE Gathering is complete + gatherComplete := webrtc.GatheringCompletePromise(peerConnection) + + // Sets the LocalDescription, and starts our UDP listeners + err = peerConnection.SetLocalDescription(answer) + if err != nil { + panic(err) + } + + // Block until ICE Gathering is complete, disabling trickle ICE + // we do this because we only can exchange one signaling message + // in a production application you should exchange ICE Candidates via OnICECandidate + <-gatherComplete + + // Output the answer in base64 so we can paste it in browser + fmt.Println(encode(peerConnection.LocalDescription())) + + // Block forever + select {} +} + +// Read from stdin until we get a newline +func readUntilNewline() (in string) { + var err error + + r := bufio.NewReader(os.Stdin) + for { + in, err = r.ReadString('\n') + if err != nil && !errors.Is(err, io.EOF) { + panic(err) + } + + if in = strings.TrimSpace(in); len(in) > 0 { + break + } + } + + fmt.Println("") + return +} + +// JSON encode + base64 a SessionDescription +func encode(obj *webrtc.SessionDescription) string { + b, err := json.Marshal(obj) + if err != nil { + panic(err) + } + + return base64.StdEncoding.EncodeToString(b) +} + +// Decode a base64 and unmarshal JSON into a SessionDescription +func decode(in string, obj *webrtc.SessionDescription) { + b, err := base64.StdEncoding.DecodeString(in) + if err != nil { + panic(err) + } + + if err = json.Unmarshal(b, obj); err != nil { + panic(err) + } +} diff --git a/pkg/media/ivfwriter/ivfwriter.go b/pkg/media/ivfwriter/ivfwriter.go index 6af42ce5414..a741ec8ed4b 100644 --- a/pkg/media/ivfwriter/ivfwriter.go +++ b/pkg/media/ivfwriter/ivfwriter.go @@ -24,6 +24,7 @@ var ( const ( mimeTypeVP8 = "video/VP8" + mimeTypeVP9 = "video/VP9" mimeTypeAV1 = "video/AV1" ivfFileHeaderSignature = "DKIF" @@ -35,9 +36,9 @@ type IVFWriter struct { count uint64 seenKeyFrame bool - isVP8, isAV1 bool + isVP8, isVP9, isAV1 bool - // VP8 + // VP8, VP9 currentFrame []byte // AV1 @@ -75,7 +76,7 @@ func NewWith(out io.Writer, opts ...Option) (*IVFWriter, error) { } } - if !writer.isAV1 && !writer.isVP8 { + if !writer.isAV1 && !writer.isVP8 && !writer.isVP9 { writer.isVP8 = true } @@ -92,9 +93,12 @@ func (i *IVFWriter) writeHeader() error { binary.LittleEndian.PutUint16(header[6:], 32) // Header size // FOURCC - if i.isVP8 { + switch { + case i.isVP8: copy(header[8:], "VP80") - } else if i.isAV1 { + case i.isVP9: + copy(header[8:], "VP90") + case i.isAV1: copy(header[8:], "AV01") } @@ -123,14 +127,15 @@ func (i *IVFWriter) writeFrame(frame []byte, timestamp uint64) error { } // WriteRTP adds a new packet and writes the appropriate headers for it -func (i *IVFWriter) WriteRTP(packet *rtp.Packet) error { +func (i *IVFWriter) WriteRTP(packet *rtp.Packet) error { //nolint:gocognit if i.ioWriter == nil { return errFileNotOpened } else if len(packet.Payload) == 0 { return nil } - if i.isVP8 { + switch { + case i.isVP8: vp8Packet := codecs.VP8Packet{} if _, err := vp8Packet.Unmarshal(packet.Payload); err != nil { return err @@ -157,7 +162,35 @@ func (i *IVFWriter) WriteRTP(packet *rtp.Packet) error { return err } i.currentFrame = nil - } else if i.isAV1 { + case i.isVP9: + vp9Packet := codecs.VP9Packet{} + if _, err := vp9Packet.Unmarshal(packet.Payload); err != nil { + return err + } + + switch { + case !i.seenKeyFrame && vp9Packet.P: + return nil + case i.currentFrame == nil && !vp9Packet.B: + return nil + } + + i.seenKeyFrame = true + i.currentFrame = append(i.currentFrame, vp9Packet.Payload[0:]...) + + if !packet.Marker { + return nil + } else if len(i.currentFrame) == 0 { + return nil + } + + // the timestamp must be sequential. webrtc mandates a clock rate of 90000 + // and we've assumed 30fps in the header. + if err := i.writeFrame(i.currentFrame, uint64(packet.Header.Timestamp)/3000); err != nil { + return err + } + i.currentFrame = nil + case i.isAV1: av1Packet := &codecs.AV1Packet{} if _, err := av1Packet.Unmarshal(packet.Payload); err != nil { return err @@ -215,13 +248,15 @@ type Option func(i *IVFWriter) error // WithCodec configures if IVFWriter is writing AV1 or VP8 packets to disk func WithCodec(mimeType string) Option { return func(i *IVFWriter) error { - if i.isVP8 || i.isAV1 { + if i.isVP8 || i.isVP9 || i.isAV1 { return errCodecAlreadySet } switch mimeType { case mimeTypeVP8: i.isVP8 = true + case mimeTypeVP9: + i.isVP9 = true case mimeTypeAV1: i.isAV1 = true default: