From 23bc58614cb64da0fd8935340629015b17146449 Mon Sep 17 00:00:00 2001 From: Gordon Klaus Date: Wed, 29 Mar 2017 10:45:46 +0200 Subject: [PATCH] Add support for sending and receiving audio, and all that entails. --- audiotrack.cc | 88 +++++++++++++++++++++++ audiotrack.go | 142 ++++++++++++++++++++++++++++++++++++++ audiotrack.h | 30 ++++++++ demo/beep/beep.go | 138 ++++++++++++++++++++++++++++++++++++ demo/beep/beep.html | 11 +++ demo/beep/beep.js | 53 ++++++++++++++ demo/echo/echo.go | 124 +++++++++++++++++++++++++++++++++ demo/echo/echo.html | 11 +++ demo/echo/echo.js | 57 +++++++++++++++ eventlistener.cc | 35 ++++++++++ eventlistener.go | 34 +++++++++ eventlistener.h | 25 +++++++ fauxaudiodevicemodule.cc | 22 ++++++ fauxaudiodevicemodule.hpp | 33 +++++++++ mediastream.cc | 17 +++++ mediastream.go | 50 ++++++++++++++ mediastream.h | 25 +++++++ mediastreamtrack.cc | 23 ++++++ mediastreamtrack.go | 64 +++++++++++++++++ mediastreamtrack.h | 27 ++++++++ peerconnection.cc | 48 ++++++++++--- peerconnection.go | 56 +++++++++++++++ peerconnection.h | 13 ++++ refptr.cc | 7 ++ refptr.go | 27 ++++++++ refptr.h | 20 ++++++ rtpreceiver.cc | 9 +++ rtpreceiver.go | 29 ++++++++ rtpreceiver.h | 26 +++++++ rtpsender.cc | 9 +++ rtpsender.go | 29 ++++++++ rtpsender.h | 26 +++++++ 32 files changed, 1298 insertions(+), 10 deletions(-) create mode 100644 audiotrack.cc create mode 100644 audiotrack.go create mode 100644 audiotrack.h create mode 100644 demo/beep/beep.go create mode 100644 demo/beep/beep.html create mode 100644 demo/beep/beep.js create mode 100644 demo/echo/echo.go create mode 100644 demo/echo/echo.html create mode 100644 demo/echo/echo.js create mode 100644 eventlistener.cc create mode 100644 eventlistener.go create mode 100644 eventlistener.h create mode 100644 fauxaudiodevicemodule.cc create mode 100644 fauxaudiodevicemodule.hpp create mode 100644 mediastream.cc create mode 100644 mediastream.go create mode 100644 mediastream.h create mode 100644 mediastreamtrack.cc create mode 100644 mediastreamtrack.go create mode 100644 mediastreamtrack.h create mode 100644 refptr.cc create mode 100644 refptr.go create mode 100644 refptr.h create mode 100644 rtpreceiver.cc create mode 100644 rtpreceiver.go create mode 100644 rtpreceiver.h create mode 100644 rtpsender.cc create mode 100644 rtpsender.go create mode 100644 rtpsender.h diff --git a/audiotrack.cc b/audiotrack.cc new file mode 100644 index 0000000..021220f --- /dev/null +++ b/audiotrack.cc @@ -0,0 +1,88 @@ +#include <_cgo_export.h> // Allow calling certain Go functions. + +#include "audiotrack.h" + +#include "webrtc/api/mediastreaminterface.h" +#include "webrtc/pc/mediastreamtrack.h" + +using namespace webrtc; + +class AudioTrack : public MediaStreamTrack { + public: + explicit AudioTrack(std::string label, CGO_GoAudioSource source) + : MediaStreamTrack(label) + , source(source) {} + + virtual std::string kind() const override { + return kAudioKind; + } + + virtual AudioSourceInterface* GetSource() const override { + return nullptr; + } + + virtual void AddSink(AudioTrackSinkInterface* sink) override { + cgoAudioSourceAddSink(source, (CGO_AudioSink)sink); + } + virtual void RemoveSink(AudioTrackSinkInterface* sink) override { + cgoAudioSourceRemoveSink(source, (CGO_AudioSink)sink); + } + + virtual bool GetSignalLevel(int* level) override { return false; } + + virtual rtc::scoped_refptr GetAudioProcessor() override { + return nullptr; + } + +protected: + ~AudioTrack() { + cgoAudioSourceDestruct(source); + } + +private: + CGO_GoAudioSource source; +}; + +CGO_AudioTrack CGO_NewAudioTrack(const char* label, CGO_GoAudioSource source) { + return new rtc::RefCountedObject(label, source); +} + +class AudioSink : public AudioTrackSinkInterface { +public: + explicit AudioSink(CGO_GoAudioSink s) : s(s) {} + + virtual void OnData(const void* audio_data, + int bits_per_sample, + int sample_rate, + size_t number_of_channels, + size_t number_of_frames) { + cgoAudioSinkOnData( + s, + (void*)audio_data, + bits_per_sample, + sample_rate, + (int)number_of_channels, + (int)number_of_frames + ); + } + + CGO_GoAudioSink s; +}; + +void CGO_AudioSinkOnData(CGO_AudioSink s, void* data, int bitsPerSample, int sampleRate, int numberOfChannels, int numberOfFrames) { + ((AudioTrackSinkInterface*)s)->OnData(data, bitsPerSample, sampleRate, (size_t)numberOfChannels, (size_t)numberOfFrames); +} + +CGO_AudioSink CGO_AudioTrack_AddSink(CGO_AudioTrack t, CGO_GoAudioSink gs) { + auto s = new AudioSink(gs); + ((AudioTrackInterface*)t)->AddSink(s); + return s; +} + +CGO_GoAudioSink CGO_AudioTrack_RemoveSink(CGO_AudioTrack t, CGO_AudioSink cs) { + auto s = (AudioSink*)cs; + ((AudioTrackInterface*)t)->RemoveSink(s); + auto gs = s->s; + delete s; + return gs; +} diff --git a/audiotrack.go b/audiotrack.go new file mode 100644 index 0000000..f85afd8 --- /dev/null +++ b/audiotrack.go @@ -0,0 +1,142 @@ +package webrtc + +/* +#include "mediastreamtrack.h" +#include "audiotrack.h" +*/ +import "C" +import ( + "math" + "reflect" + "sync" + "unsafe" +) + +/* +An AudioTrack can be obtained via NewAudioTrack (a local track) or via +PeerConnection.OnAddTrack (a remote track). Audio samples are provided or +retrieved, respectively, via an AudioSource or AudioSink. +*/ +type AudioTrack struct { + *mediaStreamTrack + t C.CGO_AudioTrack +} + +func newAudioTrack(t C.CGO_AudioTrack) *AudioTrack { + return &AudioTrack{ + mediaStreamTrack: newMediaStreamTrack(C.CGO_MediaStreamTrack(t)), + t: t, + } +} + +// NewAudioTrack creates a local audio track. +func NewAudioTrack(label string, source AudioSource) *AudioTrack { + t := C.CGO_NewAudioTrack(C.CString(label), C.CGO_GoAudioSource(audioSources.Set(source))) + return newAudioTrack(t) +} + +func (t *AudioTrack) AddSink(s AudioSink) { + cAudioSinksMu.Lock() + defer cAudioSinksMu.Unlock() + + if _, ok := cAudioSinks[s]; ok { + panic("AudioSink already added") + } + csink := C.CGO_AudioTrack_AddSink(t.t, C.CGO_GoAudioSink(audioSinks.Set(s))) + cAudioSinks[s] = csink +} + +func (t *AudioTrack) RemoveSink(s AudioSink) { + cAudioSinksMu.Lock() + defer cAudioSinksMu.Unlock() + + if csink, ok := cAudioSinks[s]; ok { + audioSinks.Delete(int(C.CGO_AudioTrack_RemoveSink(t.t, csink))) + delete(cAudioSinks, s) + } +} + +var ( + audioSources = NewCGOMap() + audioSinks = NewCGOMap() + + cAudioSinksMu sync.Mutex + cAudioSinks = map[AudioSink]C.CGO_AudioSink{} +) + +// An AudioSource pushes audio to the AudioSinks that are added to it. +type AudioSource interface { + AddAudioSink(AudioSink) + RemoveAudioSink(AudioSink) +} + +// An AudioSink receives audio, typically from an AudioSource. +type AudioSink interface { + /* + OnAudioData is called when new audio data is available. + len(data) == numberOfChannels; len(data[i]) is the same for all i. + */ + OnAudioData(data [][]float64, sampleRate float64) +} + +//export cgoAudioSourceAddSink +func cgoAudioSourceAddSink(source C.CGO_GoAudioSource, sink C.CGO_AudioSink) { + audioSources.Get(int(source)).(AudioSource).AddAudioSink(cAudioSink{sink}) +} + +//export cgoAudioSourceRemoveSink +func cgoAudioSourceRemoveSink(source C.CGO_GoAudioSource, sink C.CGO_AudioSink) { + audioSources.Get(int(source)).(AudioSource).RemoveAudioSink(cAudioSink{sink}) +} + +//export cgoAudioSourceDestruct +func cgoAudioSourceDestruct(source C.CGO_GoAudioSource) { + audioSources.Delete(int(source)) +} + +type cAudioSink struct { + s C.CGO_AudioSink +} + +func (s cAudioSink) OnAudioData(data [][]float64, sampleRate float64) { + bitsPerSample := 16 + numberOfChannels := len(data) + numberOfFrames := len(data[0]) + buf := make([]int16, numberOfChannels*numberOfFrames*bitsPerSample/2) + i := 0 + for j := range data[0] { + for _, ch := range data { + buf[i] = int16(ch[j] * float64(math.MaxInt16)) + i++ + } + } + C.CGO_AudioSinkOnData(s.s, unsafe.Pointer(&buf[0]), C.int(bitsPerSample), C.int(sampleRate), C.int(numberOfChannels), C.int(numberOfFrames)) +} + +//export cgoAudioSinkOnData +func cgoAudioSinkOnData(s C.CGO_GoAudioSink, audioData unsafe.Pointer, bitsPerSample, sampleRate, numberOfChannels, numberOfFrames int) { + if bitsPerSample != 16 { + panic("expected 16 bits per sample") + } + + var buf []int16 + sh := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + sh.Data = uintptr(audioData) + sh.Len = numberOfChannels * numberOfFrames + sh.Cap = sh.Len + + data := make([][]float64, numberOfChannels) + for i := range data { + data[i] = make([]float64, numberOfFrames) + } + + i := 0 + for j := range data[0] { + for _, ch := range data { + ch[j] = float64(buf[i]) / float64(math.MaxInt16) + i++ + } + } + + audioSinks.Get(int(s)).(AudioSink).OnAudioData(data, float64(sampleRate)) +} diff --git a/audiotrack.h b/audiotrack.h new file mode 100644 index 0000000..050fa33 --- /dev/null +++ b/audiotrack.h @@ -0,0 +1,30 @@ +#ifndef _C_AUDIOTRACK_H +#define _C_AUDIOTRACK_H + +#define WEBRTC_POSIX 1 + +#ifdef __cplusplus +extern "C" { +#endif + + // In order to present an interface cgo is happy with, nothing in this file + // can directly reference header files from libwebrtc / C++ world. All the + // casting must be hidden in the .cc file. + + typedef void* CGO_AudioTrack; // webrtc::AudioTrackInterface* + typedef int CGO_GoAudioSource; // key into Go audioSourceMap + typedef int CGO_GoAudioSink; // key into Go audioSinkMap + typedef void* CGO_AudioSink; // webrtc::AudioTrackSinkInterface + + CGO_AudioTrack CGO_NewAudioTrack(const char* label, CGO_GoAudioSource source); + + void CGO_AudioSinkOnData(CGO_AudioSink s, void* data, int bitsPerSample, int sampleRate, int numberOfChannels, int numberOfFrames); + + CGO_AudioSink CGO_AudioTrack_AddSink(CGO_AudioTrack, CGO_GoAudioSink); + CGO_GoAudioSink CGO_AudioTrack_RemoveSink(CGO_AudioTrack, CGO_AudioSink); + +#ifdef __cplusplus +} +#endif + +#endif // _C_AUDIOTRACK_H diff --git a/demo/beep/beep.go b/demo/beep/beep.go new file mode 100644 index 0000000..95b6cec --- /dev/null +++ b/demo/beep/beep.go @@ -0,0 +1,138 @@ +/* +WebRTC audio bee demo server. +See beep.html for the client. + +To use, `go run beep.go`, then open beep.html in a browser. +This server will send a tone to the browser, which will play it. +*/ +package main + +import ( + "encoding/json" + "io" + "log" + "net/http" + "sync" + "time" + + "github.com/keroserene/go-webrtc" + "golang.org/x/net/websocket" +) + +func main() { + log.SetFlags(log.LstdFlags | log.Lshortfile) + webrtc.SetLoggingVerbosity(1) + + http.Handle("/ws", websocket.Handler(handleWebSocket)) + log.Fatal(http.ListenAndServe(":49372", nil)) +} + +func handleWebSocket(ws *websocket.Conn) { + pc, err := webrtc.NewPeerConnection(webrtc.NewConfiguration()) + if err != nil { + log.Fatal(err) + } + + beep := &beep{} + pc.AddTrack(webrtc.NewAudioTrack("beep-audio", beep), nil) + go beep.run() + + pc.OnIceCandidate = func(c webrtc.IceCandidate) { + if err := websocket.JSON.Send(ws, c); err != nil { + log.Println(err) + return + } + } + + for { + var msg struct { + Type string + Body json.RawMessage + } + if err := websocket.JSON.Receive(ws, &msg); err != nil { + if err != io.EOF { + log.Println(err) + } + return + } + + switch msg.Type { + case "offer": + offer := webrtc.DeserializeSessionDescription(string(msg.Body)) + if err := pc.SetRemoteDescription(offer); err != nil { + log.Println(err) + return + } + answer, err := pc.CreateAnswer() + if err != nil { + log.Println(err) + return + } + if err := pc.SetLocalDescription(answer); err != nil { + log.Println(err) + return + } + if err := websocket.JSON.Send(ws, answer); err != nil { + log.Println(err) + return + } + case "icecandidate": + c := webrtc.DeserializeIceCandidate(string(msg.Body)) + if err := pc.AddIceCandidate(*c); err != nil { + log.Println(err) + } + default: + log.Println("unexpected message type:", msg.Type) + } + } +} + +type beep struct { + sync.Mutex + sinks []webrtc.AudioSink +} + +func (b *beep) AddAudioSink(s webrtc.AudioSink) { + b.Lock() + b.sinks = append(b.sinks, s) + b.Unlock() +} + +func (b *beep) RemoveAudioSink(s webrtc.AudioSink) { + b.Lock() + defer b.Unlock() + for i, s2 := range b.sinks { + if s2 == s { + b.sinks = append(b.sinks[:i], b.sinks[i+1:]...) + } + } +} + +func (b *beep) run() { + const ( + sampleRate = 48000 + chunkRate = 100 + numberOfFrames = sampleRate / chunkRate + toneFrequency = 256 + ) + data := [][]float64{make([]float64, numberOfFrames)} + count := 0 + x := 0.04 + for next := time.Now(); ; next = next.Add(time.Second / chunkRate) { + time.Sleep(next.Sub(time.Now())) + + for i := range data[0] { + if count%(sampleRate/toneFrequency/2) == 0 { + x = -x + } + data[0][i] = x + count++ + } + + b.Lock() + for _, sink := range b.sinks { + sink.OnAudioData(data, sampleRate) + } + b.Unlock() + } +} diff --git a/demo/beep/beep.html b/demo/beep/beep.html new file mode 100644 index 0000000..11d1540 --- /dev/null +++ b/demo/beep/beep.html @@ -0,0 +1,11 @@ + + + + + + + +

go-webrtc audio beep demo

+