From 0eeb2f04c4c3fc7fdd6c2f12869e0cd4bcb18fc3 Mon Sep 17 00:00:00 2001 From: Stefan Dej Date: Sat, 16 Sep 2023 14:08:47 +0200 Subject: [PATCH 1/8] refactor: update webcam "WebRTC MediaMTX" client Signed-off-by: Stefan Dej --- src/components/webcams/WebrtcMediaMTX.vue | 305 +++++++++++++++------- 1 file changed, 209 insertions(+), 96 deletions(-) diff --git a/src/components/webcams/WebrtcMediaMTX.vue b/src/components/webcams/WebrtcMediaMTX.vue index 8d2ce39fc..3bb202cb3 100644 --- a/src/components/webcams/WebrtcMediaMTX.vue +++ b/src/components/webcams/WebrtcMediaMTX.vue @@ -23,26 +23,37 @@ import BaseMixin from '@/components/mixins/base' import { GuiWebcamStateWebcam } from '@/store/gui/webcams/types' import WebcamMixin from '@/components/mixins/webcam' +interface OfferData { + iceUfrag: string + icePwd: string + medias: string[] +} + @Component -export default class WebrtcRTSPSimpleServer extends Mixins(BaseMixin, WebcamMixin) { +export default class WebrtcMediaMTX extends Mixins(BaseMixin, WebcamMixin) { @Prop({ required: true }) readonly camSettings!: GuiWebcamStateWebcam @Prop({ default: null }) readonly printerUrl!: string | null @Ref() declare video: HTMLVideoElement - // webrtc player vars - private terminated: boolean = false - private ws: WebSocket | null = null private pc: RTCPeerConnection | null = null - private restartTimeoutTimer: any = null + private restartTimeout: any = null private status: string = 'connecting' + private eTag: string | null = null + private queuedCandidates: RTCIceCandidate[] = [] + private offerData: OfferData = { + iceUfrag: '', + icePwd: '', + medias: [], + } + private RESTART_PAUSE = 2000 // stop the video and close the streams if the component is going to be destroyed so we don't leave hanging streams beforeDestroy() { this.terminate() // clear any potentially open restart timeout - if (this.restartTimeoutTimer) { - clearTimeout(this.restartTimeoutTimer) + if (this.restartTimeout) { + clearTimeout(this.restartTimeout) } } @@ -57,10 +68,7 @@ export default class WebrtcRTSPSimpleServer extends Mixins(BaseMixin, WebcamMixi } get url() { - let baseUrl = this.camSettings.stream_url - if (baseUrl.startsWith('http')) { - baseUrl = baseUrl.replace('http', 'ws') + 'ws' - } + let baseUrl = new URL('whep', this.camSettings.stream_url).toString() return this.convertUrl(baseUrl, this.printerUrl) } @@ -87,140 +95,245 @@ export default class WebrtcRTSPSimpleServer extends Mixins(BaseMixin, WebcamMixi this.start() } + log(msg: string, obj?: any) { + if (obj) { + window.console.log(`[WebRTC mediamtx] ${msg}`, obj) + return + } + + window.console.log(`[WebRTC mediamtx] ${msg}`) + } + // webrtc player methods - // adapated from sample player in https://github.com/mrlt8/docker-wyze-bridge - start() { - // unterminate we're starting again. - this.terminated = false + // adapated from https://github.com/bluenviron/mediamtx/blob/main/internal/core/webrtc_read_index.html - // clear any potentially open restart timeout - if (this.restartTimeoutTimer) { - clearTimeout(this.restartTimeoutTimer) - this.restartTimeoutTimer = null - } + unquoteCredential = (v: any) => JSON.parse(`"${v}"`) - window.console.log('[webcam-rtspsimpleserver] web socket connecting') + // eslint-disable-next-line no-undef + linkToIceServers(links: string | null): RTCIceServer[] { + if (links === null) return [] - // test if the url is valid - try { - const url = new URL(this.url) + return links.split(', ').map((link) => { + const m: RegExpMatchArray | null = link.match( + /^<(.+?)>; rel="ice-server"(; username="(.*?)"; credential="(.*?)"; credential-type="password")?/i + ) - // break if url protocol is not ws - if (!url.protocol.startsWith('ws')) { - console.log('[webcam-rtspsimpleserver] invalid URL (no ws protocol)') - return + // break if match is null + if (m === null) return { urls: '' } + + // eslint-disable-next-line no-undef + const ret: RTCIceServer = { + urls: [m[1]], } - } catch (err) { - console.log('[webcam-rtspsimpleserver] invalid URL') - return - } - // open websocket connection - this.ws = new WebSocket(this.url) + if (m.length > 3) { + ret.username = this.unquoteCredential(m[3]) + ret.credential = this.unquoteCredential(m[4]) + ret.credentialType = 'password' + } - this.ws.onerror = (event) => { - window.console.log('[webcam-rtspsimpleserver] websocket error', event) - this.ws?.close() - this.ws = null + return ret + }) + } + + parseOffer(offer: string) { + const ret: OfferData = { + iceUfrag: '', + icePwd: '', + medias: [], } - this.ws.onclose = (event) => { - console.log('[webcam-rtspsimpleserver] websocket closed', event) - this.ws = null - this.scheduleRestart() + for (const line of offer.split('\r\n')) { + if (line.startsWith('m=')) { + ret.medias.push(line.slice('m='.length)) + } else if (ret.iceUfrag === '' && line.startsWith('a=ice-ufrag:')) { + ret.iceUfrag = line.slice('a=ice-ufrag:'.length) + } else if (ret.icePwd === '' && line.startsWith('a=ice-pwd:')) { + ret.icePwd = line.slice('a=ice-pwd:'.length) + } } - this.ws.onmessage = (msg: MessageEvent) => this.webRtcOnIceServers(msg) + return ret } - terminate() { - this.terminated = true + generateSdpFragment(offerData: OfferData, candidates: RTCIceCandidate[]) { + // I don't found a specification for this, but it seems to be the only way to make it work + const candidatesByMedia: any = {} + for (const candidate of candidates) { + const mid = candidate.sdpMLineIndex + if (mid === null) continue - try { - this.video.pause() - } catch (err) { - // ignore -- make sure we close down the sockets anyway + // create the array if it doesn't exist + if (!(mid in candidatesByMedia)) candidatesByMedia[mid] = [] + candidatesByMedia[mid].push(candidate) } - this.ws?.close() - this.pc?.close() - } + let frag = 'a=ice-ufrag:' + offerData.iceUfrag + '\r\n' + 'a=ice-pwd:' + offerData.icePwd + '\r\n' + let mid = 0 - webRtcOnIceServers(msg: MessageEvent) { - if (this.ws === null) return + for (const media of offerData.medias) { + if (candidatesByMedia[mid] !== undefined) { + frag += 'm=' + media + '\r\n' + 'a=mid:' + mid + '\r\n' - const iceServers = JSON.parse(msg.data) - this.pc = new RTCPeerConnection({ - iceServers, - }) + for (const candidate of candidatesByMedia[mid]) { + frag += 'a=' + candidate.candidate + '\r\n' + } + } - this.ws.onmessage = (msg: MessageEvent) => this.webRtcOnRemoteDescription(msg) - this.pc.onicecandidate = (evt: RTCPeerConnectionIceEvent) => this.webRtcOnIceCandidate(evt) + mid++ + } - this.pc.oniceconnectionstatechange = () => { - if (this.pc === null) return + return frag + } - window.console.log('[webcam-rtspsimpleserver] peer connection state:', this.pc.iceConnectionState) + start() { + this.log('requesting ICE servers from ' + this.url) - this.status = (this.pc?.iceConnectionState ?? '').toString() + fetch(this.url, { + method: 'OPTIONS', + }) + .then((res) => this.onIceServers(res)) + .catch((err) => { + this.log('error: ' + err) + //this.scheduleRestart(); + }) + } - if (['failed', 'disconnected'].includes(this.status)) { - this.scheduleRestart() - } - } + onIceServers(res: Response) { + const iceServers = this.linkToIceServers(res.headers.get('Link')) + this.log('ice servers:', iceServers) - this.pc.ontrack = (evt: RTCTrackEvent) => { - window.console.log('[webcam-rtspsimpleserver] new track ' + evt.track.kind) - this.video.srcObject = evt.streams[0] - } + this.pc = new RTCPeerConnection({ + iceServers, + }) const direction = 'sendrecv' this.pc.addTransceiver('video', { direction }) this.pc.addTransceiver('audio', { direction }) - this.pc.createOffer().then((desc: any) => { - if (this.pc === null || this.ws === null) return + this.pc.onicecandidate = (evt: RTCPeerConnectionIceEvent) => this.onLocalCandidate(evt) + this.pc.oniceconnectionstatechange = () => this.onConnectionState() + + this.pc.ontrack = (evt) => { + this.log('new track:', evt.track.kind) + this.video.srcObject = evt.streams[0] + } + + this.pc.createOffer().then((offer) => this.onLocalOffer(offer)) + } - this.pc.setLocalDescription(desc) + // eslint-disable-next-line no-undef + onLocalOffer(offer: RTCSessionDescriptionInit) { + this.offerData = this.parseOffer(offer.sdp ?? '') + this.pc?.setLocalDescription(offer) - window.console.log('[webcam-rtspsimpleserver] sending offer') - this.ws.send(JSON.stringify(desc)) + this.log('sending offer', offer) + + fetch(this.url, { + method: 'POST', + headers: { + 'Content-Type': 'application/sdp', + }, + body: offer.sdp, }) + .then((res) => { + if (res.status !== 201) { + throw new Error('bad status code') + } + this.eTag = res.headers.get('E-Tag') + return res.text() + }) + .then((sdp) => { + this.onRemoteAnswer( + new RTCSessionDescription({ + type: 'answer', + sdp, + }) + ) + }) + .catch((err) => { + this.log('error: ' + err) + //this.scheduleRestart() + }) + } + + onRemoteAnswer(answer: RTCSessionDescription) { + if (this.restartTimeout !== null) return + + // this.pc.setRemoteDescription(new RTCSessionDescription(answer)); + this.pc?.setRemoteDescription(answer) + + if (this.queuedCandidates.length !== 0) { + this.sendLocalCandidates(this.queuedCandidates) + this.queuedCandidates = [] + } } - webRtcOnRemoteDescription(msg: any) { - if (this.pc === null || this.ws === null) return + onConnectionState() { + if (this.restartTimeout !== null) return - this.pc.setRemoteDescription(new RTCSessionDescription(JSON.parse(msg.data))) - this.ws.onmessage = (msg: any) => this.webRtcOnRemoteCandidate(msg) + this.status = this.pc?.iceConnectionState ?? '' + this.log('peer connection state:', this.status) + + switch (this.status) { + case 'disconnected': + this.scheduleRestart() + } } - webRtcOnIceCandidate(evt: RTCPeerConnectionIceEvent) { - if (this.ws === null) return + onLocalCandidate(evt: RTCPeerConnectionIceEvent) { + if (this.restartTimeout !== null) return + + if (evt.candidate !== null) { + if (this.eTag === '') { + this.queuedCandidates.push(evt.candidate) + return + } - if (evt.candidate?.candidate !== '') { - this.ws.send(JSON.stringify(evt.candidate)) + this.sendLocalCandidates([evt.candidate]) } } - webRtcOnRemoteCandidate(msg: any) { - if (this.pc === null) return + sendLocalCandidates(candidates: RTCIceCandidate[]) { + fetch(this.url, { + method: 'PATCH', + headers: { + 'Content-Type': 'application/trickle-ice-sdpfrag', + 'If-Match': this.eTag, + // eslint-disable-next-line no-undef + } as HeadersInit, + body: this.generateSdpFragment(this.offerData, candidates), + }) + .then((res) => { + if (res.status !== 204) throw new Error('bad status code') + }) + .catch((err) => { + this.log('error: ' + err) + //this.scheduleRestart() + }) + } + + terminate() { + this.log('terminating') - this.pc.addIceCandidate(JSON.parse(msg.data)) + if (this.pc !== null) { + this.pc.close() + this.pc = null + } } scheduleRestart() { - this.ws?.close() - this.ws = null - - this.pc?.close() - this.pc = null + if (this.restartTimeout !== null) return - if (this.terminated) return + this.terminate() - this.restartTimeoutTimer = setTimeout(() => { + this.restartTimeout = window.setTimeout(() => { + this.restartTimeout = null this.start() - }, 2000) + }, this.RESTART_PAUSE) + + this.eTag = '' + this.queuedCandidates = [] } } From f928bb10cb610a8becc55d569e89e1f600aed989 Mon Sep 17 00:00:00 2001 From: Stefan Dej Date: Sat, 16 Sep 2023 16:58:28 +0200 Subject: [PATCH 2/8] fix: always add a / at last char to stream url Signed-off-by: Stefan Dej --- src/components/webcams/WebrtcMediaMTX.vue | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/components/webcams/WebrtcMediaMTX.vue b/src/components/webcams/WebrtcMediaMTX.vue index 3bb202cb3..cabd33a48 100644 --- a/src/components/webcams/WebrtcMediaMTX.vue +++ b/src/components/webcams/WebrtcMediaMTX.vue @@ -68,7 +68,10 @@ export default class WebrtcMediaMTX extends Mixins(BaseMixin, WebcamMixin) { } get url() { - let baseUrl = new URL('whep', this.camSettings.stream_url).toString() + let baseUrl = this.camSettings.stream_url + if (!baseUrl.endsWith('/')) baseUrl += '/' + + baseUrl = new URL('whep', baseUrl).toString() return this.convertUrl(baseUrl, this.printerUrl) } From 8cdb957d54aa2e883352da8e9490ebf249638f13 Mon Sep 17 00:00:00 2001 From: Stefan Dej Date: Sat, 16 Sep 2023 19:14:45 +0200 Subject: [PATCH 3/8] refactor: cleanup code Signed-off-by: Stefan Dej --- src/components/webcams/WebrtcMediaMTX.vue | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/src/components/webcams/WebrtcMediaMTX.vue b/src/components/webcams/WebrtcMediaMTX.vue index cabd33a48..fc64f78fa 100644 --- a/src/components/webcams/WebrtcMediaMTX.vue +++ b/src/components/webcams/WebrtcMediaMTX.vue @@ -52,9 +52,7 @@ export default class WebrtcMediaMTX extends Mixins(BaseMixin, WebcamMixin) { this.terminate() // clear any potentially open restart timeout - if (this.restartTimeout) { - clearTimeout(this.restartTimeout) - } + if (this.restartTimeout) clearTimeout(this.restartTimeout) } get webcamStyle() { @@ -198,7 +196,7 @@ export default class WebrtcMediaMTX extends Mixins(BaseMixin, WebcamMixin) { .then((res) => this.onIceServers(res)) .catch((err) => { this.log('error: ' + err) - //this.scheduleRestart(); + this.scheduleRestart() }) } @@ -230,8 +228,6 @@ export default class WebrtcMediaMTX extends Mixins(BaseMixin, WebcamMixin) { this.offerData = this.parseOffer(offer.sdp ?? '') this.pc?.setLocalDescription(offer) - this.log('sending offer', offer) - fetch(this.url, { method: 'POST', headers: { @@ -240,9 +236,7 @@ export default class WebrtcMediaMTX extends Mixins(BaseMixin, WebcamMixin) { body: offer.sdp, }) .then((res) => { - if (res.status !== 201) { - throw new Error('bad status code') - } + if (res.status !== 201) throw new Error('bad status code') this.eTag = res.headers.get('E-Tag') return res.text() }) @@ -255,8 +249,8 @@ export default class WebrtcMediaMTX extends Mixins(BaseMixin, WebcamMixin) { ) }) .catch((err) => { - this.log('error: ' + err) - //this.scheduleRestart() + this.log(err) + this.scheduleRestart() }) } @@ -311,8 +305,8 @@ export default class WebrtcMediaMTX extends Mixins(BaseMixin, WebcamMixin) { if (res.status !== 204) throw new Error('bad status code') }) .catch((err) => { - this.log('error: ' + err) - //this.scheduleRestart() + this.log(err) + this.scheduleRestart() }) } @@ -331,6 +325,7 @@ export default class WebrtcMediaMTX extends Mixins(BaseMixin, WebcamMixin) { this.terminate() this.restartTimeout = window.setTimeout(() => { + this.log('scheduling restart') this.restartTimeout = null this.start() }, this.RESTART_PAUSE) From 6f9296a66a3a1c8227253084a4b854a523279859 Mon Sep 17 00:00:00 2001 From: Stefan Dej Date: Sat, 16 Sep 2023 21:11:34 +0200 Subject: [PATCH 4/8] locale: remove rtsp-simple-server in MediaMTX translation Signed-off-by: Stefan Dej --- src/locales/de.json | 2 +- src/locales/en.json | 2 +- src/locales/tr.json | 2 +- src/locales/zh.json | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/locales/de.json b/src/locales/de.json index 6d7d1b593..fd281a4a1 100644 --- a/src/locales/de.json +++ b/src/locales/de.json @@ -1133,7 +1133,7 @@ "Webcams": "Webcams", "WebrtcCameraStreamer": "WebRTC (camera-streamer)", "WebrtcJanus": "WebRTC (janus-gateway)", - "WebrtcMediaMTX": "WebRTC (MediaMTX / rtsp-simple-server)" + "WebrtcMediaMTX": "WebRTC (MediaMTX)" } }, "ScrewsTiltAdjust": { diff --git a/src/locales/en.json b/src/locales/en.json index 704471f03..7a4702cc8 100644 --- a/src/locales/en.json +++ b/src/locales/en.json @@ -1132,7 +1132,7 @@ "Vertically": "vertically", "Webcams": "Webcams", "WebrtcCameraStreamer": "WebRTC (camera-streamer)", - "WebrtcMediaMTX": "WebRTC (MediaMTX / rtsp-simple-server)" + "WebrtcMediaMTX": "WebRTC (MediaMTX)" } }, "ScrewsTiltAdjust": { diff --git a/src/locales/tr.json b/src/locales/tr.json index d8675198a..1375ccbb4 100644 --- a/src/locales/tr.json +++ b/src/locales/tr.json @@ -1125,7 +1125,7 @@ "Vertically": "dikey", "Webcams": "Web Kameraları", "WebrtcCameraStreamer": "WebRTC (kamera-streamer)", - "WebrtcMediaMTX": "WebRTC (MediaMTX / rtsp-simple-server)" + "WebrtcMediaMTX": "WebRTC (MediaMTX)" } }, "ScrewsTiltAdjust": { diff --git a/src/locales/zh.json b/src/locales/zh.json index 5ae1ff53b..0f66cc50f 100644 --- a/src/locales/zh.json +++ b/src/locales/zh.json @@ -1130,7 +1130,7 @@ "Vertically": "垂直", "Webcams": "摄像头", "WebrtcCameraStreamer": "WebRTC (camera-streamer)", - "WebrtcMediaMTX": "WebRTC (MediaMTX / rtsp-simple-server)" + "WebrtcMediaMTX": "WebRTC (MediaMTX)" } }, "ScrewsTiltAdjust": { From f2c66b459f2c789747db8a48f216dc8c7ca87b73 Mon Sep 17 00:00:00 2001 From: Stefan Dej Date: Fri, 6 Oct 2023 22:07:09 +0200 Subject: [PATCH 5/8] fix: update E-Tag to ETag for MediaMTX v1.1.1 Signed-off-by: Stefan Dej --- src/components/webcams/WebrtcMediaMTX.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/webcams/WebrtcMediaMTX.vue b/src/components/webcams/WebrtcMediaMTX.vue index fc64f78fa..1fa753ea6 100644 --- a/src/components/webcams/WebrtcMediaMTX.vue +++ b/src/components/webcams/WebrtcMediaMTX.vue @@ -237,7 +237,7 @@ export default class WebrtcMediaMTX extends Mixins(BaseMixin, WebcamMixin) { }) .then((res) => { if (res.status !== 201) throw new Error('bad status code') - this.eTag = res.headers.get('E-Tag') + this.eTag = res.headers.get('ETag') return res.text() }) .then((sdp) => { From c208dd92a17a12ed30c9dfbf8816d01e6e22410c Mon Sep 17 00:00:00 2001 From: Stefan Dej Date: Sat, 7 Oct 2023 08:23:48 +0200 Subject: [PATCH 6/8] refactor: add option to support E-Tag and ETag to support MediaMTX v1.1.0 and v1.1.1 and newer Signed-off-by: Stefan Dej --- src/components/webcams/WebrtcMediaMTX.vue | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/components/webcams/WebrtcMediaMTX.vue b/src/components/webcams/WebrtcMediaMTX.vue index 1fa753ea6..907352542 100644 --- a/src/components/webcams/WebrtcMediaMTX.vue +++ b/src/components/webcams/WebrtcMediaMTX.vue @@ -238,6 +238,10 @@ export default class WebrtcMediaMTX extends Mixins(BaseMixin, WebcamMixin) { .then((res) => { if (res.status !== 201) throw new Error('bad status code') this.eTag = res.headers.get('ETag') + + // fallback for MediaMTX v1.1.0 with broken ETag header + if (res.headers.has('E-Tag')) this.eTag = res.headers.get('E-Tag') + return res.text() }) .then((sdp) => { From 99bf1f2e0e45594a7ec7c39ea10f06a77005b2bb Mon Sep 17 00:00:00 2001 From: Stefan Dej Date: Sat, 7 Oct 2023 12:01:55 +0200 Subject: [PATCH 7/8] refactor: fix version number in comment for MediaMTX fallback Signed-off-by: Stefan Dej --- src/components/webcams/WebrtcMediaMTX.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/webcams/WebrtcMediaMTX.vue b/src/components/webcams/WebrtcMediaMTX.vue index 907352542..8556cab34 100644 --- a/src/components/webcams/WebrtcMediaMTX.vue +++ b/src/components/webcams/WebrtcMediaMTX.vue @@ -239,7 +239,7 @@ export default class WebrtcMediaMTX extends Mixins(BaseMixin, WebcamMixin) { if (res.status !== 201) throw new Error('bad status code') this.eTag = res.headers.get('ETag') - // fallback for MediaMTX v1.1.0 with broken ETag header + // fallback for MediaMTX v1.0.x with broken ETag header if (res.headers.has('E-Tag')) this.eTag = res.headers.get('E-Tag') return res.text() From e9a612cfd67b45f75e574fc70dcd013e763a10e8 Mon Sep 17 00:00:00 2001 From: Stefan Dej Date: Sat, 7 Oct 2023 12:02:39 +0200 Subject: [PATCH 8/8] refactor: fix type in comment Signed-off-by: Stefan Dej --- src/components/webcams/WebrtcMediaMTX.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/webcams/WebrtcMediaMTX.vue b/src/components/webcams/WebrtcMediaMTX.vue index 8556cab34..257a93808 100644 --- a/src/components/webcams/WebrtcMediaMTX.vue +++ b/src/components/webcams/WebrtcMediaMTX.vue @@ -106,7 +106,7 @@ export default class WebrtcMediaMTX extends Mixins(BaseMixin, WebcamMixin) { } // webrtc player methods - // adapated from https://github.com/bluenviron/mediamtx/blob/main/internal/core/webrtc_read_index.html + // adapted from https://github.com/bluenviron/mediamtx/blob/main/internal/core/webrtc_read_index.html unquoteCredential = (v: any) => JSON.parse(`"${v}"`)