Skip to content

Commit

Permalink
Merge pull request #216 from binbat/fix/web-audio-tracks
Browse files Browse the repository at this point in the history 
fix(web): handle audio tracks correctly
  • Loading branch information
@a-wing
a-wing authored Sep 2, 2024
2 parents 9a1b08b + eccc928 commit 18d5f04
Show file tree
Hide file tree
Showing 4 changed files with 71 additions and 61 deletions.
25 changes: 14 additions & 11 deletions web/shared/components/dialog-preview.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ export const PreviewDialog = forwardRef<IPreviewDialog, Props>((props, ref) => {
const [streamId, setStreamId] = useState('');
const refPeerConnection = useRef<RTCPeerConnection | null>(null);
const refWhepClient = useRef<WHEPClient | null>(null);
const refVideoTrack = useRef<MediaStreamTrack | null>(null);
const refMediaStream = useRef<MediaStream | null>();
const [connState, setConnState] = useState('');
const [videoResolution, setVideoResolution] = useState('');
const logger = useLogger();
Expand Down Expand Up @@ -47,8 +47,8 @@ export const PreviewDialog = forwardRef<IPreviewDialog, Props>((props, ref) => {
if (refVideo.current) {
refVideo.current.srcObject = null;
}
if (refVideoTrack.current) {
refVideoTrack.current = null;
if (refMediaStream.current) {
refMediaStream.current = null;
}
if (refPeerConnection.current) {
refPeerConnection.current = null;
Expand Down Expand Up @@ -88,14 +88,14 @@ export const PreviewDialog = forwardRef<IPreviewDialog, Props>((props, ref) => {
const pc = new RTCPeerConnection();
pc.addTransceiver('video', { direction: 'recvonly' });
pc.addTransceiver('audio', { direction: 'recvonly' });
const ms = new MediaStream();
refMediaStream.current = ms;
if (refVideo.current) {
refVideo.current.srcObject = ms;
}
pc.addEventListener('track', ev => {
logger.log(`track: ${ev.track.kind}`);
if (ev.track.kind === 'video' && ev.streams.length > 0) {
refVideoTrack.current = ev.track;
if (refVideo.current) {
refVideo.current.srcObject = ev.streams[0];
}
}
ms.addTrack(ev.track);
});
pc.addEventListener('iceconnectionstatechange', () => {
const state = pc.iceConnectionState;
Expand Down Expand Up @@ -136,8 +136,11 @@ export const PreviewDialog = forwardRef<IPreviewDialog, Props>((props, ref) => {
};

const handleVideoResize = (_: TargetedEvent<HTMLVideoElement>) => {
if (refVideoTrack.current) {
setVideoResolution(formatVideoTrackResolution(refVideoTrack.current));
if (refMediaStream.current) {
const videoTrack = refMediaStream.current.getVideoTracks()[0];
if (videoTrack) {
setVideoResolution(formatVideoTrackResolution(videoTrack));
}
}
};

Expand Down
11 changes: 7 additions & 4 deletions web/shared/components/dialog-web-stream.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,15 +52,18 @@ export const WebStreamDialog = forwardRef<IWebStreamDialog, Props>((props, ref)
if (refVideo.current) {
refVideo.current.srcObject = stream;
}
const videoTrack = stream.getVideoTracks()[0];
setVideoResolution(formatVideoTrackResolution(videoTrack));
updateConnState('Started');
const pc = new RTCPeerConnection();
pc.addEventListener('iceconnectionstatechange', () => {
updateConnState(pc.iceConnectionState);
});
pc.addTransceiver(videoTrack, { direction: 'sendonly' });
stream.getAudioTracks().forEach(track => pc.addTrack(track));
stream.getVideoTracks().forEach(vt => {
pc.addTransceiver(vt, { direction: 'sendonly' });
setVideoResolution(formatVideoTrackResolution(vt));
});
stream.getAudioTracks().forEach(at => {
pc.addTransceiver(at, { direction: 'sendonly' });
});
const whip = new WHIPClient();
const url = `${location.origin}/whip/${streamId}`;
const token = '';
Expand Down
40 changes: 35 additions & 5 deletions web/shared/tools/debugger/compat.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ declare module 'preact' {

declare global {
interface Window {
refreshDevice(): Promise<void>;
startWhip(): Promise<void>;
startWhep(): Promise<void>;
}
Expand Down Expand Up @@ -60,11 +59,34 @@ const WhepLayerSelect = [
{ value: 'f', text: 'HIGH' },
];

const NoneDevice = { value: '', text: 'none' };

export default function DebuggerCompat() {
const streamIdInput = useUrlParamsInput('id');
const idBearerTokenInput = useUrlParamsInput('token');

const refreshDevice = useCallback(() => window.refreshDevice(), []);
const [refreshDisabled, setRefreshDisabled] = useState(false);
const [audioDevices, setAudioDevices] = useState([NoneDevice]);
const [videoDevices, setVideoDevices] = useState([NoneDevice]);

const refreshDevice = useCallback(async () => {
try {
const mediaStream = await navigator.mediaDevices.getUserMedia({ audio: true, video: true });
mediaStream.getTracks().map(track => track.stop());
} catch (e) {
console.error('Failed to getUserMedia:', e);
}
const devices = (await navigator.mediaDevices.enumerateDevices()).filter(i => !!i.deviceId);
const audio = devices.filter(i => i.kind === 'audioinput').map(i => ({ value: i.deviceId, text: i.label }));
if (audio.length > 0) {
setAudioDevices(audio);
}
const video = devices.filter(i => i.kind === 'videoinput').map(i => ({ value: i.deviceId, text: i.label }));
if (video.length > 0) {
setVideoDevices(video);
}
setRefreshDisabled(true);
}, []);
const startWhip = useCallback(() => window.startWhip(), []);
const startWhep = useCallback(() => window.startWhep(), []);

Expand All @@ -83,9 +105,17 @@ export default function DebuggerCompat() {
<legend>WHIP</legend>
<center>
<section>
<button id="whip-device-button" onClick={refreshDevice}>Use Device</button>
<div style="margin: 0.2rem">Audio Device: <select id="whip-audio-device"><option value="">none</option></select></div>
<div style="margin: 0.2rem">Video Device: <select id="whip-video-device"><option value="">none</option></select></div>
<button id="whip-device-button" disabled={refreshDisabled} onClick={refreshDevice}>Use Device</button>
<div style="margin: 0.2rem">Audio Device:
<select id="whip-audio-device">
{audioDevices.map(d => <option value={d.value}>{d.text}</option>)}
</select>
</div>
<div style="margin: 0.2rem">Video Device:
<select id="whip-video-device">
{videoDevices.map(d => <option value={d.value}>{d.text}</option>)}
</select>
</div>
</section>

<section>
Expand Down
56 changes: 15 additions & 41 deletions web/shared/tools/debugger/debugger.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,16 +64,6 @@ const layers = [
{rid: 'f', scalabilityMode: 'L1T3'}
]

function initLayerSelect(elementId, opts) {
const selectLayer = document.getElementById(elementId)
if (selectLayer) opts.map(i => {
const opt = document.createElement("option")
opt.value = i.value
opt.text = i.text
selectLayer.appendChild(opt)
})
}

// WHIP
let whipNum = 0

Expand All @@ -87,30 +77,6 @@ const idWhipButtonStop = "whip-button-stop"
const idWhipPseudoAudio = "whip-pseudo-audio"
const idWhipDataChannel = "whip-datachannel"

// initLayerSelect(idWhipLayerSelect, [
// {value: "f", text: "Base"},
// {value: "h", text: "Base + 1/2"},
// {value: "q", text: "Base + 1/2 + 1/4"},
// ])

async function refreshDevice() {
const mediaStream = await navigator.mediaDevices.getUserMedia({audio: true, video: true})
mediaStream.getTracks().map(track => track.stop())

const devices = (await navigator.mediaDevices.enumerateDevices()).filter(i => !!i.deviceId)
initLayerSelect(idWhipAudioDevice, devices.filter(i => i.kind === 'audioinput').map(i => {
return {value: i.deviceId, text: i.label}
}))
initLayerSelect(idWhipVideoDevice, devices.filter(i => i.kind === 'videoinput').map(i => {
return {value: i.deviceId, text: i.label}
}))
}

window.refreshDevice = () => {
refreshDevice()
document.getElementById("whip-device-button").disabled = true
}

async function startWhip() {
const streamId = getElementValue(idStreamId)
if (!streamId) {
Expand All @@ -130,7 +96,10 @@ async function startWhip() {

let stream
if (!audioDevice && !videoDevice) {
stream = await navigator.mediaDevices.getDisplayMedia({audio: false, video: videoSize})
stream = await navigator.mediaDevices.getDisplayMedia({
audio: true,
video: videoSize
})
} else {
stream = await navigator.mediaDevices.getUserMedia({
audio: {deviceId: audioDevice},
Expand Down Expand Up @@ -161,7 +130,9 @@ async function startWhip() {
if (document.getElementById(idWhipPseudoAudio).checked) {
pc.addTransceiver('audio', { 'direction': 'sendonly' })
} else {
stream.getAudioTracks().map(track => pc.addTrack(track))
stream.getAudioTracks().forEach(track => pc.addTransceiver(track, {
direction: 'sendonly'
}))
}

const audioCodec = getElementValue(idWhipAudioCodec)
Expand Down Expand Up @@ -232,13 +203,16 @@ async function startWhep() {
document.getElementById(idWhepDataChannel).dataChannel = pc.createDataChannel("")

pc.oniceconnectionstatechange = e => logWhep(num, pc.iceConnectionState)
pc.addTransceiver('video', {'direction': 'recvonly'})
pc.addTransceiver('audio', {'direction': 'recvonly'})
pc.addTransceiver("video", { "direction": "recvonly" })
pc.addTransceiver("audio", { "direction": "recvonly" })

const ms = new MediaStream();
pc.ontrack = ev => {
logWhep(num, `track: ${ev.track.kind}`)
if (ev.track.kind === "video") {
if (ev.streams.length !== 0) document.getElementById("whep-video-player").srcObject = ev.streams[0]
}
ms.addTrack(ev.track);
// addtrack removetrack events won't fire when calling addTrack/removeTrack in javascript
// https://github.com/w3c/mediacapture-main/issues/517
document.getElementById("whep-video-player").srcObject = ms;
}
const whep = new WHEPClient()
const url = location.origin + "/whep/" + streamId
Expand Down

0 comments on commit 18d5f04

Please sign in to comment.