From 80f07a5454ef9ba59dd2d2b36ac678a78e5dde7c Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 15 Nov 2022 16:13:33 +0000 Subject: [PATCH 1/6] Add a 'waiting for video' state to media tiles This will show if the call is waiting for media to connect (in practice doesn't actually seem to happen all that often) but also show if the media connection is lost, with the js-sdk change. Requires https://github.com/matrix-org/matrix-js-sdk/pull/2880 Fixes: https://github.com/vector-im/element-call/issues/669 --- package.json | 2 +- src/room/GroupCallView.tsx | 2 + src/room/InCallView.tsx | 73 ++++++++++++++++++++++++++- src/video-grid/VideoGrid.stories.tsx | 5 +- src/video-grid/VideoTile.tsx | 22 ++++++-- src/video-grid/VideoTileContainer.tsx | 2 +- yarn.lock | 4 +- 7 files changed, 98 insertions(+), 12 deletions(-) diff --git a/package.json b/package.json index 6227b9155..de7eec7e3 100644 --- a/package.json +++ b/package.json @@ -45,7 +45,7 @@ "i18next": "^21.10.0", "i18next-browser-languagedetector": "^6.1.8", "i18next-http-backend": "^1.4.4", - "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#c6ee258789c9e01d328b5d9158b5b372e3a0da82", + "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#3f1c3392d45b0fc054c3788cc6c043cd5b4fb730", "matrix-widget-api": "^1.0.0", "mermaid": "^8.13.8", "normalize.css": "^8.0.1", diff --git a/src/room/GroupCallView.tsx b/src/room/GroupCallView.tsx index b05182a66..7da620b35 100644 --- a/src/room/GroupCallView.tsx +++ b/src/room/GroupCallView.tsx @@ -79,6 +79,7 @@ export function GroupCallView({ isScreensharing, screenshareFeeds, participants, + calls, unencryptedEventsFromUsers, } = useGroupCall(groupCall); @@ -235,6 +236,7 @@ export function GroupCallView({ roomName={groupCall.room.name} avatarUrl={avatarUrl} participants={participants} + calls={calls} microphoneMuted={microphoneMuted} localVideoMuted={localVideoMuted} toggleLocalVideoMuted={toggleLocalVideoMuted} diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index b4de4b9d5..d3d0fc0ca 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -14,7 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -import React, { useEffect, useCallback, useMemo, useRef } from "react"; +import React, { + useEffect, + useCallback, + useMemo, + useRef, + useState, +} from "react"; import { usePreventScroll } from "@react-aria/overlays"; import useMeasure from "react-use-measure"; import { ResizeObserver } from "@juggle/resize-observer"; @@ -25,6 +31,11 @@ import { CallFeed } from "matrix-js-sdk/src/webrtc/callFeed"; import classNames from "classnames"; import { useTranslation } from "react-i18next"; import { JoinRule } from "matrix-js-sdk/src/@types/partials"; +import { + CallEvent, + CallState, + MatrixCall, +} from "matrix-js-sdk/src/webrtc/call"; import type { IWidgetApiRequest } from "matrix-widget-api"; import styles from "./InCallView.module.css"; @@ -73,6 +84,7 @@ interface Props { client: MatrixClient; groupCall: GroupCall; participants: RoomMember[]; + calls: MatrixCall[]; roomName: string; avatarUrl: string; microphoneMuted: boolean; @@ -90,6 +102,12 @@ interface Props { hideHeader: boolean; } +export enum ConnectionState { + ESTABLISHING_CALL = "establishing call", // call hasn't been established yet + WAIT_MEDIA = "wait_media", // call is set up, waiting for ICE to connect + CONNECTED = "connected", // media is flowing +} + // Represents something that should get a tile on the layout, // ie. a user's video feed or a screen share feed. export interface TileDescriptor { @@ -99,12 +117,14 @@ export interface TileDescriptor { presenter: boolean; callFeed?: CallFeed; isLocal?: boolean; + connectionState: ConnectionState; } export function InCallView({ client, groupCall, participants, + calls, roomName, avatarUrl, microphoneMuted, @@ -154,6 +174,46 @@ export function InCallView({ const { hideScreensharing } = useUrlParams(); + const makeConnectionStatesMap = useCallback(() => { + const newConnStates = new Map(); + for (const participant of participants) { + const userCall = groupCall.getCallByUserId(participant.userId); + const feed = userMediaFeeds.find((f) => f.userId === participant.userId); + let connectionState = ConnectionState.ESTABLISHING_CALL; + if (feed && feed.isLocal()) { + connectionState = ConnectionState.CONNECTED; + } else if (userCall) { + if (userCall.state === CallState.Connected) { + connectionState = ConnectionState.CONNECTED; + } else if (userCall.state === CallState.Connecting) { + connectionState = ConnectionState.WAIT_MEDIA; + } + } + newConnStates.set(participant.userId, connectionState); + } + return newConnStates; + }, [groupCall, participants, userMediaFeeds]); + + const [connStates, setConnStates] = useState( + new Map() + ); + + const updateConnectionStates = useCallback(() => { + setConnStates(makeConnectionStatesMap()); + }, [setConnStates, makeConnectionStatesMap]); + + useEffect(() => { + for (const call of calls) { + call.on(CallEvent.State, updateConnectionStates); + } + + return () => { + for (const call of calls) { + call.off(CallEvent.State, updateConnectionStates); + } + }; + }, [calls, updateConnectionStates]); + useEffect(() => { widget?.api.transport.send( layout === "freedom" @@ -208,6 +268,7 @@ export function InCallView({ focused: screenshareFeeds.length === 0 && p.userId === activeSpeaker, isLocal: p.userId === client.getUserId(), presenter: false, + connectionState: connStates.get(p.userId), }); } @@ -231,11 +292,19 @@ export function InCallView({ focused: true, isLocal: screenshareFeed.isLocal(), presenter: false, + connectionState: ConnectionState.CONNECTED, // by definition since the screen shares arrived on the same connection }); } return tileDescriptors; - }, [client, participants, userMediaFeeds, activeSpeaker, screenshareFeeds]); + }, [ + client, + participants, + userMediaFeeds, + activeSpeaker, + screenshareFeeds, + connStates, + ]); // The maximised participant: either the participant that the user has // manually put in fullscreen, or the focused (active) participant if the diff --git a/src/video-grid/VideoGrid.stories.tsx b/src/video-grid/VideoGrid.stories.tsx index b8f367630..54f0cdfda 100644 --- a/src/video-grid/VideoGrid.stories.tsx +++ b/src/video-grid/VideoGrid.stories.tsx @@ -21,7 +21,7 @@ import { RoomMember } from "matrix-js-sdk"; import { VideoGrid, useVideoGridLayout } from "./VideoGrid"; import { VideoTile } from "./VideoTile"; import { Button } from "../button"; -import { TileDescriptor } from "../room/InCallView"; +import { ConnectionState, TileDescriptor } from "../room/InCallView"; export default { title: "VideoGrid", @@ -41,6 +41,7 @@ export const ParticipantsTest = () => { member: new RoomMember("!fake:room.id", `@user${i}:fake.dummy`), focused: false, presenter: false, + connectionState: ConnectionState.CONNECTED, })), [participantCount] ); @@ -79,7 +80,7 @@ export const ParticipantsTest = () => { key={item.id} name={`User ${item.id}`} disableSpeakingIndicator={items.length < 3} - hasFeed={true} + connectionState={ConnectionState.CONNECTED} {...rest} /> )} diff --git a/src/video-grid/VideoTile.tsx b/src/video-grid/VideoTile.tsx index 2cef8e2df..693475bb5 100644 --- a/src/video-grid/VideoTile.tsx +++ b/src/video-grid/VideoTile.tsx @@ -23,10 +23,11 @@ import styles from "./VideoTile.module.css"; import { ReactComponent as MicMutedIcon } from "../icons/MicMuted.svg"; import { ReactComponent as VideoMutedIcon } from "../icons/VideoMuted.svg"; import { AudioButton, FullscreenButton } from "../button/Button"; +import { ConnectionState } from "../room/InCallView"; interface Props { name: string; - hasFeed: Boolean; + connectionState: ConnectionState; speaking?: boolean; audioMuted?: boolean; videoMuted?: boolean; @@ -48,7 +49,7 @@ export const VideoTile = forwardRef( ( { name, - hasFeed, + connectionState, speaking, audioMuted, videoMuted, @@ -72,7 +73,7 @@ export const VideoTile = forwardRef( const { t } = useTranslation(); const toolbarButtons: JSX.Element[] = []; - if (hasFeed && !isLocal) { + if (connectionState == ConnectionState.CONNECTED && !isLocal) { toolbarButtons.push( ( } } - const caption = hasFeed ? name : t("{{name}} (Connecting...)", { name }); + let caption: string; + switch (connectionState) { + case ConnectionState.ESTABLISHING_CALL: + caption = t("{{name}} (Connecting...)", { name }); + + break; + case ConnectionState.WAIT_MEDIA: + // not strictly true, but probably easier to understand than, "Waiting for media" + caption = t("{{name}} (Waiting for video...)", { name }); + break; + case ConnectionState.CONNECTED: + caption = name; + break; + } return ( Date: Tue, 15 Nov 2022 17:19:09 +0000 Subject: [PATCH 2/6] Pass user's connection state for their screenshare feed --- src/room/InCallView.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index d3d0fc0ca..67d97750a 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -292,7 +292,7 @@ export function InCallView({ focused: true, isLocal: screenshareFeed.isLocal(), presenter: false, - connectionState: ConnectionState.CONNECTED, // by definition since the screen shares arrived on the same connection + connectionState: connStates.get(screenshareFeed.userId), }); } From 5623fa415f78334f57677c4c6fcdafef26eb20f3 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 15 Nov 2022 17:33:58 +0000 Subject: [PATCH 3/6] Also update connection states when participants change --- src/room/InCallView.tsx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 67d97750a..9935985b9 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -214,6 +214,10 @@ export function InCallView({ }; }, [calls, updateConnectionStates]); + useEffect(() => { + updateConnectionStates(); + }, [participants, updateConnectionStates]); + useEffect(() => { widget?.api.transport.send( layout === "freedom" From 432f7ef93acc4424a3213ee848f371ff690e8b57 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 15 Nov 2022 17:34:56 +0000 Subject: [PATCH 4/6] i18n update --- public/locales/en-GB/app.json | 1 + 1 file changed, 1 insertion(+) diff --git a/public/locales/en-GB/app.json b/public/locales/en-GB/app.json index 7d4ae7b31..20b595ab9 100644 --- a/public/locales/en-GB/app.json +++ b/public/locales/en-GB/app.json @@ -3,6 +3,7 @@ "{{count}} people connected|other": "{{count}} people connected", "{{displayName}}, your call is now ended": "{{displayName}}, your call is now ended", "{{name}} (Connecting...)": "{{name}} (Connecting...)", + "{{name}} (Waiting for video...)": "{{name}} (Waiting for video...)", "{{name}} is presenting": "{{name}} is presenting", "{{name}} is talking…": "{{name}} is talking…", "{{names}}, {{name}}": "{{names}}, {{name}}", From 734d330a102124be634a40c1807e74021c446492 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 16 Nov 2022 10:45:49 +0000 Subject: [PATCH 5/6] CamcelCase for enum values --- src/room/InCallView.tsx | 14 +++++++------- src/video-grid/VideoGrid.stories.tsx | 4 ++-- src/video-grid/VideoTile.tsx | 8 ++++---- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 9935985b9..a75da5f4c 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -103,9 +103,9 @@ interface Props { } export enum ConnectionState { - ESTABLISHING_CALL = "establishing call", // call hasn't been established yet - WAIT_MEDIA = "wait_media", // call is set up, waiting for ICE to connect - CONNECTED = "connected", // media is flowing + EstablishingCall = "establishing call", // call hasn't been established yet + WaitMedia = "wait_media", // call is set up, waiting for ICE to connect + Connected = "connected", // media is flowing } // Represents something that should get a tile on the layout, @@ -179,14 +179,14 @@ export function InCallView({ for (const participant of participants) { const userCall = groupCall.getCallByUserId(participant.userId); const feed = userMediaFeeds.find((f) => f.userId === participant.userId); - let connectionState = ConnectionState.ESTABLISHING_CALL; + let connectionState = ConnectionState.EstablishingCall; if (feed && feed.isLocal()) { - connectionState = ConnectionState.CONNECTED; + connectionState = ConnectionState.Connected; } else if (userCall) { if (userCall.state === CallState.Connected) { - connectionState = ConnectionState.CONNECTED; + connectionState = ConnectionState.Connected; } else if (userCall.state === CallState.Connecting) { - connectionState = ConnectionState.WAIT_MEDIA; + connectionState = ConnectionState.WaitMedia; } } newConnStates.set(participant.userId, connectionState); diff --git a/src/video-grid/VideoGrid.stories.tsx b/src/video-grid/VideoGrid.stories.tsx index 54f0cdfda..fc3317c45 100644 --- a/src/video-grid/VideoGrid.stories.tsx +++ b/src/video-grid/VideoGrid.stories.tsx @@ -41,7 +41,7 @@ export const ParticipantsTest = () => { member: new RoomMember("!fake:room.id", `@user${i}:fake.dummy`), focused: false, presenter: false, - connectionState: ConnectionState.CONNECTED, + connectionState: ConnectionState.Connected, })), [participantCount] ); @@ -80,7 +80,7 @@ export const ParticipantsTest = () => { key={item.id} name={`User ${item.id}`} disableSpeakingIndicator={items.length < 3} - connectionState={ConnectionState.CONNECTED} + connectionState={ConnectionState.Connected} {...rest} /> )} diff --git a/src/video-grid/VideoTile.tsx b/src/video-grid/VideoTile.tsx index 693475bb5..1473ec464 100644 --- a/src/video-grid/VideoTile.tsx +++ b/src/video-grid/VideoTile.tsx @@ -73,7 +73,7 @@ export const VideoTile = forwardRef( const { t } = useTranslation(); const toolbarButtons: JSX.Element[] = []; - if (connectionState == ConnectionState.CONNECTED && !isLocal) { + if (connectionState == ConnectionState.Connected && !isLocal) { toolbarButtons.push( ( let caption: string; switch (connectionState) { - case ConnectionState.ESTABLISHING_CALL: + case ConnectionState.EstablishingCall: caption = t("{{name}} (Connecting...)", { name }); break; - case ConnectionState.WAIT_MEDIA: + case ConnectionState.WaitMedia: // not strictly true, but probably easier to understand than, "Waiting for media" caption = t("{{name}} (Waiting for video...)", { name }); break; - case ConnectionState.CONNECTED: + case ConnectionState.Connected: caption = name; break; } From 93aafb141573cad413963f26fee57584f10ce003 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 16 Nov 2022 16:39:35 +0000 Subject: [PATCH 6/6] Remove mystery blank line Co-authored-by: Robin --- src/video-grid/VideoTile.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/src/video-grid/VideoTile.tsx b/src/video-grid/VideoTile.tsx index 1473ec464..365d88e56 100644 --- a/src/video-grid/VideoTile.tsx +++ b/src/video-grid/VideoTile.tsx @@ -99,7 +99,6 @@ export const VideoTile = forwardRef( switch (connectionState) { case ConnectionState.EstablishingCall: caption = t("{{name}} (Connecting...)", { name }); - break; case ConnectionState.WaitMedia: // not strictly true, but probably easier to understand than, "Waiting for media"