Show voice room participants when not connected

src/components/views/rooms/RoomTile.tsx

@@ -19,6 +19,8 @@ import React, { createRef } from "react";
 import { Room, RoomEvent } from "matrix-js-sdk/src/models/room";
 import classNames from "classnames";
 import { logger } from "matrix-js-sdk/src/logger";
+import { RoomMember } from "matrix-js-sdk/src/models/room-member";
+import { RoomStateEvent } from "matrix-js-sdk/src/models/room-state";
 
 import { RovingTabIndexWrapper } from "../../../accessibility/RovingTabIndex";
 import AccessibleButton, { ButtonEvent } from "../../views/elements/AccessibleButton";
@@ -32,6 +34,7 @@ import { ChevronFace, ContextMenuTooltipButton } from "../../structures/ContextM
 import { DefaultTagID, TagID } from "../../../stores/room-list/models";
 import { MessagePreviewStore } from "../../../stores/room-list/MessagePreviewStore";
 import BaseAvatar from "../avatars/BaseAvatar";
+import MemberAvatar from "../avatars/MemberAvatar";
 import DecoratedRoomAvatar from "../avatars/DecoratedRoomAvatar";
 import FacePile from "../elements/FacePile";
 import { RoomNotifState } from "../../../RoomNotifs";
@@ -53,6 +56,7 @@ import IconizedContextMenu, {
     IconizedContextMenuRadio,
 } from "../context_menus/IconizedContextMenu";
 import VoiceChannelStore, { VoiceChannelEvent, IJitsiParticipant } from "../../../stores/VoiceChannelStore";
+import { getConnectedMembers } from "../../../utils/VoiceChannelUtils";
 import { replaceableComponent } from "../../../utils/replaceableComponent";
 import PosthogTrackers from "../../../PosthogTrackers";
 import { ViewRoomPayload } from "../../../dispatcher/payloads/ViewRoomPayload";
@@ -80,7 +84,10 @@ interface IState {
     generalMenuPosition: PartialDOMRect;
     messagePreview?: string;
     voiceConnectionState: VoiceConnectionState;
-    voiceParticipants: IJitsiParticipant[];
+    // Active voice channel members, according to room state
+    voiceMembers: RoomMember[];
+    // Active voice channel members, according to Jitsi
+    jitsiParticipants: IJitsiParticipant[];
 }
 
 const messagePreviewId = (roomId: string) => `mx_RoomTile_messagePreview_${roomId}`;
@@ -112,7 +119,8 @@ export default class RoomTile extends React.PureComponent<IProps, IState> {
             messagePreview: "",
             voiceConnectionState: VoiceChannelStore.instance.roomId === this.props.room.roomId ?
                 VoiceConnectionState.Connected : VoiceConnectionState.Disconnected,
-            voiceParticipants: [],
+            voiceMembers: [],
+            jitsiParticipants: [],
         };
         this.generatePreview();
 
@@ -157,6 +165,8 @@ export default class RoomTile extends React.PureComponent<IProps, IState> {
                 MessagePreviewStore.getPreviewChangedEventName(this.props.room),
                 this.onRoomPreviewChanged,
             );
+            prevProps.room?.currentState?.off(RoomStateEvent.Events, this.updateVoiceMembers);
+            this.props.room?.currentState?.on(RoomStateEvent.Events, this.updateVoiceMembers);
             prevProps.room?.off(RoomEvent.Name, this.onRoomNameUpdate);
             this.props.room?.on(RoomEvent.Name, this.onRoomNameUpdate);
         }
@@ -167,6 +177,7 @@ export default class RoomTile extends React.PureComponent<IProps, IState> {
         if (this.state.selected) {
             this.scrollIntoView();
         }
+        this.updateVoiceMembers();
 
         ActiveRoomObserver.addListener(this.props.room.roomId, this.onActiveRoomUpdate);
         this.dispatcherRef = defaultDispatcher.register(this.onAction);
@@ -177,6 +188,7 @@ export default class RoomTile extends React.PureComponent<IProps, IState> {
         this.notificationState.on(NotificationStateEvents.Update, this.onNotificationUpdate);
         this.roomProps.on(PROPERTY_UPDATED, this.onRoomPropertyUpdate);
         this.props.room?.on(RoomEvent.Name, this.onRoomNameUpdate);
+        this.props.room?.currentState?.on(RoomStateEvent.Events, this.updateVoiceMembers);
     }
 
     public componentWillUnmount() {
@@ -186,6 +198,7 @@ export default class RoomTile extends React.PureComponent<IProps, IState> {
                 MessagePreviewStore.getPreviewChangedEventName(this.props.room),
                 this.onRoomPreviewChanged,
             );
+            this.props.room.currentState.off(RoomStateEvent.Events, this.updateVoiceMembers);
             this.props.room.off(RoomEvent.Name, this.onRoomNameUpdate);
         }
         ActiveRoomObserver.removeListener(this.props.room.roomId, this.onActiveRoomUpdate);
@@ -571,24 +584,40 @@ export default class RoomTile extends React.PureComponent<IProps, IState> {
         );
     }
 
-    private updateVoiceParticipants = (participants: IJitsiParticipant[]) => {
-        this.setState({ voiceParticipants: participants });
+    private updateVoiceMembers = () => {
+        this.setState({ voiceMembers: getConnectedMembers(this.props.room.currentState) });
     };
 
-    private renderVoiceChannel(): React.ReactElement {
-        if (!this.state.voiceParticipants.length) return null;
-
-        const faces = this.state.voiceParticipants.map(p =>
-            <BaseAvatar
-                key={p.participantId}
-                name={p.displayName ?? p.formattedDisplayName}
-                idName={p.participantId}
-                // This comes directly from Jitsi, so we shouldn't apply custom media routing to it
-                url={p.avatarURL}
-                width={24}
-                height={24}
-            />,
-        );
+    private updateJitsiParticipants = (participants: IJitsiParticipant[]) => {
+        this.setState({ jitsiParticipants: participants });
+    };
+
+    private renderVoiceChannel(): React.ReactElement | null {
+        let faces;
+        if (this.state.voiceConnectionState === VoiceConnectionState.Connected) {
+            faces = this.state.jitsiParticipants.map(p =>
+                <BaseAvatar
+                    key={p.participantId}
+                    name={p.displayName ?? p.formattedDisplayName}
+                    idName={p.participantId}
+                    // This comes directly from Jitsi, so we shouldn't apply custom media routing to it
+                    url={p.avatarURL}
+                    width={24}
+                    height={24}
+                />,
+            );
+        } else if (this.state.voiceMembers.length) {
+            faces = this.state.voiceMembers.map(m =>
+                <MemberAvatar
+                    key={m.userId}
+                    member={m}
+                    width={24}
+                    height={24}
+                />,
+            );
+        } else {
+            return null;
+        }
 
         // TODO: The below "join" button will eventually show up on text rooms
         // with an active voice channel, but that isn't implemented yet
@@ -615,21 +644,24 @@ export default class RoomTile extends React.PureComponent<IProps, IState> {
         // effort to solve this properly.
         await new Promise(resolve => setTimeout(resolve, 1000));
 
+        const waitForConnect = VoiceChannelStore.instance.connect(this.props.room.roomId);
+        // Participant data comes down the event channel quickly, so prepare in advance
+        VoiceChannelStore.instance.on(VoiceChannelEvent.Participants, this.updateJitsiParticipants);
         try {
-            await VoiceChannelStore.instance.connect(this.props.room.roomId);
-
+            await waitForConnect;
             this.setState({ voiceConnectionState: VoiceConnectionState.Connected });
+
             VoiceChannelStore.instance.once(VoiceChannelEvent.Disconnect, () => {
                 this.setState({
                     voiceConnectionState: VoiceConnectionState.Disconnected,
-                    voiceParticipants: [],
+                    jitsiParticipants: [],
                 }),
-                VoiceChannelStore.instance.off(VoiceChannelEvent.Participants, this.updateVoiceParticipants);
+                VoiceChannelStore.instance.off(VoiceChannelEvent.Participants, this.updateJitsiParticipants);
             });
-            VoiceChannelStore.instance.on(VoiceChannelEvent.Participants, this.updateVoiceParticipants);
         } catch (e) {
+            // If it failed, clean up our advance preparations
             logger.error("Failed to connect voice", e);
-            this.setState({ voiceConnectionState: VoiceConnectionState.Disconnected });
+            VoiceChannelStore.instance.off(VoiceChannelEvent.Participants, this.updateJitsiParticipants);
         }
     }
 

src/createRoom.ts

@@ -42,7 +42,7 @@ import { isJoinedOrNearlyJoined } from "./utils/membership";
 import { VIRTUAL_ROOM_EVENT_TYPE } from "./CallHandler";
 import SpaceStore from "./stores/spaces/SpaceStore";
 import { makeSpaceParentEvent } from "./utils/space";
-import { addVoiceChannel } from "./utils/VoiceChannelUtils";
+import { VOICE_CHANNEL_MEMBER, addVoiceChannel } from "./utils/VoiceChannelUtils";
 import { Action } from "./dispatcher/actions";
 import ErrorDialog from "./components/views/dialogs/ErrorDialog";
 import Spinner from "./components/views/elements/Spinner";
@@ -127,6 +127,24 @@ export default async function createRoom(opts: IOpts): Promise<string | null> {
             ...createOpts.creation_content,
             [RoomCreateTypeField]: opts.roomType,
         };
+
+        // In voice rooms, allow all users to send voice member updates
+        if (opts.roomType === RoomType.UnstableCall) {
+            createOpts.power_level_content_override = {
+                events: {
+                    [VOICE_CHANNEL_MEMBER]: 0,
+                    // Annoyingly, we have to reiterate all the defaults here
+                    [EventType.RoomName]: 50,
+                    [EventType.RoomAvatar]: 50,
+                    [EventType.RoomPowerLevels]: 100,
+                    [EventType.RoomHistoryVisibility]: 100,
+                    [EventType.RoomCanonicalAlias]: 50,
+                    [EventType.RoomTombstone]: 100,
+                    [EventType.RoomServerAcl]: 100,
+                    [EventType.RoomEncryption]: 100,
+                },
+            };
+        }
     }
 
     // By default, view the room after creating it

src/stores/VoiceChannelStore.ts

@@ -15,11 +15,17 @@ limitations under the License.
 */
 
 import { EventEmitter } from "events";
+import { logger } from "matrix-js-sdk/src/logger";
 import { ClientWidgetApi, IWidgetApiRequest } from "matrix-widget-api";
 
+import { MatrixClientPeg } from "../MatrixClientPeg";
 import { ElementWidgetActions } from "./widgets/ElementWidgetActions";
 import { WidgetMessagingStore } from "./widgets/WidgetMessagingStore";
-import { getVoiceChannel } from "../utils/VoiceChannelUtils";
+import {
+    VOICE_CHANNEL_MEMBER,
+    IVoiceChannelMemberContent,
+    getVoiceChannel,
+} from "../utils/VoiceChannelUtils";
 import { timeout } from "../utils/promise";
 import WidgetUtils from "../utils/WidgetUtils";
 
@@ -54,6 +60,7 @@ export default class VoiceChannelStore extends EventEmitter {
         return VoiceChannelStore._instance;
     }
 
+    private readonly cli = MatrixClientPeg.get();
     private activeChannel: ClientWidgetApi;
     private _roomId: string;
     private _participants: IJitsiParticipant[];
@@ -118,6 +125,9 @@ export default class VoiceChannelStore extends EventEmitter {
         messaging.once(`action:${ElementWidgetActions.HangupCall}`, this.onHangup);
 
         this.emit(VoiceChannelEvent.Connect);
+
+        // Tell others that we're connected, by adding our device to room state
+        await this.updateDevices(devices => Array.from(new Set(devices).add(this.cli.getDeviceId())));
     };
 
     public disconnect = async () => {
@@ -169,8 +179,8 @@ export default class VoiceChannelStore extends EventEmitter {
     private waitForAction = async (action: ElementWidgetActions) => {
         const wait = new Promise<void>(resolve =>
             this.activeChannel.once(`action:${action}`, (ev: CustomEvent<IWidgetApiRequest>) => {
-                resolve();
                 this.ack(ev);
+                resolve();
             }),
         );
         if (await timeout(wait, false, VoiceChannelStore.TIMEOUT) === false) {
@@ -182,22 +192,47 @@ export default class VoiceChannelStore extends EventEmitter {
         this.activeChannel.transport.reply(ev.detail, {});
     };
 
-    private onHangup = (ev: CustomEvent<IWidgetApiRequest>) => {
+    private updateDevices = async (fn: (devices: string[]) => string[]) => {
+        if (!this.roomId) {
+            logger.error("Tried to update devices while disconnected");
+            return;
+        }
+
+        const devices = this.cli.getRoom(this.roomId)
+            .currentState.getStateEvents(VOICE_CHANNEL_MEMBER, this.cli.getUserId())
+            ?.getContent<IVoiceChannelMemberContent>()?.devices ?? [];
+
+        await this.cli.sendStateEvent(
+            this.roomId, VOICE_CHANNEL_MEMBER, { devices: fn(devices) }, this.cli.getUserId(),
+        );
+    };
+
+    private onHangup = async (ev: CustomEvent<IWidgetApiRequest>) => {
+        this.ack(ev);
+
         this.activeChannel.off(`action:${ElementWidgetActions.CallParticipants}`, this.onParticipants);
         this.activeChannel.off(`action:${ElementWidgetActions.MuteAudio}`, this.onMuteAudio);
         this.activeChannel.off(`action:${ElementWidgetActions.UnmuteAudio}`, this.onUnmuteAudio);
         this.activeChannel.off(`action:${ElementWidgetActions.MuteVideo}`, this.onMuteVideo);
         this.activeChannel.off(`action:${ElementWidgetActions.UnmuteVideo}`, this.onUnmuteVideo);
 
-        this._roomId = null;
+        this.activeChannel = null;
         this._participants = null;
         this._audioMuted = null;
         this._videoMuted = null;
 
-        this.emit(VoiceChannelEvent.Disconnect);
-        this.ack(ev);
-        // Save this for last, since ack needs activeChannel to exist
-        this.activeChannel = null;
+        try {
+            // Tell others that we're disconnected, by removing our device from room state
+            await this.updateDevices(devices => {
+                const devicesSet = new Set(devices);
+                devicesSet.delete(this.cli.getDeviceId());
+                return Array.from(devicesSet);
+            });
+        } finally {
+            // Save this for last, since updateDevices needs the room ID
+            this._roomId = null;
+            this.emit(VoiceChannelEvent.Disconnect);
+        }
     };
 
     private onParticipants = (ev: CustomEvent<IWidgetApiRequest>) => {

src/utils/VoiceChannelUtils.ts

@@ -15,18 +15,33 @@ limitations under the License.
 */
 
 import { CallType } from "matrix-js-sdk/src/webrtc/call";
+import { RoomState } from "matrix-js-sdk/src/models/room-state";
+import { RoomMember } from "matrix-js-sdk/src/models/room-member";
 
 import WidgetStore, { IApp } from "../stores/WidgetStore";
 import { WidgetType } from "../widgets/WidgetType";
 import WidgetUtils from "./WidgetUtils";
 
-export const VOICE_CHANNEL_ID = "io.element.voice";
+export const VOICE_CHANNEL = "io.element.voice";
+export const VOICE_CHANNEL_MEMBER = "io.element.voice.member";
+
+export interface IVoiceChannelMemberContent {
+    // Connected device IDs
+    devices: string[];
+}
 
 export const getVoiceChannel = (roomId: string): IApp => {
     const apps = WidgetStore.instance.getApps(roomId);
-    return apps.find(app => WidgetType.JITSI.matches(app.type) && app.id === VOICE_CHANNEL_ID);
+    return apps.find(app => WidgetType.JITSI.matches(app.type) && app.id === VOICE_CHANNEL);
 };
 
 export const addVoiceChannel = async (roomId: string, roomName: string) => {
-    await WidgetUtils.addJitsiWidget(roomId, CallType.Voice, "Voice channel", VOICE_CHANNEL_ID, roomName);
+    await WidgetUtils.addJitsiWidget(roomId, CallType.Voice, "Voice channel", VOICE_CHANNEL, roomName);
 };
+
+export const getConnectedMembers = (state: RoomState): RoomMember[] =>
+    state.getStateEvents(VOICE_CHANNEL_MEMBER)
+        // Must have a device connected and still be joined to the room
+        .filter(e => e.getContent<IVoiceChannelMemberContent>().devices?.length)
+        .map(e => state.getMember(e.getStateKey()))
+        .filter(member => member.membership === "join");