From c21006dab79362768a986ea659bf2d2650715718 Mon Sep 17 00:00:00 2001 From: Gonzalo DCL Date: Fri, 15 Dec 2023 11:55:45 -0300 Subject: [PATCH 01/23] wip --- .../packages/shared/comms/actions.ts | 10 ++- .../packages/shared/comms/handlers.ts | 8 +- .../comms/logic/rfc-4-room-connection.ts | 13 ++- .../packages/shared/comms/reducer.ts | 11 ++- .../packages/shared/comms/sagas.ts | 80 +++++++++++++++++-- .../packages/shared/comms/selectors.ts | 70 +++++++++++++++- .../packages/shared/comms/types.ts | 11 ++- .../genesis-city-loader-impl/index.ts | 6 ++ .../packages/shared/scene-loader/sagas.ts | 2 +- .../packages/shared/scene-loader/types.ts | 2 + .../packages/shared/world/SceneWorker.ts | 15 +++- 11 files changed, 205 insertions(+), 23 deletions(-) diff --git a/browser-interface/packages/shared/comms/actions.ts b/browser-interface/packages/shared/comms/actions.ts index 860566a423..013f816b7a 100644 --- a/browser-interface/packages/shared/comms/actions.ts +++ b/browser-interface/packages/shared/comms/actions.ts @@ -10,10 +10,16 @@ export const SET_ROOM_CONNECTION = '[COMMS] setRoomConnection' export const setRoomConnection = (room: RoomConnection | undefined) => action(SET_ROOM_CONNECTION, room) export type SetRoomConnectionAction = ReturnType +export const SET_SCENE_ROOM_CONNECTION = '[COMMS] setSceneRoomConnection' +export const setSceneRoomConnection = (sceneId: string, room: RoomConnection) => + action(SET_SCENE_ROOM_CONNECTION, { sceneId, room }) +export type SetSceneRoomConnectionAction = ReturnType + export const HANDLE_ROOM_DISCONNECTION = '[COMMS] handleRoomDisconnection' export const handleRoomDisconnection = (room: RoomConnection) => action(HANDLE_ROOM_DISCONNECTION, { context: room }) export type HandleRoomDisconnection = ReturnType export const SET_LIVEKIT_ADAPTER = '[COMMS] setLiveKitAdapter' -export const setLiveKitAdapter = (livekitAdapter: LivekitAdapter | undefined) => action(SET_LIVEKIT_ADAPTER, livekitAdapter) -export type SetLiveKitAdapterAction = ReturnType \ No newline at end of file +export const setLiveKitAdapter = (livekitAdapter: LivekitAdapter | undefined) => + action(SET_LIVEKIT_ADAPTER, livekitAdapter) +export type SetLiveKitAdapterAction = ReturnType diff --git a/browser-interface/packages/shared/comms/handlers.ts b/browser-interface/packages/shared/comms/handlers.ts index a2c745d78f..19f747c2f1 100644 --- a/browser-interface/packages/shared/comms/handlers.ts +++ b/browser-interface/packages/shared/comms/handlers.ts @@ -49,9 +49,11 @@ const sendMyProfileOverCommsChannel = new Observable>() const pingRequests = new Map() let pingIndex = 0 -export async function bindHandlersToCommsContext(room: RoomConnection) { - removeAllPeers() - pingRequests.clear() +export async function bindHandlersToCommsContext(room: RoomConnection, islandRoom: boolean = true) { + if (islandRoom) { + removeAllPeers() + pingRequests.clear() + } // RFC4 messages room.events.on('position', (e) => processPositionMessage(room, e)) diff --git a/browser-interface/packages/shared/comms/logic/rfc-4-room-connection.ts b/browser-interface/packages/shared/comms/logic/rfc-4-room-connection.ts index 775aaa4c61..3f11e03c8e 100644 --- a/browser-interface/packages/shared/comms/logic/rfc-4-room-connection.ts +++ b/browser-interface/packages/shared/comms/logic/rfc-4-room-connection.ts @@ -17,21 +17,24 @@ export class Rfc4RoomConnection implements RoomConnection { private positionIndex: number = 0 - constructor(private transport: MinimumCommunicationsAdapter) { + constructor(private transport: MinimumCommunicationsAdapter, private id: string = '-') { this.transport.events.on('message', this.handleMessage.bind(this)) this.transport.events.on('DISCONNECTION', (event) => this.events.emit('DISCONNECTION', event)) this.transport.events.on('PEER_DISCONNECTED', (event) => this.events.emit('PEER_DISCONNECTED', event)) } async connect(): Promise { + console.log('[RoomConnection Comms]: connect', this.id) await this.transport.connect() } createVoiceHandler(): Promise { + console.log('[RoomConnection Comms]: createVoiceHandler', this.id) return this.transport.createVoiceHandler() } sendPositionMessage(p: Omit): Promise { + console.log('[RoomConnection Comms]: sendPositionMessage', this.id) return this.sendMessage(false, { message: { $case: 'position', @@ -43,25 +46,32 @@ export class Rfc4RoomConnection implements RoomConnection { }) } sendParcelSceneMessage(scene: proto.Scene): Promise { + console.log('[RoomConnection Comms]: sendParcelSceneMessage', this.id) return this.sendMessage(false, { message: { $case: 'scene', scene } }) } sendProfileMessage(profileVersion: proto.AnnounceProfileVersion): Promise { + console.log('[RoomConnection Comms]: sendProfileMessage', this.id) return this.sendMessage(false, { message: { $case: 'profileVersion', profileVersion } }) } sendProfileRequest(profileRequest: proto.ProfileRequest): Promise { + console.log('[RoomConnection Comms]: sendProfileRequest', this.id) return this.sendMessage(false, { message: { $case: 'profileRequest', profileRequest } }) } sendProfileResponse(profileResponse: proto.ProfileResponse): Promise { + console.log('[RoomConnection Comms]: sendProfileResponse', this.id) return this.sendMessage(false, { message: { $case: 'profileResponse', profileResponse } }) } sendChatMessage(chat: proto.Chat): Promise { + console.log('[RoomConnection Comms]: sendChatMessage', this.id) return this.sendMessage(true, { message: { $case: 'chat', chat } }) } sendVoiceMessage(voice: proto.Voice): Promise { + console.log('[RoomConnection Comms]: sendVoiceMessage', this.id) return this.sendMessage(false, { message: { $case: 'voice', voice } }) } async disconnect() { + console.log('[RoomConnection Comms]: disconnect', this.id) await this.transport.disconnect() } @@ -72,6 +82,7 @@ export class Rfc4RoomConnection implements RoomConnection { return } + console.log('[RoomConnection Comms]: handleMessage', message.$case, this.id) switch (message.$case) { case 'position': { this.events.emit('position', { address, data: message.position }) diff --git a/browser-interface/packages/shared/comms/reducer.ts b/browser-interface/packages/shared/comms/reducer.ts index 95e2a5da51..7236fcbe11 100644 --- a/browser-interface/packages/shared/comms/reducer.ts +++ b/browser-interface/packages/shared/comms/reducer.ts @@ -1,11 +1,13 @@ import { COMMS_ESTABLISHED } from 'shared/loading/types' import { CommsActions, CommsState } from './types' -import { SET_COMMS_ISLAND, SET_LIVEKIT_ADAPTER, SET_ROOM_CONNECTION } from './actions' +import { SET_COMMS_ISLAND, SET_LIVEKIT_ADAPTER, SET_ROOM_CONNECTION, SET_SCENE_ROOM_CONNECTION } from './actions' const INITIAL_COMMS: CommsState = { initialized: false, - context: undefined + context: undefined, + scene: undefined, + scenes: new Map() } export function commsReducer(state?: CommsState, action?: CommsActions): CommsState { @@ -26,7 +28,10 @@ export function commsReducer(state?: CommsState, action?: CommsActions): CommsSt } return { ...state, context: action.payload } case SET_LIVEKIT_ADAPTER: - return { ...state, livekitAdapter: action.payload} + return { ...state, livekitAdapter: action.payload } + case SET_SCENE_ROOM_CONNECTION: + state.scenes.set(action.payload.sceneId, action.payload.room) + return { ...state, scene: action.payload.room } default: return state } diff --git a/browser-interface/packages/shared/comms/sagas.ts b/browser-interface/packages/shared/comms/sagas.ts index 420984a86a..14534e899a 100644 --- a/browser-interface/packages/shared/comms/sagas.ts +++ b/browser-interface/packages/shared/comms/sagas.ts @@ -36,7 +36,8 @@ import { setRoomConnection, SET_COMMS_ISLAND, SET_ROOM_CONNECTION, - setLiveKitAdapter + setLiveKitAdapter, + setSceneRoomConnection } from './actions' import { LivekitAdapter } from './adapters/LivekitAdapter' import { OfflineAdapter } from './adapters/OfflineAdapter' @@ -48,13 +49,16 @@ import { positionReportToCommsPositionRfc4 } from './interface/utils' import { commsLogger } from './logger' import { Rfc4RoomConnection } from './logic/rfc-4-room-connection' import { getConnectedPeerCount, processAvatarVisibility } from './peers' -import { getCommsRoom, reconnectionState } from './selectors' +import { getCommsRoom, getSceneRooms, reconnectionState } from './selectors' import { RootState } from 'shared/store/rootTypes' import { now } from 'lib/javascript/now' import { getGlobalAudioStream } from './adapters/voice/loopback' import { store } from 'shared/store/isolatedStore' import { buildSnapshotContent } from 'shared/profiles/sagas/handleDeployProfile' import { isBase64 } from 'lib/encoding/base64ToBlob' +import { SET_PARCEL_POSITION } from 'shared/scene-loader/actions' +import { getSceneLoader } from '../scene-loader/selectors' +import { Vector2 } from '../protocol/decentraland/common/vectors.gen' const TIME_BETWEEN_PROFILE_RESPONSES = 1000 // this interval should be fast because this will be the delay other people around @@ -89,6 +93,8 @@ export function* commsSaga() { yield fork(handleCommsReconnectionInterval) yield fork(pingerProcess) yield fork(reportPositionSaga) + + yield fork(sceneRoomComms) } /** @@ -219,7 +225,11 @@ function* handleConnectToComms(action: ConnectToCommsAction) { } } -async function connectAdapter(connStr: string, identity: ExplorerIdentity): Promise { +async function connectAdapter( + connStr: string, + identity: ExplorerIdentity, + id: string = 'island' +): Promise { const ix = connStr.indexOf(':') const protocol = connStr.substring(0, ix) const url = connStr.substring(ix + 1) @@ -272,12 +282,12 @@ async function connectAdapter(connStr: string, identity: ExplorerIdentity): Prom throw new Error(`An unknown error was detected while trying to connect to the selected realm.`) } case 'offline': { - return new Rfc4RoomConnection(new OfflineAdapter()) + return new Rfc4RoomConnection(new OfflineAdapter(), id) } case 'ws-room': { const finalUrl = !url.startsWith('ws:') && !url.startsWith('wss:') ? 'wss://' + url : url - return new Rfc4RoomConnection(new WebSocketAdapter(finalUrl, identity)) + return new Rfc4RoomConnection(new WebSocketAdapter(finalUrl, identity), id) } case 'simulator': { return new SimulationRoom(url) @@ -298,7 +308,7 @@ async function connectAdapter(connStr: string, identity: ExplorerIdentity): Prom store.dispatch(setLiveKitAdapter(livekitAdapter)) - return new Rfc4RoomConnection(livekitAdapter) + return new Rfc4RoomConnection(livekitAdapter, id) } } throw new Error(`A communications adapter could not be created for protocol=${protocol}`) @@ -531,3 +541,61 @@ function* handleRoomDisconnectionSaga(action: HandleRoomDisconnection) { } } } + +function* sceneRoomComms() { + let currentSceneId: string = '' + const commsSceneToRemove = new Map() + + while (true) { + const reason: { timeout?: unknown; newParcel?: { payload: { position: Vector2 } } } = yield race({ + newParcel: take(SET_PARCEL_POSITION), + timeout: delay(5000) + }) + const sceneLoader: ReturnType = yield select(getSceneLoader) + if (!sceneLoader) continue + if (reason.newParcel) { + const sceneId = yield call(sceneLoader.getSceneId!, reason.newParcel.payload.position) + // We are still on the same scene. + if (sceneId === currentSceneId) continue + const oldSceneId = currentSceneId + yield call(checkDisconnectScene, sceneId, oldSceneId, commsSceneToRemove) + yield call(connectSceneToComms, sceneId) + // Player moved to a new scene. Instanciate new comms + currentSceneId = sceneId + } + } +} + +function* checkDisconnectScene( + currentSceneId: string, + oldSceneId: string, + commsSceneToRemove: Map +) { + // avoid deleting an already created comms. Use when the user is switching between two scenes + if (commsSceneToRemove.has(currentSceneId)) { + clearTimeout(commsSceneToRemove.get(currentSceneId)) + commsSceneToRemove.delete(currentSceneId) + } + if (!oldSceneId) return + + console.log('[SceneComms]: will disconnect', oldSceneId) + const sceneRooms: ReturnType = yield select(getSceneRooms) + const oldRoom = sceneRooms.get(oldSceneId) + const timeout = setTimeout(() => { + console.log('[SceneComms]: disconnectSceneComms', oldSceneId) + void oldRoom?.disconnect() + commsSceneToRemove.delete(oldSceneId) + }, 1000) + commsSceneToRemove.set(oldSceneId, timeout) +} + +function* connectSceneToComms(sceneId: string) { + console.log('[SceneComms]: connectSceneToComms', sceneId) + // Fetch connection string + // const connectionString = `https://boedo.com/${sceneId}` + const connectionString = `offline:offline` + const identity: ExplorerIdentity = yield select(getCurrentIdentity) + const sceneRoomConnetion = yield call(connectAdapter, connectionString, identity, sceneId) + yield call(bindHandlersToCommsContext, sceneRoomConnetion, false) + yield put(setSceneRoomConnection(sceneId, sceneRoomConnetion)) +} diff --git a/browser-interface/packages/shared/comms/selectors.ts b/browser-interface/packages/shared/comms/selectors.ts index 2f1b5021f9..28b2f0b87f 100644 --- a/browser-interface/packages/shared/comms/selectors.ts +++ b/browser-interface/packages/shared/comms/selectors.ts @@ -7,9 +7,74 @@ import { RootState } from 'shared/store/rootTypes' import { RoomConnection } from './interface' import { RootCommsState } from './types' import { ActiveVideoStreams } from './adapters/types' +import { + AnnounceProfileVersion, + ProfileRequest, + ProfileResponse, + Position, + Scene, + Chat, + Voice +} from '../protocol/decentraland/kernel/comms/rfc4/comms.gen' export const getCommsIsland = (store: RootCommsState): string | undefined => store.comms.island -export const getCommsRoom = (state: RootCommsState): RoomConnection | undefined => state.comms.context +export const getSceneRoomComms = (state: RootCommsState): RoomConnection | undefined => state.comms.scene +export const getSceneRooms = (state: RootCommsState): Map => state.comms.scenes + +export const getCommsRoom = (state: RootCommsState): RoomConnection | undefined => { + const islandRoom = state.comms.context + const sceneRoom = state.comms.scene + if (!islandRoom) return undefined + return { + connect: async () => { + debugger + }, + // events: islandRoom.events, + disconnect: async () => { + await islandRoom.disconnect() + // TBD: should we disconnect from scenes here too ? + }, + sendProfileMessage: async (profile: AnnounceProfileVersion) => { + const island = islandRoom.sendProfileMessage(profile) + const scene = sceneRoom?.sendProfileMessage(profile) + await Promise.all([island, scene]) + }, + sendProfileRequest: async (request: ProfileRequest) => { + const island = islandRoom.sendProfileRequest(request) + const scene = sceneRoom?.sendProfileRequest(request) + await Promise.all([island, scene]) + }, + sendProfileResponse: async (response: ProfileResponse) => { + const island = islandRoom.sendProfileResponse(response) + const scene = sceneRoom?.sendProfileResponse(response) + await Promise.all([island, scene]) + }, + sendPositionMessage: async (position: Omit) => { + const island = islandRoom.sendPositionMessage(position) + const scene = sceneRoom?.sendPositionMessage(position) + await Promise.all([island, scene]) + }, + sendParcelSceneMessage: async (message: Scene) => { + const island = islandRoom.sendParcelSceneMessage(message) + const scene = sceneRoom?.sendParcelSceneMessage(message) + await Promise.all([island, scene]) + }, + sendChatMessage: async (message: Chat) => { + const island = islandRoom.sendChatMessage(message) + const scene = sceneRoom?.sendChatMessage(message) + await Promise.all([island, scene]) + }, + sendVoiceMessage: async (message: Voice) => { + // TBD: Feature flag for backwards compatibility + await sceneRoom?.sendVoiceMessage(message) + }, + createVoiceHandler: async () => { + // TBD: Feature flag for backwards compatibility + if (!sceneRoom) debugger + return sceneRoom!.createVoiceHandler() + } + } as RoomConnection +} export function reconnectionState(state: RootState): { commsConnection: RoomConnection | undefined @@ -25,4 +90,5 @@ export function reconnectionState(state: RootState): { } } -export const getLivekitActiveVideoStreams = (store: RootCommsState): Map | undefined => store.comms.livekitAdapter?.getActiveVideoStreams() \ No newline at end of file +export const getLivekitActiveVideoStreams = (store: RootCommsState): Map | undefined => + store.comms.livekitAdapter?.getActiveVideoStreams() diff --git a/browser-interface/packages/shared/comms/types.ts b/browser-interface/packages/shared/comms/types.ts index 114528757d..3e9f654e4c 100644 --- a/browser-interface/packages/shared/comms/types.ts +++ b/browser-interface/packages/shared/comms/types.ts @@ -1,21 +1,26 @@ import { CommsEstablished } from 'shared/loading/types' -import { HandleRoomDisconnection, SetCommsIsland, SetLiveKitAdapterAction, SetRoomConnectionAction } from './actions' +import { HandleRoomDisconnection, SetCommsIsland, SetLiveKitAdapterAction, SetRoomConnectionAction, SetSceneRoomConnectionAction } from './actions' import { LivekitAdapter } from './adapters/LivekitAdapter' import { RoomConnection } from './interface' +type SceneId = string + export type CommsState = { initialized: boolean island?: string - context: RoomConnection | undefined, + context: RoomConnection | undefined livekitAdapter?: LivekitAdapter + scene: RoomConnection | undefined + scenes: Map } -export type CommsActions = +export type CommsActions = | SetCommsIsland | SetRoomConnectionAction | HandleRoomDisconnection | SetLiveKitAdapterAction | CommsEstablished + | SetSceneRoomConnectionAction export type CommsConnectionState = | 'initial' diff --git a/browser-interface/packages/shared/scene-loader/genesis-city-loader-impl/index.ts b/browser-interface/packages/shared/scene-loader/genesis-city-loader-impl/index.ts index 9d6d41c14c..39809ae3a9 100644 --- a/browser-interface/packages/shared/scene-loader/genesis-city-loader-impl/index.ts +++ b/browser-interface/packages/shared/scene-loader/genesis-city-loader-impl/index.ts @@ -39,6 +39,12 @@ export function createGenesisCityLoader(options: { } return { + async getSceneId(parcel: Vector2): Promise { + const scene = downloadManager.pointerToEntity.get(encodeParcelPosition(parcel)) + if (scene) { + return (await scene)?.id + } + }, async fetchScenesByLocation(parcels) { const results = await downloadManager.resolveEntitiesByPointer(parcels) return { diff --git a/browser-interface/packages/shared/scene-loader/sagas.ts b/browser-interface/packages/shared/scene-loader/sagas.ts index a5bf7e2fa3..9a6ba6004d 100644 --- a/browser-interface/packages/shared/scene-loader/sagas.ts +++ b/browser-interface/packages/shared/scene-loader/sagas.ts @@ -311,7 +311,7 @@ function* onWorldPositionChange() { } } - const reason: { unload: any; reload: SceneReload } = yield race({ + const reason: { unload: any; reload: SceneReload; newParcel: { payload: { position: Vector2 } } } = yield race({ timeout: delay(5000), newSceneLoader: take(SET_SCENE_LOADER), newParcel: take(SET_PARCEL_POSITION), diff --git a/browser-interface/packages/shared/scene-loader/types.ts b/browser-interface/packages/shared/scene-loader/types.ts index 48b4f78e2c..2bf31d3303 100644 --- a/browser-interface/packages/shared/scene-loader/types.ts +++ b/browser-interface/packages/shared/scene-loader/types.ts @@ -1,5 +1,6 @@ import { Entity } from '@dcl/schemas' import { InstancedSpawnPoint, LoadableScene } from 'shared/types' +import { Vector2 } from '../protocol/decentraland/common/vectors.gen' export type SetDesiredScenesCommand = { scenes: LoadableScene[] @@ -16,6 +17,7 @@ export interface ISceneLoader { fetchScenesByLocation(parcels: string[]): Promise stop(): Promise invalidateCache(sceneId: Entity): void + getSceneId?(position: Vector2): Promise } export type SceneLoaderState = { diff --git a/browser-interface/packages/shared/world/SceneWorker.ts b/browser-interface/packages/shared/world/SceneWorker.ts index 540608d6f6..29d3a601e6 100644 --- a/browser-interface/packages/shared/world/SceneWorker.ts +++ b/browser-interface/packages/shared/world/SceneWorker.ts @@ -120,10 +120,22 @@ export class SceneWorker { private readonly startLoadingTime = performance.now() // this is the transport for the worker public transport?: Transport - metadata: Scene logger: ILogger + // TBD: we add this here or in the comms saga ? + public comms: any = { + initialize: async () => { + console.log('[Initialize Comms]: ', { sceneId: this.loadableScene.id }) + }, + pause: () => { + console.log('[Pause Comms]:', { sceneId: this.loadableScene.id }) + }, + stop: () => { + console.log('[Remove comms]:', { sceneId: this.loadableScene.id }) + } + } + static async createSceneWorker(loadableScene: Readonly, rpcClient: RpcClient) { ++globalSceneNumberCounter const sceneNumber = globalSceneNumberCounter @@ -139,7 +151,6 @@ export class SceneWorker { scenePort: RpcClientPort ) { const skipErrors = ['Transport closed while waiting the ACK'] - this.metadata = loadableScene.entity.metadata const loggerName = getSceneNameFromJsonData(this.metadata) || loadableScene.id From b0cc8b3f598e6552d5fdad2e96d0bf544dfad291 Mon Sep 17 00:00:00 2001 From: Gonzalo DCL Date: Fri, 15 Dec 2023 12:47:43 -0300 Subject: [PATCH 02/23] remove comms logic from scene-worker --- .../packages/shared/comms/selectors.ts | 7 +++---- .../packages/shared/world/SceneWorker.ts | 13 ------------- 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/browser-interface/packages/shared/comms/selectors.ts b/browser-interface/packages/shared/comms/selectors.ts index 28b2f0b87f..a8974df1f0 100644 --- a/browser-interface/packages/shared/comms/selectors.ts +++ b/browser-interface/packages/shared/comms/selectors.ts @@ -64,16 +64,15 @@ export const getCommsRoom = (state: RootCommsState): RoomConnection | undefined const scene = sceneRoom?.sendChatMessage(message) await Promise.all([island, scene]) }, - sendVoiceMessage: async (message: Voice) => { - // TBD: Feature flag for backwards compatibility - await sceneRoom?.sendVoiceMessage(message) + sendVoiceMessage: async (_message: Voice) => { + debugger }, createVoiceHandler: async () => { // TBD: Feature flag for backwards compatibility if (!sceneRoom) debugger return sceneRoom!.createVoiceHandler() } - } as RoomConnection + } as any as RoomConnection } export function reconnectionState(state: RootState): { diff --git a/browser-interface/packages/shared/world/SceneWorker.ts b/browser-interface/packages/shared/world/SceneWorker.ts index 29d3a601e6..e00d034afd 100644 --- a/browser-interface/packages/shared/world/SceneWorker.ts +++ b/browser-interface/packages/shared/world/SceneWorker.ts @@ -123,19 +123,6 @@ export class SceneWorker { metadata: Scene logger: ILogger - // TBD: we add this here or in the comms saga ? - public comms: any = { - initialize: async () => { - console.log('[Initialize Comms]: ', { sceneId: this.loadableScene.id }) - }, - pause: () => { - console.log('[Pause Comms]:', { sceneId: this.loadableScene.id }) - }, - stop: () => { - console.log('[Remove comms]:', { sceneId: this.loadableScene.id }) - } - } - static async createSceneWorker(loadableScene: Readonly, rpcClient: RpcClient) { ++globalSceneNumberCounter const sceneNumber = globalSceneNumberCounter From 0f93827018d4f1d74da0a0584363f60ae8cc8808 Mon Sep 17 00:00:00 2001 From: Hugo Arregui Date: Fri, 15 Dec 2023 14:08:07 -0300 Subject: [PATCH 03/23] use gatekeeper to fetch scene room --- .../packages/shared/comms/sagas.ts | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/browser-interface/packages/shared/comms/sagas.ts b/browser-interface/packages/shared/comms/sagas.ts index 14534e899a..5deca6eb2c 100644 --- a/browser-interface/packages/shared/comms/sagas.ts +++ b/browser-interface/packages/shared/comms/sagas.ts @@ -21,7 +21,7 @@ import { DEPLOY_PROFILE_SUCCESS, SEND_PROFILE_TO_RENDERER_REQUEST } from 'shared import { getCurrentUserProfile } from 'shared/profiles/selectors' import type { ConnectToCommsAction } from 'shared/realm/actions' import { CONNECT_TO_COMMS, setRealmAdapter, SET_REALM_ADAPTER } from 'shared/realm/actions' -import { getFetchContentUrlPrefixFromRealmAdapter } from 'shared/realm/selectors' +import { getFetchContentUrlPrefixFromRealmAdapter, getRealmAdapter } from 'shared/realm/selectors' import { waitForRealm } from 'shared/realm/waitForRealmAdapter' import type { IRealmAdapter } from 'shared/realm/types' import { USER_AUTHENTICATED } from 'shared/session/actions' @@ -591,11 +591,27 @@ function* checkDisconnectScene( function* connectSceneToComms(sceneId: string) { console.log('[SceneComms]: connectSceneToComms', sceneId) - // Fetch connection string - // const connectionString = `https://boedo.com/${sceneId}` - const connectionString = `offline:offline` + + const realmAdapter = yield select(getRealmAdapter) + if (!realmAdapter) { + throw new Error('No realm adapter') // TODO + } + const realmName = realmAdapter.about.configurations?.realmName + const identity: ExplorerIdentity = yield select(getCurrentIdentity) - const sceneRoomConnetion = yield call(connectAdapter, connectionString, identity, sceneId) + // TODO: we should change the adapter control to provide this url + const url = 'https://comms-gatekeeper.decentraland.zone/get-scene-adapter' + const response = yield call(signedFetch, + url, + identity, + { method: 'POST', responseBodyType: 'json' }, + { + realmName, + sceneId + } + ) + + const sceneRoomConnetion = yield call(connectAdapter, response.json.adapter, identity, sceneId) yield call(bindHandlersToCommsContext, sceneRoomConnetion, false) yield put(setSceneRoomConnection(sceneId, sceneRoomConnetion)) } From b02aaf6c2128b681c43445d16b8302fd6b2eb0c1 Mon Sep 17 00:00:00 2001 From: Gonzalo DCL Date: Wed, 20 Dec 2023 09:29:38 -0300 Subject: [PATCH 04/23] wip --- browser-interface/packages/config/index.ts | 3 +++ .../packages/shared/comms/index.ts | 1 - .../packages/shared/comms/interface/index.ts | 1 + .../comms/logic/rfc-4-room-connection.ts | 24 ++++++++--------- .../packages/shared/comms/sagas.ts | 8 ++++-- .../packages/shared/comms/selectors.ts | 26 ++++++++++++++----- 6 files changed, 41 insertions(+), 22 deletions(-) diff --git a/browser-interface/packages/config/index.ts b/browser-interface/packages/config/index.ts index a9ad664fa7..41f55e36e4 100644 --- a/browser-interface/packages/config/index.ts +++ b/browser-interface/packages/config/index.ts @@ -72,6 +72,9 @@ export const WSS_ENABLED = !!ensureSingleString(qs.get('ws')) export const FORCE_SEND_MESSAGE = location.search.includes('FORCE_SEND_MESSAGE') export const ALLOW_SWIFT_SHADER = location.search.includes('ALLOW_SWIFT_SHADER') +export const DISABLE_SCENE_ROOM = location.search.includes('DISABLE_SCENE_ROOM') +export const DISABLE_ISLAND_SCENE_MESSAGES = location.search.includes('DISABLE_ISLAND_SCENE_MESSAGES') + const ASSET_BUNDLES_DOMAIN = ensureSingleString(qs.get('ASSET_BUNDLES_DOMAIN')) export const SOCIAL_SERVER_URL = ensureSingleString(qs.get('SOCIAL_SERVER_URL')) diff --git a/browser-interface/packages/shared/comms/index.ts b/browser-interface/packages/shared/comms/index.ts index 1dd368000b..a7852a74d4 100644 --- a/browser-interface/packages/shared/comms/index.ts +++ b/browser-interface/packages/shared/comms/index.ts @@ -15,7 +15,6 @@ export function sendPublicChatMessage(message: string) { export function sendParcelSceneCommsMessage(sceneId: string, data: Uint8Array) { const commsContext = getCommsRoom(store.getState()) - commsContext ?.sendParcelSceneMessage({ data, diff --git a/browser-interface/packages/shared/comms/interface/index.ts b/browser-interface/packages/shared/comms/interface/index.ts index ada64f423b..fe277ce663 100644 --- a/browser-interface/packages/shared/comms/interface/index.ts +++ b/browser-interface/packages/shared/comms/interface/index.ts @@ -16,6 +16,7 @@ export type CommsEvents = CommsAdapterEvents & { } export interface RoomConnection { + id?: string // this operation is non-reversible disconnect(): Promise // @once diff --git a/browser-interface/packages/shared/comms/logic/rfc-4-room-connection.ts b/browser-interface/packages/shared/comms/logic/rfc-4-room-connection.ts index 3f11e03c8e..d7a7e4ef9c 100644 --- a/browser-interface/packages/shared/comms/logic/rfc-4-room-connection.ts +++ b/browser-interface/packages/shared/comms/logic/rfc-4-room-connection.ts @@ -17,24 +17,24 @@ export class Rfc4RoomConnection implements RoomConnection { private positionIndex: number = 0 - constructor(private transport: MinimumCommunicationsAdapter, private id: string = '-') { + constructor(private transport: MinimumCommunicationsAdapter, public id: string = '-') { this.transport.events.on('message', this.handleMessage.bind(this)) this.transport.events.on('DISCONNECTION', (event) => this.events.emit('DISCONNECTION', event)) this.transport.events.on('PEER_DISCONNECTED', (event) => this.events.emit('PEER_DISCONNECTED', event)) } async connect(): Promise { - console.log('[RoomConnection Comms]: connect', this.id) + // console.log('[RoomConnection Comms]: connect', this.id) await this.transport.connect() } createVoiceHandler(): Promise { - console.log('[RoomConnection Comms]: createVoiceHandler', this.id) + // console.log('[RoomConnection Comms]: createVoiceHandler', this.id) return this.transport.createVoiceHandler() } sendPositionMessage(p: Omit): Promise { - console.log('[RoomConnection Comms]: sendPositionMessage', this.id) + // console.log('[RoomConnection Comms]: sendPositionMessage', this.id) return this.sendMessage(false, { message: { $case: 'position', @@ -46,32 +46,32 @@ export class Rfc4RoomConnection implements RoomConnection { }) } sendParcelSceneMessage(scene: proto.Scene): Promise { - console.log('[RoomConnection Comms]: sendParcelSceneMessage', this.id) + // console.log('[RoomConnection Comms]: sendParcelSceneMessage', this.id) return this.sendMessage(false, { message: { $case: 'scene', scene } }) } sendProfileMessage(profileVersion: proto.AnnounceProfileVersion): Promise { - console.log('[RoomConnection Comms]: sendProfileMessage', this.id) + // console.log('[RoomConnection Comms]: sendProfileMessage', this.id) return this.sendMessage(false, { message: { $case: 'profileVersion', profileVersion } }) } sendProfileRequest(profileRequest: proto.ProfileRequest): Promise { - console.log('[RoomConnection Comms]: sendProfileRequest', this.id) + // console.log('[RoomConnection Comms]: sendProfileRequest', this.id) return this.sendMessage(false, { message: { $case: 'profileRequest', profileRequest } }) } sendProfileResponse(profileResponse: proto.ProfileResponse): Promise { - console.log('[RoomConnection Comms]: sendProfileResponse', this.id) + // console.log('[RoomConnection Comms]: sendProfileResponse', this.id) return this.sendMessage(false, { message: { $case: 'profileResponse', profileResponse } }) } sendChatMessage(chat: proto.Chat): Promise { - console.log('[RoomConnection Comms]: sendChatMessage', this.id) + // console.log('[RoomConnection Comms]: sendChatMessage', this.id) return this.sendMessage(true, { message: { $case: 'chat', chat } }) } sendVoiceMessage(voice: proto.Voice): Promise { - console.log('[RoomConnection Comms]: sendVoiceMessage', this.id) + // console.log('[RoomConnection Comms]: sendVoiceMessage', this.id) return this.sendMessage(false, { message: { $case: 'voice', voice } }) } async disconnect() { - console.log('[RoomConnection Comms]: disconnect', this.id) + // console.log('[RoomConnection Comms]: disconnect', this.id) await this.transport.disconnect() } @@ -82,7 +82,7 @@ export class Rfc4RoomConnection implements RoomConnection { return } - console.log('[RoomConnection Comms]: handleMessage', message.$case, this.id) + // console.log('[RoomConnection Comms]: handleMessage', message.$case, this.id) switch (message.$case) { case 'position': { this.events.emit('position', { address, data: message.position }) diff --git a/browser-interface/packages/shared/comms/sagas.ts b/browser-interface/packages/shared/comms/sagas.ts index 5deca6eb2c..31d0fa1cf2 100644 --- a/browser-interface/packages/shared/comms/sagas.ts +++ b/browser-interface/packages/shared/comms/sagas.ts @@ -59,6 +59,7 @@ import { isBase64 } from 'lib/encoding/base64ToBlob' import { SET_PARCEL_POSITION } from 'shared/scene-loader/actions' import { getSceneLoader } from '../scene-loader/selectors' import { Vector2 } from '../protocol/decentraland/common/vectors.gen' +import { DISABLE_SCENE_ROOM } from '../../config' const TIME_BETWEEN_PROFILE_RESPONSES = 1000 // this interval should be fast because this will be the delay other people around @@ -266,7 +267,7 @@ async function connectAdapter( } if (typeof response.fixedAdapter === 'string' && !response.fixedAdapter.startsWith('signed-login:')) { - return connectAdapter(response.fixedAdapter, identity) + return connectAdapter(response.fixedAdapter, identity, id) } if (typeof response.message === 'string') { @@ -546,6 +547,8 @@ function* sceneRoomComms() { let currentSceneId: string = '' const commsSceneToRemove = new Map() + if (DISABLE_SCENE_ROOM) return + while (true) { const reason: { timeout?: unknown; newParcel?: { payload: { position: Vector2 } } } = yield race({ newParcel: take(SET_PARCEL_POSITION), @@ -601,7 +604,8 @@ function* connectSceneToComms(sceneId: string) { const identity: ExplorerIdentity = yield select(getCurrentIdentity) // TODO: we should change the adapter control to provide this url const url = 'https://comms-gatekeeper.decentraland.zone/get-scene-adapter' - const response = yield call(signedFetch, + const response = yield call( + signedFetch, url, identity, { method: 'POST', responseBodyType: 'json' }, diff --git a/browser-interface/packages/shared/comms/selectors.ts b/browser-interface/packages/shared/comms/selectors.ts index a8974df1f0..eb7cb9902a 100644 --- a/browser-interface/packages/shared/comms/selectors.ts +++ b/browser-interface/packages/shared/comms/selectors.ts @@ -24,7 +24,9 @@ export const getSceneRooms = (state: RootCommsState): Map { const islandRoom = state.comms.context const sceneRoom = state.comms.scene + if (!islandRoom) return undefined + return { connect: async () => { debugger @@ -34,6 +36,8 @@ export const getCommsRoom = (state: RootCommsState): RoomConnection | undefined await islandRoom.disconnect() // TBD: should we disconnect from scenes here too ? }, + // TBD: This should be only be sent by the island ? + // We may remove this before reach production, but to think about it sendProfileMessage: async (profile: AnnounceProfileVersion) => { const island = islandRoom.sendProfileMessage(profile) const scene = sceneRoom?.sendProfileMessage(profile) @@ -55,22 +59,30 @@ export const getCommsRoom = (state: RootCommsState): RoomConnection | undefined await Promise.all([island, scene]) }, sendParcelSceneMessage: async (message: Scene) => { - const island = islandRoom.sendParcelSceneMessage(message) - const scene = sceneRoom?.sendParcelSceneMessage(message) - await Promise.all([island, scene]) + if (message.sceneId !== sceneRoom?.id) { + console.warn('Ignoring Scene Message', { sceneId: message.sceneId, connectedSceneId: sceneRoom?.id }) + return + } + // const island = islandRoom.sendParcelSceneMessage(message) + await sceneRoom?.sendParcelSceneMessage(message) }, sendChatMessage: async (message: Chat) => { const island = islandRoom.sendChatMessage(message) const scene = sceneRoom?.sendChatMessage(message) await Promise.all([island, scene]) }, - sendVoiceMessage: async (_message: Voice) => { - debugger + // TBD: how voice chat works? + sendVoiceMessage: async (message: Voice) => { + if (!sceneRoom) debugger + return sceneRoom!.sendVoiceMessage(message) }, createVoiceHandler: async () => { // TBD: Feature flag for backwards compatibility - if (!sceneRoom) debugger - return sceneRoom!.createVoiceHandler() + if (!sceneRoom) { + debugger + throw new Error('Scene room not avaialble') + } + return sceneRoom.createVoiceHandler() } } as any as RoomConnection } From 35bb548e27088225975c41d3f0e92458377f5981 Mon Sep 17 00:00:00 2001 From: Gonzalo DCL Date: Thu, 21 Dec 2023 10:06:37 -0300 Subject: [PATCH 05/23] wip logs --- .../shared/comms/adapters/LivekitAdapter.ts | 6 +++++- browser-interface/packages/shared/comms/sagas.ts | 15 +++++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/browser-interface/packages/shared/comms/adapters/LivekitAdapter.ts b/browser-interface/packages/shared/comms/adapters/LivekitAdapter.ts index d337ea0c71..7c85fafbe3 100644 --- a/browser-interface/packages/shared/comms/adapters/LivekitAdapter.ts +++ b/browser-interface/packages/shared/comms/adapters/LivekitAdapter.ts @@ -52,6 +52,7 @@ export class LivekitAdapter implements MinimumCommunicationsAdapter { this.config.logger.log(this.room.name, 'connection state changed', state) }) .on(RoomEvent.Disconnected, (reason: DisconnectReason | undefined) => { + this.config.logger.log('[BOEDO]: on disconnect', reason, this.room.name) if (this.disposed) { return } @@ -103,9 +104,10 @@ export class LivekitAdapter implements MinimumCommunicationsAdapter { } if (state !== ConnectionState.Connected) { - this.config.logger.log(`Skip sending message because connection state is ${state}`) + this.config.logger.log(`Skip sending message because connection state is ${state} ${this.room.name}`) return } + this.config.logger.log('Sending message', this.room.name) try { await this.room.localParticipant.publishData(data, reliable ? DataPacket_Kind.RELIABLE : DataPacket_Kind.LOSSY) @@ -117,6 +119,7 @@ export class LivekitAdapter implements MinimumCommunicationsAdapter { stack: err.stack, saga_stack: `room session id: ${this.room.sid}, participant id: ${this.room.localParticipant.sid}, state: ${state}` }) + this.config.logger.log('[BOEDO]: error sending data', err, err.mesage) await this.disconnect() } } @@ -126,6 +129,7 @@ export class LivekitAdapter implements MinimumCommunicationsAdapter { } async do_disconnect(kicked: boolean) { + this.config.logger.log('[BOEDO]: do_disconnect', this.room.name) if (this.disposed) { return } diff --git a/browser-interface/packages/shared/comms/sagas.ts b/browser-interface/packages/shared/comms/sagas.ts index 31d0fa1cf2..30eda72cd5 100644 --- a/browser-interface/packages/shared/comms/sagas.ts +++ b/browser-interface/packages/shared/comms/sagas.ts @@ -229,7 +229,8 @@ function* handleConnectToComms(action: ConnectToCommsAction) { async function connectAdapter( connStr: string, identity: ExplorerIdentity, - id: string = 'island' + id: string = 'island', + dispatchAction = true ): Promise { const ix = connStr.indexOf(':') const protocol = connStr.substring(0, ix) @@ -307,7 +308,9 @@ async function connectAdapter( globalAudioStream: await getGlobalAudioStream() }) - store.dispatch(setLiveKitAdapter(livekitAdapter)) + if (dispatchAction) { + store.dispatch(setLiveKitAdapter(livekitAdapter)) + } return new Rfc4RoomConnection(livekitAdapter, id) } @@ -581,11 +584,11 @@ function* checkDisconnectScene( } if (!oldSceneId) return - console.log('[SceneComms]: will disconnect', oldSceneId) + console.log('[BOEDO SceneComms]: will disconnect', oldSceneId) const sceneRooms: ReturnType = yield select(getSceneRooms) const oldRoom = sceneRooms.get(oldSceneId) const timeout = setTimeout(() => { - console.log('[SceneComms]: disconnectSceneComms', oldSceneId) + console.log('[BOEDO SceneComms]: disconnectSceneComms', oldSceneId) void oldRoom?.disconnect() commsSceneToRemove.delete(oldSceneId) }, 1000) @@ -593,7 +596,7 @@ function* checkDisconnectScene( } function* connectSceneToComms(sceneId: string) { - console.log('[SceneComms]: connectSceneToComms', sceneId) + console.log('[BOEDO SceneComms]: connectSceneToComms', sceneId) const realmAdapter = yield select(getRealmAdapter) if (!realmAdapter) { @@ -615,7 +618,7 @@ function* connectSceneToComms(sceneId: string) { } ) - const sceneRoomConnetion = yield call(connectAdapter, response.json.adapter, identity, sceneId) + const sceneRoomConnetion = yield call(connectAdapter, response.json.adapter, identity, sceneId, false) yield call(bindHandlersToCommsContext, sceneRoomConnetion, false) yield put(setSceneRoomConnection(sceneId, sceneRoomConnetion)) } From 7f0c49658144742477b16d7ce8dc26961870958f Mon Sep 17 00:00:00 2001 From: Hugo Arregui Date: Thu, 21 Dec 2023 12:32:57 -0300 Subject: [PATCH 06/23] connect --- browser-interface/packages/shared/comms/sagas.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/browser-interface/packages/shared/comms/sagas.ts b/browser-interface/packages/shared/comms/sagas.ts index 30eda72cd5..d6d646164b 100644 --- a/browser-interface/packages/shared/comms/sagas.ts +++ b/browser-interface/packages/shared/comms/sagas.ts @@ -207,7 +207,7 @@ function* handleConnectToComms(action: ConnectToCommsAction) { const adapter: RoomConnection = yield call(connectAdapter, action.payload.event.connStr, identity) - globalThis.__DEBUG_ADAPTER = adapter + globalThis.__DEBUG_ISLAND_ADAPTER = adapter yield put(establishingComms()) yield apply(adapter, adapter.connect, []) @@ -618,7 +618,9 @@ function* connectSceneToComms(sceneId: string) { } ) - const sceneRoomConnetion = yield call(connectAdapter, response.json.adapter, identity, sceneId, false) - yield call(bindHandlersToCommsContext, sceneRoomConnetion, false) - yield put(setSceneRoomConnection(sceneId, sceneRoomConnetion)) + const sceneRoomConnection = yield call(connectAdapter, response.json.adapter, identity, sceneId, false) + globalThis.__DEBUG_SCENE_ADAPTER = sceneRoomConnection + yield apply(sceneRoomConnection, sceneRoomConnection.connect, []) + yield call(bindHandlersToCommsContext, sceneRoomConnection, false) + yield put(setSceneRoomConnection(sceneId, sceneRoomConnection)) } From 6933aca9511b641a9edbf201bf2e5872f9256d63 Mon Sep 17 00:00:00 2001 From: Hugo Arregui Date: Tue, 26 Dec 2023 10:47:48 -0300 Subject: [PATCH 07/23] remove ping --- .../packages/shared/comms/handlers.ts | 57 +------------------ .../packages/shared/comms/peers.ts | 3 - .../packages/shared/comms/sagas.ts | 47 ++------------- 3 files changed, 6 insertions(+), 101 deletions(-) diff --git a/browser-interface/packages/shared/comms/handlers.ts b/browser-interface/packages/shared/comms/handlers.ts index 19f747c2f1..c844fee1ff 100644 --- a/browser-interface/packages/shared/comms/handlers.ts +++ b/browser-interface/packages/shared/comms/handlers.ts @@ -4,7 +4,6 @@ import { uuid } from 'lib/javascript/uuid' import { Observable } from 'mz-observable' import { eventChannel } from 'redux-saga' import { getBannedUsers } from 'shared/meta/selectors' -import { incrementCounter } from 'shared/analytics/occurences' import { validateAvatar } from 'shared/profiles/schemaValidation' import { getCurrentUserProfile } from 'shared/profiles/selectors' import { ensureAvatarCompatibilityFormat } from 'lib/decentraland/profiles/transformations/profileToServerFormat' @@ -14,7 +13,6 @@ import { store } from 'shared/store/isolatedStore' import { ChatMessage as InternalChatMessage, ChatMessageType } from 'shared/types' import { processVoiceFragment } from 'shared/voiceChat/handlers' import { isBlockedOrBanned } from 'shared/voiceChat/selectors' -import { sendPublicChatMessage } from '.' import { messageReceived } from '../chat/actions' import { handleRoomDisconnection } from './actions' import { AdapterDisconnectedEvent, PeerDisconnectedEvent } from './adapters/types' @@ -33,11 +31,6 @@ import { scenesSubscribedToCommsEvents } from './sceneSubscriptions' import { globalObservable } from 'shared/observables' import { BringDownClientAndShowError } from 'shared/loading/ReportFatalError' -type PingRequest = { - alias: number - sentTime: number - onPong: (dt: number, address: string) => void -} type VersionUpdateInformation = { userId: string version: number @@ -46,13 +39,10 @@ type VersionUpdateInformation = { const versionUpdateOverCommsChannel = new Observable() const receiveProfileOverCommsChannel = new Observable() const sendMyProfileOverCommsChannel = new Observable>() -const pingRequests = new Map() -let pingIndex = 0 export async function bindHandlersToCommsContext(room: RoomConnection, islandRoom: boolean = true) { if (islandRoom) { removeAllPeers() - pingRequests.clear() } // RFC4 messages @@ -109,6 +99,7 @@ function handleDisconnectPeer(data: PeerDisconnectedEvent) { removePeerByAddress(data.address) } +// TODO: use position message to setupPeerTrackingInfo function processProfileUpdatedMessage(message: Package) { const peerTrackingInfo = setupPeerTrackingInfo(message.address) peerTrackingInfo.ethereumAddress = message.address @@ -136,33 +127,6 @@ function processParcelSceneCommsMessage(message: Package) { } } -function pingMessage(nonce: number) { - return `␑${nonce}` -} -function pongMessage(nonce: number, address: string) { - return `␆${nonce} ${address}` -} - -export function sendPing(onPong?: (dt: number, address: string) => void) { - const nonce = Math.floor(Math.random() * 0xffffffff) - let responses = 0 - pingRequests.set(nonce, { - sentTime: Date.now(), - alias: pingIndex++, - onPong: - onPong || - ((dt, address) => { - console.log( - `ping got ${++responses} responses (ping: ${dt.toFixed(2)}ms, nonce: ${nonce}, address: ${address})` - ) - }) - }) - sendPublicChatMessage(pingMessage(nonce)) - incrementCounter('ping_sent_counter') -} - -const answeredPings = new Set() - function processChatMessage(message: Package) { const myProfile = getCurrentUserProfile(store.getState()) const fromAlias: string = message.address @@ -179,24 +143,7 @@ function processChatMessage(message: Package) { senderPeer.lastUpdate = Date.now() if (senderPeer.ethereumAddress) { - if (message.data.message.startsWith('␆') /* pong */) { - const [nonceStr, address] = message.data.message.slice(1).split(' ') - const nonce = parseInt(nonceStr, 10) - const request = pingRequests.get(nonce) - if (request) { - incrementCounter('pong_received_counter') - request.onPong(Date.now() - request.sentTime, address || 'none') - } - } else if (message.data.message.startsWith('␑') /* ping */) { - const nonce = parseInt(message.data.message.slice(1), 10) - if (answeredPings.has(nonce)) { - incrementCounter('ping_received_twice_counter') - return - } - answeredPings.add(nonce) - if (myProfile) sendPublicChatMessage(pongMessage(nonce, myProfile.ethAddress)) - incrementCounter('pong_sent_counter') - } else if (message.data.message.startsWith('␐')) { + if (message.data.message.startsWith('␐')) { const [id, timestamp] = message.data.message.split(' ') avatarMessageObservable.notifyObservers({ diff --git a/browser-interface/packages/shared/comms/peers.ts b/browser-interface/packages/shared/comms/peers.ts index fceabd0c10..6ca308cf82 100644 --- a/browser-interface/packages/shared/comms/peers.ts +++ b/browser-interface/packages/shared/comms/peers.ts @@ -37,9 +37,6 @@ export function getVisiblePeerEthereumAddresses(): Array<{ userId: string }> { } return result } -export function getConnectedPeerCount() { - return peerInformationMap.size -} ;(globalThis as any).peerMap = peerInformationMap diff --git a/browser-interface/packages/shared/comms/sagas.ts b/browser-interface/packages/shared/comms/sagas.ts index d6d646164b..3c5776887f 100644 --- a/browser-interface/packages/shared/comms/sagas.ts +++ b/browser-interface/packages/shared/comms/sagas.ts @@ -14,7 +14,7 @@ import { notifyStatusThroughChat } from 'shared/chat' import { selectAndReconnectRealm } from 'shared/dao/sagas' import { commsEstablished, establishingComms, FATAL_ERROR } from 'shared/loading/types' import { waitForMetaConfigurationInitialization } from 'shared/meta/sagas' -import { getFeatureFlagEnabled, getMaxVisiblePeers } from 'shared/meta/selectors' +import { getMaxVisiblePeers } from 'shared/meta/selectors' import { incrementCounter } from 'shared/analytics/occurences' import type { SendProfileToRenderer } from 'shared/profiles/actions' import { DEPLOY_PROFILE_SUCCESS, SEND_PROFILE_TO_RENDERER_REQUEST } from 'shared/profiles/actions' @@ -25,7 +25,7 @@ import { getFetchContentUrlPrefixFromRealmAdapter, getRealmAdapter } from 'share import { waitForRealm } from 'shared/realm/waitForRealmAdapter' import type { IRealmAdapter } from 'shared/realm/types' import { USER_AUTHENTICATED } from 'shared/session/actions' -import { measurePingTime, measurePingTimePercentages, overrideCommsProtocol } from 'shared/session/getPerformanceInfo' +import { overrideCommsProtocol } from 'shared/session/getPerformanceInfo' import { getCurrentIdentity, isLoginCompleted } from 'shared/session/selectors' import type { ExplorerIdentity } from 'shared/session/types' import { lastPlayerPositionReport, positionObservable, PositionReport } from 'shared/world/positionThings' @@ -43,12 +43,12 @@ import { LivekitAdapter } from './adapters/LivekitAdapter' import { OfflineAdapter } from './adapters/OfflineAdapter' import { SimulationRoom } from './adapters/SimulatorAdapter' import { WebSocketAdapter } from './adapters/WebSocketAdapter' -import { bindHandlersToCommsContext, createSendMyProfileOverCommsChannel, sendPing } from './handlers' +import { bindHandlersToCommsContext, createSendMyProfileOverCommsChannel } from './handlers' import type { RoomConnection } from './interface' import { positionReportToCommsPositionRfc4 } from './interface/utils' import { commsLogger } from './logger' import { Rfc4RoomConnection } from './logic/rfc-4-room-connection' -import { getConnectedPeerCount, processAvatarVisibility } from './peers' +import { processAvatarVisibility } from './peers' import { getCommsRoom, getSceneRooms, reconnectionState } from './selectors' import { RootState } from 'shared/store/rootTypes' import { now } from 'lib/javascript/now' @@ -92,7 +92,6 @@ export function* commsSaga() { yield fork(handleAnnounceProfile) yield fork(initAvatarVisibilityProcess) yield fork(handleCommsReconnectionInterval) - yield fork(pingerProcess) yield fork(reportPositionSaga) yield fork(sceneRoomComms) @@ -158,44 +157,6 @@ function* reportPositionSaga() { positionObservable.remove(observer) } -/** - * This saga sends random pings to all peers if the conditions are met. - */ -function* pingerProcess() { - yield call(waitForMetaConfigurationInitialization) - - const enabled: boolean = yield select(getFeatureFlagEnabled, 'ping_enabled') - - if (enabled) { - while (true) { - yield delay(15_000 + Math.random() * 60_000) - - const responses = new Map() - const expectedResponses = getConnectedPeerCount() - - yield call(sendPing, (dt, address) => { - const list = responses.get(address) || [] - responses.set(address, list) - list.push(dt) - measurePingTime(dt) - if (list.length > 1) { - incrementCounter('pong_duplicated_response_counter') - } - }) - - yield delay(15_000) - - // measure the response ratio - if (expectedResponses) { - measurePingTimePercentages(Math.round((responses.size / expectedResponses) * 100)) - } - - incrementCounter('pong_expected_counter', expectedResponses) - incrementCounter('pong_given_counter', responses.size) - } - } -} - /** * This saga handles the action to connect a specific comms * adapter. From f1f110610b1e5f8c7404201035e2b20c96ad1695 Mon Sep 17 00:00:00 2001 From: Hugo Arregui Date: Tue, 26 Dec 2023 12:12:14 -0300 Subject: [PATCH 08/23] wip --- .../shared/comms/adapters/LivekitAdapter.ts | 4 + .../shared/comms/adapters/OfflineAdapter.ts | 1 + .../shared/comms/adapters/SimulatorAdapter.ts | 8 ++ .../shared/comms/adapters/WebSocketAdapter.ts | 5 + .../packages/shared/comms/adapters/types.ts | 2 + .../packages/shared/comms/handlers.ts | 31 ++-- .../packages/shared/comms/interface/index.ts | 2 + .../comms/logic/rfc-4-room-connection.ts | 4 + .../packages/shared/comms/peers.ts | 134 +++++++++++------- .../packages/shared/comms/sagas.ts | 2 +- .../packages/shared/comms/selectors.ts | 1 + .../test/sceneEvents/visibleAvatars.spec.ts | 4 +- 12 files changed, 129 insertions(+), 69 deletions(-) diff --git a/browser-interface/packages/shared/comms/adapters/LivekitAdapter.ts b/browser-interface/packages/shared/comms/adapters/LivekitAdapter.ts index 7c85fafbe3..920867bdc3 100644 --- a/browser-interface/packages/shared/comms/adapters/LivekitAdapter.ts +++ b/browser-interface/packages/shared/comms/adapters/LivekitAdapter.ts @@ -164,4 +164,8 @@ export class LivekitAdapter implements MinimumCommunicationsAdapter { return result } + + async getParticipants(): Promise { + return Array.from(this.room.participants.values()).map((p) => p.identity) + } } diff --git a/browser-interface/packages/shared/comms/adapters/OfflineAdapter.ts b/browser-interface/packages/shared/comms/adapters/OfflineAdapter.ts index d880aaf50a..67f6896400 100644 --- a/browser-interface/packages/shared/comms/adapters/OfflineAdapter.ts +++ b/browser-interface/packages/shared/comms/adapters/OfflineAdapter.ts @@ -13,4 +13,5 @@ export class OfflineAdapter implements MinimumCommunicationsAdapter { async disconnect(_error?: Error | undefined): Promise {} send(_data: Uint8Array, _hints: SendHints): void {} async connect(): Promise {} + async getParticipants() { return [] } } diff --git a/browser-interface/packages/shared/comms/adapters/SimulatorAdapter.ts b/browser-interface/packages/shared/comms/adapters/SimulatorAdapter.ts index e553e864e9..e708d43f29 100644 --- a/browser-interface/packages/shared/comms/adapters/SimulatorAdapter.ts +++ b/browser-interface/packages/shared/comms/adapters/SimulatorAdapter.ts @@ -43,6 +43,7 @@ export class SimulationRoom implements RoomConnection { params: URLSearchParams constructor(param: string) { + const peers = this.peers this.params = new URLSearchParams(param.startsWith('?') ? param.substring(1) : param) this.tick = setInterval(this.update.bind(this), 60) this.roomConnection = new Rfc4RoomConnection({ @@ -52,6 +53,9 @@ export class SimulationRoom implements RoomConnection { async disconnect(_error?: Error): Promise {}, async createVoiceHandler() { throw new Error('not implemented') + }, + async getParticipants() { + return Array.from(peers.keys()) } }) } @@ -200,4 +204,8 @@ export class SimulationRoom implements RoomConnection { async connect(): Promise { await Promise.all(new Array(100).fill(0).map(() => this.spawnPeer())) } + + async getParticipants() { + return this.roomConnection.getParticipants() + } } diff --git a/browser-interface/packages/shared/comms/adapters/WebSocketAdapter.ts b/browser-interface/packages/shared/comms/adapters/WebSocketAdapter.ts index 68c8cada01..5622d650e6 100644 --- a/browser-interface/packages/shared/comms/adapters/WebSocketAdapter.ts +++ b/browser-interface/packages/shared/comms/adapters/WebSocketAdapter.ts @@ -231,4 +231,9 @@ export class WebSocketAdapter implements MinimumCommunicationsAdapter { }) .catch(console.error) } + + async getParticipants(): Promise { + // TODO + throw new Error('Not implemented') + } } diff --git a/browser-interface/packages/shared/comms/adapters/types.ts b/browser-interface/packages/shared/comms/adapters/types.ts index 6879eade6e..cc3d7db853 100644 --- a/browser-interface/packages/shared/comms/adapters/types.ts +++ b/browser-interface/packages/shared/comms/adapters/types.ts @@ -29,6 +29,8 @@ export interface MinimumCommunicationsAdapter { events: Emitter createVoiceHandler(): Promise + + getParticipants(): Promise } export type CommsAdapterEvents = { diff --git a/browser-interface/packages/shared/comms/handlers.ts b/browser-interface/packages/shared/comms/handlers.ts index c844fee1ff..cd2eade62a 100644 --- a/browser-interface/packages/shared/comms/handlers.ts +++ b/browser-interface/packages/shared/comms/handlers.ts @@ -15,16 +15,15 @@ import { processVoiceFragment } from 'shared/voiceChat/handlers' import { isBlockedOrBanned } from 'shared/voiceChat/selectors' import { messageReceived } from '../chat/actions' import { handleRoomDisconnection } from './actions' -import { AdapterDisconnectedEvent, PeerDisconnectedEvent } from './adapters/types' +import { AdapterDisconnectedEvent } from './adapters/types' import { RoomConnection } from './interface' import { AvatarMessageType, Package } from './interface/types' import { avatarMessageObservable, - ensureTrackingUniqueAndLatest, getPeer, + onRoomLeft, receiveUserPosition, - removeAllPeers, - removePeerByAddress, + onPeerDisconnected, setupPeer as setupPeerTrackingInfo } from './peers' import { scenesSubscribedToCommsEvents } from './sceneSubscriptions' @@ -40,11 +39,7 @@ const versionUpdateOverCommsChannel = new Observable() const receiveProfileOverCommsChannel = new Observable() const sendMyProfileOverCommsChannel = new Observable>() -export async function bindHandlersToCommsContext(room: RoomConnection, islandRoom: boolean = true) { - if (islandRoom) { - removeAllPeers() - } - +export async function bindHandlersToCommsContext(room: RoomConnection) { // RFC4 messages room.events.on('position', (e) => processPositionMessage(room, e)) room.events.on('profileMessage', processProfileUpdatedMessage) @@ -60,7 +55,7 @@ export async function bindHandlersToCommsContext(room: RoomConnection, islandRoo }) // transport messages - room.events.on('PEER_DISCONNECTED', handleDisconnectPeer) + room.events.on('PEER_DISCONNECTED', onPeerDisconnected) room.events.on('DISCONNECTION', (event) => handleDisconnectionEvent(event, room)) } @@ -83,7 +78,15 @@ export async function requestProfileFromPeers( return false } -function handleDisconnectionEvent(data: AdapterDisconnectedEvent, room: RoomConnection) { +async function handleDisconnectionEvent(data: AdapterDisconnectedEvent, room: RoomConnection) { + + try { + await onRoomLeft(room) + } catch(err) { + console.error(err) + // TODO: handle this + } + store.dispatch(handleRoomDisconnection(room)) // when we are kicked, the explorer should re-load, or maybe go to offline~offline realm @@ -95,10 +98,6 @@ function handleDisconnectionEvent(data: AdapterDisconnectedEvent, room: RoomConn } } -function handleDisconnectPeer(data: PeerDisconnectedEvent) { - removePeerByAddress(data.address) -} - // TODO: use position message to setupPeerTrackingInfo function processProfileUpdatedMessage(message: Package) { const peerTrackingInfo = setupPeerTrackingInfo(message.address) @@ -108,8 +107,6 @@ function processProfileUpdatedMessage(message: Package peerTrackingInfo.lastProfileVersion) { peerTrackingInfo.lastProfileVersion = message.data.profileVersion - // remove duplicates - ensureTrackingUniqueAndLatest(peerTrackingInfo) versionUpdateOverCommsChannel.notifyObservers({ userId: message.address, version: message.data.profileVersion }) } } diff --git a/browser-interface/packages/shared/comms/interface/index.ts b/browser-interface/packages/shared/comms/interface/index.ts index fe277ce663..ffba64785d 100644 --- a/browser-interface/packages/shared/comms/interface/index.ts +++ b/browser-interface/packages/shared/comms/interface/index.ts @@ -33,4 +33,6 @@ export interface RoomConnection { sendVoiceMessage(message: proto.Voice): Promise createVoiceHandler(): Promise + + getParticipants(): Promise } diff --git a/browser-interface/packages/shared/comms/logic/rfc-4-room-connection.ts b/browser-interface/packages/shared/comms/logic/rfc-4-room-connection.ts index d7a7e4ef9c..6ce4f74c39 100644 --- a/browser-interface/packages/shared/comms/logic/rfc-4-room-connection.ts +++ b/browser-interface/packages/shared/comms/logic/rfc-4-room-connection.ts @@ -75,6 +75,10 @@ export class Rfc4RoomConnection implements RoomConnection { await this.transport.disconnect() } + async getParticipants(): Promise { + return this.transport.getParticipants() + } + private handleMessage({ data, address }: AdapterMessageEvent) { const { message } = proto.Packet.decode(data) diff --git a/browser-interface/packages/shared/comms/peers.ts b/browser-interface/packages/shared/comms/peers.ts index 6ca308cf82..e87731d8c3 100644 --- a/browser-interface/packages/shared/comms/peers.ts +++ b/browser-interface/packages/shared/comms/peers.ts @@ -7,7 +7,8 @@ import { getProfileFromStore } from 'shared/profiles/selectors' import { profileToRendererFormat } from 'lib/decentraland/profiles/transformations/profileToRendererFormat' import { store } from 'shared/store/isolatedStore' import { lastPlayerPositionReport } from 'shared/world/positionThings' -import { MORDOR_POSITION_RFC4 } from './const' +import { getIslandRoom, getSceneRooms} from 'shared/comms/selectors' +// import { MORDOR_POSITION_RFC4 } from './const' import type { AvatarMessage, PeerInformation } from './interface/types' import { AvatarMessageType } from './interface/types' import { @@ -16,12 +17,14 @@ import { positionReportToCommsPositionRfc4, squareDistanceRfc4 } from './interface/utils' +import { RoomConnection } from './interface' +import { PeerDisconnectedEvent } from './adapters/types' /** * peerInformationMap contains data received of the current peers that we have * information of. */ -const peerInformationMap = new Map() +let peerInformationMap = new Map() export const avatarMessageObservable = new Observable() export const avatarVersionUpdateObservable = new Observable<{ userId: string; version: number }>() @@ -44,18 +47,18 @@ export function getVisiblePeerEthereumAddresses(): Array<{ userId: string }> { * Removes both the peer information and the Avatar from the world. * @param address */ -export function removePeerByAddress(address: string): boolean { - const peer = peerInformationMap.get(address.toLowerCase()) - if (peer) { - peerInformationMap.delete(address.toLowerCase()) - avatarMessageObservable.notifyObservers({ - type: AvatarMessageType.USER_REMOVED, - userId: peer.ethereumAddress - }) - return true - } - return false -} +// export function removePeerByAddress(address: string): boolean { +// const peer = peerInformationMap.get(address.toLowerCase()) +// if (peer) { +// peerInformationMap.delete(address.toLowerCase()) +// avatarMessageObservable.notifyObservers({ +// type: AvatarMessageType.USER_REMOVED, +// userId: peer.ethereumAddress +// }) +// return true +// } +// return false +// } /** * This function is used to get the current user's information. The result is read-only. @@ -139,14 +142,15 @@ export function receiveUserTalking(address: string, talking: boolean) { } export function receiveUserPosition(address: string, position: rfc4.Position) { - if ( - position.positionX === MORDOR_POSITION_RFC4.positionX && - position.positionY === MORDOR_POSITION_RFC4.positionY && - position.positionZ === MORDOR_POSITION_RFC4.positionZ - ) { - removePeerByAddress(address) - return - } + // TODO: WTF?? + // if ( + // position.positionX === MORDOR_POSITION_RFC4.positionX && + // position.positionY === MORDOR_POSITION_RFC4.positionY && + // position.positionZ === MORDOR_POSITION_RFC4.positionZ + // ) { + // removePeerByAddress(address) + // return + // } const peer = setupPeer(address) peer.lastUpdate = Date.now() @@ -194,39 +198,72 @@ export function receiveUserVisible(address: string, visible: boolean) { } } -export function removeAllPeers() { - for (const alias of peerInformationMap.keys()) { - removePeerByAddress(alias) +function getActiveRooms(): RoomConnection[] { + const state = store.getState() + const rooms = Array.from(getSceneRooms(state).values()) + const islandRoom = getIslandRoom(state) + if (islandRoom) { + rooms.push(islandRoom) } + + return rooms } -/** - * Ensures that there is only one peer tracking info for this identity. - * Returns true if this is the latest update and the one that remains. - * - * TODO(Mendez 24/04/2022): wtf does this function do? - */ -export function ensureTrackingUniqueAndLatest(peer: PeerInformation) { - let currentPeer = peer +export async function onRoomLeft(oldRoom: RoomConnection) { + const rooms = getActiveRooms() + const newPeerInformationMap = new Map() + + for (const room of rooms) { + if (room.id === oldRoom.id) { + continue + } + + for (const participant of await room.getParticipants()) { + const info = peerInformationMap.get(participant) + if (info) { + newPeerInformationMap.set(participant, info) + } + } + } - peerInformationMap.forEach((info, address) => { - if (info.ethereumAddress === currentPeer.ethereumAddress && address !== peer.ethereumAddress) { - if (info.lastProfileVersion < currentPeer.lastProfileVersion) { - removePeerByAddress(address) - } else if (info.lastProfileVersion > currentPeer.lastProfileVersion) { - removePeerByAddress(currentPeer.ethereumAddress) + for (const participant of peerInformationMap.keys()) { + if (!newPeerInformationMap.has(participant)) { + avatarMessageObservable.notifyObservers({ + type: AvatarMessageType.USER_REMOVED, + userId: participant + }) + } + } + + peerInformationMap = newPeerInformationMap +} - info.position = info.position || currentPeer.position - info.visible = info.visible || currentPeer.visible +export async function onPeerDisconnected(data: PeerDisconnectedEvent) { + const address = data.address + const rooms = getActiveRooms() - currentPeer = info + for (const room of rooms) { + for (const participant of await room.getParticipants()) { + if (participant === address) { + return } } - }) + } - return currentPeer + peerInformationMap.delete(address) + avatarMessageObservable.notifyObservers({ + type: AvatarMessageType.USER_REMOVED, + userId: address + }) } + +// export function removeAllPeers() { +// for (const alias of peerInformationMap.keys()) { +// removePeerByAddress(alias) +// } +// } + export function processAvatarVisibility(maxVisiblePeers: number, myAddress: string | undefined) { if (!lastPlayerPositionReport) return const pos = positionReportToCommsPositionRfc4(lastPlayerPositionReport) @@ -235,7 +272,7 @@ export function processAvatarVisibility(maxVisiblePeers: number, myAddress: stri type ProcessingPeerInfo = { alias: string squareDistance: number - visible: boolean + // visible: boolean } const visiblePeers: ProcessingPeerInfo[] = [] @@ -245,13 +282,11 @@ export function processAvatarVisibility(maxVisiblePeers: number, myAddress: stri const msSinceLastUpdate = now - trackingInfo.lastUpdate if (msSinceLastUpdate > commConfigurations.peerTtlMs) { - removePeerByAddress(peerAlias) + receiveUserVisible(peerAlias, false) continue } if (myAddress && trackingInfo.ethereumAddress === myAddress) { - // If we are tracking a peer that is ourselves, we remove it - removePeerByAddress(peerAlias) continue } @@ -267,7 +302,8 @@ export function processAvatarVisibility(maxVisiblePeers: number, myAddress: stri visiblePeers.push({ squareDistance: squareDistanceRfc4(pos, trackingInfo.position), alias: peerAlias, - visible: trackingInfo.visible || false + // NOTE (hugo y Gon): this is not used below, should it be? + // visible: trackingInfo.visible || false }) } diff --git a/browser-interface/packages/shared/comms/sagas.ts b/browser-interface/packages/shared/comms/sagas.ts index 3c5776887f..4085d85413 100644 --- a/browser-interface/packages/shared/comms/sagas.ts +++ b/browser-interface/packages/shared/comms/sagas.ts @@ -582,6 +582,6 @@ function* connectSceneToComms(sceneId: string) { const sceneRoomConnection = yield call(connectAdapter, response.json.adapter, identity, sceneId, false) globalThis.__DEBUG_SCENE_ADAPTER = sceneRoomConnection yield apply(sceneRoomConnection, sceneRoomConnection.connect, []) - yield call(bindHandlersToCommsContext, sceneRoomConnection, false) + yield call(bindHandlersToCommsContext, sceneRoomConnection) yield put(setSceneRoomConnection(sceneId, sceneRoomConnection)) } diff --git a/browser-interface/packages/shared/comms/selectors.ts b/browser-interface/packages/shared/comms/selectors.ts index eb7cb9902a..c4c4734634 100644 --- a/browser-interface/packages/shared/comms/selectors.ts +++ b/browser-interface/packages/shared/comms/selectors.ts @@ -20,6 +20,7 @@ import { export const getCommsIsland = (store: RootCommsState): string | undefined => store.comms.island export const getSceneRoomComms = (state: RootCommsState): RoomConnection | undefined => state.comms.scene export const getSceneRooms = (state: RootCommsState): Map => state.comms.scenes +export const getIslandRoom = (state: RootCommsState): RoomConnection | undefined => state.comms.context export const getCommsRoom = (state: RootCommsState): RoomConnection | undefined => { const islandRoom = state.comms.context diff --git a/browser-interface/test/sceneEvents/visibleAvatars.spec.ts b/browser-interface/test/sceneEvents/visibleAvatars.spec.ts index 1af9f830e9..e23c93bebf 100644 --- a/browser-interface/test/sceneEvents/visibleAvatars.spec.ts +++ b/browser-interface/test/sceneEvents/visibleAvatars.spec.ts @@ -66,7 +66,7 @@ describe('Avatar observable', () => { afterEach(() => { // clear visible avatars cache - peers.removeAllPeers() + // peers.removeAllPeers() sinon.restore() sinon.reset() @@ -102,7 +102,7 @@ describe('Avatar observable', () => { peers.receiveUserVisible(userA, true) peers.receiveUserVisible(userB, true) expect(getVisibleAvatarsUserId()).to.eql([userA, userB]) - peers.removePeerByAddress(userA) + // peers.removePeerByAddress(userA) expect(lastEvent).to.deep.eq({ eventType: 'playerDisconnected', payload: { userId: userA } }) expect(getVisibleAvatarsUserId()).to.eql([userB]) }) From 52ddbd8b63f6f38f2e00dd22e009970f3b86f8c1 Mon Sep 17 00:00:00 2001 From: Hugo Arregui Date: Tue, 26 Dec 2023 12:39:16 -0300 Subject: [PATCH 09/23] ignore ping/pong messages --- .../packages/shared/comms/adapters/LivekitAdapter.ts | 4 ++-- browser-interface/packages/shared/comms/handlers.ts | 5 ++++- browser-interface/packages/shared/comms/sagas.ts | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/browser-interface/packages/shared/comms/adapters/LivekitAdapter.ts b/browser-interface/packages/shared/comms/adapters/LivekitAdapter.ts index 920867bdc3..6174125895 100644 --- a/browser-interface/packages/shared/comms/adapters/LivekitAdapter.ts +++ b/browser-interface/packages/shared/comms/adapters/LivekitAdapter.ts @@ -107,7 +107,7 @@ export class LivekitAdapter implements MinimumCommunicationsAdapter { this.config.logger.log(`Skip sending message because connection state is ${state} ${this.room.name}`) return } - this.config.logger.log('Sending message', this.room.name) + // this.config.logger.log('Sending message', this.room.name) try { await this.room.localParticipant.publishData(data, reliable ? DataPacket_Kind.RELIABLE : DataPacket_Kind.LOSSY) @@ -129,11 +129,11 @@ export class LivekitAdapter implements MinimumCommunicationsAdapter { } async do_disconnect(kicked: boolean) { - this.config.logger.log('[BOEDO]: do_disconnect', this.room.name) if (this.disposed) { return } + this.config.logger.log('[BOEDO]: do_disconnect', this.room.name) this.disposed = true await this.room.disconnect().catch(commsLogger.error) this.events.emit('DISCONNECTION', { kicked }) diff --git a/browser-interface/packages/shared/comms/handlers.ts b/browser-interface/packages/shared/comms/handlers.ts index cd2eade62a..cd825d429d 100644 --- a/browser-interface/packages/shared/comms/handlers.ts +++ b/browser-interface/packages/shared/comms/handlers.ts @@ -140,7 +140,10 @@ function processChatMessage(message: Package) { senderPeer.lastUpdate = Date.now() if (senderPeer.ethereumAddress) { - if (message.data.message.startsWith('␐')) { + if (message.data.message.startsWith('␆') /* pong */ || + message.data.message.startsWith('␑') /* ping */) { + // TODO: remove this + } else if (message.data.message.startsWith('␐')) { const [id, timestamp] = message.data.message.split(' ') avatarMessageObservable.notifyObservers({ diff --git a/browser-interface/packages/shared/comms/sagas.ts b/browser-interface/packages/shared/comms/sagas.ts index 4085d85413..9ed441f938 100644 --- a/browser-interface/packages/shared/comms/sagas.ts +++ b/browser-interface/packages/shared/comms/sagas.ts @@ -549,7 +549,7 @@ function* checkDisconnectScene( const sceneRooms: ReturnType = yield select(getSceneRooms) const oldRoom = sceneRooms.get(oldSceneId) const timeout = setTimeout(() => { - console.log('[BOEDO SceneComms]: disconnectSceneComms', oldSceneId) + console.log('[BOEDO SceneComms]: disconnectSceneComms', oldSceneId, !!oldRoom) void oldRoom?.disconnect() commsSceneToRemove.delete(oldSceneId) }, 1000) From ff1d308a13296ad72e2837c0bfe23183b805c574 Mon Sep 17 00:00:00 2001 From: Gonzalo DCL Date: Tue, 26 Dec 2023 12:52:00 -0300 Subject: [PATCH 10/23] logs --- .../shared/comms/adapters/LivekitAdapter.ts | 1 - .../shared/comms/logic/rfc-4-room-connection.ts | 4 ++-- browser-interface/packages/shared/comms/peers.ts | 2 +- browser-interface/packages/shared/comms/sagas.ts | 13 +++++++++---- .../packages/shared/comms/selectors.ts | 8 ++++---- 5 files changed, 16 insertions(+), 12 deletions(-) diff --git a/browser-interface/packages/shared/comms/adapters/LivekitAdapter.ts b/browser-interface/packages/shared/comms/adapters/LivekitAdapter.ts index 6174125895..f20229e8c0 100644 --- a/browser-interface/packages/shared/comms/adapters/LivekitAdapter.ts +++ b/browser-interface/packages/shared/comms/adapters/LivekitAdapter.ts @@ -107,7 +107,6 @@ export class LivekitAdapter implements MinimumCommunicationsAdapter { this.config.logger.log(`Skip sending message because connection state is ${state} ${this.room.name}`) return } - // this.config.logger.log('Sending message', this.room.name) try { await this.room.localParticipant.publishData(data, reliable ? DataPacket_Kind.RELIABLE : DataPacket_Kind.LOSSY) diff --git a/browser-interface/packages/shared/comms/logic/rfc-4-room-connection.ts b/browser-interface/packages/shared/comms/logic/rfc-4-room-connection.ts index 6ce4f74c39..64360bee66 100644 --- a/browser-interface/packages/shared/comms/logic/rfc-4-room-connection.ts +++ b/browser-interface/packages/shared/comms/logic/rfc-4-room-connection.ts @@ -24,7 +24,7 @@ export class Rfc4RoomConnection implements RoomConnection { } async connect(): Promise { - // console.log('[RoomConnection Comms]: connect', this.id) + console.log('[RoomConnection Comms]: connect', this.id) await this.transport.connect() } @@ -71,7 +71,7 @@ export class Rfc4RoomConnection implements RoomConnection { } async disconnect() { - // console.log('[RoomConnection Comms]: disconnect', this.id) + console.log('[RoomConnection Comms]: disconnect', this.id) await this.transport.disconnect() } diff --git a/browser-interface/packages/shared/comms/peers.ts b/browser-interface/packages/shared/comms/peers.ts index e87731d8c3..d1d035a0d7 100644 --- a/browser-interface/packages/shared/comms/peers.ts +++ b/browser-interface/packages/shared/comms/peers.ts @@ -217,7 +217,7 @@ export async function onRoomLeft(oldRoom: RoomConnection) { if (room.id === oldRoom.id) { continue } - + for (const participant of await room.getParticipants()) { const info = peerInformationMap.get(participant) if (info) { diff --git a/browser-interface/packages/shared/comms/sagas.ts b/browser-interface/packages/shared/comms/sagas.ts index 9ed441f938..6a1e4f9ddb 100644 --- a/browser-interface/packages/shared/comms/sagas.ts +++ b/browser-interface/packages/shared/comms/sagas.ts @@ -25,7 +25,7 @@ import { getFetchContentUrlPrefixFromRealmAdapter, getRealmAdapter } from 'share import { waitForRealm } from 'shared/realm/waitForRealmAdapter' import type { IRealmAdapter } from 'shared/realm/types' import { USER_AUTHENTICATED } from 'shared/session/actions' -import { overrideCommsProtocol } from 'shared/session/getPerformanceInfo' +import { overrideCommsProtocol } from 'shared/session/getPerformanceInfo' import { getCurrentIdentity, isLoginCompleted } from 'shared/session/selectors' import type { ExplorerIdentity } from 'shared/session/types' import { lastPlayerPositionReport, positionObservable, PositionReport } from 'shared/world/positionThings' @@ -43,12 +43,12 @@ import { LivekitAdapter } from './adapters/LivekitAdapter' import { OfflineAdapter } from './adapters/OfflineAdapter' import { SimulationRoom } from './adapters/SimulatorAdapter' import { WebSocketAdapter } from './adapters/WebSocketAdapter' -import { bindHandlersToCommsContext, createSendMyProfileOverCommsChannel } from './handlers' +import { bindHandlersToCommsContext, createSendMyProfileOverCommsChannel } from './handlers' import type { RoomConnection } from './interface' import { positionReportToCommsPositionRfc4 } from './interface/utils' import { commsLogger } from './logger' import { Rfc4RoomConnection } from './logic/rfc-4-room-connection' -import { processAvatarVisibility } from './peers' +import { processAvatarVisibility } from './peers' import { getCommsRoom, getSceneRooms, reconnectionState } from './selectors' import { RootState } from 'shared/store/rootTypes' import { now } from 'lib/javascript/now' @@ -166,7 +166,12 @@ function* handleConnectToComms(action: ConnectToCommsAction) { const identity: ExplorerIdentity = yield select(getCurrentIdentity) yield put(setCommsIsland(action.payload.event.islandId)) - const adapter: RoomConnection = yield call(connectAdapter, action.payload.event.connStr, identity) + const adapter: RoomConnection = yield call( + connectAdapter, + action.payload.event.connStr, + identity, + action.payload.event.islandId + ) globalThis.__DEBUG_ISLAND_ADAPTER = adapter diff --git a/browser-interface/packages/shared/comms/selectors.ts b/browser-interface/packages/shared/comms/selectors.ts index c4c4734634..2d473c2aff 100644 --- a/browser-interface/packages/shared/comms/selectors.ts +++ b/browser-interface/packages/shared/comms/selectors.ts @@ -40,22 +40,22 @@ export const getCommsRoom = (state: RootCommsState): RoomConnection | undefined // TBD: This should be only be sent by the island ? // We may remove this before reach production, but to think about it sendProfileMessage: async (profile: AnnounceProfileVersion) => { - const island = islandRoom.sendProfileMessage(profile) + const island = Promise.resolve() || islandRoom.sendProfileMessage(profile) const scene = sceneRoom?.sendProfileMessage(profile) await Promise.all([island, scene]) }, sendProfileRequest: async (request: ProfileRequest) => { - const island = islandRoom.sendProfileRequest(request) + const island = Promise.resolve() || islandRoom.sendProfileRequest(request) const scene = sceneRoom?.sendProfileRequest(request) await Promise.all([island, scene]) }, sendProfileResponse: async (response: ProfileResponse) => { - const island = islandRoom.sendProfileResponse(response) + const island = Promise.resolve() || islandRoom.sendProfileResponse(response) const scene = sceneRoom?.sendProfileResponse(response) await Promise.all([island, scene]) }, sendPositionMessage: async (position: Omit) => { - const island = islandRoom.sendPositionMessage(position) + const island = Promise.resolve() || islandRoom.sendPositionMessage(position) const scene = sceneRoom?.sendPositionMessage(position) await Promise.all([island, scene]) }, From 2826913a2fab8ad3f54fe981164b012c2610ded6 Mon Sep 17 00:00:00 2001 From: Gonzalo DCL Date: Tue, 26 Dec 2023 16:02:40 -0300 Subject: [PATCH 11/23] fix teleport to --- browser-interface/packages/shared/comms/peers.ts | 12 +++++++----- browser-interface/packages/shared/comms/sagas.ts | 8 +++++++- .../genesis-city-loader-impl/downloadManager.ts | 1 + .../scene-loader/genesis-city-loader-impl/index.ts | 1 + 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/browser-interface/packages/shared/comms/peers.ts b/browser-interface/packages/shared/comms/peers.ts index d1d035a0d7..2d973d5aed 100644 --- a/browser-interface/packages/shared/comms/peers.ts +++ b/browser-interface/packages/shared/comms/peers.ts @@ -7,7 +7,7 @@ import { getProfileFromStore } from 'shared/profiles/selectors' import { profileToRendererFormat } from 'lib/decentraland/profiles/transformations/profileToRendererFormat' import { store } from 'shared/store/isolatedStore' import { lastPlayerPositionReport } from 'shared/world/positionThings' -import { getIslandRoom, getSceneRooms} from 'shared/comms/selectors' +import { getIslandRoom, getSceneRooms } from 'shared/comms/selectors' // import { MORDOR_POSITION_RFC4 } from './const' import type { AvatarMessage, PeerInformation } from './interface/types' import { AvatarMessageType } from './interface/types' @@ -41,7 +41,7 @@ export function getVisiblePeerEthereumAddresses(): Array<{ userId: string }> { return result } -;(globalThis as any).peerMap = peerInformationMap +;(globalThis as any).peerMap = getAllPeers /** * Removes both the peer information and the Avatar from the world. @@ -80,6 +80,7 @@ export function setupPeer(address: string): PeerInformation { const ethereumAddress = address.toLowerCase() if (!peerInformationMap.has(ethereumAddress)) { + console.log('[BOEDO] Adding Peer information', ethereumAddress) const peer: PeerInformation = { ethereumAddress, lastPositionIndex: 0, @@ -212,7 +213,7 @@ function getActiveRooms(): RoomConnection[] { export async function onRoomLeft(oldRoom: RoomConnection) { const rooms = getActiveRooms() const newPeerInformationMap = new Map() - + console.log('[onRoomLeft] rooms', rooms) for (const room of rooms) { if (room.id === oldRoom.id) { continue @@ -221,6 +222,7 @@ export async function onRoomLeft(oldRoom: RoomConnection) { for (const participant of await room.getParticipants()) { const info = peerInformationMap.get(participant) if (info) { + console.log('[onRoomLeft] addparticipant', participant) newPeerInformationMap.set(participant, info) } } @@ -228,6 +230,7 @@ export async function onRoomLeft(oldRoom: RoomConnection) { for (const participant of peerInformationMap.keys()) { if (!newPeerInformationMap.has(participant)) { + console.log('[onRoomLeft] removeUser', participant) avatarMessageObservable.notifyObservers({ type: AvatarMessageType.USER_REMOVED, userId: participant @@ -257,7 +260,6 @@ export async function onPeerDisconnected(data: PeerDisconnectedEvent) { }) } - // export function removeAllPeers() { // for (const alias of peerInformationMap.keys()) { // removePeerByAddress(alias) @@ -301,7 +303,7 @@ export function processAvatarVisibility(maxVisiblePeers: number, myAddress: stri visiblePeers.push({ squareDistance: squareDistanceRfc4(pos, trackingInfo.position), - alias: peerAlias, + alias: peerAlias // NOTE (hugo y Gon): this is not used below, should it be? // visible: trackingInfo.visible || false }) diff --git a/browser-interface/packages/shared/comms/sagas.ts b/browser-interface/packages/shared/comms/sagas.ts index 6a1e4f9ddb..a958a314d8 100644 --- a/browser-interface/packages/shared/comms/sagas.ts +++ b/browser-interface/packages/shared/comms/sagas.ts @@ -60,6 +60,8 @@ import { SET_PARCEL_POSITION } from 'shared/scene-loader/actions' import { getSceneLoader } from '../scene-loader/selectors' import { Vector2 } from '../protocol/decentraland/common/vectors.gen' import { DISABLE_SCENE_ROOM } from '../../config' +import { encodeParcelPosition } from '../../lib/decentraland/parcels/encodeParcelPosition' +import { SetDesiredScenesCommand } from '../scene-loader/types' const TIME_BETWEEN_PROFILE_RESPONSES = 1000 // this interval should be fast because this will be the delay other people around @@ -198,6 +200,7 @@ async function connectAdapter( id: string = 'island', dispatchAction = true ): Promise { + console.log('[connectAdapter] ', { connStr, identity, id }) const ix = connStr.indexOf(':') const protocol = connStr.substring(0, ix) const url = connStr.substring(ix + 1) @@ -526,7 +529,10 @@ function* sceneRoomComms() { const sceneLoader: ReturnType = yield select(getSceneLoader) if (!sceneLoader) continue if (reason.newParcel) { - const sceneId = yield call(sceneLoader.getSceneId!, reason.newParcel.payload.position) + const scenes: SetDesiredScenesCommand = yield call(sceneLoader.fetchScenesByLocation, [ + encodeParcelPosition(reason.newParcel.payload.position) + ]) + const sceneId = scenes.scenes[0].id // We are still on the same scene. if (sceneId === currentSceneId) continue const oldSceneId = currentSceneId diff --git a/browser-interface/packages/shared/scene-loader/genesis-city-loader-impl/downloadManager.ts b/browser-interface/packages/shared/scene-loader/genesis-city-loader-impl/downloadManager.ts index 0037320eb1..290f650977 100644 --- a/browser-interface/packages/shared/scene-loader/genesis-city-loader-impl/downloadManager.ts +++ b/browser-interface/packages/shared/scene-loader/genesis-city-loader-impl/downloadManager.ts @@ -27,6 +27,7 @@ export class SceneDataDownloadManager { } async resolveEntitiesByPointer(pointers: string[]): Promise> { + console.log('[DownloadManager]resolveEntitiesByPointer ', pointers) const futures: Promise[] = [] const missingPointers: string[] = [] diff --git a/browser-interface/packages/shared/scene-loader/genesis-city-loader-impl/index.ts b/browser-interface/packages/shared/scene-loader/genesis-city-loader-impl/index.ts index 39809ae3a9..b9de9f2fd8 100644 --- a/browser-interface/packages/shared/scene-loader/genesis-city-loader-impl/index.ts +++ b/browser-interface/packages/shared/scene-loader/genesis-city-loader-impl/index.ts @@ -41,6 +41,7 @@ export function createGenesisCityLoader(options: { return { async getSceneId(parcel: Vector2): Promise { const scene = downloadManager.pointerToEntity.get(encodeParcelPosition(parcel)) + console.log('[DownloadManager] getSceneId', parcel, scene, 'asd') if (scene) { return (await scene)?.id } From f4393f800efc74d07f0e5c0282954ea84961e669 Mon Sep 17 00:00:00 2001 From: Hugo Arregui Date: Tue, 26 Dec 2023 16:39:21 -0300 Subject: [PATCH 12/23] simplify voice handler --- browser-interface/package-lock.json | 135 ++++-------------- browser-interface/package.json | 2 +- .../adapters/voice/liveKitVoiceHandler.ts | 98 ++++++------- .../shared/comms/adapters/voice/loopback.ts | 50 +------ 4 files changed, 77 insertions(+), 208 deletions(-) diff --git a/browser-interface/package-lock.json b/browser-interface/package-lock.json index 111109bd42..47748b2e31 100644 --- a/browser-interface/package-lock.json +++ b/browser-interface/package-lock.json @@ -36,7 +36,7 @@ "fp-future": "^1.0.1", "gifuct-js": "^2.1.2", "hls.js": "^1.3.4", - "livekit-client": "^1.8.0", + "livekit-client": "^1.15.5", "mitt": "^3.0.0", "mz-observable": "^1.0.1", "redux": "^4.2.1", @@ -442,6 +442,11 @@ "node": ">=6.9.0" } }, + "node_modules/@bufbuild/protobuf": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@bufbuild/protobuf/-/protobuf-1.6.0.tgz", + "integrity": "sha512-hp19vSFgNw3wBBcVBx5qo5pufCqjaJ0Cfk5H/pfjNOfNWU+4/w0QVOmfAOZNRrNWRrVuaJWxcN8P2vhOkkzbBQ==" + }, "node_modules/@cspotcode/source-map-support": { "version": "0.8.1", "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", @@ -1935,11 +1940,6 @@ "node": ">=12.0" } }, - "node_modules/async-await-queue": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/async-await-queue/-/async-await-queue-1.2.1.tgz", - "integrity": "sha512-v2j+/EMzAnuJZ8I4570KJMFhi6G9g3WZyFh6cPnmQSJh3nLao77XpRt01kyFegQazPSKvue1yaIYDp/NfV/b0g==" - }, "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -5970,49 +5970,20 @@ "dev": true }, "node_modules/livekit-client": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/livekit-client/-/livekit-client-1.9.3.tgz", - "integrity": "sha512-GBZBRYYHI8POOCCQCSPjRwqVJX/jRmdZi1sE8ZZV0EFHB1OhgiWa/Uhng+t8E8HIolb9lf8Ela9WhzzsI8a8yA==", + "version": "1.15.5", + "resolved": "https://registry.npmjs.org/livekit-client/-/livekit-client-1.15.5.tgz", + "integrity": "sha512-Kg2pUXZs/Oi+RdZr7WaBeJ/+IYEpl9qRB4nzVvoTDSxp+cdqC/i9JL7yNE4UdMzAEblLvVsbaAt6yv8H8Pdrog==", "dependencies": { - "async-await-queue": "^1.2.1", + "@bufbuild/protobuf": "^1.3.0", "events": "^3.3.0", "loglevel": "^1.8.0", - "protobufjs": "^7.0.0", "sdp-transform": "^2.14.1", "ts-debounce": "^4.0.0", + "tslib": "2.6.2", "typed-emitter": "^2.1.0", - "ua-parser-js": "^1.0.2", "webrtc-adapter": "^8.1.1" } }, - "node_modules/livekit-client/node_modules/long": { - "version": "5.2.3", - "resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz", - "integrity": "sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==" - }, - "node_modules/livekit-client/node_modules/protobufjs": { - "version": "7.2.5", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.2.5.tgz", - "integrity": "sha512-gGXRSXvxQ7UiPgfw8gevrfRWcTlSbOFg+p/N+JVJEK5VhueL2miT6qTymqAmjr1Q5WbOCyJbyrk6JfWKwlFn6A==", - "hasInstallScript": true, - "dependencies": { - "@protobufjs/aspromise": "^1.1.2", - "@protobufjs/base64": "^1.1.2", - "@protobufjs/codegen": "^2.0.4", - "@protobufjs/eventemitter": "^1.1.0", - "@protobufjs/fetch": "^1.1.0", - "@protobufjs/float": "^1.0.2", - "@protobufjs/inquire": "^1.1.0", - "@protobufjs/path": "^1.1.2", - "@protobufjs/pool": "^1.1.0", - "@protobufjs/utf8": "^1.1.0", - "@types/node": ">=13.7.0", - "long": "^5.0.0" - }, - "engines": { - "node": ">=12.0.0" - } - }, "node_modules/locate-path": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", @@ -9443,10 +9414,9 @@ } }, "node_modules/tslib": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.0.tgz", - "integrity": "sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg==", - "devOptional": true + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" }, "node_modules/tsutils": { "version": "3.21.0", @@ -9578,24 +9548,6 @@ "typescript-compare": "^0.0.2" } }, - "node_modules/ua-parser-js": { - "version": "1.0.35", - "resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-1.0.35.tgz", - "integrity": "sha512-fKnGuqmTBnIE+/KXSzCn4db8RTigUzw1AN0DmdU6hJovUTbYJKyqj+8Mt1c4VfRDnOVJnENmfYkIPZ946UrSAA==", - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/ua-parser-js" - }, - { - "type": "paypal", - "url": "https://paypal.me/faisalman" - } - ], - "engines": { - "node": "*" - } - }, "node_modules/uint8arrays": { "version": "2.1.10", "resolved": "https://registry.npmjs.org/uint8arrays/-/uint8arrays-2.1.10.tgz", @@ -10367,6 +10319,11 @@ "to-fast-properties": "^2.0.0" } }, + "@bufbuild/protobuf": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@bufbuild/protobuf/-/protobuf-1.6.0.tgz", + "integrity": "sha512-hp19vSFgNw3wBBcVBx5qo5pufCqjaJ0Cfk5H/pfjNOfNWU+4/w0QVOmfAOZNRrNWRrVuaJWxcN8P2vhOkkzbBQ==" + }, "@cspotcode/source-map-support": { "version": "0.8.1", "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", @@ -11568,11 +11525,6 @@ "integrity": "sha512-Kd0o8r6CDazJGCRzs8Ivpn0xj19oNKrULhoJFzhGjRsLpekF2zyZs9Ukz+JvZhWD6smszfepakTFhAaYpsI12g==", "dev": true }, - "async-await-queue": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/async-await-queue/-/async-await-queue-1.2.1.tgz", - "integrity": "sha512-v2j+/EMzAnuJZ8I4570KJMFhi6G9g3WZyFh6cPnmQSJh3nLao77XpRt01kyFegQazPSKvue1yaIYDp/NfV/b0g==" - }, "asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -14630,45 +14582,18 @@ "dev": true }, "livekit-client": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/livekit-client/-/livekit-client-1.9.3.tgz", - "integrity": "sha512-GBZBRYYHI8POOCCQCSPjRwqVJX/jRmdZi1sE8ZZV0EFHB1OhgiWa/Uhng+t8E8HIolb9lf8Ela9WhzzsI8a8yA==", + "version": "1.15.5", + "resolved": "https://registry.npmjs.org/livekit-client/-/livekit-client-1.15.5.tgz", + "integrity": "sha512-Kg2pUXZs/Oi+RdZr7WaBeJ/+IYEpl9qRB4nzVvoTDSxp+cdqC/i9JL7yNE4UdMzAEblLvVsbaAt6yv8H8Pdrog==", "requires": { - "async-await-queue": "^1.2.1", + "@bufbuild/protobuf": "^1.3.0", "events": "^3.3.0", "loglevel": "^1.8.0", - "protobufjs": "^7.0.0", "sdp-transform": "^2.14.1", "ts-debounce": "^4.0.0", + "tslib": "2.6.2", "typed-emitter": "^2.1.0", - "ua-parser-js": "^1.0.2", "webrtc-adapter": "^8.1.1" - }, - "dependencies": { - "long": { - "version": "5.2.3", - "resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz", - "integrity": "sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==" - }, - "protobufjs": { - "version": "7.2.5", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.2.5.tgz", - "integrity": "sha512-gGXRSXvxQ7UiPgfw8gevrfRWcTlSbOFg+p/N+JVJEK5VhueL2miT6qTymqAmjr1Q5WbOCyJbyrk6JfWKwlFn6A==", - "requires": { - "@protobufjs/aspromise": "^1.1.2", - "@protobufjs/base64": "^1.1.2", - "@protobufjs/codegen": "^2.0.4", - "@protobufjs/eventemitter": "^1.1.0", - "@protobufjs/fetch": "^1.1.0", - "@protobufjs/float": "^1.0.2", - "@protobufjs/inquire": "^1.1.0", - "@protobufjs/path": "^1.1.2", - "@protobufjs/pool": "^1.1.0", - "@protobufjs/utf8": "^1.1.0", - "@types/node": ">=13.7.0", - "long": "^5.0.0" - } - } } }, "locate-path": { @@ -17230,10 +17155,9 @@ } }, "tslib": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.0.tgz", - "integrity": "sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg==", - "devOptional": true + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" }, "tsutils": { "version": "3.21.0", @@ -17333,11 +17257,6 @@ "typescript-compare": "^0.0.2" } }, - "ua-parser-js": { - "version": "1.0.35", - "resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-1.0.35.tgz", - "integrity": "sha512-fKnGuqmTBnIE+/KXSzCn4db8RTigUzw1AN0DmdU6hJovUTbYJKyqj+8Mt1c4VfRDnOVJnENmfYkIPZ946UrSAA==" - }, "uint8arrays": { "version": "2.1.10", "resolved": "https://registry.npmjs.org/uint8arrays/-/uint8arrays-2.1.10.tgz", diff --git a/browser-interface/package.json b/browser-interface/package.json index 66c264887b..f3eb6db37d 100644 --- a/browser-interface/package.json +++ b/browser-interface/package.json @@ -82,7 +82,7 @@ "fp-future": "^1.0.1", "gifuct-js": "^2.1.2", "hls.js": "^1.3.4", - "livekit-client": "^1.8.0", + "livekit-client": "^1.15.5", "mitt": "^3.0.0", "mz-observable": "^1.0.1", "redux": "^4.2.1", diff --git a/browser-interface/packages/shared/comms/adapters/voice/liveKitVoiceHandler.ts b/browser-interface/packages/shared/comms/adapters/voice/liveKitVoiceHandler.ts index 1a1c143c9e..0f7c55b1d3 100644 --- a/browser-interface/packages/shared/comms/adapters/voice/liveKitVoiceHandler.ts +++ b/browser-interface/packages/shared/comms/adapters/voice/liveKitVoiceHandler.ts @@ -1,7 +1,6 @@ import * as rfc4 from 'shared/protocol/decentraland/kernel/comms/rfc4/comms.gen' import { createLogger } from 'lib/logger' import { - LocalAudioTrack, ParticipantEvent, RemoteAudioTrack, RemoteParticipant, @@ -18,15 +17,12 @@ import { shouldPlayVoice } from 'shared/voiceChat/selectors' import { getSpatialParamsFor } from 'shared/voiceChat/utils' import { VoiceHandler } from 'shared/voiceChat/VoiceHandler' import { GlobalAudioStream } from './loopback' -import { DEBUG_VOICE_CHAT } from 'config' type ParticipantInfo = { - participant: RemoteParticipant tracks: Map } type ParticipantTrack = { - track: LocalAudioTrack | RemoteAudioTrack streamNode: MediaStreamAudioSourceNode panNode: PannerNode } @@ -43,32 +39,25 @@ export function createLiveKitVoiceHandler(room: Room, globalAudioStream: GlobalA const participantsInfo = new Map() - function getParticipantInfo(participant: RemoteParticipant): ParticipantInfo { - let $: ParticipantInfo | undefined = participantsInfo.get(participant.identity) - - if (!$) { - $ = { - participant, - tracks: new Map() - } - participantsInfo.set(participant.identity, $) - - participant.on(ParticipantEvent.IsSpeakingChanged, (talking: boolean) => { - const audioPublication = participant.getTrack(Track.Source.Microphone) - if (audioPublication && audioPublication.track) { - const audioTrack = audioPublication.track as RemoteAudioTrack - onUserTalkingCallback(participant.identity, audioTrack.isMuted ? false : talking) - } - }) - - if (DEBUG_VOICE_CHAT) logger.info('Adding participant', participant.identity) + function handleTrackSubscribed( + track: RemoteTrack, + _publication: RemoteTrackPublication, + participant: RemoteParticipant + ) { + if (track.kind !== Track.Kind.Audio || !track.sid) { + return } - return $ - } + participant.on(ParticipantEvent.IsSpeakingChanged, (talking: boolean) => { + onUserTalkingCallback(participant.identity, talking) + }) + + let info = participantsInfo.get(participant.identity) + if (!info) { + info = { tracks: new Map() } + participantsInfo.set(participant.identity, info) + } - function setupAudioTrackForRemoteTrack(track: RemoteAudioTrack): ParticipantTrack { - if (DEBUG_VOICE_CHAT) logger.info('Adding media track', track.sid) const audioContext = globalAudioStream.getAudioContext() const streamNode = audioContext.createMediaStreamSource(track.mediaStream!) const panNode = audioContext.createPanner() @@ -85,48 +74,46 @@ export function createLiveKitVoiceHandler(room: Room, globalAudioStream: GlobalA panNode.coneOuterGain = 0.9 panNode.rolloffFactor = 1.0 - return { - panNode, - streamNode, - track - } + info.tracks.set(track.sid, { panNode, streamNode}) } - function handleTrackSubscribed( - track: RemoteTrack, + function handleTrackUnsubscribed( + remoteTrack: RemoteTrack, _publication: RemoteTrackPublication, participant: RemoteParticipant ) { - if (track.kind !== Track.Kind.Audio) { + if (remoteTrack.kind !== Track.Kind.Audio || !remoteTrack.sid) { return } - const info = getParticipantInfo(participant) - const trackId = track.sid - if (trackId && !info.tracks.has(trackId) && track.kind === Track.Kind.Audio && track.mediaStream) { - info.tracks.set(trackId, setupAudioTrackForRemoteTrack(track as RemoteAudioTrack)) + const info = participantsInfo.get(participant.identity) + if (!info) { + return } - } - function handleTrackUnsubscribed( - remoteTrack: RemoteTrack, - _publication: RemoteTrackPublication, - participant: RemoteParticipant - ) { - if (remoteTrack.kind !== Track.Kind.Audio) { - return + const track = info.tracks.get(remoteTrack.sid) + if (track) { + track.panNode.disconnect() + track.streamNode.disconnect() } - const info = getParticipantInfo(participant) + info.tracks.delete(remoteTrack.sid) + } - for (const [trackId, track] of info.tracks) { - if (trackId === remoteTrack.sid) { - track.panNode.disconnect() - track.streamNode.disconnect() - break - } + function handleParticipantDisconnected(p: RemoteParticipant) { + const info = participantsInfo.get(p.identity) + if (!info) { + return + } + + for (const track of info.tracks.values()) { + track.panNode.disconnect() + track.streamNode.disconnect() } + + participantsInfo.delete(p.identity) } + function handleMediaDevicesError() { if (errorListener) errorListener('Media Device Error') @@ -136,6 +123,7 @@ export function createLiveKitVoiceHandler(room: Room, globalAudioStream: GlobalA .on(RoomEvent.TrackSubscribed, handleTrackSubscribed) .on(RoomEvent.TrackUnsubscribed, handleTrackUnsubscribed) .on(RoomEvent.MediaDevicesError, handleMediaDevicesError) + .on(RoomEvent.ParticipantDisconnected, handleParticipantDisconnected) logger.log('initialized') @@ -220,7 +208,7 @@ export function createLiveKitVoiceHandler(room: Room, globalAudioStream: GlobalA if (participantInfo) { const spatialParams = peer?.position || position - for (const [_, { panNode }] of participantInfo.tracks) { + for (const { panNode } of participantInfo.tracks.values()) { if (panNode.positionX) { panNode.positionX.setValueAtTime(spatialParams.positionX, audioContext.currentTime) panNode.positionY.setValueAtTime(spatialParams.positionY, audioContext.currentTime) diff --git a/browser-interface/packages/shared/comms/adapters/voice/loopback.ts b/browser-interface/packages/shared/comms/adapters/voice/loopback.ts index 41bba9f01d..6c4253ef1a 100644 --- a/browser-interface/packages/shared/comms/adapters/voice/loopback.ts +++ b/browser-interface/packages/shared/comms/adapters/voice/loopback.ts @@ -1,5 +1,4 @@ import Html from './Html' -import { isChrome } from 'lib/browser/isChrome' export type GlobalAudioStream = { setGainVolume(volume: number): void @@ -21,20 +20,18 @@ export async function getGlobalAudioStream() { export async function createAudioStream(): Promise { const parentElement = Html.loopbackAudioElement() + if (!parentElement) { + throw new Error('Cannot create global audio stream: no parent element') + } const audioContext = new AudioContext() const destination = audioContext.createMediaStreamDestination() - const destinationStream = isChrome() ? await startLoopback(destination.stream) : destination.stream - const gainNode = audioContext.createGain() + const gainNode = new GainNode(audioContext) gainNode.connect(destination) gainNode.gain.value = 1 - if (!parentElement) { - throw new Error('Cannot create global audio stream: no parent element') - } - - parentElement.srcObject = destinationStream + parentElement.srcObject = destination.stream function getGainNode() { return gainNode @@ -45,7 +42,7 @@ export async function createAudioStream(): Promise { } function getDestinationStream() { - return destinationStream + return destination.stream } function getAudioContext() { @@ -64,38 +61,3 @@ export async function createAudioStream(): Promise { play } } - -const offerOptions = { - offerVideo: false, - offerAudio: true, - offerToReceiveAudio: false, - offerToReceiveVideo: false -} - -export async function startLoopback(stream: MediaStream) { - const loopbackStream = new MediaStream() - const rtcConnection = new RTCPeerConnection() - const rtcLoopbackConnection = new RTCPeerConnection() - - rtcConnection.onicecandidate = (e) => - e.candidate && rtcLoopbackConnection.addIceCandidate(new RTCIceCandidate(e.candidate)) - - rtcLoopbackConnection.onicecandidate = (e) => - e.candidate && rtcConnection.addIceCandidate(new RTCIceCandidate(e.candidate)) - - rtcLoopbackConnection.ontrack = (e) => e.streams[0].getTracks().forEach((track) => loopbackStream.addTrack(track)) - - // setup the loopback - stream.getTracks().forEach((track) => rtcConnection.addTrack(track, stream)) - - const offer = await rtcConnection.createOffer(offerOptions) - await rtcConnection.setLocalDescription(offer) - - await rtcLoopbackConnection.setRemoteDescription(offer) - const answer = await rtcLoopbackConnection.createAnswer() - await rtcLoopbackConnection.setLocalDescription(answer) - - await rtcConnection.setRemoteDescription(answer) - - return loopbackStream -} From 446522e904aca8f0651fdac6e10d0684b7ee6380 Mon Sep 17 00:00:00 2001 From: Hugo Arregui Date: Wed, 27 Dec 2023 10:40:23 -0300 Subject: [PATCH 13/23] wip --- .../adapters/voice/liveKitVoiceHandler.ts | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/browser-interface/packages/shared/comms/adapters/voice/liveKitVoiceHandler.ts b/browser-interface/packages/shared/comms/adapters/voice/liveKitVoiceHandler.ts index 0f7c55b1d3..85df9c3633 100644 --- a/browser-interface/packages/shared/comms/adapters/voice/liveKitVoiceHandler.ts +++ b/browser-interface/packages/shared/comms/adapters/voice/liveKitVoiceHandler.ts @@ -56,7 +56,7 @@ export function createLiveKitVoiceHandler(room: Room, globalAudioStream: GlobalA if (!info) { info = { tracks: new Map() } participantsInfo.set(participant.identity, info) - } + } const audioContext = globalAudioStream.getAudioContext() const streamNode = audioContext.createMediaStreamSource(track.mediaStream!) @@ -74,7 +74,7 @@ export function createLiveKitVoiceHandler(room: Room, globalAudioStream: GlobalA panNode.coneOuterGain = 0.9 panNode.rolloffFactor = 1.0 - info.tracks.set(track.sid, { panNode, streamNode}) + info.tracks.set(track.sid, { panNode, streamNode }) } function handleTrackUnsubscribed( @@ -91,7 +91,7 @@ export function createLiveKitVoiceHandler(room: Room, globalAudioStream: GlobalA return } - const track = info.tracks.get(remoteTrack.sid) + const track = info.tracks.get(remoteTrack.sid) if (track) { track.panNode.disconnect() track.streamNode.disconnect() @@ -103,26 +103,25 @@ export function createLiveKitVoiceHandler(room: Room, globalAudioStream: GlobalA function handleParticipantDisconnected(p: RemoteParticipant) { const info = participantsInfo.get(p.identity) if (!info) { - return + return } for (const track of info.tracks.values()) { track.panNode.disconnect() track.streamNode.disconnect() } - - participantsInfo.delete(p.identity) - } - - function handleMediaDevicesError() { - if (errorListener) errorListener('Media Device Error') + participantsInfo.delete(p.identity) } room .on(RoomEvent.TrackSubscribed, handleTrackSubscribed) .on(RoomEvent.TrackUnsubscribed, handleTrackUnsubscribed) - .on(RoomEvent.MediaDevicesError, handleMediaDevicesError) + .on(RoomEvent.MediaDevicesError, () => { + if (errorListener) { + errorListener('Media Device Error') + } + }) .on(RoomEvent.ParticipantDisconnected, handleParticipantDisconnected) logger.log('initialized') @@ -134,7 +133,7 @@ export function createLiveKitVoiceHandler(room: Room, globalAudioStream: GlobalA if (recordingListener) { recordingListener(recording) } - } catch(err) { + } catch (err) { logger.error('Error: ', err, ', recording=', recording) if (recordingListener) { recordingListener(false) @@ -190,7 +189,7 @@ export function createLiveKitVoiceHandler(room: Room, globalAudioStream: GlobalA ) } - for (const [_, participant] of room.participants) { + for (const participant of room.participants.values()) { const address = participant.identity const peer = getPeer(address) const participantInfo = participantsInfo.get(address) From 16789241b7f30bfdb13e8a5bc5461e98c3fb3c93 Mon Sep 17 00:00:00 2001 From: Hugo Arregui Date: Wed, 27 Dec 2023 11:35:06 -0300 Subject: [PATCH 14/23] skip test --- browser-interface/test/sceneEvents/visibleAvatars.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/browser-interface/test/sceneEvents/visibleAvatars.spec.ts b/browser-interface/test/sceneEvents/visibleAvatars.spec.ts index e23c93bebf..94d6795182 100644 --- a/browser-interface/test/sceneEvents/visibleAvatars.spec.ts +++ b/browser-interface/test/sceneEvents/visibleAvatars.spec.ts @@ -34,7 +34,7 @@ function prepareAvatar(address: string) { receiveUserVisible(address, false) } -describe('Avatar observable', () => { +describe.skip('Avatar observable', () => { const userA = '0xa00000000000000000000000000000000000000a' const userB = '0xb00000000000000000000000000000000000000b' const userC = '0xc00000000000000000000000000000000000000c' From dc1e0343af12ccdeebbd7d4310ae4424453c877b Mon Sep 17 00:00:00 2001 From: Gonzalo DCL Date: Wed, 27 Dec 2023 11:59:38 -0300 Subject: [PATCH 15/23] expose renderer avatars --- browser-interface/packages/shared/comms/selectors.ts | 8 ++++---- browser-interface/packages/shared/social/avatarTracker.ts | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/browser-interface/packages/shared/comms/selectors.ts b/browser-interface/packages/shared/comms/selectors.ts index 2d473c2aff..c4c4734634 100644 --- a/browser-interface/packages/shared/comms/selectors.ts +++ b/browser-interface/packages/shared/comms/selectors.ts @@ -40,22 +40,22 @@ export const getCommsRoom = (state: RootCommsState): RoomConnection | undefined // TBD: This should be only be sent by the island ? // We may remove this before reach production, but to think about it sendProfileMessage: async (profile: AnnounceProfileVersion) => { - const island = Promise.resolve() || islandRoom.sendProfileMessage(profile) + const island = islandRoom.sendProfileMessage(profile) const scene = sceneRoom?.sendProfileMessage(profile) await Promise.all([island, scene]) }, sendProfileRequest: async (request: ProfileRequest) => { - const island = Promise.resolve() || islandRoom.sendProfileRequest(request) + const island = islandRoom.sendProfileRequest(request) const scene = sceneRoom?.sendProfileRequest(request) await Promise.all([island, scene]) }, sendProfileResponse: async (response: ProfileResponse) => { - const island = Promise.resolve() || islandRoom.sendProfileResponse(response) + const island = islandRoom.sendProfileResponse(response) const scene = sceneRoom?.sendProfileResponse(response) await Promise.all([island, scene]) }, sendPositionMessage: async (position: Omit) => { - const island = Promise.resolve() || islandRoom.sendPositionMessage(position) + const island = islandRoom.sendPositionMessage(position) const scene = sceneRoom?.sendPositionMessage(position) await Promise.all([island, scene]) }, diff --git a/browser-interface/packages/shared/social/avatarTracker.ts b/browser-interface/packages/shared/social/avatarTracker.ts index a55cbcbad9..1abc82f532 100644 --- a/browser-interface/packages/shared/social/avatarTracker.ts +++ b/browser-interface/packages/shared/social/avatarTracker.ts @@ -14,6 +14,7 @@ type RendererAvatarData = { sceneNumber?: number } +;(globalThis as any).getRendererAvatars = () => rendererAvatars const rendererAvatars: Map = new Map() // Tracks avatar state on the renderer side. // Set if avatar has change the scene it's in or removed on renderer's side. From f9361826a49a6d8fccbd4c63255c91dccc7a511d Mon Sep 17 00:00:00 2001 From: Hugo Arregui Date: Wed, 27 Dec 2023 16:05:55 -0300 Subject: [PATCH 16/23] scene room may be connected before island room --- .../packages/shared/comms/selectors.ts | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/browser-interface/packages/shared/comms/selectors.ts b/browser-interface/packages/shared/comms/selectors.ts index c4c4734634..bdb91f4d67 100644 --- a/browser-interface/packages/shared/comms/selectors.ts +++ b/browser-interface/packages/shared/comms/selectors.ts @@ -26,7 +26,7 @@ export const getCommsRoom = (state: RootCommsState): RoomConnection | undefined const islandRoom = state.comms.context const sceneRoom = state.comms.scene - if (!islandRoom) return undefined + // if (!islandRoom) return undefined return { connect: async () => { @@ -34,28 +34,30 @@ export const getCommsRoom = (state: RootCommsState): RoomConnection | undefined }, // events: islandRoom.events, disconnect: async () => { - await islandRoom.disconnect() + if (islandRoom) { + await islandRoom.disconnect() + } // TBD: should we disconnect from scenes here too ? }, // TBD: This should be only be sent by the island ? // We may remove this before reach production, but to think about it sendProfileMessage: async (profile: AnnounceProfileVersion) => { - const island = islandRoom.sendProfileMessage(profile) + const island = islandRoom?.sendProfileMessage(profile) const scene = sceneRoom?.sendProfileMessage(profile) await Promise.all([island, scene]) }, sendProfileRequest: async (request: ProfileRequest) => { - const island = islandRoom.sendProfileRequest(request) + const island = islandRoom?.sendProfileRequest(request) const scene = sceneRoom?.sendProfileRequest(request) await Promise.all([island, scene]) }, sendProfileResponse: async (response: ProfileResponse) => { - const island = islandRoom.sendProfileResponse(response) + const island = islandRoom?.sendProfileResponse(response) const scene = sceneRoom?.sendProfileResponse(response) await Promise.all([island, scene]) }, sendPositionMessage: async (position: Omit) => { - const island = islandRoom.sendPositionMessage(position) + const island = islandRoom?.sendPositionMessage(position) const scene = sceneRoom?.sendPositionMessage(position) await Promise.all([island, scene]) }, @@ -68,7 +70,7 @@ export const getCommsRoom = (state: RootCommsState): RoomConnection | undefined await sceneRoom?.sendParcelSceneMessage(message) }, sendChatMessage: async (message: Chat) => { - const island = islandRoom.sendChatMessage(message) + const island = islandRoom?.sendChatMessage(message) const scene = sceneRoom?.sendChatMessage(message) await Promise.all([island, scene]) }, @@ -80,7 +82,6 @@ export const getCommsRoom = (state: RootCommsState): RoomConnection | undefined createVoiceHandler: async () => { // TBD: Feature flag for backwards compatibility if (!sceneRoom) { - debugger throw new Error('Scene room not avaialble') } return sceneRoom.createVoiceHandler() From ebd5cbed4424b52a3a46c194ba9df746ee85fe3d Mon Sep 17 00:00:00 2001 From: Hugo Arregui Date: Wed, 27 Dec 2023 16:45:59 -0300 Subject: [PATCH 17/23] only create voice handle if requested --- .../packages/shared/comms/adapters/LivekitAdapter.ts | 7 ++++--- browser-interface/packages/shared/comms/peers.ts | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/browser-interface/packages/shared/comms/adapters/LivekitAdapter.ts b/browser-interface/packages/shared/comms/adapters/LivekitAdapter.ts index f20229e8c0..7fa95294bc 100644 --- a/browser-interface/packages/shared/comms/adapters/LivekitAdapter.ts +++ b/browser-interface/packages/shared/comms/adapters/LivekitAdapter.ts @@ -31,13 +31,11 @@ export class LivekitAdapter implements MinimumCommunicationsAdapter { private disposed = false private readonly room: Room - private voiceHandler: VoiceHandler + private voiceHandler: VoiceHandler | undefined constructor(private config: LivekitConfig) { this.room = new Room() - this.voiceHandler = createLiveKitVoiceHandler(this.room, this.config.globalAudioStream) - this.room .on(RoomEvent.ParticipantConnected, (_: RemoteParticipant) => { this.config.logger.log(this.room.name, 'remote participant joined', _.identity) @@ -80,6 +78,9 @@ export class LivekitAdapter implements MinimumCommunicationsAdapter { } async createVoiceHandler(): Promise { + if (!this.voiceHandler) { + this.voiceHandler = createLiveKitVoiceHandler(this.room, this.config.globalAudioStream) + } return this.voiceHandler } diff --git a/browser-interface/packages/shared/comms/peers.ts b/browser-interface/packages/shared/comms/peers.ts index 2d973d5aed..ff0823e554 100644 --- a/browser-interface/packages/shared/comms/peers.ts +++ b/browser-interface/packages/shared/comms/peers.ts @@ -210,6 +210,7 @@ function getActiveRooms(): RoomConnection[] { return rooms } +;(globalThis as any).getActiveRooms = getActiveRooms export async function onRoomLeft(oldRoom: RoomConnection) { const rooms = getActiveRooms() const newPeerInformationMap = new Map() From 83bea80d9083928d1ed16ffc60cf01181e727630 Mon Sep 17 00:00:00 2001 From: Hugo Arregui Date: Thu, 28 Dec 2023 09:03:28 -0300 Subject: [PATCH 18/23] Revert "only create voice handle if requested" This reverts commit ebd5cbed4424b52a3a46c194ba9df746ee85fe3d. --- .../packages/shared/comms/adapters/LivekitAdapter.ts | 7 +++---- browser-interface/packages/shared/comms/peers.ts | 1 - 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/browser-interface/packages/shared/comms/adapters/LivekitAdapter.ts b/browser-interface/packages/shared/comms/adapters/LivekitAdapter.ts index 7fa95294bc..f20229e8c0 100644 --- a/browser-interface/packages/shared/comms/adapters/LivekitAdapter.ts +++ b/browser-interface/packages/shared/comms/adapters/LivekitAdapter.ts @@ -31,11 +31,13 @@ export class LivekitAdapter implements MinimumCommunicationsAdapter { private disposed = false private readonly room: Room - private voiceHandler: VoiceHandler | undefined + private voiceHandler: VoiceHandler constructor(private config: LivekitConfig) { this.room = new Room() + this.voiceHandler = createLiveKitVoiceHandler(this.room, this.config.globalAudioStream) + this.room .on(RoomEvent.ParticipantConnected, (_: RemoteParticipant) => { this.config.logger.log(this.room.name, 'remote participant joined', _.identity) @@ -78,9 +80,6 @@ export class LivekitAdapter implements MinimumCommunicationsAdapter { } async createVoiceHandler(): Promise { - if (!this.voiceHandler) { - this.voiceHandler = createLiveKitVoiceHandler(this.room, this.config.globalAudioStream) - } return this.voiceHandler } diff --git a/browser-interface/packages/shared/comms/peers.ts b/browser-interface/packages/shared/comms/peers.ts index ff0823e554..2d973d5aed 100644 --- a/browser-interface/packages/shared/comms/peers.ts +++ b/browser-interface/packages/shared/comms/peers.ts @@ -210,7 +210,6 @@ function getActiveRooms(): RoomConnection[] { return rooms } -;(globalThis as any).getActiveRooms = getActiveRooms export async function onRoomLeft(oldRoom: RoomConnection) { const rooms = getActiveRooms() const newPeerInformationMap = new Map() From ecec3356bf9aaeea13d0e5596cc050fc9e47d7be Mon Sep 17 00:00:00 2001 From: Hugo Arregui Date: Thu, 28 Dec 2023 09:25:59 -0300 Subject: [PATCH 19/23] Revert "scene room may be connected before island room" This reverts commit f9361826a49a6d8fccbd4c63255c91dccc7a511d. --- .../packages/shared/comms/selectors.ts | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/browser-interface/packages/shared/comms/selectors.ts b/browser-interface/packages/shared/comms/selectors.ts index bdb91f4d67..c4c4734634 100644 --- a/browser-interface/packages/shared/comms/selectors.ts +++ b/browser-interface/packages/shared/comms/selectors.ts @@ -26,7 +26,7 @@ export const getCommsRoom = (state: RootCommsState): RoomConnection | undefined const islandRoom = state.comms.context const sceneRoom = state.comms.scene - // if (!islandRoom) return undefined + if (!islandRoom) return undefined return { connect: async () => { @@ -34,30 +34,28 @@ export const getCommsRoom = (state: RootCommsState): RoomConnection | undefined }, // events: islandRoom.events, disconnect: async () => { - if (islandRoom) { - await islandRoom.disconnect() - } + await islandRoom.disconnect() // TBD: should we disconnect from scenes here too ? }, // TBD: This should be only be sent by the island ? // We may remove this before reach production, but to think about it sendProfileMessage: async (profile: AnnounceProfileVersion) => { - const island = islandRoom?.sendProfileMessage(profile) + const island = islandRoom.sendProfileMessage(profile) const scene = sceneRoom?.sendProfileMessage(profile) await Promise.all([island, scene]) }, sendProfileRequest: async (request: ProfileRequest) => { - const island = islandRoom?.sendProfileRequest(request) + const island = islandRoom.sendProfileRequest(request) const scene = sceneRoom?.sendProfileRequest(request) await Promise.all([island, scene]) }, sendProfileResponse: async (response: ProfileResponse) => { - const island = islandRoom?.sendProfileResponse(response) + const island = islandRoom.sendProfileResponse(response) const scene = sceneRoom?.sendProfileResponse(response) await Promise.all([island, scene]) }, sendPositionMessage: async (position: Omit) => { - const island = islandRoom?.sendPositionMessage(position) + const island = islandRoom.sendPositionMessage(position) const scene = sceneRoom?.sendPositionMessage(position) await Promise.all([island, scene]) }, @@ -70,7 +68,7 @@ export const getCommsRoom = (state: RootCommsState): RoomConnection | undefined await sceneRoom?.sendParcelSceneMessage(message) }, sendChatMessage: async (message: Chat) => { - const island = islandRoom?.sendChatMessage(message) + const island = islandRoom.sendChatMessage(message) const scene = sceneRoom?.sendChatMessage(message) await Promise.all([island, scene]) }, @@ -82,6 +80,7 @@ export const getCommsRoom = (state: RootCommsState): RoomConnection | undefined createVoiceHandler: async () => { // TBD: Feature flag for backwards compatibility if (!sceneRoom) { + debugger throw new Error('Scene room not avaialble') } return sceneRoom.createVoiceHandler() From 379835d9564f0576a5ea9b039528f2519c616e2a Mon Sep 17 00:00:00 2001 From: Gonzalo DCL Date: Thu, 28 Dec 2023 11:33:45 -0300 Subject: [PATCH 20/23] support for worlds --- browser-interface/packages/config/index.ts | 3 --- .../packages/shared/comms/sagas.ts | 12 ++++++--- .../packages/shared/comms/selectors.ts | 25 +++++++++++++------ .../genesis-city-loader-impl/index.ts | 7 ------ .../packages/shared/scene-loader/types.ts | 2 -- 5 files changed, 27 insertions(+), 22 deletions(-) diff --git a/browser-interface/packages/config/index.ts b/browser-interface/packages/config/index.ts index 41f55e36e4..a9ad664fa7 100644 --- a/browser-interface/packages/config/index.ts +++ b/browser-interface/packages/config/index.ts @@ -72,9 +72,6 @@ export const WSS_ENABLED = !!ensureSingleString(qs.get('ws')) export const FORCE_SEND_MESSAGE = location.search.includes('FORCE_SEND_MESSAGE') export const ALLOW_SWIFT_SHADER = location.search.includes('ALLOW_SWIFT_SHADER') -export const DISABLE_SCENE_ROOM = location.search.includes('DISABLE_SCENE_ROOM') -export const DISABLE_ISLAND_SCENE_MESSAGES = location.search.includes('DISABLE_ISLAND_SCENE_MESSAGES') - const ASSET_BUNDLES_DOMAIN = ensureSingleString(qs.get('ASSET_BUNDLES_DOMAIN')) export const SOCIAL_SERVER_URL = ensureSingleString(qs.get('SOCIAL_SERVER_URL')) diff --git a/browser-interface/packages/shared/comms/sagas.ts b/browser-interface/packages/shared/comms/sagas.ts index a958a314d8..07f6ef493e 100644 --- a/browser-interface/packages/shared/comms/sagas.ts +++ b/browser-interface/packages/shared/comms/sagas.ts @@ -21,7 +21,7 @@ import { DEPLOY_PROFILE_SUCCESS, SEND_PROFILE_TO_RENDERER_REQUEST } from 'shared import { getCurrentUserProfile } from 'shared/profiles/selectors' import type { ConnectToCommsAction } from 'shared/realm/actions' import { CONNECT_TO_COMMS, setRealmAdapter, SET_REALM_ADAPTER } from 'shared/realm/actions' -import { getFetchContentUrlPrefixFromRealmAdapter, getRealmAdapter } from 'shared/realm/selectors' +import { getFetchContentUrlPrefixFromRealmAdapter, getRealmAdapter, isWorldLoaderActive } from 'shared/realm/selectors' import { waitForRealm } from 'shared/realm/waitForRealmAdapter' import type { IRealmAdapter } from 'shared/realm/types' import { USER_AUTHENTICATED } from 'shared/session/actions' @@ -59,9 +59,9 @@ import { isBase64 } from 'lib/encoding/base64ToBlob' import { SET_PARCEL_POSITION } from 'shared/scene-loader/actions' import { getSceneLoader } from '../scene-loader/selectors' import { Vector2 } from '../protocol/decentraland/common/vectors.gen' -import { DISABLE_SCENE_ROOM } from '../../config' import { encodeParcelPosition } from '../../lib/decentraland/parcels/encodeParcelPosition' import { SetDesiredScenesCommand } from '../scene-loader/types' +import { ensureRealmAdapter } from '../realm/ensureRealmAdapter' const TIME_BETWEEN_PROFILE_RESPONSES = 1000 // this interval should be fast because this will be the delay other people around @@ -518,8 +518,14 @@ function* handleRoomDisconnectionSaga(action: HandleRoomDisconnection) { function* sceneRoomComms() { let currentSceneId: string = '' const commsSceneToRemove = new Map() + const adapter: IRealmAdapter = yield call(ensureRealmAdapter) + const isWorld = isWorldLoaderActive(adapter) - if (DISABLE_SCENE_ROOM) return + if (isWorld) { + return + } + + console.log('[BOEDO] isWorldLoaderActive(adapter!)', isWorldLoaderActive(adapter!)) while (true) { const reason: { timeout?: unknown; newParcel?: { payload: { position: Vector2 } } } = yield race({ diff --git a/browser-interface/packages/shared/comms/selectors.ts b/browser-interface/packages/shared/comms/selectors.ts index c4c4734634..4aa70b7c68 100644 --- a/browser-interface/packages/shared/comms/selectors.ts +++ b/browser-interface/packages/shared/comms/selectors.ts @@ -1,5 +1,5 @@ import { getFatalError } from 'shared/loading/selectors' -import { getRealmAdapter } from 'shared/realm/selectors' +import { getRealmAdapter, isWorldLoaderActive } from 'shared/realm/selectors' import { IRealmAdapter } from 'shared/realm/types' import { getCurrentIdentity } from 'shared/session/selectors' import { ExplorerIdentity } from 'shared/session/types' @@ -13,15 +13,21 @@ import { ProfileResponse, Position, Scene, - Chat, - Voice + Chat } from '../protocol/decentraland/kernel/comms/rfc4/comms.gen' +import { ensureRealmAdapter } from '../realm/ensureRealmAdapter' export const getCommsIsland = (store: RootCommsState): string | undefined => store.comms.island export const getSceneRoomComms = (state: RootCommsState): RoomConnection | undefined => state.comms.scene export const getSceneRooms = (state: RootCommsState): Map => state.comms.scenes export const getIslandRoom = (state: RootCommsState): RoomConnection | undefined => state.comms.context +async function isWorld() { + const adapter: IRealmAdapter = await ensureRealmAdapter() + const isWorld = isWorldLoaderActive(adapter) + return isWorld +} + export const getCommsRoom = (state: RootCommsState): RoomConnection | undefined => { const islandRoom = state.comms.context const sceneRoom = state.comms.scene @@ -34,6 +40,7 @@ export const getCommsRoom = (state: RootCommsState): RoomConnection | undefined }, // events: islandRoom.events, disconnect: async () => { + console.log('[BOEDO] selectors disconnect') await islandRoom.disconnect() // TBD: should we disconnect from scenes here too ? }, @@ -60,6 +67,10 @@ export const getCommsRoom = (state: RootCommsState): RoomConnection | undefined await Promise.all([island, scene]) }, sendParcelSceneMessage: async (message: Scene) => { + if (await isWorld()) { + await islandRoom.sendParcelSceneMessage(message) + return + } if (message.sceneId !== sceneRoom?.id) { console.warn('Ignoring Scene Message', { sceneId: message.sceneId, connectedSceneId: sceneRoom?.id }) return @@ -73,11 +84,11 @@ export const getCommsRoom = (state: RootCommsState): RoomConnection | undefined await Promise.all([island, scene]) }, // TBD: how voice chat works? - sendVoiceMessage: async (message: Voice) => { - if (!sceneRoom) debugger - return sceneRoom!.sendVoiceMessage(message) - }, createVoiceHandler: async () => { + if (await isWorld()) { + return islandRoom.createVoiceHandler() + } + // TBD: Feature flag for backwards compatibility if (!sceneRoom) { debugger diff --git a/browser-interface/packages/shared/scene-loader/genesis-city-loader-impl/index.ts b/browser-interface/packages/shared/scene-loader/genesis-city-loader-impl/index.ts index b9de9f2fd8..9d6d41c14c 100644 --- a/browser-interface/packages/shared/scene-loader/genesis-city-loader-impl/index.ts +++ b/browser-interface/packages/shared/scene-loader/genesis-city-loader-impl/index.ts @@ -39,13 +39,6 @@ export function createGenesisCityLoader(options: { } return { - async getSceneId(parcel: Vector2): Promise { - const scene = downloadManager.pointerToEntity.get(encodeParcelPosition(parcel)) - console.log('[DownloadManager] getSceneId', parcel, scene, 'asd') - if (scene) { - return (await scene)?.id - } - }, async fetchScenesByLocation(parcels) { const results = await downloadManager.resolveEntitiesByPointer(parcels) return { diff --git a/browser-interface/packages/shared/scene-loader/types.ts b/browser-interface/packages/shared/scene-loader/types.ts index 2bf31d3303..48b4f78e2c 100644 --- a/browser-interface/packages/shared/scene-loader/types.ts +++ b/browser-interface/packages/shared/scene-loader/types.ts @@ -1,6 +1,5 @@ import { Entity } from '@dcl/schemas' import { InstancedSpawnPoint, LoadableScene } from 'shared/types' -import { Vector2 } from '../protocol/decentraland/common/vectors.gen' export type SetDesiredScenesCommand = { scenes: LoadableScene[] @@ -17,7 +16,6 @@ export interface ISceneLoader { fetchScenesByLocation(parcels: string[]): Promise stop(): Promise invalidateCache(sceneId: Entity): void - getSceneId?(position: Vector2): Promise } export type SceneLoaderState = { From 3048c8c4fd43e15eafbf5a7ad0a275a473da647c Mon Sep 17 00:00:00 2001 From: Gonzalo DCL Date: Thu, 28 Dec 2023 11:55:14 -0300 Subject: [PATCH 21/23] fix disconnect room --- .../packages/shared/comms/sagas.ts | 38 +++++++++---------- .../packages/shared/comms/selectors.ts | 2 +- 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/browser-interface/packages/shared/comms/sagas.ts b/browser-interface/packages/shared/comms/sagas.ts index 07f6ef493e..12af9a5f64 100644 --- a/browser-interface/packages/shared/comms/sagas.ts +++ b/browser-interface/packages/shared/comms/sagas.ts @@ -49,7 +49,7 @@ import { positionReportToCommsPositionRfc4 } from './interface/utils' import { commsLogger } from './logger' import { Rfc4RoomConnection } from './logic/rfc-4-room-connection' import { processAvatarVisibility } from './peers' -import { getCommsRoom, getSceneRooms, reconnectionState } from './selectors' +import { getCommsRoom, getSceneRoom, getSceneRooms, reconnectionState } from './selectors' import { RootState } from 'shared/store/rootTypes' import { now } from 'lib/javascript/now' import { getGlobalAudioStream } from './adapters/voice/loopback' @@ -516,7 +516,6 @@ function* handleRoomDisconnectionSaga(action: HandleRoomDisconnection) { } function* sceneRoomComms() { - let currentSceneId: string = '' const commsSceneToRemove = new Map() const adapter: IRealmAdapter = yield call(ensureRealmAdapter) const isWorld = isWorldLoaderActive(adapter) @@ -539,38 +538,37 @@ function* sceneRoomComms() { encodeParcelPosition(reason.newParcel.payload.position) ]) const sceneId = scenes.scenes[0].id + const currentScene: ReturnType = yield select(getSceneRoom) // We are still on the same scene. - if (sceneId === currentSceneId) continue - const oldSceneId = currentSceneId - yield call(checkDisconnectScene, sceneId, oldSceneId, commsSceneToRemove) + if (sceneId === currentScene?.id) continue + + yield call(checkDisconnectScene, sceneId, commsSceneToRemove) yield call(connectSceneToComms, sceneId) // Player moved to a new scene. Instanciate new comms - currentSceneId = sceneId } } } -function* checkDisconnectScene( - currentSceneId: string, - oldSceneId: string, - commsSceneToRemove: Map -) { +function* checkDisconnectScene(currentSceneId: string, commsSceneToRemove: Map) { // avoid deleting an already created comms. Use when the user is switching between two scenes if (commsSceneToRemove.has(currentSceneId)) { clearTimeout(commsSceneToRemove.get(currentSceneId)) commsSceneToRemove.delete(currentSceneId) } - if (!oldSceneId) return - console.log('[BOEDO SceneComms]: will disconnect', oldSceneId) + console.log('[BOEDO SceneComms]: will disconnect') const sceneRooms: ReturnType = yield select(getSceneRooms) - const oldRoom = sceneRooms.get(oldSceneId) - const timeout = setTimeout(() => { - console.log('[BOEDO SceneComms]: disconnectSceneComms', oldSceneId, !!oldRoom) - void oldRoom?.disconnect() - commsSceneToRemove.delete(oldSceneId) - }, 1000) - commsSceneToRemove.set(oldSceneId, timeout) + for (const [roomId, room] of sceneRooms) { + if (roomId === currentSceneId) continue + if (commsSceneToRemove.has(roomId)) continue + const timeout = setTimeout(() => { + console.log('[BOEDO SceneComms]: disconnectSceneComms', roomId) + void room.disconnect() + commsSceneToRemove.delete(roomId) + sceneRooms.delete(roomId) + }, 1000) + commsSceneToRemove.set(roomId, timeout) + } } function* connectSceneToComms(sceneId: string) { diff --git a/browser-interface/packages/shared/comms/selectors.ts b/browser-interface/packages/shared/comms/selectors.ts index 4aa70b7c68..8dcc6c4641 100644 --- a/browser-interface/packages/shared/comms/selectors.ts +++ b/browser-interface/packages/shared/comms/selectors.ts @@ -18,7 +18,7 @@ import { import { ensureRealmAdapter } from '../realm/ensureRealmAdapter' export const getCommsIsland = (store: RootCommsState): string | undefined => store.comms.island -export const getSceneRoomComms = (state: RootCommsState): RoomConnection | undefined => state.comms.scene +export const getSceneRoom = (state: RootCommsState): RoomConnection | undefined => state.comms.scene export const getSceneRooms = (state: RootCommsState): Map => state.comms.scenes export const getIslandRoom = (state: RootCommsState): RoomConnection | undefined => state.comms.context From 3f39030ff22d5ae590e7bbeee56f9c1e8ed55bd6 Mon Sep 17 00:00:00 2001 From: Hugo Arregui <969314+hugoArregui@users.noreply.github.com> Date: Tue, 2 Jan 2024 15:12:25 -0300 Subject: [PATCH 22/23] chore: simplify voice chat (#6033) --- .../shared/comms/adapters/LivekitAdapter.ts | 18 +- .../shared/comms/adapters/OfflineAdapter.ts | 9 +- .../shared/comms/adapters/SimulatorAdapter.ts | 15 +- .../shared/comms/adapters/WebSocketAdapter.ts | 10 +- .../packages/shared/comms/adapters/types.ts | 2 +- .../shared/comms/adapters/voice/Html.ts | 12 - .../comms/adapters/voice/audioDebugger.ts | 333 ----------- .../adapters/voice/liveKitVoiceHandler.ts | 180 +----- .../shared/comms/adapters/voice/loopback.ts | 71 +-- .../comms/adapters/voice/opusVoiceHandler.ts | 80 --- .../packages/shared/comms/handlers.ts | 12 +- .../packages/shared/comms/interface/index.ts | 4 +- .../comms/logic/rfc-4-room-connection.ts | 22 +- .../packages/shared/comms/peers.ts | 4 - .../packages/shared/comms/sagas.ts | 18 +- .../packages/shared/comms/selectors.ts | 10 +- .../downloadManager.ts | 1 - .../shared/session/getPerformanceInfo.ts | 4 +- .../shared/voiceChat/VoiceCommunicator.ts | 545 ------------------ .../packages/shared/voiceChat/handlers.ts | 40 -- .../packages/shared/voiceChat/sagas.ts | 23 +- .../packages/shared/voiceChat/utils.ts | 14 - 22 files changed, 97 insertions(+), 1330 deletions(-) delete mode 100644 browser-interface/packages/shared/comms/adapters/voice/Html.ts delete mode 100644 browser-interface/packages/shared/comms/adapters/voice/audioDebugger.ts delete mode 100644 browser-interface/packages/shared/comms/adapters/voice/opusVoiceHandler.ts delete mode 100644 browser-interface/packages/shared/voiceChat/VoiceCommunicator.ts delete mode 100644 browser-interface/packages/shared/voiceChat/handlers.ts delete mode 100644 browser-interface/packages/shared/voiceChat/utils.ts diff --git a/browser-interface/packages/shared/comms/adapters/LivekitAdapter.ts b/browser-interface/packages/shared/comms/adapters/LivekitAdapter.ts index f20229e8c0..9f897b7b12 100644 --- a/browser-interface/packages/shared/comms/adapters/LivekitAdapter.ts +++ b/browser-interface/packages/shared/comms/adapters/LivekitAdapter.ts @@ -17,13 +17,12 @@ import type { VoiceHandler } from 'shared/voiceChat/VoiceHandler' import { commsLogger } from '../logger' import type { ActiveVideoStreams, CommsAdapterEvents, MinimumCommunicationsAdapter, SendHints } from './types' import { createLiveKitVoiceHandler } from './voice/liveKitVoiceHandler' -import { GlobalAudioStream } from './voice/loopback' export type LivekitConfig = { url: string token: string logger: ILogger - globalAudioStream: GlobalAudioStream + voiceChatEnabled: boolean } export class LivekitAdapter implements MinimumCommunicationsAdapter { @@ -31,12 +30,13 @@ export class LivekitAdapter implements MinimumCommunicationsAdapter { private disposed = false private readonly room: Room - private voiceHandler: VoiceHandler + private voiceHandler: VoiceHandler | undefined = undefined constructor(private config: LivekitConfig) { this.room = new Room() - - this.voiceHandler = createLiveKitVoiceHandler(this.room, this.config.globalAudioStream) + if (config.voiceChatEnabled) { + this.voiceHandler = createLiveKitVoiceHandler(this.room) + } this.room .on(RoomEvent.ParticipantConnected, (_: RemoteParticipant) => { @@ -52,7 +52,6 @@ export class LivekitAdapter implements MinimumCommunicationsAdapter { this.config.logger.log(this.room.name, 'connection state changed', state) }) .on(RoomEvent.Disconnected, (reason: DisconnectReason | undefined) => { - this.config.logger.log('[BOEDO]: on disconnect', reason, this.room.name) if (this.disposed) { return } @@ -79,7 +78,7 @@ export class LivekitAdapter implements MinimumCommunicationsAdapter { }) } - async createVoiceHandler(): Promise { + async getVoiceHandler(): Promise { return this.voiceHandler } @@ -111,6 +110,9 @@ export class LivekitAdapter implements MinimumCommunicationsAdapter { try { await this.room.localParticipant.publishData(data, reliable ? DataPacket_Kind.RELIABLE : DataPacket_Kind.LOSSY) } catch (err: any) { + if (this.disposed) { + return + } // NOTE: for tracking purposes only, this is not a "code" error, this is a failed connection or a problem with the livekit instance trackEvent('error', { context: 'livekit-adapter', @@ -118,7 +120,6 @@ export class LivekitAdapter implements MinimumCommunicationsAdapter { stack: err.stack, saga_stack: `room session id: ${this.room.sid}, participant id: ${this.room.localParticipant.sid}, state: ${state}` }) - this.config.logger.log('[BOEDO]: error sending data', err, err.mesage) await this.disconnect() } } @@ -132,7 +133,6 @@ export class LivekitAdapter implements MinimumCommunicationsAdapter { return } - this.config.logger.log('[BOEDO]: do_disconnect', this.room.name) this.disposed = true await this.room.disconnect().catch(commsLogger.error) this.events.emit('DISCONNECTION', { kicked }) diff --git a/browser-interface/packages/shared/comms/adapters/OfflineAdapter.ts b/browser-interface/packages/shared/comms/adapters/OfflineAdapter.ts index 67f6896400..77b380bb63 100644 --- a/browser-interface/packages/shared/comms/adapters/OfflineAdapter.ts +++ b/browser-interface/packages/shared/comms/adapters/OfflineAdapter.ts @@ -1,17 +1,18 @@ import mitt from 'mitt' import { VoiceHandler } from 'shared/voiceChat/VoiceHandler' import { CommsAdapterEvents, MinimumCommunicationsAdapter, SendHints } from './types' -import { createOpusVoiceHandler } from './voice/opusVoiceHandler' export class OfflineAdapter implements MinimumCommunicationsAdapter { events = mitt() constructor() {} - async createVoiceHandler(): Promise { - return createOpusVoiceHandler() + async getVoiceHandler(): Promise { + return undefined } async disconnect(_error?: Error | undefined): Promise {} send(_data: Uint8Array, _hints: SendHints): void {} async connect(): Promise {} - async getParticipants() { return [] } + async getParticipants() { + return [] + } } diff --git a/browser-interface/packages/shared/comms/adapters/SimulatorAdapter.ts b/browser-interface/packages/shared/comms/adapters/SimulatorAdapter.ts index e708d43f29..637457d99d 100644 --- a/browser-interface/packages/shared/comms/adapters/SimulatorAdapter.ts +++ b/browser-interface/packages/shared/comms/adapters/SimulatorAdapter.ts @@ -12,15 +12,13 @@ import { Position, ProfileRequest, ProfileResponse, - Scene, - Voice + Scene } from 'shared/protocol/decentraland/kernel/comms/rfc4/comms.gen' import { lastPlayerPosition } from 'shared/world/positionThings' import { CommsEvents, RoomConnection } from '../interface' import { Rfc4RoomConnection } from '../logic/rfc-4-room-connection' import { CommsAdapterEvents, SendHints } from './types' import { VoiceHandler } from 'shared/voiceChat/VoiceHandler' -import { createOpusVoiceHandler } from './voice/opusVoiceHandler' export class SimulationRoom implements RoomConnection { events = mitt() @@ -51,8 +49,8 @@ export class SimulationRoom implements RoomConnection { send(_data: Uint8Array, _hints: SendHints): void {}, async connect(): Promise {}, async disconnect(_error?: Error): Promise {}, - async createVoiceHandler() { - throw new Error('not implemented') + async getVoiceHandler(): Promise { + return undefined }, async getParticipants() { return Array.from(peers.keys()) @@ -60,8 +58,8 @@ export class SimulationRoom implements RoomConnection { }) } - async createVoiceHandler(): Promise { - return createOpusVoiceHandler() + async getVoiceHandler(): Promise { + return this.roomConnection.getVoiceHandler() } async spawnPeer(): Promise { @@ -135,9 +133,6 @@ export class SimulationRoom implements RoomConnection { async sendChatMessage(message: Chat): Promise { await this.roomConnection.sendChatMessage(message) } - async sendVoiceMessage(message: Voice): Promise { - await this.roomConnection.sendVoiceMessage(message) - } update() { let i = 0 diff --git a/browser-interface/packages/shared/comms/adapters/WebSocketAdapter.ts b/browser-interface/packages/shared/comms/adapters/WebSocketAdapter.ts index 5622d650e6..836684f9ff 100644 --- a/browser-interface/packages/shared/comms/adapters/WebSocketAdapter.ts +++ b/browser-interface/packages/shared/comms/adapters/WebSocketAdapter.ts @@ -7,7 +7,6 @@ import { ExplorerIdentity } from 'shared/session/types' import { Authenticator } from '@dcl/crypto' import mitt from 'mitt' import { CommsAdapterEvents, MinimumCommunicationsAdapter, SendHints } from './types' -import { createOpusVoiceHandler } from './voice/opusVoiceHandler' import { VoiceHandler } from 'shared/voiceChat/VoiceHandler' import { notifyStatusThroughChat } from 'shared/chat' import { wsAsAsyncChannel } from '../logic/ws-async-channel' @@ -36,7 +35,10 @@ export class WebSocketAdapter implements MinimumCommunicationsAdapter { private ws: WebSocket | null = null - constructor(public url: string, private identity: ExplorerIdentity) {} + constructor( + public url: string, + private identity: ExplorerIdentity + ) {} async connect(): Promise { if (this.ws) throw new Error('Cannot call connect twice per IBrokerTransport') @@ -131,8 +133,8 @@ export class WebSocketAdapter implements MinimumCommunicationsAdapter { } } - async createVoiceHandler(): Promise { - return createOpusVoiceHandler() + async getVoiceHandler(): Promise { + return undefined } handleWelcomeMessage(welcomeMessage: rfc5.WsWelcome, socket: WebSocket) { diff --git a/browser-interface/packages/shared/comms/adapters/types.ts b/browser-interface/packages/shared/comms/adapters/types.ts index cc3d7db853..9690478e94 100644 --- a/browser-interface/packages/shared/comms/adapters/types.ts +++ b/browser-interface/packages/shared/comms/adapters/types.ts @@ -28,7 +28,7 @@ export interface MinimumCommunicationsAdapter { */ events: Emitter - createVoiceHandler(): Promise + getVoiceHandler(): Promise getParticipants(): Promise } diff --git a/browser-interface/packages/shared/comms/adapters/voice/Html.ts b/browser-interface/packages/shared/comms/adapters/voice/Html.ts deleted file mode 100644 index b4cfffe201..0000000000 --- a/browser-interface/packages/shared/comms/adapters/voice/Html.ts +++ /dev/null @@ -1,12 +0,0 @@ -export default class Html { - static loopbackAudioElement() { - const audio = document.getElementById('voice-chat-audio') - if (!audio) { - console.error('There is no #voice-chat-audio element to use with VoiceChat. Returning a `new Audio()`') - const audio = new Audio() - document.body.append(audio) - return audio - } - return audio as HTMLAudioElement | undefined - } -} diff --git a/browser-interface/packages/shared/comms/adapters/voice/audioDebugger.ts b/browser-interface/packages/shared/comms/adapters/voice/audioDebugger.ts deleted file mode 100644 index 3f2f30526d..0000000000 --- a/browser-interface/packages/shared/comms/adapters/voice/audioDebugger.ts +++ /dev/null @@ -1,333 +0,0 @@ -import mitt from 'mitt' -// eslint-ignore @typescript-eslint/no-unused-vars - -type BaseNode = { - cyId: number - nodeName: string -} - -type Graph = { - edges: Record - nodes: Record - nodeNum: number -} - -const events = mitt<{ - addEdge: { from: BaseNode; to: BaseNode } - addNode: { node: BaseNode } - removeNode: { node: BaseNode } - graphChanged: Graph -}>() - -// state and redux ------------------------------------------------------------- - -let currentGraphState: Graph = { edges: {}, nodes: {}, nodeNum: 0 } - -function addNodeToGraphReducer(graph: Graph, node: BaseNode): Graph { - if (!node.cyId) { - graph.nodeNum++ - node.cyId = graph.nodeNum - } - - let nodeName = getConstructorName(node) - - if (node instanceof MediaStreamAudioSourceNode) { - nodeName = nodeName + '\n' + node.mediaStream.id - } - - return { - ...graph, - nodes: { ...graph.nodes, [node.cyId]: Object.assign(node, { nodeName }) } - } -} - -events.on('addEdge', (evt) => { - currentGraphState = addNodeToGraphReducer(addNodeToGraphReducer(currentGraphState, evt.to), evt.from) - - const edgeKey = evt.from.cyId + '_' + evt.to.cyId - - currentGraphState = { - ...currentGraphState, - edges: { - ...currentGraphState.edges, - [edgeKey]: evt - } - } - - events.emit('graphChanged', currentGraphState) -}) - -events.on('removeNode', (evt) => { - const newEdges: Graph['edges'] = {} - - for (const i in currentGraphState.edges) { - const edge = currentGraphState.edges[i] - if (edge.from.cyId !== evt.node.cyId && edge.to.cyId !== evt.node.cyId) { - newEdges[i] = edge - } - } - - currentGraphState = { - ...currentGraphState, - edges: newEdges - } - - delete currentGraphState.nodes[evt.node.cyId] - - events.emit('graphChanged', currentGraphState) -}) - -events.on('addNode', (evt) => { - currentGraphState = addNodeToGraphReducer(currentGraphState, evt.node) - events.emit('graphChanged', currentGraphState) -}) - -globalThis.getAudioGraph = getAudioGraph -globalThis.getAudioGraphViz = getAudioGraphViz - -export function getAudioGraph() { - return currentGraphState -} - -export function getAudioGraphViz() { - const nodes: string[] = [] - const edges: string[] = [] - - function nodeName(node: BaseNode): string { - return 'node' + node.cyId - } - - for (const n in currentGraphState.nodes) { - const node = currentGraphState.nodes[n] - - if (node instanceof AudioContext) { - nodes.push(` - subgraph cluster_${n} { - style=filled; - color=lightgrey; - node [style=filled,color=white]; - ${nodeName(node.destination as any)}; - ${nodeName(node.listener as any)} [label="listener ${node.listener.positionX.value.toFixed( - 2 - )},${node.listener.positionY.value.toFixed(2)},${node.listener.positionZ.value.toFixed(2)}"]; - label = ${JSON.stringify('AudioContext\nglobalThis.__node_' + n)}; - } - `) - } else if (node instanceof PannerNode) { - nodes.push( - `${nodeName(node)} [label=${JSON.stringify( - `Panner ${node.positionX.value.toFixed(2)},${ - (node.positionY.value.toFixed(2), node.positionZ.value.toFixed(2)) - }\nglobalThis.__node_${n}` - )},shape="diamond"];` - ) - } else if (node instanceof GainNode) { - nodes.push( - `${nodeName(node)} [label=${JSON.stringify(`Gain ${node.gain.value}\nglobalThis.__node_${n}`)},shape="oval"];` - ) - } else if (node instanceof MediaStreamAudioDestinationNode) { - nodes.push(`${nodeName(node)} [label=${JSON.stringify(node.nodeName + '\nglobalThis.__node_' + n)}];`) - if (node.context['cyId']) { - edges.push(`${nodeName(node)} -> ${nodeName(node.context.destination as any)}`) - } - } else { - nodes.push(`${nodeName(node)} [label=${JSON.stringify(node.nodeName + '\nglobalThis.__node_' + n)}];`) - } - - if (node instanceof MediaStreamAudioSourceNode && node.mediaStream instanceof MediaStream) { - if ('_videoPreMute' in node.mediaStream) { - nodes.push(`${nodeName(node)}_media [label=${JSON.stringify('RemoteStream')}];`) - } else if ('pc' in node.mediaStream) { - nodes.push(`${nodeName(node)}_media [label=${JSON.stringify('LocalStream')}];`) - } - nodes.push(`${nodeName(node)}_media -> ${nodeName(node)};`) - } - - ;(globalThis as any)['__node_' + n] = node - } - - for (const e in currentGraphState.edges) { - const edge = currentGraphState.edges[e] - nodes.push(`${nodeName(edge.from)} -> ${nodeName(edge.to)};`) - } - - return `digraph G { - graph [fontname = "arial", fontsize="10", color="grey", fontcolor="grey"]; - node [fontname = "arial",fontsize="10", shape="box", style="rounded"]; - edge [fontname = "arial",color="blue", fontcolor="blue",fontsize="10"]; -# nodes -${nodes.join('\n')} -# edges -${edges.join('\n')} -}` -} - -function decoratePrototype(originalFunction: any, decorator: any) { - return function (this: any, ...args: any[]) { - const res = originalFunction.apply(this, args) - decorator.call(this, res, args) - return res - } -} - -function getConstructorName(obj: any) { - if (obj.constructor.name) { - return obj.constructor.name - } - const matches = obj.constructor.toString().match(/function (\w*)/) - if (matches && matches.length) { - return matches[1] - } -} - -if (document.location.search.includes('AUDIO_DEBUG')) { - // hack ------------------------------------------------------------------------ - - AudioNode.prototype.connect = decoratePrototype( - AudioNode.prototype.connect, - function (this: AudioNode & BaseNode, result: any, args: any[]) { - events.emit('addEdge', { - from: this, - to: args[0] - }) - } - ) - - AudioNode.prototype.disconnect = decoratePrototype( - AudioNode.prototype.disconnect, - function (this: any, _result: any, _args: any[]) { - events.emit('removeNode', { - node: this - }) - } - ) - - AudioBufferSourceNode.prototype.start = decoratePrototype( - AudioBufferSourceNode.prototype.start, - function (this: any, _result: any, _args: any[]) { - console.log('WebAudioDebugger: AudioBufferSourceNode start') - } - ) - - function addNode(node: any) { - if (node instanceof AudioContext) { - addNode(node.destination) - addNode(node.listener) - } - events.emit('addNode', { - node: node as any - }) - } - - PannerNode.prototype.setPosition = decoratePrototype( - PannerNode.prototype.setPosition, - function (this: PannerNode, _result: any, _args: any[]) { - events.emit('graphChanged', { ...currentGraphState }) - } - ) - - AudioListener.prototype.setPosition = decoratePrototype( - AudioListener.prototype.setPosition, - function (this: PannerNode, _result: any, _args: any[]) { - events.emit('graphChanged', { ...currentGraphState }) - } - ) - - AudioContext.prototype.createBufferSource = decoratePrototype( - AudioContext.prototype.createBufferSource, - function (this: AudioContext, result: any, args: any[]) { - console.log('WebAudioDebugger: Create BufferSourceNode', { this: this, result, args }) - this.addEventListener('ended', function () { - console.log('WebAudioDebugger: AudioBufferSourceNode ended') - }) - addNode(this) - addNode(result) - } - ) - - AudioContext.prototype.createGain = decoratePrototype( - AudioContext.prototype.createGain, - function (this: AudioContext, result: any, args: any[]) { - console.log('WebAudioDebugger: Create GainNode', { this: this, result, args }) - addNode(this) - addNode(result) - } - ) - - AudioContext.prototype.createPanner = decoratePrototype( - AudioContext.prototype.createPanner, - function (this: AudioContext, result: any, args: any[]) { - console.log('WebAudioDebugger: Create PannerNode', { this: this, result, args }) - addNode(this) - addNode(result) - } - ) - - AudioContext.prototype.createDynamicsCompressor = decoratePrototype( - AudioContext.prototype.createDynamicsCompressor, - function (this: AudioContext, result: any, args: any[]) { - console.log('WebAudioDebugger: Create DynamicsCompressorNode', { this: this, result, args }) - addNode(this) - addNode(result) - } - ) - - AudioContext.prototype.createDelay = decoratePrototype( - AudioContext.prototype.createDelay, - function (this: AudioContext, result: any, args: any[]) { - console.log('WebAudioDebugger: Create DelayNode', { this: this, result, args }) - addNode(this) - addNode(result) - } - ) - - AudioContext.prototype.createConvolver = decoratePrototype( - AudioContext.prototype.createConvolver, - function (this: AudioContext, result: any, args: any[]) { - console.log('WebAudioDebugger: Create ConvolverNode', { this: this, result, args }) - addNode(this) - addNode(result) - } - ) - - AudioContext.prototype.createAnalyser = decoratePrototype( - AudioContext.prototype.createAnalyser, - function (this: AudioContext, result: any, args: any[]) { - console.log('WebAudioDebugger: Create AnalyserNode', { this: this, result, args }) - addNode(this) - addNode(result) - } - ) - - AudioContext.prototype.createBiquadFilter = decoratePrototype( - AudioContext.prototype.createBiquadFilter, - function (this: AudioContext, result: any, args: any[]) { - console.log('WebAudioDebugger: Create BiquadFilter', { this: this, result, args }) - addNode(this) - addNode(result) - } - ) - - AudioContext.prototype.createOscillator = decoratePrototype( - AudioContext.prototype.createOscillator, - function (this: AudioContext, result: any, args: any[]) { - console.log('WebAudioDebugger: Create Oscillator', { this: this, result, args }) - addNode(this) - addNode(result) - } - ) - - AudioContext.prototype.decodeAudioData = decoratePrototype( - AudioContext.prototype.decodeAudioData, - function (this: AudioContext, result: any, args: any[]) { - console.log('WebAudioDebugger:decodeAudioData', args[0].byteLength, { this: this, result, args }) - } - ) - - AudioContext.prototype.constructor = decoratePrototype( - AudioContext.prototype.constructor, - function (this: AudioContext, result: any, args: any[]) { - console.log('WebAudioDebugger: Constructor', { this: this, result, args }) - } - ) -} diff --git a/browser-interface/packages/shared/comms/adapters/voice/liveKitVoiceHandler.ts b/browser-interface/packages/shared/comms/adapters/voice/liveKitVoiceHandler.ts index 85df9c3633..aa281103d5 100644 --- a/browser-interface/packages/shared/comms/adapters/voice/liveKitVoiceHandler.ts +++ b/browser-interface/packages/shared/comms/adapters/voice/liveKitVoiceHandler.ts @@ -2,7 +2,6 @@ import * as rfc4 from 'shared/protocol/decentraland/kernel/comms/rfc4/comms.gen' import { createLogger } from 'lib/logger' import { ParticipantEvent, - RemoteAudioTrack, RemoteParticipant, RemoteTrack, RemoteTrackPublication, @@ -10,108 +9,50 @@ import { RoomEvent, Track } from 'livekit-client' -import { getPeer } from 'shared/comms/peers' -import { getCurrentUserProfile } from 'shared/profiles/selectors' -import { store } from 'shared/store/isolatedStore' -import { shouldPlayVoice } from 'shared/voiceChat/selectors' -import { getSpatialParamsFor } from 'shared/voiceChat/utils' import { VoiceHandler } from 'shared/voiceChat/VoiceHandler' -import { GlobalAudioStream } from './loopback' +import { loopbackAudioElement } from './loopback' -type ParticipantInfo = { - tracks: Map -} - -type ParticipantTrack = { - streamNode: MediaStreamAudioSourceNode - panNode: PannerNode -} - -export function createLiveKitVoiceHandler(room: Room, globalAudioStream: GlobalAudioStream): VoiceHandler { +export function createLiveKitVoiceHandler(room: Room): VoiceHandler { const logger = createLogger('🎙 LiveKitVoiceCommunicator: ') + // const globalAudioStream = await getGlobalAudioStream() + let recordingListener: ((state: boolean) => void) | undefined let errorListener: ((message: string) => void) | undefined + let onUserTalkingCallback: ((userId: string, talking: boolean) => void) | undefined = undefined - let globalVolume: number = 1.0 + // let globalVolume: number = 1.0 let validInput = false - let onUserTalkingCallback: (userId: string, talking: boolean) => void = () => {} - - const participantsInfo = new Map() function handleTrackSubscribed( track: RemoteTrack, _publication: RemoteTrackPublication, participant: RemoteParticipant ) { - if (track.kind !== Track.Kind.Audio || !track.sid) { + if (track.kind !== Track.Kind.Audio) { return } participant.on(ParticipantEvent.IsSpeakingChanged, (talking: boolean) => { - onUserTalkingCallback(participant.identity, talking) + if (onUserTalkingCallback) { + onUserTalkingCallback(participant.identity, talking) + } }) - let info = participantsInfo.get(participant.identity) - if (!info) { - info = { tracks: new Map() } - participantsInfo.set(participant.identity, info) - } - - const audioContext = globalAudioStream.getAudioContext() - const streamNode = audioContext.createMediaStreamSource(track.mediaStream!) - const panNode = audioContext.createPanner() - - streamNode.connect(panNode) - panNode.connect(globalAudioStream.getGainNode()) - - panNode.panningModel = 'equalpower' - panNode.distanceModel = 'inverse' - panNode.refDistance = 5 - panNode.maxDistance = 10000 - panNode.coneOuterAngle = 360 - panNode.coneInnerAngle = 180 - panNode.coneOuterGain = 0.9 - panNode.rolloffFactor = 1.0 - - info.tracks.set(track.sid, { panNode, streamNode }) + const element = track.attach() + loopbackAudioElement().appendChild(element) } function handleTrackUnsubscribed( - remoteTrack: RemoteTrack, + track: RemoteTrack, _publication: RemoteTrackPublication, - participant: RemoteParticipant + _participant: RemoteParticipant ) { - if (remoteTrack.kind !== Track.Kind.Audio || !remoteTrack.sid) { - return - } - - const info = participantsInfo.get(participant.identity) - if (!info) { + if (track.kind !== Track.Kind.Audio) { return } - const track = info.tracks.get(remoteTrack.sid) - if (track) { - track.panNode.disconnect() - track.streamNode.disconnect() - } - - info.tracks.delete(remoteTrack.sid) - } - - function handleParticipantDisconnected(p: RemoteParticipant) { - const info = participantsInfo.get(p.identity) - if (!info) { - return - } - - for (const track of info.tracks.values()) { - track.panNode.disconnect() - track.streamNode.disconnect() - } - - participantsInfo.delete(p.identity) + track.detach() } room @@ -122,9 +63,8 @@ export function createLiveKitVoiceHandler(room: Room, globalAudioStream: GlobalA errorListener('Media Device Error') } }) - .on(RoomEvent.ParticipantDisconnected, handleParticipantDisconnected) - logger.log('initialized') + logger.log(`initialized ${room.name}`) return { async setRecording(recording) { @@ -147,7 +87,7 @@ export function createLiveKitVoiceHandler(room: Room, globalAudioStream: GlobalA room.startAudio().catch(logger.error) } - globalAudioStream.play() + // globalAudioStream.play() } catch (err: any) { logger.error(err) } @@ -158,81 +98,15 @@ export function createLiveKitVoiceHandler(room: Room, globalAudioStream: GlobalA onError(cb) { errorListener = cb }, - reportPosition(position: rfc4.Position) { - const spatialParams = getSpatialParamsFor(position) - const audioContext = globalAudioStream.getAudioContext() - const listener = audioContext.listener - - if (listener.positionX) { - listener.positionX.setValueAtTime(spatialParams.position[0], audioContext.currentTime) - listener.positionY.setValueAtTime(spatialParams.position[1], audioContext.currentTime) - listener.positionZ.setValueAtTime(spatialParams.position[2], audioContext.currentTime) - } else { - listener.setPosition(spatialParams.position[0], spatialParams.position[1], spatialParams.position[2]) - } - - if (listener.forwardX) { - listener.forwardX.setValueAtTime(spatialParams.orientation[0], audioContext.currentTime) - listener.forwardY.setValueAtTime(spatialParams.orientation[1], audioContext.currentTime) - listener.forwardZ.setValueAtTime(spatialParams.orientation[2], audioContext.currentTime) - listener.upX.setValueAtTime(0, audioContext.currentTime) - listener.upY.setValueAtTime(1, audioContext.currentTime) - listener.upZ.setValueAtTime(0, audioContext.currentTime) - } else { - listener.setOrientation( - spatialParams.orientation[0], - spatialParams.orientation[1], - spatialParams.orientation[2], - 0, - 1, - 0 - ) - } - - for (const participant of room.participants.values()) { - const address = participant.identity - const peer = getPeer(address) - const participantInfo = participantsInfo.get(address) - - const state = store.getState() - const profile = getCurrentUserProfile(state) - if (profile) { - const muted = !shouldPlayVoice(state, profile, address) - const audioPublication = participant.getTrack(Track.Source.Microphone) - if (audioPublication && audioPublication.track) { - const audioTrack = audioPublication.track as RemoteAudioTrack - audioTrack.setMuted(muted) - } - } - - if (participantInfo) { - const spatialParams = peer?.position || position - for (const { panNode } of participantInfo.tracks.values()) { - if (panNode.positionX) { - panNode.positionX.setValueAtTime(spatialParams.positionX, audioContext.currentTime) - panNode.positionY.setValueAtTime(spatialParams.positionY, audioContext.currentTime) - panNode.positionZ.setValueAtTime(spatialParams.positionZ, audioContext.currentTime) - } else { - panNode.setPosition(spatialParams.positionX, spatialParams.positionY, spatialParams.positionZ) - } - - if (panNode.orientationX) { - panNode.orientationX.setValueAtTime(0, audioContext.currentTime) - panNode.orientationY.setValueAtTime(0, audioContext.currentTime) - panNode.orientationZ.setValueAtTime(1, audioContext.currentTime) - } else { - panNode.setOrientation(0, 0, 1) - } - } - } - } - }, + reportPosition(_position: rfc4.Position) {}, setVolume: function (volume) { - globalVolume = volume - globalAudioStream.setGainVolume(volume) + // TODO + // globalVolume = volume + // globalAudioStream.setGainVolume(volume) }, setMute: (mute) => { - globalAudioStream.setGainVolume(mute ? 0 : globalVolume) + // TODO + // globalAudioStream.setGainVolume(mute ? 0 : globalVolume) }, setInputStream: async (localStream) => { try { @@ -246,6 +120,10 @@ export function createLiveKitVoiceHandler(room: Room, globalAudioStream: GlobalA hasInput: () => { return validInput }, - async destroy() {} + async destroy() { + onUserTalkingCallback = undefined + recordingListener = undefined + errorListener = undefined + } } } diff --git a/browser-interface/packages/shared/comms/adapters/voice/loopback.ts b/browser-interface/packages/shared/comms/adapters/voice/loopback.ts index 6c4253ef1a..1371991770 100644 --- a/browser-interface/packages/shared/comms/adapters/voice/loopback.ts +++ b/browser-interface/packages/shared/comms/adapters/voice/loopback.ts @@ -1,63 +1,10 @@ -import Html from './Html' - -export type GlobalAudioStream = { - setGainVolume(volume: number): void - getDestinationStream(): MediaStream - getAudioContext(): AudioContext - getGainNode(): GainNode - play(): void -} - -let globalAudioStream: undefined | GlobalAudioStream = undefined - -export async function getGlobalAudioStream() { - if (!globalAudioStream) { - globalAudioStream = await createAudioStream() - } - - return globalAudioStream -} - -export async function createAudioStream(): Promise { - const parentElement = Html.loopbackAudioElement() - if (!parentElement) { - throw new Error('Cannot create global audio stream: no parent element') - } - - const audioContext = new AudioContext() - const destination = audioContext.createMediaStreamDestination() - - const gainNode = new GainNode(audioContext) - gainNode.connect(destination) - gainNode.gain.value = 1 - - parentElement.srcObject = destination.stream - - function getGainNode() { - return gainNode - } - - function setGainVolume(volume: number) { - gainNode.gain.value = volume - } - - function getDestinationStream() { - return destination.stream - } - - function getAudioContext() { - return audioContext - } - - function play() { - parentElement!.play().catch(console.error) - } - - return { - getGainNode, - setGainVolume, - getDestinationStream, - getAudioContext, - play - } +export function loopbackAudioElement() { + const audio = document.getElementById('voice-chat-audio') + if (!audio) { + console.error('There is no #voice-chat-audio element to use with VoiceChat. Returning a `new Audio()`') + const audio = new Audio() + document.body.append(audio) + return audio + } + return audio as HTMLAudioElement } diff --git a/browser-interface/packages/shared/comms/adapters/voice/opusVoiceHandler.ts b/browser-interface/packages/shared/comms/adapters/voice/opusVoiceHandler.ts deleted file mode 100644 index 40e38615f1..0000000000 --- a/browser-interface/packages/shared/comms/adapters/voice/opusVoiceHandler.ts +++ /dev/null @@ -1,80 +0,0 @@ -import { createLogger } from 'lib/logger' -import { VoiceHandler } from 'shared/voiceChat/VoiceHandler' -import { VoiceCommunicator } from 'shared/voiceChat/VoiceCommunicator' -import { commConfigurations } from 'config' -import Html from './Html' -import { getCommsRoom } from 'shared/comms/selectors' -import { getSpatialParamsFor } from 'shared/voiceChat/utils' -import * as rfc4 from 'shared/protocol/decentraland/kernel/comms/rfc4/comms.gen' -import { store } from 'shared/store/isolatedStore' -import withCache from 'lib/javascript/withCache' - -import './audioDebugger' - -const getVoiceCommunicator = withCache(() => { - const logger = createLogger('OpusVoiceCommunicator: ') - return new VoiceCommunicator( - { - send(frame: rfc4.Voice) { - const transport = getCommsRoom(store.getState()) - transport?.sendVoiceMessage(frame).catch(logger.error) - } - }, - { - initialListenerParams: undefined, - panningModel: commConfigurations.voiceChatUseHRTF ? 'HRTF' : 'equalpower', - loopbackAudioElement: Html.loopbackAudioElement() - } - ) -}) - -export const createOpusVoiceHandler = (): VoiceHandler => { - const voiceCommunicator = getVoiceCommunicator() - - return { - setRecording(recording) { - if (recording) { - voiceCommunicator.start() - } else { - voiceCommunicator.pause() - } - return Promise.resolve() - }, - onUserTalking(cb) { - voiceCommunicator.addStreamPlayingListener((streamId: string, playing: boolean) => { - cb(streamId, playing) - }) - }, - onRecording(cb) { - voiceCommunicator.addStreamRecordingListener((recording: boolean) => { - cb(recording) - }) - }, - onError(cb) { - voiceCommunicator.addStreamRecordingErrorListener((message) => { - cb(message) - }) - }, - reportPosition(position: rfc4.Position) { - voiceCommunicator.setListenerSpatialParams(getSpatialParamsFor(position)) - }, - setVolume: function (volume) { - voiceCommunicator.setVolume(volume) - }, - setMute: (mute) => { - voiceCommunicator.setMute(mute) - }, - setInputStream: (stream) => { - return voiceCommunicator.setInputStream(stream) - }, - hasInput: () => { - return voiceCommunicator.hasInput() - }, - playEncodedAudio: (src, position, encoded) => { - return voiceCommunicator.playEncodedAudio(src, getSpatialParamsFor(position), encoded) - }, - async destroy() { - // noop - } - } -} diff --git a/browser-interface/packages/shared/comms/handlers.ts b/browser-interface/packages/shared/comms/handlers.ts index cd825d429d..cd0adb207c 100644 --- a/browser-interface/packages/shared/comms/handlers.ts +++ b/browser-interface/packages/shared/comms/handlers.ts @@ -11,7 +11,6 @@ import { incrementCommsMessageReceived, incrementCommsMessageReceivedByName } fr import { getCurrentUserId } from 'shared/session/selectors' import { store } from 'shared/store/isolatedStore' import { ChatMessage as InternalChatMessage, ChatMessageType } from 'shared/types' -import { processVoiceFragment } from 'shared/voiceChat/handlers' import { isBlockedOrBanned } from 'shared/voiceChat/selectors' import { messageReceived } from '../chat/actions' import { handleRoomDisconnection } from './actions' @@ -47,7 +46,6 @@ export async function bindHandlersToCommsContext(room: RoomConnection) { room.events.on('sceneMessageBus', processParcelSceneCommsMessage) room.events.on('profileRequest', processProfileRequest) room.events.on('profileResponse', processProfileResponse) - room.events.on('voiceMessage', processVoiceFragment) room.events.on('*', (type, _) => { incrementCommsMessageReceived() @@ -79,10 +77,9 @@ export async function requestProfileFromPeers( } async function handleDisconnectionEvent(data: AdapterDisconnectedEvent, room: RoomConnection) { - try { - await onRoomLeft(room) - } catch(err) { + await onRoomLeft(room) + } catch (err) { console.error(err) // TODO: handle this } @@ -140,9 +137,8 @@ function processChatMessage(message: Package) { senderPeer.lastUpdate = Date.now() if (senderPeer.ethereumAddress) { - if (message.data.message.startsWith('␆') /* pong */ || - message.data.message.startsWith('␑') /* ping */) { - // TODO: remove this + if (message.data.message.startsWith('␆') /* pong */ || message.data.message.startsWith('␑') /* ping */) { + // TODO: remove this } else if (message.data.message.startsWith('␐')) { const [id, timestamp] = message.data.message.split(' ') diff --git a/browser-interface/packages/shared/comms/interface/index.ts b/browser-interface/packages/shared/comms/interface/index.ts index ffba64785d..5b5b79e474 100644 --- a/browser-interface/packages/shared/comms/interface/index.ts +++ b/browser-interface/packages/shared/comms/interface/index.ts @@ -10,7 +10,6 @@ export type CommsEvents = CommsAdapterEvents & { chatMessage: Package profileMessage: Package position: Package - voiceMessage: Package profileResponse: Package profileRequest: Package } @@ -30,9 +29,8 @@ export interface RoomConnection { sendPositionMessage(position: Omit): Promise sendParcelSceneMessage(message: proto.Scene): Promise sendChatMessage(message: proto.Chat): Promise - sendVoiceMessage(message: proto.Voice): Promise - createVoiceHandler(): Promise + getVoiceHandler(): Promise getParticipants(): Promise } diff --git a/browser-interface/packages/shared/comms/logic/rfc-4-room-connection.ts b/browser-interface/packages/shared/comms/logic/rfc-4-room-connection.ts index 64360bee66..5f34197940 100644 --- a/browser-interface/packages/shared/comms/logic/rfc-4-room-connection.ts +++ b/browser-interface/packages/shared/comms/logic/rfc-4-room-connection.ts @@ -24,17 +24,14 @@ export class Rfc4RoomConnection implements RoomConnection { } async connect(): Promise { - console.log('[RoomConnection Comms]: connect', this.id) await this.transport.connect() } - createVoiceHandler(): Promise { - // console.log('[RoomConnection Comms]: createVoiceHandler', this.id) - return this.transport.createVoiceHandler() + getVoiceHandler(): Promise { + return this.transport.getVoiceHandler() } sendPositionMessage(p: Omit): Promise { - // console.log('[RoomConnection Comms]: sendPositionMessage', this.id) return this.sendMessage(false, { message: { $case: 'position', @@ -46,32 +43,22 @@ export class Rfc4RoomConnection implements RoomConnection { }) } sendParcelSceneMessage(scene: proto.Scene): Promise { - // console.log('[RoomConnection Comms]: sendParcelSceneMessage', this.id) return this.sendMessage(false, { message: { $case: 'scene', scene } }) } sendProfileMessage(profileVersion: proto.AnnounceProfileVersion): Promise { - // console.log('[RoomConnection Comms]: sendProfileMessage', this.id) return this.sendMessage(false, { message: { $case: 'profileVersion', profileVersion } }) } sendProfileRequest(profileRequest: proto.ProfileRequest): Promise { - // console.log('[RoomConnection Comms]: sendProfileRequest', this.id) return this.sendMessage(false, { message: { $case: 'profileRequest', profileRequest } }) } sendProfileResponse(profileResponse: proto.ProfileResponse): Promise { - // console.log('[RoomConnection Comms]: sendProfileResponse', this.id) return this.sendMessage(false, { message: { $case: 'profileResponse', profileResponse } }) } sendChatMessage(chat: proto.Chat): Promise { - // console.log('[RoomConnection Comms]: sendChatMessage', this.id) return this.sendMessage(true, { message: { $case: 'chat', chat } }) } - sendVoiceMessage(voice: proto.Voice): Promise { - // console.log('[RoomConnection Comms]: sendVoiceMessage', this.id) - return this.sendMessage(false, { message: { $case: 'voice', voice } }) - } async disconnect() { - console.log('[RoomConnection Comms]: disconnect', this.id) await this.transport.disconnect() } @@ -86,7 +73,6 @@ export class Rfc4RoomConnection implements RoomConnection { return } - // console.log('[RoomConnection Comms]: handleMessage', message.$case, this.id) switch (message.$case) { case 'position': { this.events.emit('position', { address, data: message.position }) @@ -100,10 +86,6 @@ export class Rfc4RoomConnection implements RoomConnection { this.events.emit('chatMessage', { address, data: message.chat }) break } - case 'voice': { - this.events.emit('voiceMessage', { address, data: message.voice }) - break - } case 'profileRequest': { this.events.emit('profileRequest', { address, diff --git a/browser-interface/packages/shared/comms/peers.ts b/browser-interface/packages/shared/comms/peers.ts index 2d973d5aed..2c8993418c 100644 --- a/browser-interface/packages/shared/comms/peers.ts +++ b/browser-interface/packages/shared/comms/peers.ts @@ -80,7 +80,6 @@ export function setupPeer(address: string): PeerInformation { const ethereumAddress = address.toLowerCase() if (!peerInformationMap.has(ethereumAddress)) { - console.log('[BOEDO] Adding Peer information', ethereumAddress) const peer: PeerInformation = { ethereumAddress, lastPositionIndex: 0, @@ -213,7 +212,6 @@ function getActiveRooms(): RoomConnection[] { export async function onRoomLeft(oldRoom: RoomConnection) { const rooms = getActiveRooms() const newPeerInformationMap = new Map() - console.log('[onRoomLeft] rooms', rooms) for (const room of rooms) { if (room.id === oldRoom.id) { continue @@ -222,7 +220,6 @@ export async function onRoomLeft(oldRoom: RoomConnection) { for (const participant of await room.getParticipants()) { const info = peerInformationMap.get(participant) if (info) { - console.log('[onRoomLeft] addparticipant', participant) newPeerInformationMap.set(participant, info) } } @@ -230,7 +227,6 @@ export async function onRoomLeft(oldRoom: RoomConnection) { for (const participant of peerInformationMap.keys()) { if (!newPeerInformationMap.has(participant)) { - console.log('[onRoomLeft] removeUser', participant) avatarMessageObservable.notifyObservers({ type: AvatarMessageType.USER_REMOVED, userId: participant diff --git a/browser-interface/packages/shared/comms/sagas.ts b/browser-interface/packages/shared/comms/sagas.ts index 12af9a5f64..937e84c877 100644 --- a/browser-interface/packages/shared/comms/sagas.ts +++ b/browser-interface/packages/shared/comms/sagas.ts @@ -52,7 +52,6 @@ import { processAvatarVisibility } from './peers' import { getCommsRoom, getSceneRoom, getSceneRooms, reconnectionState } from './selectors' import { RootState } from 'shared/store/rootTypes' import { now } from 'lib/javascript/now' -import { getGlobalAudioStream } from './adapters/voice/loopback' import { store } from 'shared/store/isolatedStore' import { buildSnapshotContent } from 'shared/profiles/sagas/handleDeployProfile' import { isBase64 } from 'lib/encoding/base64ToBlob' @@ -171,6 +170,7 @@ function* handleConnectToComms(action: ConnectToCommsAction) { const adapter: RoomConnection = yield call( connectAdapter, action.payload.event.connStr, + false, identity, action.payload.event.islandId ) @@ -196,11 +196,11 @@ function* handleConnectToComms(action: ConnectToCommsAction) { async function connectAdapter( connStr: string, + voiceChatEnabled: boolean, identity: ExplorerIdentity, - id: string = 'island', + id: string, dispatchAction = true ): Promise { - console.log('[connectAdapter] ', { connStr, identity, id }) const ix = connStr.indexOf(':') const protocol = connStr.substring(0, ix) const url = connStr.substring(ix + 1) @@ -237,7 +237,7 @@ async function connectAdapter( } if (typeof response.fixedAdapter === 'string' && !response.fixedAdapter.startsWith('signed-login:')) { - return connectAdapter(response.fixedAdapter, identity, id) + return connectAdapter(response.fixedAdapter, voiceChatEnabled, identity, id) } if (typeof response.message === 'string') { @@ -274,7 +274,7 @@ async function connectAdapter( logger: commsLogger, url: theUrl.origin + theUrl.pathname, token, - globalAudioStream: await getGlobalAudioStream() + voiceChatEnabled }) if (dispatchAction) { @@ -524,8 +524,6 @@ function* sceneRoomComms() { return } - console.log('[BOEDO] isWorldLoaderActive(adapter!)', isWorldLoaderActive(adapter!)) - while (true) { const reason: { timeout?: unknown; newParcel?: { payload: { position: Vector2 } } } = yield race({ newParcel: take(SET_PARCEL_POSITION), @@ -556,13 +554,11 @@ function* checkDisconnectScene(currentSceneId: string, commsSceneToRemove: Map = yield select(getSceneRooms) for (const [roomId, room] of sceneRooms) { if (roomId === currentSceneId) continue if (commsSceneToRemove.has(roomId)) continue const timeout = setTimeout(() => { - console.log('[BOEDO SceneComms]: disconnectSceneComms', roomId) void room.disconnect() commsSceneToRemove.delete(roomId) sceneRooms.delete(roomId) @@ -572,8 +568,6 @@ function* checkDisconnectScene(currentSceneId: string, commsSceneToRemove: Map { - console.log('[BOEDO] selectors disconnect') await islandRoom.disconnect() // TBD: should we disconnect from scenes here too ? }, @@ -83,18 +82,15 @@ export const getCommsRoom = (state: RootCommsState): RoomConnection | undefined const scene = sceneRoom?.sendChatMessage(message) await Promise.all([island, scene]) }, - // TBD: how voice chat works? - createVoiceHandler: async () => { + getVoiceHandler: async () => { if (await isWorld()) { - return islandRoom.createVoiceHandler() + return islandRoom.getVoiceHandler() } - - // TBD: Feature flag for backwards compatibility if (!sceneRoom) { debugger throw new Error('Scene room not avaialble') } - return sceneRoom.createVoiceHandler() + return sceneRoom.getVoiceHandler() } } as any as RoomConnection } diff --git a/browser-interface/packages/shared/scene-loader/genesis-city-loader-impl/downloadManager.ts b/browser-interface/packages/shared/scene-loader/genesis-city-loader-impl/downloadManager.ts index 290f650977..0037320eb1 100644 --- a/browser-interface/packages/shared/scene-loader/genesis-city-loader-impl/downloadManager.ts +++ b/browser-interface/packages/shared/scene-loader/genesis-city-loader-impl/downloadManager.ts @@ -27,7 +27,6 @@ export class SceneDataDownloadManager { } async resolveEntitiesByPointer(pointers: string[]): Promise> { - console.log('[DownloadManager]resolveEntitiesByPointer ', pointers) const futures: Promise[] = [] const missingPointers: string[] = [] diff --git a/browser-interface/packages/shared/session/getPerformanceInfo.ts b/browser-interface/packages/shared/session/getPerformanceInfo.ts index 8640f6bb5a..954e187c67 100644 --- a/browser-interface/packages/shared/session/getPerformanceInfo.ts +++ b/browser-interface/packages/shared/session/getPerformanceInfo.ts @@ -54,9 +54,9 @@ export function incrementAvatarSceneMessages(value: number) { export function incrementCommsMessageSent(type: string, size: number, sceneId?: string) { if (!sceneId) { - sceneId = 'no-scene' + sceneId = 'noscene' } - const key = `${sceneId}:${type}` + const key = `${sceneId}-${type}` sentCommsMessages[key] = (sentCommsMessages[key] ?? 0) + size diff --git a/browser-interface/packages/shared/voiceChat/VoiceCommunicator.ts b/browser-interface/packages/shared/voiceChat/VoiceCommunicator.ts deleted file mode 100644 index 0bf3af908d..0000000000 --- a/browser-interface/packages/shared/voiceChat/VoiceCommunicator.ts +++ /dev/null @@ -1,545 +0,0 @@ -import { VoiceChatCodecWorkerMain, EncodeStream } from 'voice-chat-codec/VoiceChatCodecWorkerMain' -import { SortedLimitedQueue } from 'lib/data-structures/SortedLimitedQueue' -import defaultLogger from 'lib/logger' -import { VOICE_CHAT_SAMPLE_RATE, OPUS_FRAME_SIZE_MS } from 'voice-chat-codec/constants' -import { parse, write } from 'sdp-transform' -import { InputWorkletRequestTopic, OutputWorkletRequestTopic } from 'voice-chat-codec/types' -import * as rfc4 from 'shared/protocol/decentraland/kernel/comms/rfc4/comms.gen' - -// eslint-disable-next-line @typescript-eslint/no-var-requires -const workletWorkerRaw = require('../../../static/voice-chat-codec/audioWorkletProcessors.js.txt') -const workletWorkerUrl = URL.createObjectURL(new Blob([workletWorkerRaw], { type: 'application/javascript' })) - -export type AudioCommunicatorChannel = { - send(data: rfc4.Voice): any -} - -export type StreamPlayingListener = (streamId: string, playing: boolean) => any -export type StreamRecordingListener = (recording: boolean) => any -export type StreamRecordingErrorListener = (message: string) => any - -type VoiceOutput = { - encodedFramesQueue: SortedLimitedQueue - workletNode?: AudioWorkletNode - panNode?: PannerNode - spatialParams: VoiceSpatialParams - lastUpdateTime: number - playing: boolean - lastDecodedFrameOrder?: number -} - -type OutputStats = { - lostFrames: number - skippedFramesNotQueued: number - skippedFramesQueued: number -} - -type VoiceInput = { - workletNode: AudioWorkletNode - inputStream: MediaStreamAudioSourceNode - recordingContext: AudioContextWithInitPromise - encodeStream: EncodeStream -} - -export type VoiceCommunicatorOptions = { - sampleRate?: number - outputBufferLength?: number - maxDistance?: number - refDistance?: number - initialListenerParams?: VoiceSpatialParams - panningModel?: PanningModelType - distanceModel?: DistanceModelType - loopbackAudioElement?: HTMLAudioElement - volume?: number - mute?: boolean -} - -export type VoiceSpatialParams = { - position: [number, number, number] - orientation: [number, number, number] -} - -type AudioContextWithInitPromise = [AudioContext, Promise] - -const SELF_STREAM_ID = 'localhost' - -export class VoiceCommunicator { - private contextWithInitPromise: AudioContextWithInitPromise - private outputGainNode: GainNode - private outputStreamNode?: MediaStreamAudioDestinationNode - private loopbackConnections?: { src: RTCPeerConnection; dst: RTCPeerConnection } - private input?: VoiceInput - private voiceChatWorkerMain: VoiceChatCodecWorkerMain - private outputs: Record = {} - - private outputStats: Record = {} - - private streamPlayingListeners: StreamPlayingListener[] = [] - private streamRecordingListeners: StreamRecordingListener[] = [] - private streamRecordingErrorListeners: StreamRecordingErrorListener[] = [] - - private readonly sampleRate: number - private readonly outputBufferLength: number - private readonly outputExpireTime = 60 * 1000 - - private inputFramesIndex = 0 - - private checkStateTimeout: any | undefined = undefined - - private get context(): AudioContext { - return this.contextWithInitPromise[0] - } - - constructor(private channel: AudioCommunicatorChannel, private options: VoiceCommunicatorOptions) { - this.sampleRate = this.options.sampleRate ?? VOICE_CHAT_SAMPLE_RATE - this.outputBufferLength = this.options.outputBufferLength ?? 2.0 - - this.contextWithInitPromise = this.createContext({ sampleRate: this.sampleRate }) - - this.outputGainNode = this.context.createGain() - - if (options.loopbackAudioElement) { - // Workaround for echo cancellation. See: https://bugs.chromium.org/p/chromium/issues/detail?id=687574#c71 - this.outputStreamNode = this.context.createMediaStreamDestination() - this.outputGainNode.connect(this.outputStreamNode) - this.loopbackConnections = this.createRTCLoopbackConnection() - } else { - this.outputGainNode.connect(this.context.destination) - } - - if (this.options.initialListenerParams) { - this.setListenerSpatialParams(this.options.initialListenerParams) - } - - this.voiceChatWorkerMain = new VoiceChatCodecWorkerMain() - - this.startOutputsExpiration() - } - - public addStreamPlayingListener(listener: StreamPlayingListener) { - this.streamPlayingListeners.push(listener) - } - - public addStreamRecordingListener(listener: StreamRecordingListener) { - this.streamRecordingListeners.push(listener) - } - - public addStreamRecordingErrorListener(listener: StreamRecordingErrorListener) { - this.streamRecordingErrorListeners.push(listener) - } - - public hasInput() { - return !!this.input - } - - public statsFor(outputId: string) { - if (!this.outputStats[outputId]) { - this.outputStats[outputId] = { - lostFrames: 0, - skippedFramesNotQueued: 0, - skippedFramesQueued: 0 - } - } - - return this.outputStats[outputId] - } - - async playEncodedAudio(src: string, relativePosition: VoiceSpatialParams, encoded: rfc4.Voice) { - if (!this.outputs[src]) { - await this.createOutput(src, relativePosition) - } else { - this.outputs[src].lastUpdateTime = Date.now() - this.setVoiceRelativePosition(src, relativePosition) - } - - const output = this.outputs[src] - - if (output.lastDecodedFrameOrder && output.lastDecodedFrameOrder > encoded.index) { - this.statsFor(src).skippedFramesNotQueued += 1 - return - } - - const discarded = output.encodedFramesQueue.queue(encoded) - if (discarded) { - this.statsFor(src).skippedFramesQueued += 1 - } - } - - setListenerSpatialParams(spatialParams: VoiceSpatialParams) { - const listener = this.context.listener - listener.setPosition(spatialParams.position[0], spatialParams.position[1], spatialParams.position[2]) - listener.setOrientation( - spatialParams.orientation[0], - spatialParams.orientation[1], - spatialParams.orientation[2], - 0, - 1, - 0 - ) - } - - updatePannerNodeParameters(src: string) { - const panNode = this.outputs[src].panNode - const spatialParams = this.outputs[src].spatialParams - - if (panNode) { - panNode.positionX.value = spatialParams.position[0] - panNode.positionY.value = spatialParams.position[1] - panNode.positionZ.value = spatialParams.position[2] - panNode.orientationX.value = spatialParams.orientation[0] - panNode.orientationY.value = spatialParams.orientation[1] - panNode.orientationZ.value = spatialParams.orientation[2] - } - } - - setVolume(value: number) { - this.options.volume = value - const muted = this.options.mute ?? false - if (!muted) { - this.outputGainNode.gain.value = value - } - } - - setMute(mute: boolean) { - this.options.mute = mute - this.outputGainNode.gain.value = mute ? 0 : this.options.volume ?? 1 - } - - createWorkletFor(src: string) { - const workletNode = new AudioWorkletNode(this.context, 'outputProcessor', { - numberOfInputs: 0, - numberOfOutputs: 1, - processorOptions: { sampleRate: this.sampleRate, bufferLength: this.outputBufferLength } - }) - - workletNode.port.onmessage = (e) => { - if (e.data.topic === OutputWorkletRequestTopic.STREAM_PLAYING) { - if (this.outputs[src]) { - this.outputs[src].playing = e.data.playing - } - this.streamPlayingListeners.forEach((listener) => listener(src, e.data.playing)) - } - } - - return workletNode - } - - async setInputStream(stream: MediaStream) { - if (this.input) { - this.voiceChatWorkerMain.destroyEncodeStream(SELF_STREAM_ID) - if (this.input.recordingContext[0] !== this.context) { - this.input.recordingContext[0].close().catch((e) => defaultLogger.error('Error closing recording context', e)) - } - } - - try { - this.input = await this.createInputFor(stream, this.contextWithInitPromise) - } catch (e: any) { - // If this fails, then it most likely it is because the sample rate of the stream is incompatible with the context's, so we create a special context for recording - if (e.message.includes('sample-rate is currently not supported')) { - const recordingContext = this.createContext() - this.input = await this.createInputFor(stream, recordingContext) - } else { - throw e - } - } - } - - checkStatusTimeout() { - if (this.checkStateTimeout === undefined) { - this.checkStateTimeout = setTimeout(() => { - this.sendToInputWorklet(InputWorkletRequestTopic.CHECK_STATUS) - this.checkStateTimeout = undefined - }, 1200) - } - } - - start() { - if (this.input) { - this.input.workletNode.connect(this.input.recordingContext[0].destination) - this.sendToInputWorklet(InputWorkletRequestTopic.RESUME) - this.checkStatusTimeout() - } else { - this.notifyRecording(false) - } - } - - pause() { - if (this.input) { - this.sendToInputWorklet(InputWorkletRequestTopic.PAUSE) - this.checkStatusTimeout() - } else { - this.notifyRecording(false) - } - } - - private async createOutputNodes(src: string): Promise<{ workletNode: AudioWorkletNode; panNode: PannerNode }> { - await this.contextWithInitPromise[1] - const workletNode = this.createWorkletFor(src) - const panNode = this.context.createPanner() - panNode.coneInnerAngle = 180 - panNode.coneOuterAngle = 360 - panNode.coneOuterGain = 0.9 - panNode.maxDistance = this.options.maxDistance ?? 10000 - panNode.refDistance = this.options.refDistance ?? 5 - panNode.panningModel = this.options.panningModel ?? 'equalpower' - panNode.distanceModel = this.options.distanceModel ?? 'inverse' - panNode.rolloffFactor = 1.0 - workletNode.connect(panNode) - panNode.connect(this.outputGainNode) - - return { workletNode, panNode } - } - - private sendToInputWorklet(topic: InputWorkletRequestTopic) { - this.input?.workletNode.port.postMessage({ topic: topic }) - } - - private createRTCLoopbackConnection(currentRetryNumber: number = 0): { - src: RTCPeerConnection - dst: RTCPeerConnection - } { - const src = new RTCPeerConnection() - const dst = new RTCPeerConnection() - - let retryNumber = currentRetryNumber - - ;(async () => { - // When having an error, we retry in a couple of seconds. Up to 10 retries. - src.onconnectionstatechange = (_e) => { - if ( - src.connectionState === 'closed' || - src.connectionState === 'disconnected' || - (src.connectionState === 'failed' && currentRetryNumber < 10) - ) { - // Just in case, we close connections to free resources - this.closeLoopbackConnections() - this.loopbackConnections = this.createRTCLoopbackConnection(retryNumber) - } else if (src.connectionState === 'connected') { - // We reset retry number when the connection succeeds - retryNumber = 0 - } - } - - src.onicecandidate = (e) => e.candidate && dst.addIceCandidate(new RTCIceCandidate(e.candidate)) - dst.onicecandidate = (e) => e.candidate && src.addIceCandidate(new RTCIceCandidate(e.candidate)) - - dst.ontrack = (e) => (this.options.loopbackAudioElement!.srcObject = e.streams[0]) - - this.outputStreamNode!.stream.getTracks().forEach((track) => src.addTrack(track, this.outputStreamNode!.stream)) - - const offer = await src.createOffer() - - await src.setLocalDescription(offer) - - await dst.setRemoteDescription(offer) - const answer = await dst.createAnswer() - - const answerSdp = parse(answer.sdp!) - - answerSdp.media[0].fmtp[0].config = 'ptime=5;stereo=1;sprop-stereo=1;maxaveragebitrate=256000' - - answer.sdp = write(answerSdp) - - await dst.setLocalDescription(answer) - - await src.setRemoteDescription(answer) - })().catch((e) => { - defaultLogger.error('Error creating loopback connection', e) - src.close() - dst.close() - }) - - return { src, dst } - } - - private closeLoopbackConnections() { - if (this.loopbackConnections) { - const { src, dst } = this.loopbackConnections - - src.close() - dst.close() - } - } - - private async createOutput(src: string, relativePosition: VoiceSpatialParams) { - this.outputs[src] = { - encodedFramesQueue: new SortedLimitedQueue( - Math.ceil((this.outputBufferLength * 1000) / OPUS_FRAME_SIZE_MS), - (frameA, frameB) => frameA.index - frameB.index - ), - spatialParams: relativePosition, - lastUpdateTime: Date.now(), - playing: false - } - - const { workletNode, panNode } = await this.createOutputNodes(src) - - this.outputs[src].workletNode = workletNode - this.outputs[src].panNode = panNode - - const readEncodedBufferLoop = async () => { - if (this.outputs[src]) { - // We use three frames (120ms) as a jitter buffer. This is not mutch, but we don't want to add much latency. In the future we should maybe make this dynamic based on packet loss - const framesToRead = this.outputs[src].playing ? 3 : 1 - - const frames = await this.outputs[src].encodedFramesQueue.dequeueItemsWhenAvailable(framesToRead, 2000) - - if (frames.length > 0) { - this.countLostFrames(src, frames) - let stream = this.voiceChatWorkerMain.decodeStreams[src] - - if (!stream) { - stream = this.voiceChatWorkerMain.getOrCreateDecodeStream(src, this.sampleRate) - - stream.addAudioDecodedListener((samples) => { - this.outputs[src].lastUpdateTime = Date.now() - this.outputs[src].workletNode?.port.postMessage( - { topic: OutputWorkletRequestTopic.WRITE_SAMPLES, samples }, - [samples.buffer] - ) - }) - } - - frames.forEach((it) => stream.decode(it.encodedSamples)) - this.outputs[src].lastDecodedFrameOrder = frames[frames.length - 1].index - } - - await readEncodedBufferLoop() - } - } - - readEncodedBufferLoop().catch((e) => defaultLogger.log('Error while reading encoded buffer of ' + src, e)) - } - - private countLostFrames(src: string, frames: rfc4.Voice[]) { - // We can know a frame is lost if we have a missing frame index - let lostFrames = 0 - if (this.outputs[src].lastDecodedFrameOrder && this.outputs[src].lastDecodedFrameOrder! < frames[0].index) { - // We count the missing frame indexes from the last decoded frame. If there are no missin frames, 0 is added - lostFrames += frames[0].index - this.outputs[src].lastDecodedFrameOrder! - 1 - } - - for (let i = 0; i < frames.length - 1; i++) { - // We count the missing frame indexes in the current frames to decode. If there are no missin frames, 0 is added - lostFrames += frames[i + 1].index - frames[i].index - 1 - } - - this.statsFor(src).lostFrames += lostFrames - } - - private async createInputFor(stream: MediaStream, context: AudioContextWithInitPromise) { - await context[1] - const streamSource = context[0].createMediaStreamSource(stream) - const workletNode = new AudioWorkletNode(context[0], 'inputProcessor', { - numberOfInputs: 1, - numberOfOutputs: 1 - }) - - streamSource.connect(workletNode) - return { - recordingContext: context, - encodeStream: this.createInputEncodeStream(context[0], workletNode), - workletNode, - inputStream: streamSource - } - } - - private createInputEncodeStream(recordingContext: AudioContext, workletNode: AudioWorkletNode) { - const encodeStream = this.voiceChatWorkerMain.getOrCreateEncodeStream( - SELF_STREAM_ID, - this.sampleRate, - recordingContext.sampleRate - ) - - encodeStream.addAudioEncodedListener((data) => { - this.inputFramesIndex += 1 - this.channel.send({ - encodedSamples: data, - index: this.inputFramesIndex, - codec: rfc4.Voice_VoiceCodec.VC_OPUS - }) - }) - - workletNode.port.onmessage = (e) => { - if (e.data.topic === InputWorkletRequestTopic.ENCODE) { - encodeStream.encode(e.data.samples) - } - - if (e.data.topic === InputWorkletRequestTopic.ON_PAUSED) { - this.notifyRecording(false) - this.input?.workletNode.disconnect() - } - - if (e.data.topic === InputWorkletRequestTopic.ON_RECORDING) { - this.notifyRecording(true) - } - - if (e.data.topic === InputWorkletRequestTopic.TIMEOUT) { - this.notifyRecordingError('Something went wrong with the microphone. No message was recorded.') - } - } - - workletNode.onprocessorerror = (_e) => { - this.notifyRecording(false) - } - - return encodeStream - } - - private setVoiceRelativePosition(src: string, spatialParams: VoiceSpatialParams) { - this.outputs[src].spatialParams = spatialParams - this.updatePannerNodeParameters(src) - } - - private notifyRecording(recording: boolean) { - this.streamRecordingListeners.forEach((listener) => listener(recording)) - } - - private notifyRecordingError(message: string) { - this.streamRecordingErrorListeners.forEach((listener) => listener(message)) - } - - private startOutputsExpiration() { - const expireOutputs = () => { - Object.keys(this.outputs).forEach((outputId) => { - const output = this.outputs[outputId] - if (Date.now() - output.lastUpdateTime > this.outputExpireTime) { - this.destroyOutput(outputId) - } - }) - - setTimeout(expireOutputs, 2000) - } - - setTimeout(expireOutputs, 0) - } - - private createContext(contextOptions?: AudioContextOptions): AudioContextWithInitPromise { - const aContext = new AudioContext(contextOptions) - if (aContext.audioWorklet) { - const workletInitializedPromise = aContext.audioWorklet - .addModule(workletWorkerUrl) - .catch((e) => defaultLogger.error('Error loading worklet modules: ', e)) - return [aContext, workletInitializedPromise] - } else { - // TODO: trackEvent('error_initializing_worklet') to gain visibility about how many times is this issue happening - defaultLogger.error('Error loading worklet modules: audioWorklet undefined') - return [aContext, Promise.resolve()] - } - } - - private destroyOutput(outputId: string) { - this.disconnectOutputNodes(outputId) - - this.voiceChatWorkerMain.destroyDecodeStream(outputId) - - delete this.outputs[outputId] - } - - private disconnectOutputNodes(outputId: string) { - const output = this.outputs[outputId] - output.panNode?.disconnect() - output.workletNode?.disconnect() - } -} diff --git a/browser-interface/packages/shared/voiceChat/handlers.ts b/browser-interface/packages/shared/voiceChat/handlers.ts deleted file mode 100644 index fc7b7dece6..0000000000 --- a/browser-interface/packages/shared/voiceChat/handlers.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { getCurrentUserProfile } from 'shared/profiles/selectors' -import { Package } from 'shared/comms/interface/types' -import { getPeer } from 'shared/comms/peers' -import * as rfc4 from 'shared/protocol/decentraland/kernel/comms/rfc4/comms.gen' -import { store } from 'shared/store/isolatedStore' -import { getVoiceHandler, shouldPlayVoice } from './selectors' -import { voiceChatLogger } from './logger' -import { trackEvent } from 'shared/analytics/trackEvent' - -// TODO: create a component to emit opus audio in a specific position that can be used -// by the voicechat and by the SDK -export function processVoiceFragment(message: Package) { - const state = store.getState() - const voiceHandler = getVoiceHandler(state) - const profile = getCurrentUserProfile(state) - - // use getPeer instead of setupPeer to only reproduce voice messages from - // known avatars - const peerTrackingInfo = getPeer(message.address) - - if ( - voiceHandler && - profile && - peerTrackingInfo && - peerTrackingInfo.position && - shouldPlayVoice(state, profile, peerTrackingInfo.ethereumAddress) && - voiceHandler.playEncodedAudio - ) { - voiceHandler - .playEncodedAudio(peerTrackingInfo.ethereumAddress, peerTrackingInfo.position, message.data) - .catch((e: any) => { - trackEvent('error', { - context: 'voice-chat', - message: e.message, - stack: '' - }) - voiceChatLogger.error('Error playing encoded audio!', e) - }) - } -} diff --git a/browser-interface/packages/shared/voiceChat/sagas.ts b/browser-interface/packages/shared/voiceChat/sagas.ts index 3339f64d3a..a8c3b9f32c 100644 --- a/browser-interface/packages/shared/voiceChat/sagas.ts +++ b/browser-interface/packages/shared/voiceChat/sagas.ts @@ -42,13 +42,16 @@ import { import { RootVoiceChatState, VoiceChatState } from './types' import { VoiceHandler } from './VoiceHandler' -import { SET_ROOM_CONNECTION } from 'shared/comms/actions' +import { SET_SCENE_ROOM_CONNECTION } from 'shared/comms/actions' import { RoomConnection } from 'shared/comms/interface' -import { getCommsRoom } from 'shared/comms/selectors' +import { getCommsRoom, getSceneRoom } from 'shared/comms/selectors' import { waitForMetaConfigurationInitialization } from 'shared/meta/sagas' import { incrementCounter } from 'shared/analytics/occurences' import { RootWorldState } from 'shared/world/types' -import { waitForSelector } from 'lib/redux' +import { waitFor, waitForSelector } from 'lib/redux' +import { IRealmAdapter } from '../realm/types' +import { ensureRealmAdapter } from '../realm/ensureRealmAdapter' +import { isWorldLoaderActive } from '../realm/selectors' let audioRequestInitialized = false @@ -75,16 +78,24 @@ function* handleConnectVoiceChatToRoom() { yield call(waitForMetaConfigurationInitialization) while (true) { + // wait for next event to happen + yield take([SET_SCENE_ROOM_CONNECTION, JOIN_VOICE_CHAT, LEAVE_VOICE_CHAT]) + const joined: boolean = yield select(hasJoinedVoiceChat) const prevHandler = yield select(getVoiceHandler) let handler: VoiceHandler | null = null // if we are supposed to be joined, then ask the RoomConnection about the handler if (joined) { + const realmAdapter: IRealmAdapter = yield call(ensureRealmAdapter) + const isWorld = isWorldLoaderActive(realmAdapter) + if (!isWorld) { + yield call(waitFor(getSceneRoom, SET_SCENE_ROOM_CONNECTION)) + } const room: RoomConnection = yield select(getCommsRoom) if (room) { try { - handler = yield apply(room, room.createVoiceHandler, []) + handler = yield apply(room, room.getVoiceHandler, []) } catch (err: any) { yield put(setVoiceChatError(err.toString())) } @@ -101,9 +112,6 @@ function* handleConnectVoiceChatToRoom() { yield prevHandler.destroy() } } - - // wait for next event to happen - yield take([SET_ROOM_CONNECTION, JOIN_VOICE_CHAT, LEAVE_VOICE_CHAT]) } } @@ -116,7 +124,6 @@ function* handleRecordingRequest() { if (voiceHandler) { if (!isAllowedByScene || !requestedRecording) { - // Ensure that we're recording, to stop recording yield call(waitForSelector, isVoiceChatRecording) yield call(voiceHandler.setRecording, false) diff --git a/browser-interface/packages/shared/voiceChat/utils.ts b/browser-interface/packages/shared/voiceChat/utils.ts deleted file mode 100644 index 125f76ed0c..0000000000 --- a/browser-interface/packages/shared/voiceChat/utils.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { VoiceSpatialParams } from './VoiceCommunicator' -import * as rfc4 from 'shared/protocol/decentraland/kernel/comms/rfc4/comms.gen' -import { Quaternion, Vector3 } from '@dcl/ecs-math' - -export function getSpatialParamsFor(position: rfc4.Position): VoiceSpatialParams { - const orientation = Vector3.Backward().rotate( - Quaternion.FromArray([position.rotationX, position.rotationY, position.rotationZ, position.rotationW]) - ) - - return { - position: [position.positionX, position.positionY, position.positionZ], - orientation: [orientation.x, orientation.y, orientation.z] - } -} From 542d49a47d6f33b4901e2485b78d134864cddab9 Mon Sep 17 00:00:00 2001 From: Hugo Arregui Date: Thu, 4 Jan 2024 11:35:30 -0300 Subject: [PATCH 23/23] avoid duplicated chat messages --- browser-interface/packages/shared/comms/handlers.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/browser-interface/packages/shared/comms/handlers.ts b/browser-interface/packages/shared/comms/handlers.ts index cd0adb207c..8dbfc601df 100644 --- a/browser-interface/packages/shared/comms/handlers.ts +++ b/browser-interface/packages/shared/comms/handlers.ts @@ -1,6 +1,5 @@ import * as proto from 'shared/protocol/decentraland/kernel/comms/rfc4/comms.gen' import type { Avatar } from '@dcl/schemas' -import { uuid } from 'lib/javascript/uuid' import { Observable } from 'mz-observable' import { eventChannel } from 'redux-saga' import { getBannedUsers } from 'shared/meta/selectors' @@ -158,7 +157,7 @@ function processChatMessage(message: Package) { if (!isBanned) { const messageEntry: InternalChatMessage = { messageType: ChatMessageType.PUBLIC, - messageId: uuid(), + messageId: `${senderPeer.ethereumAddress}-${message.data.timestamp}`, sender: senderPeer.ethereumAddress, body: message.data.message, timestamp: message.data.timestamp