diff --git a/res/css/_components.scss b/res/css/_components.scss index ec3af8655e2..1517527034a 100644 --- a/res/css/_components.scss +++ b/res/css/_components.scss @@ -37,6 +37,11 @@ @import "./structures/_ViewSource.scss"; @import "./structures/auth/_CompleteSecurity.scss"; @import "./structures/auth/_Login.scss"; +@import "./views/audio_messages/_AudioPlayer.scss"; +@import "./views/audio_messages/_PlayPauseButton.scss"; +@import "./views/audio_messages/_PlaybackContainer.scss"; +@import "./views/audio_messages/_SeekBar.scss"; +@import "./views/audio_messages/_Waveform.scss"; @import "./views/auth/_AuthBody.scss"; @import "./views/auth/_AuthButtons.scss"; @import "./views/auth/_AuthFooter.scss"; @@ -165,6 +170,7 @@ @import "./views/messages/_MTextBody.scss"; @import "./views/messages/_MVideoBody.scss"; @import "./views/messages/_MVoiceMessageBody.scss"; +@import "./views/messages/_MediaBody.scss"; @import "./views/messages/_MessageActionBar.scss"; @import "./views/messages/_MessageTimestamp.scss"; @import "./views/messages/_MjolnirBody.scss"; @@ -253,9 +259,6 @@ @import "./views/toasts/_AnalyticsToast.scss"; @import "./views/toasts/_NonUrgentEchoFailureToast.scss"; @import "./views/verification/_VerificationShowSas.scss"; -@import "./views/voice_messages/_PlayPauseButton.scss"; -@import "./views/voice_messages/_PlaybackContainer.scss"; -@import "./views/voice_messages/_Waveform.scss"; @import "./views/voip/_CallContainer.scss"; @import "./views/voip/_CallView.scss"; @import "./views/voip/_CallViewForRoom.scss"; diff --git a/res/css/views/audio_messages/_AudioPlayer.scss b/res/css/views/audio_messages/_AudioPlayer.scss new file mode 100644 index 00000000000..9a65ad008f6 --- /dev/null +++ b/res/css/views/audio_messages/_AudioPlayer.scss @@ -0,0 +1,68 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +.mx_AudioPlayer_container { + padding: 16px 12px 12px 12px; + max-width: 267px; // use max to make the control fit in the files/pinned panels + + .mx_AudioPlayer_primaryContainer { + display: flex; + + .mx_PlayPauseButton { + margin-right: 8px; + } + + .mx_AudioPlayer_mediaInfo { + flex: 1; + overflow: hidden; // makes the ellipsis on the file name work + + & > * { + display: block; + } + + .mx_AudioPlayer_mediaName { + color: $primary-fg-color; + font-size: $font-15px; + line-height: $font-15px; + text-overflow: ellipsis; + white-space: nowrap; + overflow: hidden; + padding-bottom: 4px; // mimics the line-height differences in the Figma + } + + .mx_AudioPlayer_byline { + font-size: $font-12px; + line-height: $font-12px; + } + } + } + + .mx_AudioPlayer_seek { + display: flex; + align-items: center; + + .mx_SeekBar { + flex: 1; + } + + .mx_Clock { + width: $font-42px; // we're not using a monospace font, so fake it + min-width: $font-42px; // for flexbox + padding-left: 4px; // isolate from seek bar + text-align: right; + } + } +} diff --git a/res/css/views/voice_messages/_PlayPauseButton.scss b/res/css/views/audio_messages/_PlayPauseButton.scss similarity index 91% rename from res/css/views/voice_messages/_PlayPauseButton.scss rename to res/css/views/audio_messages/_PlayPauseButton.scss index 6caedafa29b..714da3e6053 100644 --- a/res/css/views/voice_messages/_PlayPauseButton.scss +++ b/res/css/views/audio_messages/_PlayPauseButton.scss @@ -18,6 +18,8 @@ limitations under the License. position: relative; width: 32px; height: 32px; + min-width: 32px; // for when the button is used in a flexbox + min-height: 32px; // for when the button is used in a flexbox border-radius: 32px; background-color: $voice-playback-button-bg-color; diff --git a/res/css/views/voice_messages/_PlaybackContainer.scss b/res/css/views/audio_messages/_PlaybackContainer.scss similarity index 88% rename from res/css/views/voice_messages/_PlaybackContainer.scss rename to res/css/views/audio_messages/_PlaybackContainer.scss index f0e29900ab4..fd01864bba5 100644 --- a/res/css/views/voice_messages/_PlaybackContainer.scss +++ b/res/css/views/audio_messages/_PlaybackContainer.scss @@ -22,17 +22,11 @@ limitations under the License. // 7px top and bottom for visual design. 12px left & right, but the waveform (right) // has a 1px padding on it that we want to account for. padding: 7px 12px 7px 11px; - background-color: $voice-record-waveform-bg-color; - border-radius: 12px; // Cheat at alignment a bit display: flex; align-items: center; - color: $voice-record-waveform-fg-color; - font-size: $font-14px; - line-height: $font-24px; - contain: content; .mx_Waveform { @@ -45,7 +39,7 @@ limitations under the License. &.mx_Waveform_bar_100pct { // Small animation to remove the mechanical feel of progress transition: background-color 250ms ease; - background-color: $voice-record-waveform-fg-color; + background-color: $message-body-panel-fg-color; } } } diff --git a/res/css/views/audio_messages/_SeekBar.scss b/res/css/views/audio_messages/_SeekBar.scss new file mode 100644 index 00000000000..d13fe4ac6a6 --- /dev/null +++ b/res/css/views/audio_messages/_SeekBar.scss @@ -0,0 +1,103 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// CSS inspiration from: +// * https://www.w3schools.com/howto/howto_js_rangeslider.asp +// * https://stackoverflow.com/a/28283806 +// * https://css-tricks.com/styling-cross-browser-compatible-range-inputs-css/ + +.mx_SeekBar { + // Dev note: we deliberately do not have the -ms-track (and friends) selectors because we don't + // need to support IE. + + appearance: none; // default style override + + width: 100%; + height: 1px; + background: $quaternary-fg-color; + outline: none; // remove blue selection border + position: relative; // for before+after pseudo elements later on + + cursor: pointer; + + &::-webkit-slider-thumb { + appearance: none; // default style override + + // Dev note: This needs to be duplicated with the -moz-range-thumb selector + // because otherwise Edge (webkit) will fail to see the styles and just refuse + // to apply them. + width: 8px; + height: 8px; + border-radius: 8px; + background-color: $tertiary-fg-color; + cursor: pointer; + } + + &::-moz-range-thumb { + width: 8px; + height: 8px; + border-radius: 8px; + background-color: $tertiary-fg-color; + cursor: pointer; + + // Firefox adds a border on the thumb + border: none; + } + + // This is for webkit support, but we can't limit the functionality of it to just webkit + // browsers. Firefox responds to webkit-prefixed values now, which means we can't use media + // or support queries to selectively apply the rule. An upside is that this CSS doesn't work + // in firefox, so it's just wasted CPU/GPU time. + &::before { // ::before to ensure it ends up under the thumb + content: ''; + background-color: $tertiary-fg-color; + + // Absolute positioning to ensure it overlaps with the existing bar + position: absolute; + top: 0; + left: 0; + + // Sizing to match the bar + width: 100%; + height: 1px; + + // And finally dynamic width without overly hurting the rendering engine. + transform-origin: 0 100%; + transform: scaleX(var(--fillTo)); + } + + // This is firefox's built-in support for the above, with 100% less hacks. + &::-moz-range-progress { + background-color: $tertiary-fg-color; + height: 1px; + } + + &:disabled { + opacity: 0.5; + } + + // Increase clickable area for the slider (approximately same size as browser default) + // We do it this way to keep the same padding and margins of the element, avoiding margin math. + // Source: https://front-back.com/expand-clickable-areas-for-a-better-touch-experience/ + &::after { + content: ''; + position: absolute; + top: -6px; + bottom: -6px; + left: 0; + right: 0; + } +} diff --git a/res/css/views/voice_messages/_Waveform.scss b/res/css/views/audio_messages/_Waveform.scss similarity index 100% rename from res/css/views/voice_messages/_Waveform.scss rename to res/css/views/audio_messages/_Waveform.scss diff --git a/res/css/views/messages/_MediaBody.scss b/res/css/views/messages/_MediaBody.scss new file mode 100644 index 00000000000..12e441750cf --- /dev/null +++ b/res/css/views/messages/_MediaBody.scss @@ -0,0 +1,28 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// A "media body" is any file upload looking thing, apart from images and videos (they +// have unique styles). + +.mx_MediaBody { + background-color: $message-body-panel-bg-color; + border-radius: 12px; + + color: $message-body-panel-fg-color; + font-size: $font-14px; + line-height: $font-24px; +} + diff --git a/res/themes/dark/css/_dark.scss b/res/themes/dark/css/_dark.scss index 8b5fde3bd10..81fd3c892a3 100644 --- a/res/themes/dark/css/_dark.scss +++ b/res/themes/dark/css/_dark.scss @@ -215,8 +215,6 @@ $message-body-panel-icon-fg-color: #21262C; // "Separator" $message-body-panel-icon-bg-color: $tertiary-fg-color; $voice-record-stop-border-color: $quaternary-fg-color; -$voice-record-waveform-bg-color: $message-body-panel-bg-color; -$voice-record-waveform-fg-color: $message-body-panel-fg-color; $voice-record-waveform-incomplete-fg-color: $quaternary-fg-color; $voice-record-icon-color: $quaternary-fg-color; $voice-playback-button-bg-color: $message-body-panel-icon-bg-color; diff --git a/res/themes/legacy-dark/css/_legacy-dark.scss b/res/themes/legacy-dark/css/_legacy-dark.scss index eb6dc40599c..df01efbe1e6 100644 --- a/res/themes/legacy-dark/css/_legacy-dark.scss +++ b/res/themes/legacy-dark/css/_legacy-dark.scss @@ -20,6 +20,9 @@ $tertiary-fg-color: $primary-fg-color; $primary-bg-color: $bg-color; $muted-fg-color: $header-panel-text-primary-color; +// Legacy theme backports +$quaternary-fg-color: #6F7882; + // used for dialog box text $light-fg-color: $header-panel-text-secondary-color; @@ -209,8 +212,6 @@ $message-body-panel-icon-bg-color: $secondary-fg-color; // See non-legacy dark for variable information $voice-record-stop-border-color: #6F7882; -$voice-record-waveform-bg-color: $message-body-panel-bg-color; -$voice-record-waveform-fg-color: $message-body-panel-fg-color; $voice-record-waveform-incomplete-fg-color: #6F7882; $voice-record-icon-color: #6F7882; $voice-playback-button-bg-color: $tertiary-fg-color; diff --git a/res/themes/legacy-light/css/_legacy-light.scss b/res/themes/legacy-light/css/_legacy-light.scss index a6b180bab4b..c7debcdabef 100644 --- a/res/themes/legacy-light/css/_legacy-light.scss +++ b/res/themes/legacy-light/css/_legacy-light.scss @@ -28,6 +28,9 @@ $tertiary-fg-color: $primary-fg-color; $primary-bg-color: #ffffff; $muted-fg-color: #61708b; // Commonly used in headings and relevant alt text +// Legacy theme backports +$quaternary-fg-color: #C1C6CD; + // used for dialog box text $light-fg-color: #747474; @@ -334,8 +337,6 @@ $message-body-panel-icon-bg-color: $primary-bg-color; $voice-record-stop-symbol-color: #ff4b55; $voice-record-live-circle-color: #ff4b55; $voice-record-stop-border-color: #E3E8F0; -$voice-record-waveform-bg-color: $message-body-panel-bg-color; -$voice-record-waveform-fg-color: $message-body-panel-fg-color; $voice-record-waveform-incomplete-fg-color: #C1C6CD; $voice-record-icon-color: $tertiary-fg-color; $voice-playback-button-bg-color: $message-body-panel-icon-bg-color; diff --git a/res/themes/light/css/_light.scss b/res/themes/light/css/_light.scss index d8dab9c9c4c..7e958c2af63 100644 --- a/res/themes/light/css/_light.scss +++ b/res/themes/light/css/_light.scss @@ -335,8 +335,6 @@ $voice-record-stop-symbol-color: #ff4b55; $voice-record-live-circle-color: #ff4b55; $voice-record-stop-border-color: #E3E8F0; // "Separator" -$voice-record-waveform-bg-color: $message-body-panel-bg-color; -$voice-record-waveform-fg-color: $message-body-panel-fg-color; $voice-record-waveform-incomplete-fg-color: $quaternary-fg-color; $voice-record-icon-color: $tertiary-fg-color; $voice-playback-button-bg-color: $message-body-panel-icon-bg-color; diff --git a/src/components/views/audio_messages/AudioPlayer.tsx b/src/components/views/audio_messages/AudioPlayer.tsx new file mode 100644 index 00000000000..66efa646585 --- /dev/null +++ b/src/components/views/audio_messages/AudioPlayer.tsx @@ -0,0 +1,124 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { Playback, PlaybackState } from "../../../voice/Playback"; +import React, { createRef, ReactNode, RefObject } from "react"; +import { UPDATE_EVENT } from "../../../stores/AsyncStore"; +import PlayPauseButton from "./PlayPauseButton"; +import { replaceableComponent } from "../../../utils/replaceableComponent"; +import { formatBytes } from "../../../utils/FormattingUtils"; +import DurationClock from "./DurationClock"; +import { Key } from "../../../Keyboard"; +import { _t } from "../../../languageHandler"; +import SeekBar from "./SeekBar"; +import PlaybackClock from "./PlaybackClock"; + +interface IProps { + // Playback instance to render. Cannot change during component lifecycle: create + // an all-new component instead. + playback: Playback; + + mediaName: string; +} + +interface IState { + playbackPhase: PlaybackState; +} + +@replaceableComponent("views.audio_messages.AudioPlayer") +export default class AudioPlayer extends React.PureComponent { + private playPauseRef: RefObject = createRef(); + private seekRef: RefObject = createRef(); + + constructor(props: IProps) { + super(props); + + this.state = { + playbackPhase: PlaybackState.Decoding, // default assumption + }; + + // We don't need to de-register: the class handles this for us internally + this.props.playback.on(UPDATE_EVENT, this.onPlaybackUpdate); + + // Don't wait for the promise to complete - it will emit a progress update when it + // is done, and it's not meant to take long anyhow. + // noinspection JSIgnoredPromiseFromCall + this.props.playback.prepare(); + } + + private onPlaybackUpdate = (ev: PlaybackState) => { + this.setState({ playbackPhase: ev }); + }; + + private onKeyDown = (ev: React.KeyboardEvent) => { + // stopPropagation() prevents the FocusComposer catch-all from triggering, + // but we need to do it on key down instead of press (even though the user + // interaction is typically on press). + if (ev.key === Key.SPACE) { + ev.stopPropagation(); + this.playPauseRef.current?.toggleState(); + } else if (ev.key === Key.ARROW_LEFT) { + ev.stopPropagation(); + this.seekRef.current?.left(); + } else if (ev.key === Key.ARROW_RIGHT) { + ev.stopPropagation(); + this.seekRef.current?.right(); + } + }; + + protected renderFileSize(): string { + const bytes = this.props.playback.sizeBytes; + if (!bytes) return null; + + // Not translated here - we're just presenting the data which should already + // be translated if needed. + return `(${formatBytes(bytes)})`; + } + + public render(): ReactNode { + // tabIndex=0 to ensure that the whole component becomes a tab stop, where we handle keyboard + // events for accessibility + return
+
+ +
+ + {this.props.mediaName || _t("Unnamed audio")} + +
+ +   {/* easiest way to introduce a gap between the components */} + { this.renderFileSize() } +
+
+
+
+ + +
+
; + } +} diff --git a/src/components/views/voice_messages/Clock.tsx b/src/components/views/audio_messages/Clock.tsx similarity index 96% rename from src/components/views/voice_messages/Clock.tsx rename to src/components/views/audio_messages/Clock.tsx index 1e78cc7bbd5..7f387715f89 100644 --- a/src/components/views/voice_messages/Clock.tsx +++ b/src/components/views/audio_messages/Clock.tsx @@ -28,7 +28,7 @@ interface IState { * Simply converts seconds into minutes and seconds. Note that hours will not be * displayed, making it possible to see "82:29". */ -@replaceableComponent("views.voice_messages.Clock") +@replaceableComponent("views.audio_messages.Clock") export default class Clock extends React.Component { public constructor(props) { super(props); diff --git a/src/components/views/audio_messages/DurationClock.tsx b/src/components/views/audio_messages/DurationClock.tsx new file mode 100644 index 00000000000..81852b59447 --- /dev/null +++ b/src/components/views/audio_messages/DurationClock.tsx @@ -0,0 +1,55 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import React from "react"; +import { replaceableComponent } from "../../../utils/replaceableComponent"; +import Clock from "./Clock"; +import { Playback } from "../../../voice/Playback"; + +interface IProps { + playback: Playback; +} + +interface IState { + durationSeconds: number; +} + +/** + * A clock which shows a clip's maximum duration. + */ +@replaceableComponent("views.audio_messages.DurationClock") +export default class DurationClock extends React.PureComponent { + public constructor(props) { + super(props); + + this.state = { + // we track the duration on state because we won't really know what the clip duration + // is until the first time update, and as a PureComponent we are trying to dedupe state + // updates as much as possible. This is just the easiest way to avoid a forceUpdate() or + // member property to track "did we get a duration". + durationSeconds: this.props.playback.clockInfo.durationSeconds, + }; + this.props.playback.clockInfo.liveData.onUpdate(this.onTimeUpdate); + } + + private onTimeUpdate = (time: number[]) => { + this.setState({ durationSeconds: time[1] }); + }; + + public render() { + return ; + } +} diff --git a/src/components/views/voice_messages/LiveRecordingClock.tsx b/src/components/views/audio_messages/LiveRecordingClock.tsx similarity index 90% rename from src/components/views/voice_messages/LiveRecordingClock.tsx rename to src/components/views/audio_messages/LiveRecordingClock.tsx index 2a20e9bfec6..a9dbd3c52fa 100644 --- a/src/components/views/voice_messages/LiveRecordingClock.tsx +++ b/src/components/views/audio_messages/LiveRecordingClock.tsx @@ -1,9 +1,12 @@ /* Copyright 2021 The Matrix.org Foundation C.I.C. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -12,16 +15,13 @@ limitations under the License. */ import React from "react"; -import Clock from "./Clock"; +import { IRecordingUpdate, VoiceRecording } from "../../../voice/VoiceRecording"; import { replaceableComponent } from "../../../utils/replaceableComponent"; +import Clock from "./Clock"; import { MarkedExecution } from "../../../utils/MarkedExecution"; -import { - IRecordingUpdate, - VoiceRecording, -} from "../../../voice/VoiceRecording"; interface IProps { - recorder?: VoiceRecording; + recorder: VoiceRecording; } interface IState { @@ -31,7 +31,7 @@ interface IState { /** * A clock for a live recording. */ -@replaceableComponent("views.voice_messages.LiveRecordingClock") +@replaceableComponent("views.audio_messages.LiveRecordingClock") export default class LiveRecordingClock extends React.PureComponent { private seconds = 0; private scheduledUpdate = new MarkedExecution( diff --git a/src/components/views/voice_messages/LiveRecordingWaveform.tsx b/src/components/views/audio_messages/LiveRecordingWaveform.tsx similarity index 66% rename from src/components/views/voice_messages/LiveRecordingWaveform.tsx rename to src/components/views/audio_messages/LiveRecordingWaveform.tsx index 1e3440fbade..b9c5f80f05d 100644 --- a/src/components/views/voice_messages/LiveRecordingWaveform.tsx +++ b/src/components/views/audio_messages/LiveRecordingWaveform.tsx @@ -1,9 +1,12 @@ /* Copyright 2021 The Matrix.org Foundation C.I.C. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -12,16 +15,15 @@ limitations under the License. */ import React from "react"; -import Waveform from "./Waveform"; +import { IRecordingUpdate, RECORDING_PLAYBACK_SAMPLES, VoiceRecording } from "../../../voice/VoiceRecording"; import { replaceableComponent } from "../../../utils/replaceableComponent"; +import { arrayFastResample } from "../../../utils/arrays"; +import { percentageOf } from "../../../utils/numbers"; +import Waveform from "./Waveform"; import { MarkedExecution } from "../../../utils/MarkedExecution"; -import { - IRecordingUpdate, - VoiceRecording, -} from "../../../voice/VoiceRecording"; interface IProps { - recorder?: VoiceRecording; + recorder: VoiceRecording; } interface IState { @@ -31,7 +33,7 @@ interface IState { /** * A waveform which shows the waveform of a live recording */ -@replaceableComponent("views.voice_messages.LiveRecordingWaveform") +@replaceableComponent("views.audio_messages.LiveRecordingWaveform") export default class LiveRecordingWaveform extends React.PureComponent { public static defaultProps = { progress: 1, @@ -52,15 +54,18 @@ export default class LiveRecordingWaveform extends React.PureComponent { - this.waveform = update.waveform; + const bars = arrayFastResample(Array.from(update.waveform), RECORDING_PLAYBACK_SAMPLES); + // The incoming data is between zero and one, but typically even screaming into a + // microphone won't send you over 0.6, so we artificially adjust the gain for the + // waveform. This results in a slightly more cinematic/animated waveform for the + // user. + this.waveform = bars.map(b => percentageOf(b, 0, 0.50)); this.scheduledUpdate.mark(); }); } private updateWaveform() { - this.setState({ - waveform: this.waveform, - }); + this.setState({ waveform: this.waveform }); } public render() { diff --git a/src/components/views/voice_messages/PlayPauseButton.tsx b/src/components/views/audio_messages/PlayPauseButton.tsx similarity index 76% rename from src/components/views/voice_messages/PlayPauseButton.tsx rename to src/components/views/audio_messages/PlayPauseButton.tsx index 1ca05a2cd0c..a4f1e770f24 100644 --- a/src/components/views/voice_messages/PlayPauseButton.tsx +++ b/src/components/views/audio_messages/PlayPauseButton.tsx @@ -21,7 +21,8 @@ import { _t } from "../../../languageHandler"; import { Playback, PlaybackState } from "../../../voice/Playback"; import classNames from "classnames"; -interface IProps { +// omitted props are handled by render function +interface IProps extends Omit, "title" | "onClick" | "disabled"> { // Playback instance to manipulate. Cannot change during the component lifecycle. playback: Playback; @@ -33,19 +34,25 @@ interface IProps { * Displays a play/pause button (activating the play/pause function of the recorder) * to be displayed in reference to a recording. */ -@replaceableComponent("views.voice_messages.PlayPauseButton") +@replaceableComponent("views.audio_messages.PlayPauseButton") export default class PlayPauseButton extends React.PureComponent { public constructor(props) { super(props); } - private onClick = async () => { - await this.props.playback.toggle(); + private onClick = () => { + // noinspection JSIgnoredPromiseFromCall + this.toggleState(); }; + public async toggleState() { + await this.props.playback.toggle(); + } + public render(): ReactNode { - const isPlaying = this.props.playback.isPlaying; - const isDisabled = this.props.playbackPhase === PlaybackState.Decoding; + const { playback, playbackPhase, ...restProps } = this.props; + const isPlaying = playback.isPlaying; + const isDisabled = playbackPhase === PlaybackState.Decoding; const classes = classNames('mx_PlayPauseButton', { 'mx_PlayPauseButton_play': !isPlaying, 'mx_PlayPauseButton_pause': isPlaying, @@ -56,6 +63,7 @@ export default class PlayPauseButton extends React.PureComponent { title={isPlaying ? _t("Pause") : _t("Play")} onClick={this.onClick} disabled={isDisabled} + {...restProps} />; } } diff --git a/src/components/views/voice_messages/PlaybackClock.tsx b/src/components/views/audio_messages/PlaybackClock.tsx similarity index 83% rename from src/components/views/voice_messages/PlaybackClock.tsx rename to src/components/views/audio_messages/PlaybackClock.tsx index 9c9298f764c..374d47c31db 100644 --- a/src/components/views/voice_messages/PlaybackClock.tsx +++ b/src/components/views/audio_messages/PlaybackClock.tsx @@ -22,6 +22,11 @@ import { UPDATE_EVENT } from "../../../stores/AsyncStore"; interface IProps { playback: Playback; + + // The default number of seconds to show when the playback has completed or + // has not started. Not used during playback, even when paused. Defaults to + // clip duration length. + defaultDisplaySeconds?: number; } interface IState { @@ -33,7 +38,7 @@ interface IState { /** * A clock for a playback of a recording. */ -@replaceableComponent("views.voice_messages.PlaybackClock") +@replaceableComponent("views.audio_messages.PlaybackClock") export default class PlaybackClock extends React.PureComponent { public constructor(props) { super(props); @@ -64,7 +69,11 @@ export default class PlaybackClock extends React.PureComponent { public render() { let seconds = this.state.seconds; if (this.state.playbackPhase === PlaybackState.Stopped) { - seconds = this.state.durationSeconds; + if (Number.isFinite(this.props.defaultDisplaySeconds)) { + seconds = this.props.defaultDisplaySeconds; + } else { + seconds = this.state.durationSeconds; + } } return ; } diff --git a/src/components/views/voice_messages/PlaybackWaveform.tsx b/src/components/views/audio_messages/PlaybackWaveform.tsx similarity index 97% rename from src/components/views/voice_messages/PlaybackWaveform.tsx rename to src/components/views/audio_messages/PlaybackWaveform.tsx index 096ba8199c1..ea1b846c015 100644 --- a/src/components/views/voice_messages/PlaybackWaveform.tsx +++ b/src/components/views/audio_messages/PlaybackWaveform.tsx @@ -33,7 +33,7 @@ interface IState { /** * A waveform which shows the waveform of a previously recorded recording */ -@replaceableComponent("views.voice_messages.PlaybackWaveform") +@replaceableComponent("views.audio_messages.PlaybackWaveform") export default class PlaybackWaveform extends React.PureComponent { public constructor(props) { super(props); diff --git a/src/components/views/voice_messages/RecordingPlayback.tsx b/src/components/views/audio_messages/RecordingPlayback.tsx similarity index 90% rename from src/components/views/voice_messages/RecordingPlayback.tsx rename to src/components/views/audio_messages/RecordingPlayback.tsx index 63e823200e6..a0dea1c6db2 100644 --- a/src/components/views/voice_messages/RecordingPlayback.tsx +++ b/src/components/views/audio_messages/RecordingPlayback.tsx @@ -20,6 +20,7 @@ import { UPDATE_EVENT } from "../../../stores/AsyncStore"; import PlaybackWaveform from "./PlaybackWaveform"; import PlayPauseButton from "./PlayPauseButton"; import PlaybackClock from "./PlaybackClock"; +import { replaceableComponent } from "../../../utils/replaceableComponent"; interface IProps { // Playback instance to render. Cannot change during component lifecycle: create @@ -31,6 +32,7 @@ interface IState { playbackPhase: PlaybackState; } +@replaceableComponent("views.audio_messages.RecordingPlayback") export default class RecordingPlayback extends React.PureComponent { constructor(props: IProps) { super(props); @@ -53,7 +55,7 @@ export default class RecordingPlayback extends React.PureComponent + return
diff --git a/src/components/views/audio_messages/SeekBar.tsx b/src/components/views/audio_messages/SeekBar.tsx new file mode 100644 index 00000000000..5231a2fb797 --- /dev/null +++ b/src/components/views/audio_messages/SeekBar.tsx @@ -0,0 +1,112 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { Playback, PlaybackState } from "../../../voice/Playback"; +import React, { ChangeEvent, CSSProperties, ReactNode } from "react"; +import { replaceableComponent } from "../../../utils/replaceableComponent"; +import { MarkedExecution } from "../../../utils/MarkedExecution"; +import { percentageOf } from "../../../utils/numbers"; + +interface IProps { + // Playback instance to render. Cannot change during component lifecycle: create + // an all-new component instead. + playback: Playback; + + // Tab index for the underlying component. Useful if the seek bar is in a managed state. + // Defaults to zero. + tabIndex?: number; + + playbackPhase: PlaybackState; +} + +interface IState { + percentage: number; +} + +interface ISeekCSS extends CSSProperties { + '--fillTo': number; +} + +const ARROW_SKIP_SECONDS = 5; // arbitrary + +@replaceableComponent("views.audio_messages.SeekBar") +export default class SeekBar extends React.PureComponent { + // We use an animation frame request to avoid overly spamming prop updates, even if we aren't + // really using anything demanding on the CSS front. + + private animationFrameFn = new MarkedExecution( + () => this.doUpdate(), + () => requestAnimationFrame(() => this.animationFrameFn.trigger())); + + public static defaultProps = { + tabIndex: 0, + }; + + constructor(props: IProps) { + super(props); + + this.state = { + percentage: 0, + }; + + // We don't need to de-register: the class handles this for us internally + this.props.playback.clockInfo.liveData.onUpdate(() => this.animationFrameFn.mark()); + } + + private doUpdate() { + this.setState({ + percentage: percentageOf( + this.props.playback.clockInfo.timeSeconds, + 0, + this.props.playback.clockInfo.durationSeconds), + }); + } + + public left() { + // noinspection JSIgnoredPromiseFromCall + this.props.playback.skipTo(this.props.playback.clockInfo.timeSeconds - ARROW_SKIP_SECONDS); + } + + public right() { + // noinspection JSIgnoredPromiseFromCall + this.props.playback.skipTo(this.props.playback.clockInfo.timeSeconds + ARROW_SKIP_SECONDS); + } + + private onChange = (ev: ChangeEvent) => { + // Thankfully, onChange is only called when the user changes the value, not when we + // change the value on the component. We can use this as a reliable "skip to X" function. + // + // noinspection JSIgnoredPromiseFromCall + this.props.playback.skipTo(Number(ev.target.value) * this.props.playback.clockInfo.durationSeconds); + }; + + public render(): ReactNode { + // We use a range input to avoid having to re-invent accessibility handling on + // a custom set of divs. + return ; + } +} diff --git a/src/components/views/voice_messages/Waveform.tsx b/src/components/views/audio_messages/Waveform.tsx similarity index 93% rename from src/components/views/voice_messages/Waveform.tsx rename to src/components/views/audio_messages/Waveform.tsx index 5a4447065a9..3b7a881754f 100644 --- a/src/components/views/voice_messages/Waveform.tsx +++ b/src/components/views/audio_messages/Waveform.tsx @@ -17,8 +17,13 @@ limitations under the License. import React from "react"; import { replaceableComponent } from "../../../utils/replaceableComponent"; import classNames from "classnames"; +import { CSSProperties } from "react"; -export interface IProps { +interface WaveformCSSProperties extends CSSProperties { + '--barHeight': number; +} + +interface IProps { relHeights: number[]; // relative heights (0-1) progress: number; // percent complete, 0-1, default 100% } @@ -34,14 +39,7 @@ interface IState { * For CSS purposes, a mx_Waveform_bar_100pct class is added when the bar should be * "filled", as a demonstration of the progress property. */ - -import { CSSProperties } from "react"; - -export interface WaveformCSSProperties extends CSSProperties { - '--barHeight': number; -} - -@replaceableComponent("views.voice_messages.Waveform") +@replaceableComponent("views.audio_messages.Waveform") export default class Waveform extends React.PureComponent { public static defaultProps = { progress: 1, diff --git a/src/components/views/messages/MAudioBody.js b/src/components/views/messages/MAudioBody.js deleted file mode 100644 index 977c88448be..00000000000 --- a/src/components/views/messages/MAudioBody.js +++ /dev/null @@ -1,112 +0,0 @@ -/* - Copyright 2016 OpenMarket Ltd - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -import React from 'react'; -import MFileBody from './MFileBody'; - -import { decryptFile } from '../../../utils/DecryptFile'; -import { _t } from '../../../languageHandler'; -import InlineSpinner from '../elements/InlineSpinner'; -import { replaceableComponent } from "../../../utils/replaceableComponent"; -import { mediaFromContent } from "../../../customisations/Media"; - -@replaceableComponent("views.messages.MAudioBody") -export default class MAudioBody extends React.Component { - constructor(props) { - super(props); - this.state = { - playing: false, - decryptedUrl: null, - decryptedBlob: null, - error: null, - }; - } - onPlayToggle() { - this.setState({ - playing: !this.state.playing, - }); - } - - _getContentUrl() { - const media = mediaFromContent(this.props.mxEvent.getContent()); - if (media.isEncrypted) { - return this.state.decryptedUrl; - } else { - return media.srcHttp; - } - } - - componentDidMount() { - const content = this.props.mxEvent.getContent(); - if (content.file !== undefined && this.state.decryptedUrl === null) { - let decryptedBlob; - decryptFile(content.file).then(function(blob) { - decryptedBlob = blob; - return URL.createObjectURL(decryptedBlob); - }).then((url) => { - this.setState({ - decryptedUrl: url, - decryptedBlob: decryptedBlob, - }); - }, (err) => { - console.warn("Unable to decrypt attachment: ", err); - this.setState({ - error: err, - }); - }); - } - } - - componentWillUnmount() { - if (this.state.decryptedUrl) { - URL.revokeObjectURL(this.state.decryptedUrl); - } - } - - render() { - const content = this.props.mxEvent.getContent(); - - if (this.state.error !== null) { - return ( - - - { _t("Error decrypting audio") } - - ); - } - - if (content.file !== undefined && this.state.decryptedUrl === null) { - // Need to decrypt the attachment - // The attachment is decrypted in componentDidMount. - // For now add an img tag with a 16x16 spinner. - // Not sure how tall the audio player is so not sure how tall it should actually be. - return ( - - - - ); - } - - const contentUrl = this._getContentUrl(); - - return ( - - - ); - } -} diff --git a/src/components/views/messages/MAudioBody.tsx b/src/components/views/messages/MAudioBody.tsx new file mode 100644 index 00000000000..bc7216f42c8 --- /dev/null +++ b/src/components/views/messages/MAudioBody.tsx @@ -0,0 +1,110 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import React from "react"; +import { MatrixEvent } from "matrix-js-sdk/src/models/event"; +import { replaceableComponent } from "../../../utils/replaceableComponent"; +import { Playback } from "../../../voice/Playback"; +import MFileBody from "./MFileBody"; +import InlineSpinner from '../elements/InlineSpinner'; +import { _t } from "../../../languageHandler"; +import { mediaFromContent } from "../../../customisations/Media"; +import { decryptFile } from "../../../utils/DecryptFile"; +import { IMediaEventContent } from "../../../customisations/models/IMediaEventContent"; +import AudioPlayer from "../audio_messages/AudioPlayer"; + +interface IProps { + mxEvent: MatrixEvent; +} + +interface IState { + error?: Error; + playback?: Playback; + decryptedBlob?: Blob; +} + +@replaceableComponent("views.messages.MAudioBody") +export default class MAudioBody extends React.PureComponent { + constructor(props: IProps) { + super(props); + + this.state = {}; + } + + public async componentDidMount() { + let buffer: ArrayBuffer; + const content: IMediaEventContent = this.props.mxEvent.getContent(); + const media = mediaFromContent(content); + if (media.isEncrypted) { + try { + const blob = await decryptFile(content.file); + buffer = await blob.arrayBuffer(); + this.setState({ decryptedBlob: blob }); + } catch (e) { + this.setState({ error: e }); + console.warn("Unable to decrypt audio message", e); + return; // stop processing the audio file + } + } else { + try { + buffer = await media.downloadSource().then(r => r.blob()).then(r => r.arrayBuffer()); + } catch (e) { + this.setState({ error: e }); + console.warn("Unable to download audio message", e); + return; // stop processing the audio file + } + } + + // We should have a buffer to work with now: let's set it up + const playback = new Playback(buffer); + playback.clockInfo.populatePlaceholdersFrom(this.props.mxEvent); + this.setState({ playback }); + // Note: the RecordingPlayback component will handle preparing the Playback class for us. + } + + public componentWillUnmount() { + this.state.playback?.destroy(); + } + + public render() { + if (this.state.error) { + // TODO: @@TR: Verify error state + return ( + + + { _t("Error processing audio message") } + + ); + } + + if (!this.state.playback) { + // TODO: @@TR: Verify loading/decrypting state + return ( + + + + ); + } + + // At this point we should have a playable state + return ( + + + + + ); + } +} diff --git a/src/components/views/messages/MVoiceMessageBody.tsx b/src/components/views/messages/MVoiceMessageBody.tsx index edb791f7ee5..2edd42f2e46 100644 --- a/src/components/views/messages/MVoiceMessageBody.tsx +++ b/src/components/views/messages/MVoiceMessageBody.tsx @@ -23,7 +23,7 @@ import InlineSpinner from '../elements/InlineSpinner'; import { _t } from "../../../languageHandler"; import { mediaFromContent } from "../../../customisations/Media"; import { decryptFile } from "../../../utils/DecryptFile"; -import RecordingPlayback from "../voice_messages/RecordingPlayback"; +import RecordingPlayback from "../audio_messages/RecordingPlayback"; import { IMediaEventContent } from "../../../customisations/models/IMediaEventContent"; interface IProps { diff --git a/src/components/views/rooms/VoiceRecordComposerTile.tsx b/src/components/views/rooms/VoiceRecordComposerTile.tsx index 1c4bdceba5d..f08c8fe6dff 100644 --- a/src/components/views/rooms/VoiceRecordComposerTile.tsx +++ b/src/components/views/rooms/VoiceRecordComposerTile.tsx @@ -18,22 +18,18 @@ import AccessibleTooltipButton from "../elements/AccessibleTooltipButton"; import { _t } from "../../../languageHandler"; import React, { ReactNode } from "react"; import { - IRecordingUpdate, - RECORDING_PLAYBACK_SAMPLES, RecordingState, VoiceRecording, } from "../../../voice/VoiceRecording"; import { Room } from "matrix-js-sdk/src/models/room"; import { MatrixClientPeg } from "../../../MatrixClientPeg"; import classNames from "classnames"; -import LiveRecordingWaveform from "../voice_messages/LiveRecordingWaveform"; +import LiveRecordingWaveform from "../audio_messages/LiveRecordingWaveform"; import { replaceableComponent } from "../../../utils/replaceableComponent"; -import { arrayFastResample, arraySeed } from "../../../utils/arrays"; -import { percentageOf } from "../../../utils/numbers"; -import LiveRecordingClock from "../voice_messages/LiveRecordingClock"; +import LiveRecordingClock from "../audio_messages/LiveRecordingClock"; import { VoiceRecordingStore } from "../../../stores/VoiceRecordingStore"; import { UPDATE_EVENT } from "../../../stores/AsyncStore"; -import RecordingPlayback from "../voice_messages/RecordingPlayback"; +import RecordingPlayback from "../audio_messages/RecordingPlayback"; import { MsgType } from "matrix-js-sdk/src/@types/event"; import Modal from "../../../Modal"; import ErrorDialog from "../dialogs/ErrorDialog"; @@ -46,8 +42,6 @@ interface IProps { interface IState { recorder?: VoiceRecording; recordingPhase?: RecordingState; - relHeights: number[]; - seconds: number; } /** @@ -55,58 +49,18 @@ interface IState { */ @replaceableComponent("views.rooms.VoiceRecordComposerTile") export default class VoiceRecordComposerTile extends React.PureComponent { - private waveform: number[] = []; - private seconds = 0; - private scheduledAnimationFrame = false; - public constructor(props) { super(props); this.state = { recorder: null, // no recording started by default - seconds: 0, - relHeights: arraySeed(0, RECORDING_PLAYBACK_SAMPLES), }; } - public componentDidUpdate(prevProps, prevState) { - if (!prevState.recorder && this.state.recorder) { - this.state.recorder.liveData.onUpdate(this.onRecordingUpdate); - } - } - public async componentWillUnmount() { await VoiceRecordingStore.instance.disposeRecording(); } - private onRecordingUpdate = (update: IRecordingUpdate): void => { - this.waveform = update.waveform; - this.seconds = update.timeSeconds; - - if (this.scheduledAnimationFrame) { - return; - } - - this.scheduledAnimationFrame = true; - // The audio recorder flushes data faster than the screen refresh rate - // Using requestAnimationFrame makes sure that we only flush the data - // to react once per tick to avoid unneeded work. - requestAnimationFrame(() => { - // The waveform and the downsample target are pretty close, so we should be fine to - // do this, despite the docs on arrayFastResample. - const bars = arrayFastResample(Array.from(this.waveform), RECORDING_PLAYBACK_SAMPLES); - this.setState({ - // The incoming data is between zero and one, but typically even screaming into a - // microphone won't send you over 0.6, so we artificially adjust the gain for the - // waveform. This results in a slightly more cinematic/animated waveform for the - // user. - relHeights: bars.map(b => percentageOf(b, 0, 0.50)), - seconds: this.seconds, - }); - this.scheduledAnimationFrame = false; - }); - }; - // called by composer public async send() { if (!this.state.recorder) { @@ -228,7 +182,7 @@ export default class VoiceRecordComposerTile extends React.PureComponent + return
; diff --git a/src/i18n/strings/en_EN.json b/src/i18n/strings/en_EN.json index f0599c7e49d..6f06e3d6a40 100644 --- a/src/i18n/strings/en_EN.json +++ b/src/i18n/strings/en_EN.json @@ -918,8 +918,6 @@ "Silence call": "Silence call", "Decline": "Decline", "Accept": "Accept", - "Pause": "Pause", - "Play": "Play", "The other party cancelled the verification.": "The other party cancelled the verification.", "Verified!": "Verified!", "You've successfully verified this user.": "You've successfully verified this user.", @@ -1868,7 +1866,7 @@ "Ignored attempt to disable encryption": "Ignored attempt to disable encryption", "Encryption not enabled": "Encryption not enabled", "The encryption used by this room isn't supported.": "The encryption used by this room isn't supported.", - "Error decrypting audio": "Error decrypting audio", + "Error processing audio message": "Error processing audio message", "React": "React", "Edit": "Edit", "Retry": "Retry", @@ -2602,6 +2600,9 @@ "Use email or phone to optionally be discoverable by existing contacts.": "Use email or phone to optionally be discoverable by existing contacts.", "Use email to optionally be discoverable by existing contacts.": "Use email to optionally be discoverable by existing contacts.", "Sign in with SSO": "Sign in with SSO", + "Unnamed audio": "Unnamed audio", + "Pause": "Pause", + "Play": "Play", "Couldn't load page": "Couldn't load page", "You must register to use this functionality": "You must register to use this functionality", "You must join the room to see its files": "You must join the room to see its files", diff --git a/src/voice/Playback.ts b/src/voice/Playback.ts index 61da435151c..6a120bf924f 100644 --- a/src/voice/Playback.ts +++ b/src/voice/Playback.ts @@ -58,6 +58,7 @@ export class Playback extends EventEmitter implements IDestroyable { private resampledWaveform: number[]; private waveformObservable = new SimpleObservable(); private readonly clock: PlaybackClock; + private readonly fileSize: number; /** * Creates a new playback instance from a buffer. @@ -67,12 +68,22 @@ export class Playback extends EventEmitter implements IDestroyable { */ constructor(private buf: ArrayBuffer, seedWaveform = DEFAULT_WAVEFORM) { super(); + // Capture the file size early as reading the buffer will result in a 0-length buffer left behind + this.fileSize = this.buf.byteLength; this.context = createAudioContext(); this.resampledWaveform = arrayFastResample(seedWaveform ?? DEFAULT_WAVEFORM, PLAYBACK_WAVEFORM_SAMPLES); this.waveformObservable.update(this.resampledWaveform); this.clock = new PlaybackClock(this.context); } + /** + * Size of the audio clip in bytes. May be zero if unknown. This is updated + * when the playback goes through phase changes. + */ + public get sizeBytes(): number { + return this.fileSize; + } + /** * Stable waveform for the playback. Values are guaranteed to be between * zero and one, inclusive. @@ -150,16 +161,9 @@ export class Playback extends EventEmitter implements IDestroyable { public async play() { // We can't restart a buffer source, so we need to create a new one if we hit the end if (this.state === PlaybackState.Stopped) { - if (this.source) { - this.source.disconnect(); - this.source.removeEventListener("ended", this.onPlaybackEnd); - } - - this.source = this.context.createBufferSource(); - this.source.connect(this.context.destination); - this.source.buffer = this.audioBuf; - this.source.start(); // start immediately - this.source.addEventListener("ended", this.onPlaybackEnd); + this.disconnectSource(); + this.makeNewSourceBuffer(); + this.source.start(); } // We use the context suspend/resume functions because it allows us to pause a source @@ -169,6 +173,18 @@ export class Playback extends EventEmitter implements IDestroyable { this.emit(PlaybackState.Playing); } + private disconnectSource() { + this.source?.disconnect(); + this.source?.removeEventListener("ended", this.onPlaybackEnd); + } + + private makeNewSourceBuffer() { + this.source = this.context.createBufferSource(); + this.source.buffer = this.audioBuf; + this.source.addEventListener("ended", this.onPlaybackEnd); + this.source.connect(this.context.destination); + } + public async pause() { await this.context.suspend(); this.emit(PlaybackState.Paused); @@ -183,4 +199,60 @@ export class Playback extends EventEmitter implements IDestroyable { if (this.isPlaying) await this.pause(); else await this.play(); } + + public async skipTo(timeSeconds: number) { + // Dev note: this function talks a lot about clock desyncs. There is a clock running + // independently to the audio context and buffer so that accurate human-perceptible + // time can be exposed. The PlaybackClock class has more information, but the short + // version is that we need to line up the useful time (clip position) with the context + // time, and avoid as many deviations as possible as otherwise the user could see the + // wrong time, and we stop playback at the wrong time, etc. + + timeSeconds = clamp(timeSeconds, 0, this.clock.durationSeconds); + + // Track playing state so we don't cause seeking to start playing the track. + const isPlaying = this.isPlaying; + + if (isPlaying) { + // Pause first so we can get an accurate measurement of time + await this.context.suspend(); + } + + // We can't simply tell the context/buffer to jump to a time, so we have to + // start a whole new buffer and start it from the new time offset. + const now = this.context.currentTime; + this.disconnectSource(); + this.makeNewSourceBuffer(); + + // We have to resync the clock because it can get confused about where we're + // at in the audio clip. + this.clock.syncTo(now, timeSeconds); + + // Always start the source to queue it up. We have to do this now (and pause + // quickly if we're not supposed to be playing) as otherwise the clock can desync + // when it comes time to the user hitting play. After a couple jumps, the user + // will have desynced the clock enough to be about 10-15 seconds off, while this + // keeps it as close to perfect as humans can perceive. + this.source.start(now, timeSeconds); + + // Dev note: it's critical that the code gap between `this.source.start()` and + // `this.pause()` is as small as possible: we do not want to delay *anything* + // as that could cause a clock desync, or a buggy feeling as a single note plays + // during seeking. + + if (isPlaying) { + // If we were playing before, continue the context so the clock doesn't desync. + await this.context.resume(); + } else { + // As mentioned above, we'll have to pause the clip if we weren't supposed to + // be playing it just yet. If we didn't have this, the audio clip plays but all + // the states will be wrong: clock won't advance, pause state doesn't match the + // blaring noise leaving the user's speakers, etc. + // + // Also as mentioned, if the code gap is small enough then this should be + // executed immediately after the start time, leaving no feasible time for the + // user's speakers to play any sound. + await this.pause(); + } + } } diff --git a/src/voice/PlaybackClock.ts b/src/voice/PlaybackClock.ts index 1eee921429d..e3f41930de0 100644 --- a/src/voice/PlaybackClock.ts +++ b/src/voice/PlaybackClock.ts @@ -16,8 +16,44 @@ limitations under the License. import { SimpleObservable } from "matrix-widget-api"; import { IDestroyable } from "../utils/IDestroyable"; - -// Because keeping track of time is sufficiently complicated... +import { MatrixEvent } from "matrix-js-sdk/src/models/event"; + +/** + * Tracks accurate human-perceptible time for an audio clip, as informed + * by managed playback. This clock is tightly coupled with the operation + * of the Playback class, making assumptions about how the provided + * AudioContext will be used (suspended/resumed to preserve time, etc). + * + * But why do we need a clock? The AudioContext exposes time information, + * and so does the audio buffer, but not in a way that is useful for humans + * to perceive. The audio buffer time is often lagged behind the context + * time due to internal processing delays of the audio API. Additionally, + * the context's time is tracked from when it was first initialized/started, + * not related to positioning within the clip. However, the context time + * is the most accurate time we can use to determine position within the + * clip if we're fast enough to track the pauses and stops. + * + * As a result, we track every play, pause, stop, and seek event from the + * Playback class (kinda: it calls us, which is close enough to the same + * thing). These events are then tracked on the AudioContext time scale, + * with assumptions that code execution will result in negligible desync + * of the clock, or at least no perceptible difference in time. It's + * extremely important that the calling code, and the clock's own code, + * is extremely fast between the event happening and the clock time being + * tracked - anything more than a dozen milliseconds is likely to stack up + * poorly, leading to clock desync. + * + * Clock desync can be dangerous for the stability of the playback controls: + * if the clock thinks the user is somewhere else in the clip, it could + * inform the playback of the wrong place in time, leading to dead air in + * the output or, if severe enough, a clock that won't stop running while + * the audio is paused/stopped. Other examples include the clip stopping at + * 90% time due to playback ending, the clip playing from the wrong spot + * relative to the time, and negative clock time. + * + * Note that the clip duration is fed to the clock: this is to ensure that + * we have the most accurate time possible to present. + */ export class PlaybackClock implements IDestroyable { private clipStart = 0; private stopped = true; @@ -25,12 +61,13 @@ export class PlaybackClock implements IDestroyable { private observable = new SimpleObservable(); private timerId: number; private clipDuration = 0; + private placeholderDuration = 0; public constructor(private context: AudioContext) { } public get durationSeconds(): number { - return this.clipDuration; + return this.clipDuration || this.placeholderDuration; } public set durationSeconds(val: number) { @@ -39,6 +76,12 @@ export class PlaybackClock implements IDestroyable { } public get timeSeconds(): number { + // The modulo is to ensure that we're only looking at the most recent clip + // time, as the context is long-running and multiple plays might not be + // informed to us (if the control is looping, for example). By taking the + // remainder of the division operation, we're assuming that playback is + // incomplete or stopped, thus giving an accurate position within the active + // clip segment. return (this.context.currentTime - this.clipStart) % this.clipDuration; } @@ -47,13 +90,23 @@ export class PlaybackClock implements IDestroyable { } private checkTime = () => { - const now = this.timeSeconds; + const now = this.timeSeconds; // calculated dynamically if (this.lastCheck !== now) { this.observable.update([now, this.durationSeconds]); this.lastCheck = now; } }; + /** + * Populates default information about the audio clip from the event body. + * The placeholders will be overridden once known. + * @param {MatrixEvent} event The event to use for placeholders. + */ + public populatePlaceholdersFrom(event: MatrixEvent) { + const durationSeconds = Number(event.getContent()['info']?.['duration']); + if (Number.isFinite(durationSeconds)) this.placeholderDuration = durationSeconds; + } + /** * Mark the time in the audio context where the clip starts/has been loaded. * This is to ensure the clock isn't skewed into thinking it is ~0.5s into @@ -70,8 +123,9 @@ export class PlaybackClock implements IDestroyable { } if (!this.timerId) { - // case to number because the types are wrong - // 100ms interval to make sure the time is as accurate as possible + // cast to number because the types are wrong + // 100ms interval to make sure the time is as accurate as possible without + // being overly insane this.timerId = setInterval(this.checkTime, 100); } } @@ -80,6 +134,12 @@ export class PlaybackClock implements IDestroyable { this.stopped = true; } + public syncTo(contextTime: number, clipTime: number) { + this.clipStart = contextTime - clipTime; + this.stopped = false; // count as a mid-stream pause (if we were stopped) + this.checkTime(); + } + public destroy() { this.observable.close(); if (this.timerId) clearInterval(this.timerId);