From 4515654555b892c917446ff0db8f50fcd9eb0fb4 Mon Sep 17 00:00:00 2001 From: William Wong Date: Mon, 5 Jul 2021 01:29:00 -0700 Subject: [PATCH 01/19] Add AudioContext priming --- packages/bundle/package-lock.json | 20 +- packages/bundle/package.json | 1 + ...veServicesSpeechServicesPonyfillFactory.ts | 37 ++- .../src/speech/CustomAudioInputStream.ts | 282 ++++++++++++++++++ .../src/speech/MicrophoneAudioInputStream.ts | 170 +++++++++++ packages/bundle/src/speech/bytesPerSample.ts | 4 + .../bundle/src/speech/createAudioContext.ts | 14 + packages/bundle/src/speech/getUserMedia.ts | 14 + packages/bundle/webpack.config.js | 26 +- packages/component/package-lock.json | 14 +- .../directlinespeech/src/createAdapters.js | 8 +- packages/directlinespeech/webpack.config.js | 11 +- 12 files changed, 571 insertions(+), 30 deletions(-) create mode 100644 packages/bundle/src/speech/CustomAudioInputStream.ts create mode 100644 packages/bundle/src/speech/MicrophoneAudioInputStream.ts create mode 100644 packages/bundle/src/speech/bytesPerSample.ts create mode 100644 packages/bundle/src/speech/createAudioContext.ts create mode 100644 packages/bundle/src/speech/getUserMedia.ts diff --git a/packages/bundle/package-lock.json b/packages/bundle/package-lock.json index db2ac26b03..70cc160e59 100644 --- a/packages/bundle/package-lock.json +++ b/packages/bundle/package-lock.json @@ -3167,6 +3167,13 @@ "@types/ws": "^6.0.3", "uuid": "^3.4.0", "ws": "^7.1.2" + }, + "dependencies": { + "uuid": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" + } } }, "brace-expansion": { @@ -6603,6 +6610,13 @@ "uuid": "^3.3.3", "ws": "^7.3.1", "xmlhttprequest-ts": "^1.0.1" + }, + "dependencies": { + "uuid": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" + } } }, "miller-rabin": { @@ -9274,9 +9288,9 @@ "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" }, "uuid": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", - "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==" }, "v8-compile-cache": { "version": "2.2.0", diff --git a/packages/bundle/package.json b/packages/bundle/package.json index 191fd34c52..b44fc35abf 100644 --- a/packages/bundle/package.json +++ b/packages/bundle/package.json @@ -51,6 +51,7 @@ "prop-types": "15.7.2", "sanitize-html": "1.27.5", "url-search-params-polyfill": "8.1.1", + "uuid": "8.3.2", "web-speech-cognitive-services": "7.1.0", "whatwg-fetch": "3.6.2" }, diff --git a/packages/bundle/src/createCognitiveServicesSpeechServicesPonyfillFactory.ts b/packages/bundle/src/createCognitiveServicesSpeechServicesPonyfillFactory.ts index 3e53142435..a6957eca90 100644 --- a/packages/bundle/src/createCognitiveServicesSpeechServicesPonyfillFactory.ts +++ b/packages/bundle/src/createCognitiveServicesSpeechServicesPonyfillFactory.ts @@ -1,8 +1,11 @@ -import { AudioConfig } from 'microsoft-cognitiveservices-speech-sdk/distrib/lib/src/sdk/Audio/AudioConfig'; +import { AudioConfig } from 'microsoft-cognitiveservices-speech-sdk'; import { WebSpeechPonyfillFactory } from 'botframework-webchat-api'; import createPonyfill from 'web-speech-cognitive-services/lib/SpeechServices'; import CognitiveServicesCredentials from './types/CognitiveServicesCredentials'; +// import createMicrophoneAudioConfig from './createMicrophoneAudioConfig'; +import MicrophoneAudioInputStream from './speech/MicrophoneAudioInputStream'; +import createAudioContext from './speech/createAudioContext'; type CognitiveServicesAudioOutputFormat = | 'audio-16khz-128kbitrate-mono-mp3' @@ -31,6 +34,16 @@ type CognitiveServicesAudioOutputFormat = | 'webm-16khz-16bit-mono-opus' | 'webm-24khz-16bit-mono-opus'; +function appendPrimeButton(onClick) { + const primeButton = document.createElement('button'); + + primeButton.addEventListener('click', onClick); + primeButton.setAttribute('style', 'position: absolute; left: 10px; top: 10px; z-index: 1;'); + primeButton.textContent = 'Prime MicrophoneAudioInputStream'; + + document.body.appendChild(primeButton); +} + export default function createCognitiveServicesSpeechServicesPonyfillFactory({ audioConfig, audioContext, @@ -66,13 +79,29 @@ export default function createCognitiveServicesSpeechServicesPonyfillFactory({ ); } + audioContext || (audioContext = createAudioContext()); + + appendPrimeButton(async () => { + if (audioContext.state === 'suspended') { + await audioContext.resume(); + } + }); + // WORKAROUND: We should prevent AudioContext object from being recreated because they may be blessed and UX-wise expensive to recreate. // In Cognitive Services SDK, if they detect the "end" function is falsy, they will not call "end" but "suspend" instead. // And on next recognition, they will re-use the AudioContext object. if (!audioConfig) { - audioConfig = audioInputDeviceId - ? AudioConfig.fromMicrophoneInput(audioInputDeviceId) - : AudioConfig.fromDefaultMicrophoneInput(); + // audioConfig = audioInputDeviceId + // ? AudioConfig.fromMicrophoneInput(audioInputDeviceId) + // : AudioConfig.fromDefaultMicrophoneInput(); + // audioConfig = createMicrophoneAudioConfig({ audioInputDeviceId }); + audioConfig = AudioConfig.fromStreamInput( + new MicrophoneAudioInputStream({ + audioConstraints: { deviceId: audioInputDeviceId }, + audioContext, + telemetry: enableTelemetry ? true : undefined + }) + ); } return ({ referenceGrammarID } = {}) => { diff --git a/packages/bundle/src/speech/CustomAudioInputStream.ts b/packages/bundle/src/speech/CustomAudioInputStream.ts new file mode 100644 index 0000000000..f3eb777dc5 --- /dev/null +++ b/packages/bundle/src/speech/CustomAudioInputStream.ts @@ -0,0 +1,282 @@ +// TODO: We should export this type of AudioInputStream to allow web developers to bring in their own microphone. +// For example, it should enable React Native devs to bring in their microphone implementation and use Cognitive Services Speech Services. +import { AudioInputStream } from 'microsoft-cognitiveservices-speech-sdk'; + +// TODO: Revisit all imports from internals of Speech SDK. +// It should works with React TypeScript projects without modifying its internal Webpack configuration. +import { + AudioSourceErrorEvent, + AudioSourceEvent, + AudioSourceInitializingEvent, + AudioSourceOffEvent, + AudioSourceReadyEvent, + AudioStreamNodeAttachedEvent, + AudioStreamNodeAttachingEvent, + AudioStreamNodeDetachedEvent, + AudioStreamNodeErrorEvent, + Events, + EventSource +} from 'microsoft-cognitiveservices-speech-sdk/distrib/lib/src/common/Exports'; + +import { AudioStreamFormatImpl } from 'microsoft-cognitiveservices-speech-sdk/distrib/lib/src/sdk/Audio/AudioStreamFormat'; + +import { + connectivity as Connectivity, + ISpeechConfigAudioDevice, + type as Type +} from 'microsoft-cognitiveservices-speech-sdk/distrib/lib/src/common.speech/Exports'; + +import { v4 } from 'uuid'; + +const SYMBOL_DEVICE_INFO = Symbol('deviceInfo'); +const SYMBOL_EVENTS = Symbol('events'); +const SYMBOL_FORMAT = Symbol('format'); +const SYMBOL_OPTIONS = Symbol('options'); + +type AudioStreamNode = { + detach: () => Promise; + id: () => string; + read: () => Promise>; +}; + +type DeviceInfo = { + connectivity?: Connectivity | 'Bluetooth' | 'Wired' | 'WiFi' | 'Cellular' | 'InBuilt' | 'Unknown'; + manufacturer?: string; + model?: string; + type?: + | Type + | 'Phone' + | 'Speaker' + | 'Car' + | 'Headset' + | 'Thermostat' + | 'Microphones' + | 'Deskphone' + | 'RemoteControl' + | 'Unknown' + | 'File' + | 'Stream'; +}; + +type Format = { + bitsPerSample: number; + channels: number; + samplesPerSec: number; +}; + +type NormalizedOptions = Required> & { + debug: boolean; +}; + +type Options = { + debug?: true; + id?: string; +}; + +type StreamChunk = { + isEnd: boolean; + buffer: T; + timeReceived: number; +}; + +// Speech SDK quirks: Only 2 lifecycle functions are actually used. +// They are: attach() and turnOff(). +// Others are not used, including: blob(), close(), detach(), turnOn(). +abstract class CustomAudioInputStream extends AudioInputStream { + constructor(options: Options) { + super(); + + const normalizedOptions: NormalizedOptions = { + debug: options.debug || false, + id: options.id || v4().replace(/-/gu, '') + }; + + this[SYMBOL_EVENTS] = new EventSource(); + this[SYMBOL_OPTIONS] = normalizedOptions; + } + + [SYMBOL_DEVICE_INFO]: DeviceInfo; + [SYMBOL_EVENTS]: EventSource; + [SYMBOL_FORMAT]: Format; + [SYMBOL_OPTIONS]: NormalizedOptions; + + // This code will only works in browsers other than IE11. Only works in ES5 is okay. + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore Accessors are only available when targeting ECMAScript 5 and higher.ts(1056) + get events(): EventSource { + return this[SYMBOL_EVENTS]; + } + + // Speech SDK quirks: AudioStreamFormatImpl is internal implementation while AudioStreamFormat is public. + // It is weird to expose AudioStreamFormatImpl instead of AudioStreamFormat. + // Speech SDK quirks: It is weird to return a Promise in a property. + // Especially this is audio format. Setup options should be initialized synchronously. + // This code will only works in browsers other than IE11. Only works in ES5 is okay. + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore Accessors are only available when targeting ECMAScript 5 and higher.ts(1056) + get format(): Promise { + const format = this[SYMBOL_FORMAT]; + + if (!format) { + throw new Error('"format" is not available until attach() is called.'); + } + + return Promise.resolve(new AudioStreamFormatImpl(format.samplesPerSec, format.bitsPerSample, format.channels)); + } + + id(): string { + return this[SYMBOL_OPTIONS].id; + } + + // Speech SDK quirks: in JavaScript, onXxx means "listen to event XXX". + // instead, in Speech SDK, it means "emit event XXX". + protected onEvent(event: AudioSourceEvent): void { + this[SYMBOL_EVENTS].onEvent(event); + Events.instance.onEvent(event); + } + + protected emitInitializing(): void { + this.debug('Emitting "AudioSourceInitializingEvent".'); + this.onEvent(new AudioSourceInitializingEvent(this.id())); + } + + protected emitReady(): void { + this.debug('Emitting "AudioSourceReadyEvent".'); + this.onEvent(new AudioSourceReadyEvent(this.id())); + } + + // Speech SDK quirks: "error" is a string, instead of an Error object. + protected emitError(error: string): void { + this.debug('Emitting "AudioSourceErrorEvent".', { error }); + this.onEvent(new AudioSourceErrorEvent(this.id(), error)); + } + + protected emitNodeAttaching(audioNodeId: string): void { + this.debug(`Emitting "AudioStreamNodeAttachingEvent" for node "${audioNodeId}".`); + this.onEvent(new AudioStreamNodeAttachingEvent(this.id(), audioNodeId)); + } + + protected emitNodeAttached(audioNodeId: string): void { + this.debug(`Emitting "AudioStreamNodeAttachedEvent" for node "${audioNodeId}".`); + this.onEvent(new AudioStreamNodeAttachedEvent(this.id(), audioNodeId)); + } + + // Speech SDK quirks: "error" is a string, instead of an Error object. + protected emitNodeError(audioNodeId: string, error: string): void { + this.debug(`Emitting "AudioStreamNodeErrorEvent" for node "${audioNodeId}".`, { error }); + this.onEvent(new AudioStreamNodeErrorEvent(this.id(), audioNodeId, error)); + } + + protected emitNodeDetached(audioNodeId: string): void { + this.debug('Emitting "AudioStreamNodeDetachedEvent".'); + this.onEvent(new AudioStreamNodeDetachedEvent(this.id(), audioNodeId)); + } + + protected emitOff(): void { + this.debug('Emitting "AudioSourceOffEvent".'); + this.onEvent(new AudioSourceOffEvent(this.id())); + } + + // Speech SDK quirks: It seems close() is never called, despite, it is marked as abstract. + // Speech SDK requires this function. + // eslint-disable-next-line class-methods-use-this + close(): void { + throw new Error('Not implemented'); + } + + private debug(message, ...args) { + // eslint-disable-next-line no-console + this[SYMBOL_OPTIONS].debug && console.info(`CustomAudioInputStream: ${message}`, ...args); + } + + /** Implements this function. When called, it should start recording and return an `IAudioStreamNode`. */ + protected abstract performAttach( + audioNodeId: string + ): Promise<{ + audioStreamNode: AudioStreamNode; + deviceInfo: DeviceInfo; + format: Format; + }>; + + attach(audioNodeId: string): Promise { + this.debug(`Callback for "attach" with "${audioNodeId}".`); + + this.emitNodeAttaching(audioNodeId); + + return Promise.resolve().then(async () => { + this.emitInitializing(); + + try { + const { audioStreamNode, deviceInfo, format } = await this.performAttach(audioNodeId); + + this[SYMBOL_DEVICE_INFO] = deviceInfo; + this[SYMBOL_FORMAT] = format; + + this.emitReady(); + this.emitNodeAttached(audioNodeId); + + return { + detach: async () => { + this.debug(`Detaching audio node "${audioNodeId}".`); + + await audioStreamNode.detach(); + + this.emitNodeDetached(audioNodeId); + }, + id: () => audioStreamNode.id(), + read: () => { + this.debug('Reading'); + + return audioStreamNode.read(); + } + }; + } catch (error) { + this.emitNodeError(audioNodeId, error); + + throw error; + } + }); + } + + /** Implements this function. When called, it should stop recording. This is called before the `IAudioStreamNode.detach` function. */ + protected abstract performTurnOff(): Promise; + + async turnOff(): Promise { + this.debug(`Callback for "turnOff".`); + + await this.performTurnOff(); + + this.emitOff(); + } + + // This code will only works in browsers other than IE11. Only works in ES5 is okay. + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore Accessors are only available when targeting ECMAScript 5 and higher.ts(1056) + get deviceInfo(): Promise { + this.debug(`Getting "deviceInfo".`); + + const deviceInfo = this[SYMBOL_DEVICE_INFO]; + + if (!deviceInfo) { + throw new Error('"deviceInfo" is not available until attach() is called.'); + } + + const { connectivity, manufacturer, model, type } = deviceInfo; + const { bitsPerSample, channels, samplesPerSec } = this[SYMBOL_FORMAT]; + + return Promise.resolve({ + bitspersample: bitsPerSample, + channelcount: channels, + connectivity: + typeof connectivity === 'string' ? Connectivity[connectivity] : connectivity || Connectivity.Unknown, + manufacturer: manufacturer || '', + model: model || '', + samplerate: samplesPerSec, + type: typeof type === 'string' ? Type[type] : type || Type.Unknown + }); + } +} + +export default CustomAudioInputStream; + +export type { AudioStreamNode, DeviceInfo, Format, Options }; diff --git a/packages/bundle/src/speech/MicrophoneAudioInputStream.ts b/packages/bundle/src/speech/MicrophoneAudioInputStream.ts new file mode 100644 index 0000000000..e7c7b4cc0e --- /dev/null +++ b/packages/bundle/src/speech/MicrophoneAudioInputStream.ts @@ -0,0 +1,170 @@ +import { ChunkedArrayBufferStream } from 'microsoft-cognitiveservices-speech-sdk/distrib/lib/src/common/Exports'; +import { PcmRecorder } from 'microsoft-cognitiveservices-speech-sdk/distrib/lib/src/common.browser/Exports'; + +import bytesPerSample from './bytesPerSample'; +import CustomAudioInputStream, { AudioStreamNode, DeviceInfo, Format } from './CustomAudioInputStream'; +import getUserMedia from './getUserMedia'; + +// This is how often we are flushing audio buffer to the network. Modify this value will affect latency. +const DEFAULT_BUFFER_DURATION_IN_MS = 100; + +// PcmRecorder always downscale to 16000 Hz. We cannot use the dynamic value from MediaConstraints or MediaTrackSettings. +const PCM_RECORDER_HARDCODED_SETTINGS: MediaTrackSettings = Object.freeze({ + channelCount: 1, + sampleRate: 16000, + sampleSize: 16 +}); + +const PCM_RECORDER_HARDCODED_FORMAT: Format = Object.freeze({ + bitsPerSample: PCM_RECORDER_HARDCODED_SETTINGS.sampleSize, + channels: PCM_RECORDER_HARDCODED_SETTINGS.channelCount, + samplesPerSec: PCM_RECORDER_HARDCODED_SETTINGS.sampleRate +}); + +type MicrophoneAudioInputStreamOptions = { + /** Specifies the constraints for selecting an audio device. */ + audioConstraints?: true | MediaTrackConstraints; + + /** Specifies the `AudioContext` to use. This object must be primed and ready to use. */ + audioContext: AudioContext; + + /** Specifies the buffering delay on how often to flush audio data to network. Increasing the value will increase audio latency. Default is 100 ms. */ + bufferDurationInMS?: number; + + /** Specifies the `AudioWorklet` URL for `PcmRecorder`. If not specified, will use script processor on UI thread instead. */ + pcmRecorderWorkletUrl?: string; + + /** Specifies if telemetry data should be sent. If not specified, telemetry data will NOT be sent. */ + telemetry?: true; +}; + +const SYMBOL_AUDIO_CONSTRAINTS = Symbol('audioConstraints'); +const SYMBOL_AUDIO_CONTEXT = Symbol('audioContext'); +const SYMBOL_BUFFER_DURATION_IN_MS = Symbol('bufferDurationInMS'); +const SYMBOL_OUTPUT_STREAM = Symbol('outputStream'); +const SYMBOL_PCM_RECORDER = Symbol('pcmRecorder'); +const SYMBOL_TELEMETRY = Symbol('telemetry'); + +export default class MicrophoneAudioInputStream extends CustomAudioInputStream { + constructor(options: MicrophoneAudioInputStreamOptions) { + super({ debug: true }); + + const { audioConstraints, audioContext, bufferDurationInMS, pcmRecorderWorkletUrl } = options; + + this[SYMBOL_AUDIO_CONSTRAINTS] = audioConstraints === 'boolean' || audioConstraints; + this[SYMBOL_AUDIO_CONTEXT] = audioContext; + this[SYMBOL_BUFFER_DURATION_IN_MS] = bufferDurationInMS || DEFAULT_BUFFER_DURATION_IN_MS; + + const pcmRecorder = (this[SYMBOL_PCM_RECORDER] = new PcmRecorder()); + + pcmRecorderWorkletUrl && pcmRecorder.setWorkletUrl(pcmRecorderWorkletUrl); + } + + [SYMBOL_AUDIO_CONSTRAINTS]: true | MediaTrackConstraints; + [SYMBOL_AUDIO_CONTEXT]: AudioContext; + [SYMBOL_BUFFER_DURATION_IN_MS]: number; + [SYMBOL_OUTPUT_STREAM]?: ChunkedArrayBufferStream; + [SYMBOL_PCM_RECORDER]?: PcmRecorder; + [SYMBOL_TELEMETRY]?: true; + + // This code will only works in browsers other than IE11. Only works in ES5 is okay. + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore Accessors are only available when targeting ECMAScript 5 and higher.ts(1056) + get audioConstraints(): true | MediaTrackConstraints { + return this[SYMBOL_AUDIO_CONSTRAINTS]; + } + + // This code will only works in browsers other than IE11. Only works in ES5 is okay. + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore Accessors are only available when targeting ECMAScript 5 and higher.ts(1056) + get audioContext(): AudioContext { + return this[SYMBOL_AUDIO_CONTEXT]; + } + + // This code will only works in browsers other than IE11. Only works in ES5 is okay. + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore Accessors are only available when targeting ECMAScript 5 and higher.ts(1056) + get bufferDurationInMS(): number { + return this[SYMBOL_BUFFER_DURATION_IN_MS]; + } + + // Speech SDK quirks: It is confused to have both "turnOff" and "detach". "turnOff" is called before "detach". + // Why don't we put all logics at "detach"? + // Speech SDK quirks: event "source off" is sent before event "node detached". + // Shouldn't source "bigger" (in terms of responsibilities) and include nodes? + // Why we "close the source" before "close the node"? + performTurnOff(): Promise { + // Speech SDK quirks: in SDK, it call outputStream.close() in turnOff() before outputStream.readEnded() in detach(). + // I think it make sense to call readEnded() before close(). + this[SYMBOL_OUTPUT_STREAM].readEnded(); + this[SYMBOL_OUTPUT_STREAM].close(); + + // PcmRecorder.releaseMediaResources() will disconnect/stop the MediaStream. + // We cannot use MediaStream again after turned off. + this[SYMBOL_PCM_RECORDER].releaseMediaResources(this.audioContext); + + // Required by TypeScript + // eslint-disable-next-line no-useless-return + return; + } + + async performAttach( + audioNodeId: string + ): Promise<{ + audioStreamNode: AudioStreamNode; + deviceInfo: DeviceInfo; + format: Format; + }> { + const { + [SYMBOL_AUDIO_CONTEXT]: audioContext, + [SYMBOL_BUFFER_DURATION_IN_MS]: bufferDurationInMS, + [SYMBOL_PCM_RECORDER]: pcmRecorder + } = this; + + // We need to get new MediaStream on every attach(). + // This is because PcmRecorder.releaseMediaResources() disconnected/stopped them. + const mediaStream = await getUserMedia({ audio: this.audioConstraints, video: false }); + + const [firstAudioTrack] = mediaStream.getAudioTracks(); + + if (!firstAudioTrack) { + throw new Error('No audio device is found.'); + } + + const outputStream = (this[SYMBOL_OUTPUT_STREAM] = new ChunkedArrayBufferStream( + // Speech SDK quirks: PcmRecorder hardcoded sample rate of 16000 Hz. + // eslint-disable-next-line no-magic-numbers + bytesPerSample(PCM_RECORDER_HARDCODED_SETTINGS) * ((bufferDurationInMS || DEFAULT_BUFFER_DURATION_IN_MS) / 1000), + audioNodeId + )); + + pcmRecorder.record(audioContext, mediaStream, outputStream); + + return { + audioStreamNode: { + // Speech SDK quirks: in SDK's MicAudioSource, it call turnOff() during detach(). + // That means, it call turnOff(), then detach(), then turnOff() again. Seems redundant. + detach: (): Promise => { + // MediaStream will become inactive after all tracks are removed. + mediaStream.getTracks().forEach(track => mediaStream.removeTrack(track)); + + // Required by TypeScript + // eslint-disable-next-line no-useless-return + return; + }, + id: () => audioNodeId, + read: () => outputStream.read() + }, + deviceInfo: { + manufacturer: 'Bot Framework Web Chat', + model: this[SYMBOL_TELEMETRY] ? firstAudioTrack.label : '', + type: this[SYMBOL_TELEMETRY] ? 'Microphones' : 'Unknown' + }, + // Speech SDK quirks: PcmRecorder hardcoded sample rate of 16000 Hz. + // We cannot obtain this number other than looking at their source code. + // I.e. no getter property. + // PcmRecorder always downscale to 16000 Hz. We cannot use the dynamic value from MediaConstraints or MediaTrackSettings. + format: PCM_RECORDER_HARDCODED_FORMAT + }; + } +} diff --git a/packages/bundle/src/speech/bytesPerSample.ts b/packages/bundle/src/speech/bytesPerSample.ts new file mode 100644 index 0000000000..62503ae0c2 --- /dev/null +++ b/packages/bundle/src/speech/bytesPerSample.ts @@ -0,0 +1,4 @@ +export default function bytesPerSample(settings: MediaTrackSettings) { + // eslint-disable-next-line no-magic-numbers + return ((settings.sampleSize as number) >> 3) * (settings.channelCount as number) * (settings.sampleRate as number); +} diff --git a/packages/bundle/src/speech/createAudioContext.ts b/packages/bundle/src/speech/createAudioContext.ts new file mode 100644 index 0000000000..4e0574f627 --- /dev/null +++ b/packages/bundle/src/speech/createAudioContext.ts @@ -0,0 +1,14 @@ +export default function createAudioContext(): AudioContext { + if (typeof window.AudioContext !== 'undefined') { + return new window.AudioContext(); + + // Required by TypeScript. + // eslint-disable-next-line dot-notation + } else if (typeof window['webkitAudioContext'] !== 'undefined') { + // eslint-disable-next-line dot-notation + return new window['webkitAudioContext'](); + } + + // TODO: Fix this. + throw new Error('This browser does not support Web Audio API.'); +} diff --git a/packages/bundle/src/speech/getUserMedia.ts b/packages/bundle/src/speech/getUserMedia.ts new file mode 100644 index 0000000000..2ac9c6f7cd --- /dev/null +++ b/packages/bundle/src/speech/getUserMedia.ts @@ -0,0 +1,14 @@ +export default function getUserMedia(constraints: MediaStreamConstraints): Promise { + const { navigator } = window; + + if (typeof navigator.mediaDevices !== 'undefined') { + return navigator.mediaDevices.getUserMedia(constraints); + } + + // TODO: Does it need vendor prefix? + if (typeof navigator.getUserMedia !== 'undefined') { + return new Promise((resolve, reject) => navigator.getUserMedia(constraints, resolve, reject)); + } + + throw new Error('This browser does not support Web Audio API.'); +} diff --git a/packages/bundle/webpack.config.js b/packages/bundle/webpack.config.js index bc861e130f..7332035cf2 100644 --- a/packages/bundle/webpack.config.js +++ b/packages/bundle/webpack.config.js @@ -42,17 +42,35 @@ let config = { ], resolve: { alias: { - 'microsoft-cognitiveservices-speech-sdk/distrib/lib/src/sdk/Audio/AudioConfig': resolve( + // It is smaller to use /lib/ instead of /es2015/. + // Verifies if /es2015/ is better when moving to esbuild. + 'microsoft-cognitiveservices-speech-sdk': resolve( __dirname, - 'node_modules/microsoft-cognitiveservices-speech-sdk/distrib/lib/src/sdk/Audio/AudioConfig.js' + 'node_modules/microsoft-cognitiveservices-speech-sdk/distrib/lib/microsoft.cognitiveservices.speech.sdk.js' ), 'microsoft-cognitiveservices-speech-sdk/distrib/lib/microsoft.cognitiveservices.speech.sdk': resolve( __dirname, 'node_modules/microsoft-cognitiveservices-speech-sdk/distrib/lib/microsoft.cognitiveservices.speech.sdk.js' ), - 'microsoft-cognitiveservices-speech-sdk': resolve( + 'microsoft-cognitiveservices-speech-sdk/distrib/lib/src/common.browser/Exports': resolve( __dirname, - 'node_modules/microsoft-cognitiveservices-speech-sdk/distrib/lib/microsoft.cognitiveservices.speech.sdk.js' + 'node_modules/microsoft-cognitiveservices-speech-sdk/distrib/lib/src/common.browser/Exports.js' + ), + 'microsoft-cognitiveservices-speech-sdk/distrib/lib/src/common.speech/Exports': resolve( + __dirname, + 'node_modules/microsoft-cognitiveservices-speech-sdk/distrib/lib/src/common.speech/Exports.js' + ), + 'microsoft-cognitiveservices-speech-sdk/distrib/lib/src/common/Exports': resolve( + __dirname, + 'node_modules/microsoft-cognitiveservices-speech-sdk/distrib/lib/src/common/Exports.js' + ), + 'microsoft-cognitiveservices-speech-sdk/distrib/lib/src/sdk/Audio/AudioStreamFormat': resolve( + __dirname, + 'node_modules/microsoft-cognitiveservices-speech-sdk/distrib/lib/src/sdk/Audio/AudioStreamFormat.js' + ), + 'microsoft-cognitiveservices-speech-sdk/distrib/lib/src/sdk/Exports': resolve( + __dirname, + 'node_modules/microsoft-cognitiveservices-speech-sdk/distrib/lib/src/sdk/Exports.js' ), react: resolve(__dirname, 'node_modules/isomorphic-react/dist/react.js'), 'react-dom': resolve(__dirname, 'node_modules/isomorphic-react-dom/dist/react-dom.js') diff --git a/packages/component/package-lock.json b/packages/component/package-lock.json index e9e81b3514..805ab489f4 100644 --- a/packages/component/package-lock.json +++ b/packages/component/package-lock.json @@ -2056,11 +2056,11 @@ } }, "@babel/runtime-corejs3": { - "version": "7.14.0", - "resolved": "https://registry.npmjs.org/@babel/runtime-corejs3/-/runtime-corejs3-7.14.0.tgz", - "integrity": "sha512-0R0HTZWHLk6G8jIk0FtoX+AatCtKnswS98VhXwGImFc759PJRp4Tru0PQYZofyijTFUr+gT8Mu7sgXVJLQ0ceg==", + "version": "7.14.7", + "resolved": "https://registry.npmjs.org/@babel/runtime-corejs3/-/runtime-corejs3-7.14.7.tgz", + "integrity": "sha512-Wvzcw4mBYbTagyBVZpAJWI06auSIj033T/yNE0Zn1xcup83MieCddZA7ls3kme17L4NOGBrQ09Q+nKB41RLWBA==", "requires": { - "core-js-pure": "^3.0.0", + "core-js-pure": "^3.15.0", "regenerator-runtime": "^0.13.4" } }, @@ -3921,9 +3921,9 @@ } }, "core-js-pure": { - "version": "3.13.1", - "resolved": "https://registry.npmjs.org/core-js-pure/-/core-js-pure-3.13.1.tgz", - "integrity": "sha512-wVlh0IAi2t1iOEh16y4u1TRk6ubd4KvLE8dlMi+3QUI6SfKphQUh7tAwihGGSQ8affxEXpVIPpOdf9kjR4v4Pw==" + "version": "3.15.2", + "resolved": "https://registry.npmjs.org/core-js-pure/-/core-js-pure-3.15.2.tgz", + "integrity": "sha512-D42L7RYh1J2grW8ttxoY1+17Y4wXZeKe7uyplAI3FkNQyI5OgBIAjUfFiTPfL1rs0qLpxaabITNbjKl1Sp82tA==" }, "core-util-is": { "version": "1.0.2", diff --git a/packages/directlinespeech/src/createAdapters.js b/packages/directlinespeech/src/createAdapters.js index b00fa3e373..ad1932947c 100644 --- a/packages/directlinespeech/src/createAdapters.js +++ b/packages/directlinespeech/src/createAdapters.js @@ -1,7 +1,11 @@ /* eslint complexity: ["error", 33] */ -import { AudioConfig } from 'microsoft-cognitiveservices-speech-sdk/distrib/lib/src/sdk/Audio/AudioConfig'; -import { BotFrameworkConfig, DialogServiceConnector, PropertyId } from 'microsoft-cognitiveservices-speech-sdk'; +import { + AudioConfig, + BotFrameworkConfig, + DialogServiceConnector, + PropertyId +} from 'microsoft-cognitiveservices-speech-sdk'; import createWebSpeechPonyfillFactory from './createWebSpeechPonyfillFactory'; import DirectLineSpeech from './DirectLineSpeech'; diff --git a/packages/directlinespeech/webpack.config.js b/packages/directlinespeech/webpack.config.js index 051acab1a8..71e5f0855f 100644 --- a/packages/directlinespeech/webpack.config.js +++ b/packages/directlinespeech/webpack.config.js @@ -15,16 +15,7 @@ let config = { filename: 'stats.json', transform: (_, opts) => JSON.stringify(opts.compiler.getStats().toJson({ chunkModules: true }), null, 2) }) - ], - resolve: { - alias: { - // TODO: [P1] #3575 Remove the following line when bumping to Speech SDK 1.14.0 or higher - 'microsoft-cognitiveservices-speech-sdk/distrib/lib/src/common.browser/MicAudioSource': resolve( - __dirname, - 'node_modules/microsoft-cognitiveservices-speech-sdk/distrib/lib/src/common.browser/MicAudioSource.js' - ) - } - } + ] }; // VSTS always emits uppercase environment variables. From bbb182c7751ec7a28f54af785e2d54fae47c71e9 Mon Sep 17 00:00:00 2001 From: William Wong Date: Mon, 5 Jul 2021 04:25:17 -0700 Subject: [PATCH 02/19] Fix build break --- packages/bundle/webpack.config.js | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/packages/bundle/webpack.config.js b/packages/bundle/webpack.config.js index 7332035cf2..38fc4954c7 100644 --- a/packages/bundle/webpack.config.js +++ b/packages/bundle/webpack.config.js @@ -44,14 +44,6 @@ let config = { alias: { // It is smaller to use /lib/ instead of /es2015/. // Verifies if /es2015/ is better when moving to esbuild. - 'microsoft-cognitiveservices-speech-sdk': resolve( - __dirname, - 'node_modules/microsoft-cognitiveservices-speech-sdk/distrib/lib/microsoft.cognitiveservices.speech.sdk.js' - ), - 'microsoft-cognitiveservices-speech-sdk/distrib/lib/microsoft.cognitiveservices.speech.sdk': resolve( - __dirname, - 'node_modules/microsoft-cognitiveservices-speech-sdk/distrib/lib/microsoft.cognitiveservices.speech.sdk.js' - ), 'microsoft-cognitiveservices-speech-sdk/distrib/lib/src/common.browser/Exports': resolve( __dirname, 'node_modules/microsoft-cognitiveservices-speech-sdk/distrib/lib/src/common.browser/Exports.js' @@ -72,6 +64,16 @@ let config = { __dirname, 'node_modules/microsoft-cognitiveservices-speech-sdk/distrib/lib/src/sdk/Exports.js' ), + 'microsoft-cognitiveservices-speech-sdk/distrib/lib/microsoft.cognitiveservices.speech.sdk': resolve( + __dirname, + 'node_modules/microsoft-cognitiveservices-speech-sdk/distrib/lib/microsoft.cognitiveservices.speech.sdk.js' + ), + + // This line must be placed after other specific imports. + 'microsoft-cognitiveservices-speech-sdk': resolve( + __dirname, + 'node_modules/microsoft-cognitiveservices-speech-sdk/distrib/lib/microsoft.cognitiveservices.speech.sdk.js' + ), react: resolve(__dirname, 'node_modules/isomorphic-react/dist/react.js'), 'react-dom': resolve(__dirname, 'node_modules/isomorphic-react-dom/dist/react-dom.js') } From 85bcc600c4950a5f0240942e0eaa98c91b107b26 Mon Sep 17 00:00:00 2001 From: William Wong Date: Mon, 5 Jul 2021 04:47:30 -0700 Subject: [PATCH 03/19] Update comment --- packages/bundle/webpack.config.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/bundle/webpack.config.js b/packages/bundle/webpack.config.js index 38fc4954c7..bd9f594bf5 100644 --- a/packages/bundle/webpack.config.js +++ b/packages/bundle/webpack.config.js @@ -42,8 +42,8 @@ let config = { ], resolve: { alias: { - // It is smaller to use /lib/ instead of /es2015/. - // Verifies if /es2015/ is better when moving to esbuild. + // TODO: [P1] #3914 It is smaller to use /lib/ instead of /es2015/ with Webpack. + // Verifies if /es2015/ is better when moving to esbuild. 'microsoft-cognitiveservices-speech-sdk/distrib/lib/src/common.browser/Exports': resolve( __dirname, 'node_modules/microsoft-cognitiveservices-speech-sdk/distrib/lib/src/common.browser/Exports.js' From 452c583ebfe843bb2617d9998d16076933a49cf0 Mon Sep 17 00:00:00 2001 From: William Wong Date: Mon, 5 Jul 2021 06:25:37 -0700 Subject: [PATCH 04/19] Add resumeAudioContext --- packages/api/src/types/WebSpeechPonyfill.ts | 1 + ...veServicesSpeechServicesPonyfillFactory.ts | 50 +++++++------------ .../src/SendBox/MicrophoneButton.tsx | 4 ++ .../hooks/internal/useResumeAudioContext.ts | 7 +++ 4 files changed, 30 insertions(+), 32 deletions(-) create mode 100644 packages/component/src/hooks/internal/useResumeAudioContext.ts diff --git a/packages/api/src/types/WebSpeechPonyfill.ts b/packages/api/src/types/WebSpeechPonyfill.ts index 3834c86dd6..8c354a5cd6 100644 --- a/packages/api/src/types/WebSpeechPonyfill.ts +++ b/packages/api/src/types/WebSpeechPonyfill.ts @@ -1,6 +1,7 @@ /* globals SpeechGrammarList, SpeechRecognition, SpeechSynthesis */ type WebSpeechPonyfill = { + resumeAudioContext?: () => Promise; SpeechGrammarList?: typeof SpeechGrammarList; SpeechRecognition?: typeof SpeechRecognition; speechSynthesis?: SpeechSynthesis; diff --git a/packages/bundle/src/createCognitiveServicesSpeechServicesPonyfillFactory.ts b/packages/bundle/src/createCognitiveServicesSpeechServicesPonyfillFactory.ts index a6957eca90..da2b3578de 100644 --- a/packages/bundle/src/createCognitiveServicesSpeechServicesPonyfillFactory.ts +++ b/packages/bundle/src/createCognitiveServicesSpeechServicesPonyfillFactory.ts @@ -3,9 +3,8 @@ import { WebSpeechPonyfillFactory } from 'botframework-webchat-api'; import createPonyfill from 'web-speech-cognitive-services/lib/SpeechServices'; import CognitiveServicesCredentials from './types/CognitiveServicesCredentials'; -// import createMicrophoneAudioConfig from './createMicrophoneAudioConfig'; -import MicrophoneAudioInputStream from './speech/MicrophoneAudioInputStream'; import createAudioContext from './speech/createAudioContext'; +import MicrophoneAudioInputStream from './speech/MicrophoneAudioInputStream'; type CognitiveServicesAudioOutputFormat = | 'audio-16khz-128kbitrate-mono-mp3' @@ -34,16 +33,6 @@ type CognitiveServicesAudioOutputFormat = | 'webm-16khz-16bit-mono-opus' | 'webm-24khz-16bit-mono-opus'; -function appendPrimeButton(onClick) { - const primeButton = document.createElement('button'); - - primeButton.addEventListener('click', onClick); - primeButton.setAttribute('style', 'position: absolute; left: 10px; top: 10px; z-index: 1;'); - primeButton.textContent = 'Prime MicrophoneAudioInputStream'; - - document.body.appendChild(primeButton); -} - export default function createCognitiveServicesSpeechServicesPonyfillFactory({ audioConfig, audioContext, @@ -73,31 +62,27 @@ export default function createCognitiveServicesSpeechServicesPonyfillFactory({ return () => ({}); } - if (audioConfig && audioInputDeviceId) { - console.warn( - 'botframework-webchat: "audioConfig" and "audioInputDeviceId" cannot be set at the same time; ignoring "audioInputDeviceId".' - ); - } - - audioContext || (audioContext = createAudioContext()); + if (audioConfig) { + if (audioInputDeviceId) { + console.warn( + 'botframework-webchat: "audioConfig" and "audioInputDeviceId" cannot be set at the same time; ignoring "audioInputDeviceId".' + ); + } - appendPrimeButton(async () => { - if (audioContext.state === 'suspended') { - await audioContext.resume(); + if (audioContext) { + console.warn( + 'botframework-webchat: "audioConfig" and "audioContext" cannot be set at the same time; ignoring "audioContext" for speech recognition.' + ); } - }); + } else { + // WORKAROUND: We should prevent AudioContext object from being recreated because they may be blessed and UX-wise expensive to recreate. + // In Cognitive Services SDK, if they detect the "end" function is falsy, they will not call "end" but "suspend" instead. + // And on next recognition, they will re-use the AudioContext object. - // WORKAROUND: We should prevent AudioContext object from being recreated because they may be blessed and UX-wise expensive to recreate. - // In Cognitive Services SDK, if they detect the "end" function is falsy, they will not call "end" but "suspend" instead. - // And on next recognition, they will re-use the AudioContext object. - if (!audioConfig) { - // audioConfig = audioInputDeviceId - // ? AudioConfig.fromMicrophoneInput(audioInputDeviceId) - // : AudioConfig.fromDefaultMicrophoneInput(); - // audioConfig = createMicrophoneAudioConfig({ audioInputDeviceId }); + audioContext || (audioContext = createAudioContext()); audioConfig = AudioConfig.fromStreamInput( new MicrophoneAudioInputStream({ - audioConstraints: { deviceId: audioInputDeviceId }, + audioConstraints: audioInputDeviceId ? { deviceId: audioInputDeviceId } : true, audioContext, telemetry: enableTelemetry ? true : undefined }) @@ -118,6 +103,7 @@ export default function createCognitiveServicesSpeechServicesPonyfillFactory({ }); return { + resumeAudioContext: () => audioContext.state === 'suspended' && audioContext.resume(), SpeechGrammarList, SpeechRecognition, speechSynthesis, diff --git a/packages/component/src/SendBox/MicrophoneButton.tsx b/packages/component/src/SendBox/MicrophoneButton.tsx index ce4c1f40b9..4cbe0c415d 100644 --- a/packages/component/src/SendBox/MicrophoneButton.tsx +++ b/packages/component/src/SendBox/MicrophoneButton.tsx @@ -12,6 +12,7 @@ import connectToWebChat from '../connectToWebChat'; import IconButton from './IconButton'; import MicrophoneIcon from './Assets/MicrophoneIcon'; import useDictateAbortable from '../hooks/useDictateAbortable'; +import useResumeAudioContext from '../hooks/internal/useResumeAudioContext'; import useStyleSet from '../hooks/useStyleSet'; import useStyleToEmotionObject from '../hooks/internal/useStyleToEmotionObject'; import useWebSpeechPonyfill from '../hooks/useWebSpeechPonyfill'; @@ -99,6 +100,7 @@ function useMicrophoneButtonClick(): () => void { const [dictateInterims] = useDictateInterims(); const [dictateState] = useDictateState(); const [webSpeechPonyfill] = useWebSpeechPonyfill(); + const resumeAudioContext = useResumeAudioContext(); const startDictate = useStartDictate(); const stopDictate = useStopDictate(); @@ -118,6 +120,8 @@ function useMicrophoneButtonClick(): () => void { // TODO: [P2] We should revisit this function later // The click() logic seems local to the component, but may not be generalized across all implementations. return useCallback(() => { + resumeAudioContext(); + if (dictateState === DictateState.WILL_START) { setShouldSpeakIncomingActivity(false); } else if (dictateState === DictateState.DICTATING) { diff --git a/packages/component/src/hooks/internal/useResumeAudioContext.ts b/packages/component/src/hooks/internal/useResumeAudioContext.ts new file mode 100644 index 0000000000..05a2d82da8 --- /dev/null +++ b/packages/component/src/hooks/internal/useResumeAudioContext.ts @@ -0,0 +1,7 @@ +import useWebSpeechPonyfill from '../useWebSpeechPonyfill'; + +export default function useResumeAudioContext(): () => Promise { + const [{ resumeAudioContext }] = useWebSpeechPonyfill(); + + return () => resumeAudioContext && resumeAudioContext(); +} From 146dab35aef162e17f5eee9aab808fac92b44995 Mon Sep 17 00:00:00 2001 From: William Wong Date: Mon, 5 Jul 2021 06:47:18 -0700 Subject: [PATCH 05/19] Add descriptions --- packages/api/src/types/WebSpeechPonyfill.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/packages/api/src/types/WebSpeechPonyfill.ts b/packages/api/src/types/WebSpeechPonyfill.ts index 8c354a5cd6..baef831438 100644 --- a/packages/api/src/types/WebSpeechPonyfill.ts +++ b/packages/api/src/types/WebSpeechPonyfill.ts @@ -1,10 +1,23 @@ /* globals SpeechGrammarList, SpeechRecognition, SpeechSynthesis */ type WebSpeechPonyfill = { + /** + * Function to resume `AudioContext` object when called. + * + * Web Chat will call this function on user gestures to resume suspended `AudioContext`. + */ resumeAudioContext?: () => Promise; + + /** Polyfill for Web Speech API `SpeechGrammarList` class. */ SpeechGrammarList?: typeof SpeechGrammarList; + + /** Polyfill for Web Speech API `SpeechRecognition` class. */ SpeechRecognition?: typeof SpeechRecognition; + + /** Polyfill for Web Speech API `speechSynthesis` instance. */ speechSynthesis?: SpeechSynthesis; + + /** Polyfill for Web Speech API `SpeechSynthesisUtterance` class. */ SpeechSynthesisUtterance?: typeof SpeechSynthesisUtterance; }; From 1c4f5d663926394f008b7bfa72b97faac889da51 Mon Sep 17 00:00:00 2001 From: William Wong Date: Mon, 5 Jul 2021 07:49:13 -0700 Subject: [PATCH 06/19] Move resumeAudioContext to Dictation --- packages/component/src/Dictation.js | 10 +++++++++- packages/component/src/SendBox/MicrophoneButton.tsx | 4 ---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/packages/component/src/Dictation.js b/packages/component/src/Dictation.js index dffc34083c..1133691b23 100644 --- a/packages/component/src/Dictation.js +++ b/packages/component/src/Dictation.js @@ -2,8 +2,9 @@ import { Composer as DictateComposer } from 'react-dictate-button'; import { Constants } from 'botframework-webchat-core'; import { hooks } from 'botframework-webchat-api'; import PropTypes from 'prop-types'; -import React, { useCallback, useMemo } from 'react'; +import React, { useCallback, useEffect, useMemo } from 'react'; +import useResumeAudioContext from './hooks/internal/useResumeAudioContext'; import useSettableDictateAbortable from './hooks/internal/useSettableDictateAbortable'; import useWebSpeechPonyfill from './hooks/useWebSpeechPonyfill'; @@ -40,6 +41,7 @@ const Dictation = ({ onError }) => { const [sendTypingIndicator] = useSendTypingIndicator(); const [speechLanguage] = useLanguage('speech'); const emitTypingIndicator = useEmitTypingIndicator(); + const resumeAudioContext = useResumeAudioContext(); const setDictateState = useSetDictateState(); const stopDictate = useStopDictate(); const submitSendBox = useSubmitSendBox(); @@ -97,6 +99,12 @@ const Dictation = ({ onError }) => { [dictateState, onError, setDictateState, stopDictate] ); + useEffect(() => { + window.addEventListener('pointerdown', resumeAudioContext); + + return () => window.removeEventListener('pointerdown', resumeAudioContext); + }, [resumeAudioContext]); + return ( void { const [dictateInterims] = useDictateInterims(); const [dictateState] = useDictateState(); const [webSpeechPonyfill] = useWebSpeechPonyfill(); - const resumeAudioContext = useResumeAudioContext(); const startDictate = useStartDictate(); const stopDictate = useStopDictate(); @@ -120,8 +118,6 @@ function useMicrophoneButtonClick(): () => void { // TODO: [P2] We should revisit this function later // The click() logic seems local to the component, but may not be generalized across all implementations. return useCallback(() => { - resumeAudioContext(); - if (dictateState === DictateState.WILL_START) { setShouldSpeakIncomingActivity(false); } else if (dictateState === DictateState.DICTATING) { From f543d9aa8e314b10e5267cef9e4ddb46bd4ac438 Mon Sep 17 00:00:00 2001 From: William Wong Date: Mon, 5 Jul 2021 22:52:16 -0700 Subject: [PATCH 07/19] Support Direct Line Speech --- ...veServicesSpeechServicesPonyfillFactory.ts | 59 +++--------- .../src/createDirectLineSpeechAdapters.ts | 76 ++++++++++++++- .../src/speech/CustomAudioInputStream.ts | 96 +++++++++++-------- .../src/speech/MicrophoneAudioInputStream.ts | 50 +++++----- .../src/speech/createMicrophoneAudioConfig.ts | 30 ++++++ .../CognitiveServicesAudioOutputFormat.ts | 28 ++++++ .../CognitiveServicesTextNormalization.ts | 3 + 7 files changed, 232 insertions(+), 110 deletions(-) create mode 100644 packages/bundle/src/speech/createMicrophoneAudioConfig.ts create mode 100644 packages/bundle/src/types/CognitiveServicesAudioOutputFormat.ts create mode 100644 packages/bundle/src/types/CognitiveServicesTextNormalization.ts diff --git a/packages/bundle/src/createCognitiveServicesSpeechServicesPonyfillFactory.ts b/packages/bundle/src/createCognitiveServicesSpeechServicesPonyfillFactory.ts index da2b3578de..b170c4f284 100644 --- a/packages/bundle/src/createCognitiveServicesSpeechServicesPonyfillFactory.ts +++ b/packages/bundle/src/createCognitiveServicesSpeechServicesPonyfillFactory.ts @@ -2,36 +2,10 @@ import { AudioConfig } from 'microsoft-cognitiveservices-speech-sdk'; import { WebSpeechPonyfillFactory } from 'botframework-webchat-api'; import createPonyfill from 'web-speech-cognitive-services/lib/SpeechServices'; +import CognitiveServicesAudioOutputFormat from './types/CognitiveServicesAudioOutputFormat'; import CognitiveServicesCredentials from './types/CognitiveServicesCredentials'; -import createAudioContext from './speech/createAudioContext'; -import MicrophoneAudioInputStream from './speech/MicrophoneAudioInputStream'; - -type CognitiveServicesAudioOutputFormat = - | 'audio-16khz-128kbitrate-mono-mp3' - | 'audio-16khz-32kbitrate-mono-mp3' - | 'audio-16khz-64kbitrate-mono-mp3' - | 'audio-24khz-160kbitrate-mono-mp3' - | 'audio-24khz-48kbitrate-mono-mp3' - | 'audio-24khz-96kbitrate-mono-mp3' - | 'audio-48khz-192kbitrate-mono-mp3' - | 'audio-48khz-96kbitrate-mono-mp3' - | 'ogg-16khz-16bit-mono-opus' - | 'ogg-24khz-16bit-mono-opus' - | 'ogg-48khz-16bit-mono-opus' - | 'raw-16khz-16bit-mono-pcm' - | 'raw-16khz-16bit-mono-truesilk' - | 'raw-24khz-16bit-mono-pcm' - | 'raw-24khz-16bit-mono-truesilk' - | 'raw-48khz-16bit-mono-pcm' - | 'raw-8khz-8bit-mono-alaw' - | 'raw-8khz-8bit-mono-mulaw' - | 'riff-16khz-16bit-mono-pcm' - | 'riff-24khz-16bit-mono-pcm' - | 'riff-48khz-16bit-mono-pcm' - | 'riff-8khz-8bit-mono-alaw' - | 'riff-8khz-8bit-mono-mulaw' - | 'webm-16khz-16bit-mono-opus' - | 'webm-24khz-16bit-mono-opus'; +import CognitiveServicesTextNormalization from './types/CognitiveServicesTextNormalization'; +import createMicrophoneAudioConfig from './speech/createMicrophoneAudioConfig'; export default function createCognitiveServicesSpeechServicesPonyfillFactory({ audioConfig, @@ -48,11 +22,11 @@ export default function createCognitiveServicesSpeechServicesPonyfillFactory({ audioContext?: AudioContext; audioInputDeviceId?: string; credentials: CognitiveServicesCredentials; - enableTelemetry?: boolean; + enableTelemetry?: true; speechRecognitionEndpointId?: string; speechSynthesisDeploymentId?: string; speechSynthesisOutputFormat?: CognitiveServicesAudioOutputFormat; - textNormalization?: 'display' | 'itn' | 'lexical' | 'maskeditn'; + textNormalization?: CognitiveServicesTextNormalization; }): WebSpeechPonyfillFactory { if (!window.navigator.mediaDevices && !audioConfig) { console.warn( @@ -63,30 +37,23 @@ export default function createCognitiveServicesSpeechServicesPonyfillFactory({ } if (audioConfig) { - if (audioInputDeviceId) { + audioInputDeviceId && console.warn( 'botframework-webchat: "audioConfig" and "audioInputDeviceId" cannot be set at the same time; ignoring "audioInputDeviceId".' ); - } - if (audioContext) { + audioContext && console.warn( 'botframework-webchat: "audioConfig" and "audioContext" cannot be set at the same time; ignoring "audioContext" for speech recognition.' ); - } } else { - // WORKAROUND: We should prevent AudioContext object from being recreated because they may be blessed and UX-wise expensive to recreate. - // In Cognitive Services SDK, if they detect the "end" function is falsy, they will not call "end" but "suspend" instead. - // And on next recognition, they will re-use the AudioContext object. + const result = createMicrophoneAudioConfig({ + audioContext, + audioInputDeviceId, + enableTelemetry + }); - audioContext || (audioContext = createAudioContext()); - audioConfig = AudioConfig.fromStreamInput( - new MicrophoneAudioInputStream({ - audioConstraints: audioInputDeviceId ? { deviceId: audioInputDeviceId } : true, - audioContext, - telemetry: enableTelemetry ? true : undefined - }) - ); + ({ audioConfig, audioContext } = result); } return ({ referenceGrammarID } = {}) => { diff --git a/packages/bundle/src/createDirectLineSpeechAdapters.ts b/packages/bundle/src/createDirectLineSpeechAdapters.ts index a884cef428..7a75d274de 100644 --- a/packages/bundle/src/createDirectLineSpeechAdapters.ts +++ b/packages/bundle/src/createDirectLineSpeechAdapters.ts @@ -1,12 +1,80 @@ +import { AudioConfig } from 'microsoft-cognitiveservices-speech-sdk'; import { createAdapters } from 'botframework-directlinespeech-sdk'; import { DirectLineJSBotConnection } from 'botframework-webchat-core'; import { WebSpeechPonyfill } from 'botframework-webchat-api'; -export default function createDirectLineSpeechAdapters( - ...args -): { +import CognitiveServicesAudioOutputFormat from './types/CognitiveServicesAudioOutputFormat'; +import CognitiveServicesCredentials from './types/CognitiveServicesCredentials'; +import CognitiveServicesTextNormalization from './types/CognitiveServicesTextNormalization'; +import createMicrophoneAudioConfig from './speech/createMicrophoneAudioConfig'; + +const DEFAULT_LANGUAGE = 'en-US'; + +// TODO: When using DLSpeech via bundle, we will add our own MicrophoneAudioConfig. +export default function createDirectLineSpeechAdapters({ + audioConfig, + audioContext, + audioInputDeviceId, + enableInternalHTTPSupport, + enableTelemetry, + fetchCredentials, + speechRecognitionEndpointId, + speechRecognitionLanguage = window?.navigator?.language || DEFAULT_LANGUAGE, + speechSynthesisDeploymentId, + speechSynthesisOutputFormat, + textNormalization, + userID, + username +}: { + audioConfig?: AudioConfig; + audioContext?: AudioContext; + audioInputDeviceId?: string; + enableInternalHTTPSupport?: true; + enableTelemetry?: true; + fetchCredentials: CognitiveServicesCredentials; + speechRecognitionEndpointId?: string; + speechRecognitionLanguage?: string; + speechSynthesisDeploymentId?: string; + speechSynthesisOutputFormat?: CognitiveServicesAudioOutputFormat; + textNormalization?: CognitiveServicesTextNormalization; + userID?: string; + username?: string; +}): { directLine: DirectLineJSBotConnection; webSpeechPonyfill: WebSpeechPonyfill; } { - return createAdapters(...args); + if (audioConfig) { + audioInputDeviceId && + console.warn( + 'botframework-webchat: "audioConfig" and "audioInputDeviceId" cannot be set at the same time; ignoring "audioInputDeviceId".' + ); + + audioContext && + console.warn( + 'botframework-webchat: "audioConfig" and "audioContext" cannot be set at the same time; ignoring "audioContext" for speech recognition.' + ); + } else { + const result = createMicrophoneAudioConfig({ + audioContext, + audioInputDeviceId, + enableTelemetry + }); + + ({ audioConfig, audioContext } = result); + } + + return createAdapters({ + audioConfig, + audioContext, + enableInternalHTTPSupport, + enableTelemetry, + fetchCredentials, + speechRecognitionEndpointId, + speechRecognitionLanguage, + speechSynthesisDeploymentId, + speechSynthesisOutputFormat, + textNormalization, + userID, + username + }); } diff --git a/packages/bundle/src/speech/CustomAudioInputStream.ts b/packages/bundle/src/speech/CustomAudioInputStream.ts index f3eb777dc5..d2a8fd5c5a 100644 --- a/packages/bundle/src/speech/CustomAudioInputStream.ts +++ b/packages/bundle/src/speech/CustomAudioInputStream.ts @@ -27,10 +27,11 @@ import { } from 'microsoft-cognitiveservices-speech-sdk/distrib/lib/src/common.speech/Exports'; import { v4 } from 'uuid'; +import createDeferred, { DeferredPromise } from 'p-defer'; -const SYMBOL_DEVICE_INFO = Symbol('deviceInfo'); +const SYMBOL_DEVICE_INFO_DEFERRED = Symbol('deviceInfoDeferred'); const SYMBOL_EVENTS = Symbol('events'); -const SYMBOL_FORMAT = Symbol('format'); +const SYMBOL_FORMAT_DEFERRED = Symbol('formatDeferred'); const SYMBOL_OPTIONS = Symbol('options'); type AudioStreamNode = { @@ -91,16 +92,18 @@ abstract class CustomAudioInputStream extends AudioInputStream { id: options.id || v4().replace(/-/gu, '') }; + this[SYMBOL_DEVICE_INFO_DEFERRED] = createDeferred(); this[SYMBOL_EVENTS] = new EventSource(); this[SYMBOL_OPTIONS] = normalizedOptions; + this[SYMBOL_FORMAT_DEFERRED] = createDeferred(); } - [SYMBOL_DEVICE_INFO]: DeviceInfo; + [SYMBOL_DEVICE_INFO_DEFERRED]: DeferredPromise; [SYMBOL_EVENTS]: EventSource; - [SYMBOL_FORMAT]: Format; + [SYMBOL_FORMAT_DEFERRED]: DeferredPromise; [SYMBOL_OPTIONS]: NormalizedOptions; - // This code will only works in browsers other than IE11. Only works in ES5 is okay. + // ESLint: This code will only works in browsers other than IE11. Only works in ES5 is okay. // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore Accessors are only available when targeting ECMAScript 5 and higher.ts(1056) get events(): EventSource { @@ -111,17 +114,15 @@ abstract class CustomAudioInputStream extends AudioInputStream { // It is weird to expose AudioStreamFormatImpl instead of AudioStreamFormat. // Speech SDK quirks: It is weird to return a Promise in a property. // Especially this is audio format. Setup options should be initialized synchronously. - // This code will only works in browsers other than IE11. Only works in ES5 is okay. + // Speech SDK quirks: In normal speech recognition, getter of "format" is called only after "attach". + // But in Direct Line Speech, it is called before "attach". + // ESLint: This code will only works in browsers other than IE11. Only works in ES5 is okay. // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore Accessors are only available when targeting ECMAScript 5 and higher.ts(1056) get format(): Promise { - const format = this[SYMBOL_FORMAT]; + this.debug('Getting "format".'); - if (!format) { - throw new Error('"format" is not available until attach() is called.'); - } - - return Promise.resolve(new AudioStreamFormatImpl(format.samplesPerSec, format.bitsPerSample, format.channels)); + return this[SYMBOL_FORMAT_DEFERRED].promise; } id(): string { @@ -178,13 +179,30 @@ abstract class CustomAudioInputStream extends AudioInputStream { } // Speech SDK quirks: It seems close() is never called, despite, it is marked as abstract. - // Speech SDK requires this function. + // ESLint: Speech SDK requires this function. // eslint-disable-next-line class-methods-use-this close(): void { + this.debug('Callback for "close".'); + + throw new Error('Not implemented'); + } + + // Speech SDK quirks: Although "turnOn" is implemented in XxxAudioInputStream, they are never called. + turnOn(): void { + this.debug('Callback for "turnOn".'); + + throw new Error('Not implemented'); + } + + // Speech SDK quirks: Although "detach" is implemented in XxxAudioInputStream, they are never called. + detach(): void { + this.debug('Callback for "detach".'); + throw new Error('Not implemented'); } private debug(message, ...args) { + // ESLint: For debugging, will only log when "debug" is set to "true". // eslint-disable-next-line no-console this[SYMBOL_OPTIONS].debug && console.info(`CustomAudioInputStream: ${message}`, ...args); } @@ -209,8 +227,12 @@ abstract class CustomAudioInputStream extends AudioInputStream { try { const { audioStreamNode, deviceInfo, format } = await this.performAttach(audioNodeId); - this[SYMBOL_DEVICE_INFO] = deviceInfo; - this[SYMBOL_FORMAT] = format; + // Although only getter of "format" is called before "attach" (in Direct Line Speech), + // we are handling both "deviceInfo" and "format" in similar way for uniformity. + this[SYMBOL_DEVICE_INFO_DEFERRED].resolve(deviceInfo); + this[SYMBOL_FORMAT_DEFERRED].resolve( + new AudioStreamFormatImpl(format.samplesPerSec, format.bitsPerSample, format.channels) + ); this.emitReady(); this.emitNodeAttached(audioNodeId); @@ -221,6 +243,8 @@ abstract class CustomAudioInputStream extends AudioInputStream { await audioStreamNode.detach(); + // Speech SDK quirks: Since "turnOff" is never called, we will emit event "source off" here instead. + this.emitOff(); this.emitNodeDetached(audioNodeId); }, id: () => audioStreamNode.id(), @@ -238,42 +262,38 @@ abstract class CustomAudioInputStream extends AudioInputStream { }); } - /** Implements this function. When called, it should stop recording. This is called before the `IAudioStreamNode.detach` function. */ + /** + * Implements this function. When called, it should stop recording. This is called before the `IAudioStreamNode.detach` function. + * + * Note: when using with Direct Line Speech, this function is never called. + */ protected abstract performTurnOff(): Promise; + // Speech SDK quirks: When using with Direct Line Speech, "turnOff" is never called. async turnOff(): Promise { this.debug(`Callback for "turnOff".`); await this.performTurnOff(); - - this.emitOff(); } - // This code will only works in browsers other than IE11. Only works in ES5 is okay. + // ESLint: This code will only works in browsers other than IE11. Only works in ES5 is okay. // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore Accessors are only available when targeting ECMAScript 5 and higher.ts(1056) get deviceInfo(): Promise { this.debug(`Getting "deviceInfo".`); - const deviceInfo = this[SYMBOL_DEVICE_INFO]; - - if (!deviceInfo) { - throw new Error('"deviceInfo" is not available until attach() is called.'); - } - - const { connectivity, manufacturer, model, type } = deviceInfo; - const { bitsPerSample, channels, samplesPerSec } = this[SYMBOL_FORMAT]; - - return Promise.resolve({ - bitspersample: bitsPerSample, - channelcount: channels, - connectivity: - typeof connectivity === 'string' ? Connectivity[connectivity] : connectivity || Connectivity.Unknown, - manufacturer: manufacturer || '', - model: model || '', - samplerate: samplesPerSec, - type: typeof type === 'string' ? Type[type] : type || Type.Unknown - }); + return Promise.all([this[SYMBOL_DEVICE_INFO_DEFERRED].promise, this[SYMBOL_FORMAT_DEFERRED].promise]).then( + ([{ connectivity, manufacturer, model, type }, { bitsPerSample, channels, samplesPerSec }]) => ({ + bitspersample: bitsPerSample, + channelcount: channels, + connectivity: + typeof connectivity === 'string' ? Connectivity[connectivity] : connectivity || Connectivity.Unknown, + manufacturer: manufacturer || '', + model: model || '', + samplerate: samplesPerSec, + type: typeof type === 'string' ? Type[type] : type || Type.Unknown + }) + ); } } diff --git a/packages/bundle/src/speech/MicrophoneAudioInputStream.ts b/packages/bundle/src/speech/MicrophoneAudioInputStream.ts index e7c7b4cc0e..8f9a845aa2 100644 --- a/packages/bundle/src/speech/MicrophoneAudioInputStream.ts +++ b/packages/bundle/src/speech/MicrophoneAudioInputStream.ts @@ -31,19 +31,19 @@ type MicrophoneAudioInputStreamOptions = { /** Specifies the buffering delay on how often to flush audio data to network. Increasing the value will increase audio latency. Default is 100 ms. */ bufferDurationInMS?: number; + /** Specifies if telemetry data should be sent. If not specified, telemetry data will NOT be sent. */ + enableTelemetry?: true; + /** Specifies the `AudioWorklet` URL for `PcmRecorder`. If not specified, will use script processor on UI thread instead. */ pcmRecorderWorkletUrl?: string; - - /** Specifies if telemetry data should be sent. If not specified, telemetry data will NOT be sent. */ - telemetry?: true; }; const SYMBOL_AUDIO_CONSTRAINTS = Symbol('audioConstraints'); const SYMBOL_AUDIO_CONTEXT = Symbol('audioContext'); const SYMBOL_BUFFER_DURATION_IN_MS = Symbol('bufferDurationInMS'); +const SYMBOL_ENABLE_TELEMETRY = Symbol('enableTelemetry'); const SYMBOL_OUTPUT_STREAM = Symbol('outputStream'); const SYMBOL_PCM_RECORDER = Symbol('pcmRecorder'); -const SYMBOL_TELEMETRY = Symbol('telemetry'); export default class MicrophoneAudioInputStream extends CustomAudioInputStream { constructor(options: MicrophoneAudioInputStreamOptions) { @@ -65,23 +65,23 @@ export default class MicrophoneAudioInputStream extends CustomAudioInputStream { [SYMBOL_BUFFER_DURATION_IN_MS]: number; [SYMBOL_OUTPUT_STREAM]?: ChunkedArrayBufferStream; [SYMBOL_PCM_RECORDER]?: PcmRecorder; - [SYMBOL_TELEMETRY]?: true; + [SYMBOL_ENABLE_TELEMETRY]?: true; - // This code will only works in browsers other than IE11. Only works in ES5 is okay. + // ESLint: This code will only works in browsers other than IE11. Only works in ES5 is okay. // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore Accessors are only available when targeting ECMAScript 5 and higher.ts(1056) get audioConstraints(): true | MediaTrackConstraints { return this[SYMBOL_AUDIO_CONSTRAINTS]; } - // This code will only works in browsers other than IE11. Only works in ES5 is okay. + // ESLint: This code will only works in browsers other than IE11. Only works in ES5 is okay. // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore Accessors are only available when targeting ECMAScript 5 and higher.ts(1056) get audioContext(): AudioContext { return this[SYMBOL_AUDIO_CONTEXT]; } - // This code will only works in browsers other than IE11. Only works in ES5 is okay. + // ESLint: This code will only works in browsers other than IE11. Only works in ES5 is okay. // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore Accessors are only available when targeting ECMAScript 5 and higher.ts(1056) get bufferDurationInMS(): number { @@ -93,17 +93,13 @@ export default class MicrophoneAudioInputStream extends CustomAudioInputStream { // Speech SDK quirks: event "source off" is sent before event "node detached". // Shouldn't source "bigger" (in terms of responsibilities) and include nodes? // Why we "close the source" before "close the node"? - performTurnOff(): Promise { - // Speech SDK quirks: in SDK, it call outputStream.close() in turnOff() before outputStream.readEnded() in detach(). - // I think it make sense to call readEnded() before close(). - this[SYMBOL_OUTPUT_STREAM].readEnded(); - this[SYMBOL_OUTPUT_STREAM].close(); + // Speech SDK quirks: Direct Line Speech never call "turnOff". Event "source off" need to be emitted during "detach". + // Also for ending and closing output streams. - // PcmRecorder.releaseMediaResources() will disconnect/stop the MediaStream. - // We cannot use MediaStream again after turned off. - this[SYMBOL_PCM_RECORDER].releaseMediaResources(this.audioContext); - - // Required by TypeScript + // ESLint: We are not implementing this function because it is not called by Direct Line Speech. + // eslint-disable-next-line class-methods-use-this + performTurnOff(): Promise { + // ESLint: "return" is required by TypeScript // eslint-disable-next-line no-useless-return return; } @@ -142,13 +138,23 @@ export default class MicrophoneAudioInputStream extends CustomAudioInputStream { return { audioStreamNode: { - // Speech SDK quirks: in SDK's MicAudioSource, it call turnOff() during detach(). + // Speech SDK quirks: In SDK's original MicAudioSource implementation, it call turnOff() during detach(). // That means, it call turnOff(), then detach(), then turnOff() again. Seems redundant. + // When using with Direct Line Speech, turnOff() is never called. detach: (): Promise => { + // Speech SDK quirks: In SDK, it call outputStream.close() in turnOff() before outputStream.readEnded() in detach(). + // I think it make sense to call readEnded() before close(). + this[SYMBOL_OUTPUT_STREAM].readEnded(); + this[SYMBOL_OUTPUT_STREAM].close(); + + // PcmRecorder.releaseMediaResources() will disconnect/stop the MediaStream. + // We cannot use MediaStream again after turned off. + this[SYMBOL_PCM_RECORDER].releaseMediaResources(this.audioContext); + // MediaStream will become inactive after all tracks are removed. mediaStream.getTracks().forEach(track => mediaStream.removeTrack(track)); - // Required by TypeScript + // ESLint: "return" is required by TypeScript // eslint-disable-next-line no-useless-return return; }, @@ -157,8 +163,8 @@ export default class MicrophoneAudioInputStream extends CustomAudioInputStream { }, deviceInfo: { manufacturer: 'Bot Framework Web Chat', - model: this[SYMBOL_TELEMETRY] ? firstAudioTrack.label : '', - type: this[SYMBOL_TELEMETRY] ? 'Microphones' : 'Unknown' + model: this[SYMBOL_ENABLE_TELEMETRY] ? firstAudioTrack.label : '', + type: this[SYMBOL_ENABLE_TELEMETRY] ? 'Microphones' : 'Unknown' }, // Speech SDK quirks: PcmRecorder hardcoded sample rate of 16000 Hz. // We cannot obtain this number other than looking at their source code. diff --git a/packages/bundle/src/speech/createMicrophoneAudioConfig.ts b/packages/bundle/src/speech/createMicrophoneAudioConfig.ts new file mode 100644 index 0000000000..62e2176518 --- /dev/null +++ b/packages/bundle/src/speech/createMicrophoneAudioConfig.ts @@ -0,0 +1,30 @@ +import { AudioConfig } from 'microsoft-cognitiveservices-speech-sdk'; + +import createAudioContext from './createAudioContext'; +import MicrophoneAudioInputStream from './MicrophoneAudioInputStream'; + +export default function createMicrophoneAudioConfig({ + audioContext, + audioInputDeviceId, + enableTelemetry +}: { + audioContext?: AudioContext; + audioInputDeviceId?: string; + enableTelemetry?: true; +}) { + // Web Chat has an implementation of AudioConfig for microphone that would enable better support on Safari: + // - Maintain same instance of `AudioContext` across recognitions; + // - Resume suspended `AudioContext` on user gestures. + audioContext || (audioContext = createAudioContext()); + + return { + audioConfig: AudioConfig.fromStreamInput( + new MicrophoneAudioInputStream({ + audioConstraints: audioInputDeviceId ? { deviceId: audioInputDeviceId } : true, + audioContext, + enableTelemetry: enableTelemetry ? true : undefined + }) + ), + audioContext + }; +} diff --git a/packages/bundle/src/types/CognitiveServicesAudioOutputFormat.ts b/packages/bundle/src/types/CognitiveServicesAudioOutputFormat.ts new file mode 100644 index 0000000000..622a2fe0f8 --- /dev/null +++ b/packages/bundle/src/types/CognitiveServicesAudioOutputFormat.ts @@ -0,0 +1,28 @@ +type CognitiveServicesAudioOutputFormat = + | 'audio-16khz-128kbitrate-mono-mp3' + | 'audio-16khz-32kbitrate-mono-mp3' + | 'audio-16khz-64kbitrate-mono-mp3' + | 'audio-24khz-160kbitrate-mono-mp3' + | 'audio-24khz-48kbitrate-mono-mp3' + | 'audio-24khz-96kbitrate-mono-mp3' + | 'audio-48khz-192kbitrate-mono-mp3' + | 'audio-48khz-96kbitrate-mono-mp3' + | 'ogg-16khz-16bit-mono-opus' + | 'ogg-24khz-16bit-mono-opus' + | 'ogg-48khz-16bit-mono-opus' + | 'raw-16khz-16bit-mono-pcm' + | 'raw-16khz-16bit-mono-truesilk' + | 'raw-24khz-16bit-mono-pcm' + | 'raw-24khz-16bit-mono-truesilk' + | 'raw-48khz-16bit-mono-pcm' + | 'raw-8khz-8bit-mono-alaw' + | 'raw-8khz-8bit-mono-mulaw' + | 'riff-16khz-16bit-mono-pcm' + | 'riff-24khz-16bit-mono-pcm' + | 'riff-48khz-16bit-mono-pcm' + | 'riff-8khz-8bit-mono-alaw' + | 'riff-8khz-8bit-mono-mulaw' + | 'webm-16khz-16bit-mono-opus' + | 'webm-24khz-16bit-mono-opus'; + +export default CognitiveServicesAudioOutputFormat; diff --git a/packages/bundle/src/types/CognitiveServicesTextNormalization.ts b/packages/bundle/src/types/CognitiveServicesTextNormalization.ts new file mode 100644 index 0000000000..84f8c9274f --- /dev/null +++ b/packages/bundle/src/types/CognitiveServicesTextNormalization.ts @@ -0,0 +1,3 @@ +type CognitiveServicesTextNormalization = 'display' | 'itn' | 'lexical' | 'maskeditn'; + +export default CognitiveServicesTextNormalization; From 90848be2b9d7a1e19f873ba6bc36364ccfa989c1 Mon Sep 17 00:00:00 2001 From: William Wong Date: Mon, 5 Jul 2021 23:03:57 -0700 Subject: [PATCH 08/19] Clean up --- .../src/speech/CustomAudioInputStream.ts | 59 +++++++++++-------- .../src/speech/MicrophoneAudioInputStream.ts | 43 +------------- 2 files changed, 39 insertions(+), 63 deletions(-) diff --git a/packages/bundle/src/speech/CustomAudioInputStream.ts b/packages/bundle/src/speech/CustomAudioInputStream.ts index d2a8fd5c5a..ee7caa2b91 100644 --- a/packages/bundle/src/speech/CustomAudioInputStream.ts +++ b/packages/bundle/src/speech/CustomAudioInputStream.ts @@ -1,9 +1,7 @@ -// TODO: We should export this type of AudioInputStream to allow web developers to bring in their own microphone. +// TODO: [P2] #XXX We should export this type of AudioInputStream to allow web developers to bring in their own microphone. // For example, it should enable React Native devs to bring in their microphone implementation and use Cognitive Services Speech Services. import { AudioInputStream } from 'microsoft-cognitiveservices-speech-sdk'; -// TODO: Revisit all imports from internals of Speech SDK. -// It should works with React TypeScript projects without modifying its internal Webpack configuration. import { AudioSourceErrorEvent, AudioSourceEvent, @@ -29,11 +27,6 @@ import { import { v4 } from 'uuid'; import createDeferred, { DeferredPromise } from 'p-defer'; -const SYMBOL_DEVICE_INFO_DEFERRED = Symbol('deviceInfoDeferred'); -const SYMBOL_EVENTS = Symbol('events'); -const SYMBOL_FORMAT_DEFERRED = Symbol('formatDeferred'); -const SYMBOL_OPTIONS = Symbol('options'); - type AudioStreamNode = { detach: () => Promise; id: () => string; @@ -80,6 +73,11 @@ type StreamChunk = { timeReceived: number; }; +const SYMBOL_DEVICE_INFO_DEFERRED = Symbol('deviceInfoDeferred'); +const SYMBOL_EVENTS = Symbol('events'); +const SYMBOL_FORMAT_DEFERRED = Symbol('formatDeferred'); +const SYMBOL_OPTIONS = Symbol('options'); + // Speech SDK quirks: Only 2 lifecycle functions are actually used. // They are: attach() and turnOff(). // Others are not used, including: blob(), close(), detach(), turnOn(). @@ -129,8 +127,8 @@ abstract class CustomAudioInputStream extends AudioInputStream { return this[SYMBOL_OPTIONS].id; } - // Speech SDK quirks: in JavaScript, onXxx means "listen to event XXX". - // instead, in Speech SDK, it means "emit event XXX". + // Speech SDK quirks: In JavaScript, onXxx means "listen to event XXX". + // Instead, in Speech SDK, it means "emit event XXX". protected onEvent(event: AudioSourceEvent): void { this[SYMBOL_EVENTS].onEvent(event); Events.instance.onEvent(event); @@ -146,10 +144,13 @@ abstract class CustomAudioInputStream extends AudioInputStream { this.onEvent(new AudioSourceReadyEvent(this.id())); } - // Speech SDK quirks: "error" is a string, instead of an Error object. - protected emitError(error: string): void { + // Speech SDK quirks: Since "turnOn" is never called and "turnOff" does not work in Direct Line Speech, the "source error" event is not emitted at all. + // Instead, we only emit "node error" event. + protected emitError(error: Error): void { this.debug('Emitting "AudioSourceErrorEvent".', { error }); - this.onEvent(new AudioSourceErrorEvent(this.id(), error)); + + // Speech SDK quirks: "error" is a string, instead of object of type "Error". + this.onEvent(new AudioSourceErrorEvent(this.id(), error.message)); } protected emitNodeAttaching(audioNodeId: string): void { @@ -162,10 +163,11 @@ abstract class CustomAudioInputStream extends AudioInputStream { this.onEvent(new AudioStreamNodeAttachedEvent(this.id(), audioNodeId)); } - // Speech SDK quirks: "error" is a string, instead of an Error object. - protected emitNodeError(audioNodeId: string, error: string): void { + protected emitNodeError(audioNodeId: string, error: Error): void { this.debug(`Emitting "AudioStreamNodeErrorEvent" for node "${audioNodeId}".`, { error }); - this.onEvent(new AudioStreamNodeErrorEvent(this.id(), audioNodeId, error)); + + // Speech SDK quirks: "error" is a string, instead of object of type "Error". + this.onEvent(new AudioStreamNodeErrorEvent(this.id(), audioNodeId, error.message)); } protected emitNodeDetached(audioNodeId: string): void { @@ -178,8 +180,9 @@ abstract class CustomAudioInputStream extends AudioInputStream { this.onEvent(new AudioSourceOffEvent(this.id())); } - // Speech SDK quirks: It seems close() is never called, despite, it is marked as abstract. - // ESLint: Speech SDK requires this function. + // Speech SDK quirks: Although "close" is marked as abstract, it is never called in our observations. + + // ESLint: Speech SDK requires this function, but we are not implementing it. // eslint-disable-next-line class-methods-use-this close(): void { this.debug('Callback for "close".'); @@ -187,14 +190,14 @@ abstract class CustomAudioInputStream extends AudioInputStream { throw new Error('Not implemented'); } - // Speech SDK quirks: Although "turnOn" is implemented in XxxAudioInputStream, they are never called. + // Speech SDK quirks: Although "turnOn" is implemented in XxxAudioInputStream, it is never called in our observations. turnOn(): void { this.debug('Callback for "turnOn".'); throw new Error('Not implemented'); } - // Speech SDK quirks: Although "detach" is implemented in XxxAudioInputStream, they are never called. + // Speech SDK quirks: Although "detach" is implemented in XxxAudioInputStream, it is never called in our observations. detach(): void { this.debug('Callback for "detach".'); @@ -243,7 +246,7 @@ abstract class CustomAudioInputStream extends AudioInputStream { await audioStreamNode.detach(); - // Speech SDK quirks: Since "turnOff" is never called, we will emit event "source off" here instead. + // Speech SDK quirks: Since "turnOff" is not called in Direct Line Speech, we will emit event "source off" here instead. this.emitOff(); this.emitNodeDetached(audioNodeId); }, @@ -267,9 +270,19 @@ abstract class CustomAudioInputStream extends AudioInputStream { * * Note: when using with Direct Line Speech, this function is never called. */ - protected abstract performTurnOff(): Promise; - // Speech SDK quirks: When using with Direct Line Speech, "turnOff" is never called. + // ESLint: We are not implementing this function because it is not called by Direct Line Speech. + // eslint-disable-next-line class-methods-use-this + protected performTurnOff(): Promise { + // ESLint: "return" is required by TypeScript + // eslint-disable-next-line no-useless-return + return; + } + + // Speech SDK quirks: It is confused to have both "turnOff" and "detach". "turnOff" is called before "detach". + // Why don't we put all logics at "detach"? + // Speech SDK quirks: Direct Line Speech never call "turnOff". "Source off" event need to be emitted during "detach" instead. + // Also, custom implementation should be done at "detach" instead, such as ending and closing output streams. async turnOff(): Promise { this.debug(`Callback for "turnOff".`); diff --git a/packages/bundle/src/speech/MicrophoneAudioInputStream.ts b/packages/bundle/src/speech/MicrophoneAudioInputStream.ts index 8f9a845aa2..5263ac82c5 100644 --- a/packages/bundle/src/speech/MicrophoneAudioInputStream.ts +++ b/packages/bundle/src/speech/MicrophoneAudioInputStream.ts @@ -63,46 +63,9 @@ export default class MicrophoneAudioInputStream extends CustomAudioInputStream { [SYMBOL_AUDIO_CONSTRAINTS]: true | MediaTrackConstraints; [SYMBOL_AUDIO_CONTEXT]: AudioContext; [SYMBOL_BUFFER_DURATION_IN_MS]: number; + [SYMBOL_ENABLE_TELEMETRY]?: true; [SYMBOL_OUTPUT_STREAM]?: ChunkedArrayBufferStream; [SYMBOL_PCM_RECORDER]?: PcmRecorder; - [SYMBOL_ENABLE_TELEMETRY]?: true; - - // ESLint: This code will only works in browsers other than IE11. Only works in ES5 is okay. - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore Accessors are only available when targeting ECMAScript 5 and higher.ts(1056) - get audioConstraints(): true | MediaTrackConstraints { - return this[SYMBOL_AUDIO_CONSTRAINTS]; - } - - // ESLint: This code will only works in browsers other than IE11. Only works in ES5 is okay. - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore Accessors are only available when targeting ECMAScript 5 and higher.ts(1056) - get audioContext(): AudioContext { - return this[SYMBOL_AUDIO_CONTEXT]; - } - - // ESLint: This code will only works in browsers other than IE11. Only works in ES5 is okay. - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore Accessors are only available when targeting ECMAScript 5 and higher.ts(1056) - get bufferDurationInMS(): number { - return this[SYMBOL_BUFFER_DURATION_IN_MS]; - } - - // Speech SDK quirks: It is confused to have both "turnOff" and "detach". "turnOff" is called before "detach". - // Why don't we put all logics at "detach"? - // Speech SDK quirks: event "source off" is sent before event "node detached". - // Shouldn't source "bigger" (in terms of responsibilities) and include nodes? - // Why we "close the source" before "close the node"? - // Speech SDK quirks: Direct Line Speech never call "turnOff". Event "source off" need to be emitted during "detach". - // Also for ending and closing output streams. - - // ESLint: We are not implementing this function because it is not called by Direct Line Speech. - // eslint-disable-next-line class-methods-use-this - performTurnOff(): Promise { - // ESLint: "return" is required by TypeScript - // eslint-disable-next-line no-useless-return - return; - } async performAttach( audioNodeId: string @@ -119,7 +82,7 @@ export default class MicrophoneAudioInputStream extends CustomAudioInputStream { // We need to get new MediaStream on every attach(). // This is because PcmRecorder.releaseMediaResources() disconnected/stopped them. - const mediaStream = await getUserMedia({ audio: this.audioConstraints, video: false }); + const mediaStream = await getUserMedia({ audio: this[SYMBOL_AUDIO_CONSTRAINTS], video: false }); const [firstAudioTrack] = mediaStream.getAudioTracks(); @@ -149,7 +112,7 @@ export default class MicrophoneAudioInputStream extends CustomAudioInputStream { // PcmRecorder.releaseMediaResources() will disconnect/stop the MediaStream. // We cannot use MediaStream again after turned off. - this[SYMBOL_PCM_RECORDER].releaseMediaResources(this.audioContext); + this[SYMBOL_PCM_RECORDER].releaseMediaResources(this[SYMBOL_AUDIO_CONTEXT]); // MediaStream will become inactive after all tracks are removed. mediaStream.getTracks().forEach(track => mediaStream.removeTrack(track)); From 8cfa4752cc90923e01767ea5e347dd277c5716fd Mon Sep 17 00:00:00 2001 From: William Wong Date: Tue, 6 Jul 2021 09:45:02 -0700 Subject: [PATCH 09/19] Add tests --- __tests__/html/assets/hello-world.wav | Bin 0 -> 103758 bytes __tests__/html/speech.customAudioConfig.html | 74 +++++++++ ...veServicesSpeechServicesPonyfillFactory.ts | 10 +- .../src/createDirectLineSpeechAdapters.ts | 8 +- .../src/speech/CustomAudioInputStream.ts | 4 +- .../src/speech/MicrophoneAudioInputStream.ts | 139 ----------------- .../bundle/src/speech/createAudioConfig.ts | 57 +++++++ .../src/speech/createMicrophoneAudioConfig.ts | 30 ---- ...ateMicrophoneAudioConfigAndAudioContext.ts | 145 ++++++++++++++++++ packages/bundle/src/speech/getUserMedia.ts | 1 - packages/test/page-object/package-lock.json | 30 ++-- packages/test/page-object/package.json | 2 +- .../src/globals/testHelpers/index.js | 2 + ...eAudioInputStreamFromRiffWavArrayBuffer.js | 71 +++++++++ 14 files changed, 373 insertions(+), 200 deletions(-) create mode 100644 __tests__/html/assets/hello-world.wav create mode 100644 __tests__/html/speech.customAudioConfig.html delete mode 100644 packages/bundle/src/speech/MicrophoneAudioInputStream.ts create mode 100644 packages/bundle/src/speech/createAudioConfig.ts delete mode 100644 packages/bundle/src/speech/createMicrophoneAudioConfig.ts create mode 100644 packages/bundle/src/speech/createMicrophoneAudioConfigAndAudioContext.ts create mode 100644 packages/test/page-object/src/globals/testHelpers/speech/audioConfig/createAudioInputStreamFromRiffWavArrayBuffer.js diff --git a/__tests__/html/assets/hello-world.wav b/__tests__/html/assets/hello-world.wav new file mode 100644 index 0000000000000000000000000000000000000000..e5f209d2e016e7c6aee4dc06de46aa9bd4ed46fd GIT binary patch literal 103758 zcmeFZg_j#g6FAtzJc1>!nPO&U$IR?FhBd^@j4{N_kl3+fW@cuJnIUFaly=1{?TTS$ zuE+Z6obT@4KX5Ojw^C0-QCD|YS66TMcCA~t-iUyooAqineB9LHS^xk7Kezh>F#j3= zDA0nA?RxYq2=m)@Zr!C_kJdd!nC|$?^x?JZRjEcjb^qN?^+QFW#M(xGV;-mfg{zh+@R8vsia{+`|5O@d z5Tb3P-~Y~yQGr05(MSDN6!hDlN`;#Gf47TK9fSY>{TsuE{XeQiVMa6mosL3|(iO{( z{mPJ%7(A8A*qfTK@{YhiRSIey^%n&aUxfWTn=RQK`aL-_-Y5UJROA7JH)QF$^mFSRcgN#W+;4s;$&IF^*K;qB&9CqEMsU z#AKlAMU1Z)))<^9Kk8ddGSS+xdQsd_`lC?)4J%4dv~G;YSj*UNG++Hz^+|oOZ@AQy z`X1GY7|$^%|HcxdQ7!upEh^b)yO?%ug6n=;FznZ4D|M&B~RE3~Y5T)wh z@S-KrQuRCf9?SiA&F`gZ>iZLG@xR#A+-Se3Fu%i#rlK6E>3^3*-__c&zErV9`%q2A z`V#*aWHmop62*i3-@5c2YqVWV$71lJm}59<f#7FN^W;osRDbRup<{ZjX)g*o+jLCBD~>!cnPESFGr-ieFa0zF|?*>ddCrjOKm&`Iale zr1XEsb6bp%GsqBO%7_^9oo_|&)Ww5WKa^7wcD_i{Biszp)$qIjd< zQCd{mqf%09sPuowtI9Si`N%i?vJ!#sDlbuL)ZbW*Zw0VK{Y2Zus8CCyy`$!+DRteA zmZ~(U^}n}>)``&&#Q;DQzgiY;r>@iL450EB&H2XYHyOoXtNBq5ROnGGDmFD0g%guN z%npdw_T1SNyZ4vFGXssyBSc_=gD4kJ^QNE&(RESX-#iS6+kG5A!)fp{XPo+oI zK^20U8`YZFuUZ-{QDLZh6Kx%vHPm*|x@tT1@89EJZL79WA*fH3*XXxe_MLk*PkoC) zQQy@$Il4AQ%Tx%_nLIXIM|0HgC{?jv6_ScG)>~?Om107z0H5fa+Ow*~Awm8DwAIxb zs;#c3qI9T|Q6CcWt&VV&J_Nq0Pm~K4VvLU{4{FOOezk6FY^yM1JjQ5MYpRbb!Dv0T zL>+NzuKKO!qTe(^O{)|{=bZ02W4L3okCv;PsL-Q5tF}@fwU__>sIpWbhtZF!l?1F2 zi4l^3=DR2rkQi&7L()cR_!N>@}{RLa%)L*+5L8$zK3)pS%h zRHeiVp=8{ENy=-D9aw^ex(ZDz(vFS9IozYN*=lDrR*) zP+O?Es)ndDesl&?VZ{18DsdG`G^MswTc~nT`&Xr#gtCbBw<<{-mZ^NGGKyiK0r!nN z3ev1nEh*{>pnj_mqI<+BEh^-wmZ?~yka55O8iG>mMyXR%s`S+-O0~MqsCc6|RoSb( zph`8`XX@&s_Ce(T!|XI;*H8vg{*3!%3pL{P$@wH z3qKsBQl*)MzxcO0Q5scwk&tgyQqg`zVU1`V2J*%OJ$xhKN99gkCDkVg=}>7ps>WP1Yq4L)e)EY6c{JYpcq|2QjF< zuJ){|nGuLdO-B^3VpCk-O2bfEIWVnCFamQ^l`JSHRZ1$S(NV`kJUFl^UzJaa1(s0o zJs$oEKm%#e!*2m3gG!(p{FHE)3;qQ9ddk z6g!l&%2^mn#{p~Dl{849I^so06Ad930R`Zv2q+3lKukW!nJO7K#PJ?Nc%ocXt|_-+ zD@92L#bNz;*iKe>P!7}swLx`I1vCV;;9GInS1NR`qKEJbL5XBTZjIkGAs13r98xa; zvvNZ@tejS!LfE-V5y)W~Pyy1x0VnJ&9Lo1c&KEWtoea0i&cb<=fTy%cByrKc#F$}LFuQRTj}N7)MH zp#kH;A}|rs;f0X5D5sQHP`(JP>xP)$D+`pi$|&WF(giF4OF(ld{}aj|%1u~Hfj!KF zil7hJ3wD4%z%a;vJ;*^FD1*z&d}X9ENclzC1Zk)X%PK=U@4(#dN*ASvvIh2Ve{d9> z2CJbqU{IcAA&;}bN-zNY0C8VaE<+A3!Ov>tG1S}&uxvc!4+mG2CCYD5?vG#(T~Urf zPL?XIl}?b`2EYtl;4NqYp26JNkj{|O2aE(Kz;4*P9bvzbAQ$rQgPJr1(t91u0jm_R z++SWVmr+J21)+Z3h4PyQCV?lgPjOHRq<~dm9B2t)MwF7Ubyd&+&Ja(OZcqa6U<&|7 zDhrj(kh25Ib7g|k6Uxu28~{$x7U_qG$a2_c6_r%Ev@!X42)pa;~xB*^t6 zsQol32)TDE@lZob!G5j;N`noMH?vY0d;q_L$B_2j5cU~4By-9Xr5mJ(MKZuwunU}4 zUddi5POc^6$}Z)h(iKW^KiC4sg5h8g*Z}&%aj;bRNhtyLfSyQiP33(_7`PL~OIZkP>4260)9csl5 zr3P3BMnfo5;iwr427*UGhkOK$!33yVHK8UPg<5u6SqaN`gG-3}8!rdIJ4H~=%LU|? z@;k)<#z8KU5gPd&EQO<}BUlJFf=!B6c_UAj2g=LjZg6}q1OenXvmzYe4`@ORSH6N%hyzW){zj{y0y1COA!SOU zavW+?bz~PfreuOD$P%DePD%x&za#DCTFMJ03mk`7TY^oXB@z#%UJUe6KFeuxMc@Os z(L-oD=&n$5Z|QMFBll7!gWr)Z$Q;mB!JrOI21k&1%z>6dMI=+XDZh}%D3sDwDGg-M z4#5$>(pLT`4OXs#`N%%x6nF(Zkd~`ZyUxHF$gTVU`)iVPGtyKp3kD+*5RZ(9kRQuH z`3yBGtbBm-U5PA4S|hXJSn76u8)+%+t-$+C33}!|y$h=%fS&qCwUqH%+ z!7(3J?!h^$AaVdLgWW_1fa&ssuq#w39E{9VJ|I`I3wRr195xtfq3oA}VSV_1&RBA2VA|LLcKjF@0S`$h%y{#k>+sRO$A|a1J0495F_{kXM!$h6)c3z zP@YCAL@bds@CP;sFNUQfCgcKeNuNVK!sQ}orTxkcIG=BYv(|Lwotz@)DxVQMG7Fsq z=SWT&5b=bLh8~0tN}1pRDiDc8H^PbTl3qpzMjA?O6*E!_AB%q`K4LA9R5=c6)d46O zMP3c({ciGvaM)iXxII!>xq@hj$<$s(W?altY7({p9R!Zbk0Ky4LAoxrLaw2`l`WCu z!Rvv`!F3@qqLYdP8=8r|C0Vi`N`j3_BA}oQSnwxYv9ALisspDb5aB~DgYQCaxdGIG z3@Fv1U=*5+_rp4Y74rN@fzZpaLH?+W0wyS{naU-!9QlTbx+PUL+AiF65l^8XN+9qJ-& zMR#B&u?U(39xMI8X4x(u2O+d0F&#UGjFy*3=jFqZEs-B3LEZsgBC`+|SR<8=tO;8J zJHt!mXYwwP4tAm)@dWY+g;1H~d%QlD4i+H>q!6?e9dco~_P$4Yq4klwNEu>L8l`Vp!K4YlENFqxbz0R#%^Jw@Rs-{Xg3~F^wP3WBv{km$}b1E zhE9feglyqsk&|E-wh8~4_zP=~Qu1WU9_}0|6Uhpe4z%}F_P_II`KkuWL0-E>8bXVs zInoeMq}tI1NSw&T1Z*IhC}#&UJth4fsV$Moci<+F1U46rRsyx*EN;bC5Y@YZet*w3|HWXb;KslOZ&@z`Hw`ZX4TGBlYl36K`@>Vi{!q5Rr#IKfh3-fxiM~Z6!fIDJ_&H zK?{2|K9a7(|0Ooy70yL}VBhdFg&iDCB@@fg4p=?>5Hk|dZ`vgZ_ zrQ+%1^f@wxIE|c%bO;p=H4R9fa_&{Gy1CZe`HsDgQ4XK2hkc{{xns1`;%?@R_s!nA~j`&R}n(mr%Q7)ty_>_9TmR^$X~7x5bH z2D%}g!B*@rzK^&=Tq8D9>xe(;2AqrQ%gC}Ave|rou;#h8kZy+Nljg9diRQg#jaXa!Q5Ylc z(p(f};Sk%9sYYqY(L_T$9^Z^Lz=vbq(L2fn>0M~QANO_iZ1?=_E#?0j>KExEU5Pvm zUiF@HhHXjN4^q2-Ecm5F)<9E|D=)u+XGqYgWaAS65#i(~;UDBMCejbtnQTvHK9&hB z7iXkIu#)$@XP0-Re^+FYJO+D3R-{TXlc^-ePPb+&@<}|Rd8MtWUlFH`uc?>#)yx`d z9T_5C6K$zVR1()p^Hf(yPwJ*=4-1{Suk?@PA^ZwvN3Vbh^2^A(P^I7kZ-M+5E-ZhW zd%CYcXmR9FWKXDYuuQ()A{l3=YTh?|-ywBjc0F@S?snhDP;KNS^_>0;VGPDsVV_Zq zTF3s#v?SMKO^`Rr7U6^-<6u*+?R4HQze(Y_4Fbgt zVb>GAAcZyYzCdDNRGRfYlr4yXadSdLEIbzUWPSI8~S@v^(R^=+|ji zYg!1eG@Ep-^*c19`FwT{^MT$#kD>Mx3-EKqYx=gZPSrX+3GV;}{pT zL_X00GD!RB`*adL4z6oY(8u@^Y=lx4S&4kc7h&PZ7HMvvNYL$@;!lmtj8w<|!4{BS zezvw?++f{Q4KA)@7W0xQi5K|;bT4KY)s3CPd6`P=VdfB9jCVu##Yb_SW}tY3Pvct& zV+Eci$i3JVa7o%8eiwY}f9hH9u9G*yX>|_I-Q<{SpKa@9>ttzeTAp$6OX077W=t^d zGVHcC%KhNGs|>bZsGRP!xEc`-{4UlZa2gC*2J%3C{7wyN|ibT7xwiNOxay|kqtI%|Hsk_QR;m{I z$2Y<=+qca#)F1Lqj1)qqQE&JM+VgQobTf6I#U6q~I4|UL7&o2X!(3udaogGIOd8dN zzQ-(K&+%67B40_^D!dep;w)ir7#_mtjOVI!9u?Jo@V)lUH>>7*1DEb z=8HMqObd+{GwH8OKTQ1SN!@D5&pB^<<9rzY87;-t&|cO|)#Zq5g@VF0W*~i+a$qt# z9zB4MMW%u$((GUt|7L%i02;U$U_%d-TS!4>rchLSRud3+iw3^0a8Ve;EuprOMF@eu zMPFsQ(H&@qA<1{GQ5?Wm;S00H*lx^UR2i}&{un8( zT#j@OuJ%{+oyouHYVEMv%GfSgEatV@H?qR1Yd%eVyX(WUFJ05d8vAFr%&imr3+c}t z;^y!-b*D8CwI{{y9N^B-47HOiLvN?P0HN@Nt2WK~LBZCg2yCTI@V- zF*lXp&+laeRB>V@ev8;h>ZyuMH?}?BOb7^v#QU1@;%ngsHm-6@6NnAzV z#ZBXEw1$!}9^I_$jnoNk^SeAt-M#V(xyI&xv7NWxGW}xwF|+j7#qXDXDDYw7*9EDU zGTNGt*lYREN_DXlWJz`a*G_oHKM+0(ZTMvF54I|IjW-HTwmW0qn}5m8;LdZq1(s{d zb|uq@7Gw{63|bhu3O%vEgPW)k9fy9w#$iYBg7_M83~>fuj<3Tj5fAVz@-SJST1H)G z#&U1?isE)LOBfGVi(;Cy;$x`ouekR7VZI0dh0SDAm;~w&u?<^;NYc^JD}Pnb$UIl> zCVOcsYW~OM$Zn9?C_O3ld}@)5&*@*&duI&H?rhuOtd-y0i-!BiN5FmbE%pNyp_kK> zsm4@Uu7dD|FVAjedA_btU3|&vm|k>C<}dmeYB}j6cH)^>JKTsFu&Y=REE!n>ZKA>W zV{9|D$s6I#q19f7Nn}1V7g+xOzKx+51^%n+oO}H9XIJoi;q3%Xpl*+OXO1)Wlg!y2`kJ@(vCZ z57(6kBI6JrI-2-QyrqAoQ<;fueQqhYmmk2z@lAPxTLo>z9;};wME^pspr=xUsBOdm zVm6V2m&2!`?a>m*NaQl|C%Ozx!E&)_#C?*dU(!D^HCcwc$2H_%^XIuzdJ- z{#+r}PB){=&^O6T_+IFd`z$?(90@H6obhTrL-STR)9qo)anpLkgshbrO|!0LzRRj^ zJZG$7ZsZv69GKtMbI$7w)(q8>TFYJKCV<8UW2MNK)EBx1dx32xoa6Tih?M|G=dqcrmmR_8vKQHPECc6ej$S~nCq7`OkWvZ@4}~&<+JNY5?5~=}vl0D1U z6Gn)QG&{tAFiI>U?hzJ>gwS0`cq9%r@0%HkLB z3B(lAPVS(7p)Ax>>H!UyO-yxW0mHDh*>P-JHj#yC<_;y3KM{5DLMRIg$jc%tLbU>G zyh-j^c`b8?+EPpfvrig37+z*27@B6a$zEsLWhQM}TW0P?*Cf~EeAZLLHzIH)xF)H>D-R>DG-VtRAy`3gcO;ecQjjtK~~cHgxW`)CuZVP&{XJQ^F#`V2Lvwr7I{AA`>HyZkZisqt#|jb$IM&bBj1PJMZN`bT{kxw;d=duqtgQ*s1HX?u@CkTZyd7Q> z$MAZ12#d$PI8G#xMM#WlNtJ@T(t^xvx)J@HIzi?V4e%M*E~F7cJ*4qpW621KVVKj^mai$8p$!X?o3HDkr*#Xp3&mc-ZSEx#pm};7^#)4y zE%BJ_OB%?sWC%Zx&%^gZPr(_iEr#Ovv152yVhhy4B2;y%2~~|gM3;oSlr}^$d@Z&H z9Sf$((<0@mp#Yq|TXcU|Bd^qQ?!79d^luf%C8n_k5n<$mNdg>9l&EUi7HU7~BNck2AQCA!Ai zcbYrUf^W{%XIC-(84Z(6_n>x>3#tCpO!6X8kvND?hAUA|xI@SBarj+)3Ne@rkTMx0 zM^baBX4GSNB>Du7jo;9(V7X$JW=7729tI})O`eMG*UrJYvh4@kQfoVl!E()f+3L33 zvd*w|vahnQvRQ2L_WHRZ+|#&Sgj?%f8~7>YiOf)L0}c=4KT;FvAcJvDxaGoDak1v6 zHd$9hzfxZ|u9yC~u9LPUwD)xUEY8C8W46%a=(_YuswQ=t%p#kRw+M;|<4&wNULJ3Z z{{|)R$3GDB;Ve^}YD%@Gex(?C0$GMc2s2LLKcdOVA!UlZC$cS6Ffh|w)@^akb-c9K zwRN+NvtpJ8*05!;t)RW8ZMD^HU1m$Sue9mxMI38$TjmwcPw^b`UJee9RENGr9a0g$ zPRyk?(&w2DTt2@+Y^<%KtEoSx@1p-nUqwGhe_FRoGgK%JSEhm7YxW*9iGD==PS>Z0 zK`Ui8`H?^f4$54@AK~lqR(L<62h^W~gB7+@p5Ct+Z{RwW6hqwXgM##c11Sv)LxuI$9grQ*GPry>j0;a-2i+ zUwi8KI|Sy1ak&Jzfn3K@iN$0MW;bJCC-Zv2uc@rvqP6RE`g8gu{Tit0g*0}dBwvVI z26uxKnB8yFi5?6^*uh@q`!VYj>wfDW zw!yZkwmSB8j?208T@~DYednP!@vDSF+rK=XOwJ?wP{o@vQDa7v@qZPrEfDSD$m zDQ>Xdr0pQ?;xo9iToKkwYnT!AV7er;k2cXPU5y$>J|P+td+}~Kg;&F0<12{iWD{xw zb&`Ha&xDr5VcJNQrD~GN#LxI>jKdhLC(;S7niO2^cLs|2PUJPnO|qS_CEFub(l*AN zZvJQq+s*cewwBhN<`iqTZI<0??QC0WukX5_-^O#&H!{#NoCv+IA`EL-Mh=HIzngi( zZ4_P!Uo;=JF5P)uPkn;Ek`C1`)~*$c^4aVOHi4VPlw}6dTKYNEa)zz~*P{t=?yf>? z!9U`IoWZd0k$0y>K(=na&c+D;9lNR#_Ho`!*q9#$l!%?}C+6Zf0hx?}i(8hU-&Bg|! zQ;{~v1n8%39IhX{?Mv|_yAxas?9VNfeW%@RTW>pTEoOIHx7jDyL$(ySD!DBt^JrT? z+j9Fo`!CLw`9r;319O7ukzUFWxM~-J!50TBjVvo zwTOH`o`AZTLXl)$Vhvsy8;+Rax%GYdWMoV5p8tfmhi75_@6KuV5w`m_8IDGib*pug zwXD@_ldP<5h~=JHF*mc6vz@i>vVF1da~ktzd;aj(4z`KxQ|=)ibTV0luF4+bCJKV4 zsiwJ}hW1EVLvMqTX zTF4EEw}g-AO*|&P5)aAqP;ZXHaXOC9fcmqTx=RJV?VS5y%aMXg!$@lIuy2<8l9RVh zH+3;S%$%QbD6MmvKP@?PeU>q+L{8hBTbBB^8TLI6(w5S(=+LF4Cnj~?TIGx?W^dURpO|b{cN#!#2&~^)3wc zM>h~RsS)&BcA?ukj_Lf`0m5L;NXDU4q%Fb0{)+xvzJ$QY zz_P%e5FVKzzml)R^{$>YKeAZf54IsY2$8PE|Ebl+wNC1lTrs&`!Sn)03!F&mo-`zJ zPP{elP@FG*Oxz`XG2QRtQ2uZFFCq;|m#T&^-`V_o&Iyizwyx&JId2VxvT9}wOa1hD z*ZZ>X9{!_wwf5zxH-<0Q(#m98ZJcj`GM2ib*`#@^YZf=Fzz+qEB=m}(pik9x);-tv z);-cz6+WC?&tsEy5^jdf6bfkZxjkiIZ`fi6I(=(wA&+)R}F&Xpv0^vkw>wXO7G4kWnS=ZECNt#*fF}C4HEd3ev6^^x3N|Ii61;lnJv| zF`}QB(4de#X;xCDI1txZOymgu2Wk;Dhr|dMs4Op((jx~0n>0`W2cSdYarjsRL{KX$&lRO&T7ugy4 zJ9H&nH9Ri#Dj4TI;2Y_e!u!BTv=aF{J(NAGN!Md>vhI+09qxPgawWLk%olnhQCSdy*KiT~0=W#UgP`oaATI{F&9M>}L zM}3dPn%Z^zHgXT~9-o8#Lrg~#k+I5HrHC?6+8r(zUI|amlKhK<#z=D|2v?UxW)pWu z(@S?iYtwz!^b#L)YlH;R%hUWg=o=U)j?`4q;+oOW6Hx>1ql{c8dzY~??b!8nQDy|& zAMQbRP&wphbS5%Vei-cHz2O{?D_X0XQnJ??wZ`>XX!c0s4MUx*K53z^=1&XKR;S0M zZ^`7$^SoQbmkAp?O?xHbK~mGgzb5t8C2F4YTd8Si1JDVX4|Yp8C8r!O-IR4Q9=;lA z;;Zhj7&zYtGg3UK zEv~t&(F-zvh~3Q2U{A9pprycaow>*C38o>n3!8z4r4ON0-#B*_*D>2k%i5f-IaQ3D zA=lU>$C#6ExR9|etyTKR)DDJDhWyO^Cc8Z$vJy;YD(V99&69JI|0+B)(V{!Y{Yjg! zdQx$zZ)9QUkI=S&D|j$CG591@KKRK$*c^ z9;7UHLcAW=J^po4a>0IaPjpkbOH36+R_=wXh04oz-@R}f|D3?^@O^)!(AeNtNPE9< zt|FpW@lVVK-lrL)sg~F(p?ciFxO$q+VkdSsd!5tL=P8*wPk&~QaA^W59E5hmFwqBf zsh;LN{|mQ--@(u2uX9JaAK2r>D*O^=0w*F3gS$N^;cEYb^|IwRqtT!-W@i3lSZ+9A zxM#>qSH8YVePJkWyklx$5p47QoHU#Goj=cy&^JoFl;DqlB(@M%u*1o5N?{n=krL`F z*9d(G-V0w1y_FsEq)6|`jz|kh0vgncz9Wt@rE!@xuPGlJ~`jYEmy)KK5ZlHfoXU2sgfg*U`UF=nbC zKTbpI2Iwi>8%;rBjApv1aDTJe^Z|MvU5^d$kGXf;Bq2rfi>9vTwMdC$xFhTfrYOCd z?nS*MClUjoCu{PmQ6x#35o5^(*%V z`j*Ym#?05W&^*(Y*8a(dnK9%(d<9+)TK+i%OFYBuSS#!x+DQ2;ILo^xuZ3foWm!(& zY*Usl>$c&YaePkAoY6T$j6WDMjVDagb22SXYni;afhA#{`pW#E+ZZ2CxSv=uVQt(9 zp*ve1dn^|Wl?x;UCVQ=3$+tf6AhbVBD{~}Dst-mWd(eK^8Jr~>FvVD#NJF{biaV&^ zqQf+s#HsvR_800;vO7MBWa(ziQ+5sCP%}}3=z3{?)m{_!@?Y5pbQ5Yb+{?WrijfR) z1oxqXkiX?!p(O8@Jga@5IX&m8(QXiod$T*3`k7Cfnwk@{_hf9(x?w16Xl~qNOmq~> zza6?l6k<5tjJP|6{K=Z6r}6E?gMxq;L}!Fsg>pk?|MkF1->Bg1kX9ZneTn2nT1vIR zb!8&xh6>mwViEm`P0t*jX)-hxmzve8@nP)DM z^M|pkVTZAC_TPqKCdEuSjCnr)TD&}yqq~yuB;js>CIym`8|uq*M~Gh$XDC0I=ez7# z>$SO;`FHp>29u;O@-wNsJQGHfHO3xeHsS};L-pbxi!R+8eFwd$f219ynA$f|N4td+eoq}We9~e6QNGvYoFjeXK^a5%Sc?Lr;3hjc8M$v=!Pf zMf3$Vzc5vZ)zYx=40rqd0r~FyBc9&w;elDf&XkLkCPGaMjmGnTOG~7e1pq|ignL@%{VU+faPOsN$uZR?W@bzeO(tVm^9c)p`?qX+ z@BH5)2c`bxDPb-@DlS7;Ey1pnco$of=m~m+|Bf{BulN4sJK}xdZRC3#zAZJ4BrE%% zj-SJ>5t&p7zeGCdWM($In$6+M3YlDppCp{;FLCoZ2Q!aeOFv+OOnv4x+XedD2aD4* zxMr=;N_Z<&;!m@!m{V}yu@N7F+hE*Omi!_zJlNMa)uXs>=3cP%w+759X4HDey4qqk zx6GNAbv0{vc5&k?<0C^aQ;qz}J`qeJFmAc7VxqCo^yCxqpLB}0C*2e~78x1%;O&*S z-~A35!o+ifjih8tT{P}$iU~2Z>bXWJjTW><&1(^Q(ku;o>dc?I+~BX zpEXiNsgh(I8HOV(z=Gl1rE_i_c){+7DYp2Y!O}}K{&i>VKGkcq51@y&@cXv{d!Q?=&(bwY z+z?+~x0ZQG+(8#a#Bj^NRIl6p+*8QA&3nk(E98^j%8S7>FdSP>tc1JdHS}51Pk(`@ z7Keo|>`d+&v==3gc_4Kq2Tu^f_Jjuhu8lnbzvozbqkhx}~ngYZ;fb+t4NJxGB!^m$`_u zS6++Y&&mP1x+V?o2Bsz^C$^5Op#99=BI+rpB1-}*z2)w zj1vXTHF37Km-e*KpKHnfMQtF8V9k+S`9!#8xIpl|PwzFjujKvas^T=|x*QhAI!ANc zcyl2WWtwZel2tupVAd$ZJJaIa+WF1kS>bj1SIuv_Ch^k}Hzw3cpyJw#aD70ADF;L2 z{7by?-evB8+*dtod<#SRNV<%|i1XWs2Kr%^ll_T-#5SrRvkYFs_=}&;oB66Dt~Cg) z`Il@C)e}Z0z;iEZ5pjVenM|f3Uj&|ET@g!anrRnkE{ns(OZ;}`Z>kfK2Is?vk#WKC zzCmswzm&`9XqH>jQ7HGVBgHv1x0|D&?JskMoKuGWhFcjIGkO~TuuO6^%g+v+#b&TT z*DO9e{zl^0#9@gYw9^6ctor# z784!_ub~AtkgrK+5*&U2TGid9LZR;dPu`)Pe(tAvDXtC9V!3aem7EsGG>2ecV|{Ph zW^8XD>+i_>+NaQO_nrsJyXa$&)h$|q`8vSom<7d+P6p^h`ykfim6&m z|2i%`ZjtT;-;Wtbu0n@Mn?uzDQ$5YRKYJE>tNJA0^!bkjq|g|T7S-W`A#V_Njp756CXc0@oUoO_*QWjG##0uWJj#JG$*jhU(%D|W<7Pi zkNqb9`*3UJFQpk0!DkXLh(*LgqC2^Q_?g&5z9x%9A2Ug}f>9mQxL*8zj_2C)|8ORD zD*FPSYn6pQ_CYX8pc8YMt;Zeal7(r)5+RlE!2iUxW6DwsiHm4+a9ES@qZCKv8U_a#rI*V;24vXuS z*drkvm#pc*w`XR+n3|fA@xfZY9iA257v8%5`+>yBJSjn5fYgHL&NxvN`uBM}9m^oT zVny(Y#9m?t)t*j*YitYN#-|C_g_Xi(VKJv=FEbyhHS{T}9`w&0C1pC5EybPYJHr!c zR{R6TxRm9KGg|T;)($k1n}vG^CiqTzhPg+&Kf8~)i+E1BAGZ5?gAfpBfC5+BxnPWYU>rC`khT@vxQ%Hl)nN2I>AD3Ija=Kd+K zlB=!zhOb`G8}2C8MQ>p@iTl)McqUPT)Irb3ZL|-z9$rZsfX~Ork|n8 zt%b#cncv4ffsuN9=~nbRUQxVxIC97?yua^+g#^uR-JA2^2h``-FW2dhZeh(uo6;ZYSJ|o1)cehp}hzS<3Qo zIJm;gdX1hn-gNKKV0t)LTCS`{TjD;fCp;l-45PTNU@OsjXdK4j`7lC0M3$rXG5PE- zkg|XH5&U?*58H`3!Te26CHLSBi2cM>3a8uCdioyQS(qpOF7^?o3y1jUd^0|g@#90W zkw~f{O4~xW0#kgydUHH;J;U9?OJuYYt=fOhggt7M;UP<(%9A;V!fsLR?p_1v{6yOK+o^ z!!zPU@+6rAeFzkDfnLDg=bFMu7>?h@E@#fc$iMBxBlIWap}a%-Iov-qE0E!reT1*4 z_jgZ4&z1a-c|~12bEjG7n@bpfGfdBnq-SKz$|#+E*EGP<-%W>(fU4v}ZkMLA{zSYj zu5g@R!|)dRA=V7~RlkJ7fos01zRJFSffK);8;Bjyd8#dYF$a2kFF$8pQqwoDvVlX!x^#kb%#c+Oo3_Yw^4XCAR#xqf^D z;RZb4$An@W@-5bcC+fnp;ay5m`Bp>-UkJJaPyNe$OT246vOC9J)ZHq7nk$f-?Z9B9 zPc?I`oJ&SBdqqxZbI`Kbk(cN3b&Lc7N%p57aVBAfctPyUKWA|oCp735c)oGGO& zbN{=*6c|TLOV6a|@M0TKz`b=JGrymO71vY zm~FxIp!buviDV)TL-6XjhS-kxBL~3vpQ&sEPRkeMm%#qJ!bKP-Z6<4yjqm~RtYIC{ zDU0Q$@Fvo;$j9*TaP^QicrFkQ%=H)bzwlg#Uf|QY1MLkgg>%rXLRrHz8ykijtLI!b z56NBaxf~Ke4mOHT;wEbjYv*Z4ij{;v+4htUpN7;_CP#LKXN9f=mj~a%i2K9xAf+AX zg_J-yUL3+fwS?wWabXxrclX z-lJLp<7`43nD(ZuvVW%Xrp!I_r8?F9QdC&HHU7^S63-1~18# zFhJ|r`Y@L8sW4d>&39(gsewc~ynFG3;)Wie=8;q3w1_k8mVZ`wc*_YzSD?Mo$`}sm zOhdAeYVbbMbz~eChi$}<5H4~cJUwj8t!Jxno7pYwNTv*}rM?m!@MX{jn}a&hEm$(K zkr+*Nq-#M7#>9Q$N^!f{5lnZwCwU#Wz}pKnIH#n`nec?fAXibA!&|Ut;0>jba!?u` zITU^y{2_3~3p`lo>*L>1m!u7#>H3*}2K?Pz6`;>bj z^b}u+KM2p@dYwgHz!Q=FN_pv8sAFhpuxIdA=xq20sgvwgmZS05Ko|{g#t-59;EwDh zdK&$LSkcF55Dj5B@NLjvF%b5~?`#Y160{>nup8(h)G=ZM^b=1&*CFo!f)>SAz_`D~ zWD2xrKCtbewbq%vKsThDkT>u&^fKtLRF-q3YSN)dWoYFcgg2UAC_lja!w*54GFd4q zm5i(p-4DF>HGp7O3rsmt0->mx_bzOTseS_Iy7kGi5quMYB zxpIOiEEG<#IJ=en2d|9Mpd38qITE@8{is{RTO&|6log-`Ru~Txe-h)#dxRE7cdf$1 z*l@_>W;6w@3ullw&|5@M^XL=sykjD(V^1>+=}Pnp5+Mr`2COE$<&h3QL$QbW9z2(* zNlv8n&}Z3?E=;{3TN3MFJc%1oqxJrlYsz0GSz_eg@^*Qj+zH-N{G$A<803ibB=R!c zF?2TY!AE!zPqq9)d9R#nbFGe^_BS@4Wu&EsrKa_kEz>?Ucbu!e=SZNsG#&`pJ>m)T z4?jd4Aa)Sj2?)QD?N09?W}#+amVuNN?hz>;c^Ao){#0gx%SdBf#>9t>JuHUq1seNB-%(GxyP$i1{=2+9*BsYqC*tIDN9OLxZQ`U{Me=&P)4l%$ zwuXnvufa6D9#w?>#E#%@a4Wd-++rq&vO`VG1+NqUW226QlOjta@1?@qD=yvb1)-bopNO-7reLy!kZb+j#Z9;=P>gq?U#Zl+kcHy!~aKw48jk_(A0 z_-!l)&NIc(4@f$Kp&Q}2Cb2nKWqct1Gp@(qU@Nh%*a0{KBrp@+8dq=WZ-%!HpDUfg z1bFXmCpZrdgURq-^Fk#T-ktqJ8Xwsmz8LBsS`xew_~5_m|J|?kH}ZY<+;h)`cKTEo z?`-JkU@zfV;?OzII#crYc$_{W{ENH{31j<-I!roK5XMmB9L1va78rk63?GXgN4|is z@a}G3@Df zk~hc-kY9>AMiwAf5`&4JxEAk*wT7Qg*fwk?_8A+4zktzG7hnX;bjbA$>>4T~RpHFa zgT9I^pO-Jm_hm1ZhO;jUpaygO7CdT@UY zwhk^2Z1J!0jq*-+56*j-d&P0j_RN}X)maDFhd2hgPPqH~u7_^OcC-WWget}zcYLLF^|O)pZ!Ff-OYP!4v;fr60V@ z+Xe2@4=AJI{hAgqGL<8Ck$=Jy>V9-jdLy)iXHw_M>hMm20rLA86;LOV3nSz_=ve5P z{1vxAi>?RJnz#n-fr?lucw6qJa$EjeBBf4|PvNs+ZRGbzQ|W`WOzxnRfp@VxAj9Fk z=n}{ac-Q^DlA!z`|0R`>UPSsveBtZio8gn;zTrSTRmU>}KYbQ!oO{8K|~eiyak|HajNz&TNU@59MV zdM}$~`%*+eKzav3q$46l=^!AW0wNtLf}kLRQba^SiULvuL=gl8X$p#Tw)fuqW>aP+ zd7rcQn?Jt4%qNp6x14s%OwN5yncx({TTyPIosHX)t%I?THtY}gxE0pV168yAIs?*eq;Da+> zwMOZHEM8XZ=Ns{t(Z=s{x43GqmPeCX>q4Y> zOf+GolNoCkXJd7J3!nrnmaTylvVtvwGXsfI4{40}K6>1?{%YS|-$dWv-fz4=dh5Iw zyo zf0A9o%K7v1!T8T+5eR zZZtQ9J0Wi_-v#`H$yi(bhvbqLV-4M{1HY5 z{|Zz~I;o$yNZ94y>AT_8c&B-Ot@pWyyVurjtKCpDr20%%*Xqx!$Jd;%xl%j0{x$Cv zp;nqIQ}drH^;mr}6K`V%8ecNrGBmwo6IS>1(>%S0FJ^HjJz`WKwvUhfQa#B@GyGwaZ;f;05gsL!X5wL z{wlxQKU26V%o3lKPDv{QcLUSmkKRZ2nQR$WT~7uMg;RDhI3QRM*n&BjGVqNw5|V2p z*|2|B>LPtA?Uas5hosfgyHYEZJW2dq5dEY4`MzGhZCIl*$}`FHg~#Qc;EnV(@=q6X z#7_e=Whdn4`A3Srs^(ZtC~5xERBA-^Z9LVEQ|*FJ(>>n8r*m7lU%1b>zqp^d_UO;{ z@R!lXr|<{S`~4y}%fFDF4$cf74~&Ap$P0msf%?D-U~&9|b-C|xo$=&jg-=(u;<%y+ z*1+Kn**=T_pI}|!TR`+U3uK9|L4V+nz|UCO8xb%CevmT2R~P9Iaj7^_?1*n0u_u1N z79WVsq>rR5DJHN2y=7@Yhd1zC@JIM<=SrWUC;0$#$FagM{waRRch9%n_lj?|Z{Wc|0jQnpT)nN0`9l2f{&*jM9lb6eO0L|nquoSKb-oT9PrnF0%BMp&Wkorm;fK%{>v_x7UZIhNs zA4{)GuS${9Me&GuL0B!!^%wZsVI}E6PpzA;f5$z={g(S@_tN?so|(P`|5niuI3sJq z^--)>{-Ej%pVdL|6}|~y_&Mr!>WA=N3|BPAh*d1#ix%sUx0JstUoKaJ_SxK9m{-5c z`Q>@&g^Fc;WH*B^0~aCz(shpqS|3_s|Jrog;WLU=7NGWr4{-4R8iF0Hwl+-fxL) zAX>o@*+$t#w4tN2y=WC{Fb>bh)z{ERE&%q)e%Uc#Ds0DU`%73!eHblf2v9~I04HV) zW`*HECrJk;gE?qGO*4Y;0X3s6*jzRg*fPUqy<{!X3)}{pM|AK`U^V8I-%1(cPO*u2 zT{tZ?6WR+_VG_n3i?~+|h(Ahm1Kohta}ziqRq~~r7f)MD_!|Bye~tf*f1lsa`?%-$ zE8H?}BBw$B@tgcQ5Fft7Q_bh{59Ax=bL4a7OXbhVhstw-0r3qGAco5R0{X;qpu_YD zx`J^*0Vpa1fm8B5X0*{jANf|6Bzq36W;fQsk0ePTzU{GLf;B+7} z&>K$%H-OaBQ8o?r-2i2EG7cyhg+S=I2Naf5z_!_e`RF^qTZsa8O%bq*=B7W0lYOqzz@&C+f30RzyME(Qt$VZoPRKkG1WTMI0g z1$Y}j2FNBa01e_L{0_xjYCWXt0ZPF;pr=4)1^&Vk`Fz;!4rpSndOM4NBGllrqVi)E{uLOHz zem($W)GQ$UYyuw9RUiW8Li!JZY;y=2n29t^fDw`r_$6>4uqNM$>L%0Bk>(^ow!*%B_@eYq(7xNSndcQ_dLMdq!YK7)9~~8$9!zV z_*%pN%YV-=z;p7S+zPHY=GnfaKg%y+I|!aXLSN8RUW)edzN{*Y<+kBvoNF_jz4*|}TtRgg+CBpHD| zV8oCsXTaT4yE(LavCSa1+DR0mrEf8}jx>{Hm%x4<>phxWb>_OlLGcEARA0|(^{ z>@!Ekqi5?Oe@(tpzFmF-T;G=eEx(1Dtxc8(TR4KgY7ID^ z6P$!?RB$}_m<_vm2Up&O9jyX_#|L;9uoiOp2>TVdJ_EG966}ZPt#-loKp<)#?1HDS zDQKZUCQWOK=(Jc{$h%NI9*8&!T_pfj)IAYTF78|A{`N0%&6f^bL(bWeenOgxnV3 z{)j+uDu>K-@hyafRRVQ^aLCl)Aq*H#4&ZtjfpJD?Y6VD{h&l3o*!8{Ozu@pz@DYB~ zfl-!=+|(2Jf?DKo08K3t{V8EFMc_Fm3`cg+dbdU@wJVuXrAGzXDVit0if@g+MsYQKoK!u$U3Ln)?*n#!XYbEM0 z#8w3DR>Qiy(1i+mBpzcxX*29&kk19~NNW<(5JFoywj$)CeHH4efvuFG#!}qP$JK0< zn~N?S^q017gxv* z315(~1qrh+x`EE74Yoxxc7Rd~?r9pngfFRRI3mqhz=s`Q7xd*qUJE!P3`D{h^`Z7^ zP+khI3h>QGUBpdggJkL&?10)IVJ#BIrys|J%vcA?Np|?_4}4VNtQOqVffGtgD4`4` z60ZSZ6AVp}P!cIkZ9|<^sI>xhmEpIlp{#n$5*Ugi>5p=fRtN=@xFKvu!uO;PyBFe8 zjeCSONeGYhCEir9KLw5yuotpl+5ltFVC7aEkyP}s36c^km5>svKpE+{1acsGlwc$3 z>KiOdfs*vdN1Gk;CYz#KsKt;xsLvpqD)`@)(;!hQrx^E$+X|#}qck_lV_1(2sV6sq2Kv&Gp&=dNXc9su!}Vm?m;qP>Nssyq zW*rPClkA4!V%DSXx&|4MMUjrlmdP&Y!zg6OGxL<8*!Q#awBX_`Vz_~?Mb5Sm*@{+f)Yh!$3zwBoA#_;vVN+ST0%$$R0ipcl|cM3 znUHpf5^4|BW|-X&XQX{!gZ0(H{;FZE)b^+i(wkAzkFP-=l!wY-vSl?;KT4KJ_M~f| zfyyQeU=&k3C2F~bUpbD5W+qeC_UV`*v@%YJPpW}5Lb9=9qkh_kjihD7eI{miP zC7Y$*C(uA6{gSm25-VY_l64chFWn_Oq&|vz8roCOO7=#UNvVh)*1M3583rxwi9h;M zy()0Rs8=*t1?ih%?Gl%y1+qiZ1@TY!*oUYh%MR5^7R*{4@jxw+_M}a+Hlmnu!deYc z8_G|mP)lM6wS=}ClI{Phf>fwv;+54#{R-)jxFvo)xWif`^)%Ghh5SPtSzbtgR05roMyVF|%W9_UL;?G-bc_R*mT8nx$Y>zT zAdXmjB5jafXwNt#-LZ}7hRqBJi<#XannOCIvWRL{@>A|vi)Azsb!4R^71lOb8zQWbaffCZOdrGpNg>qwsV1VG+6JS6Xs1~VwTt?OjkOyrs%=QcxO$qNQBM+l zDgkB%te;^zBq@_Ev9SZygAOJ`+EYq8XTMC^A_M*OjjjZ}n*O&qhnm98=V>4=rXax=PFi6Oa%`Y1*P ztBXcxrV~bIh?Y>FKtcllW#5VY4pEM^rQEF*(r{k}8uMlQ*-bkR<>6wD44~p%I(OlIo>d63zXo z-2bZwc1}FeJ<=AXqdlXTD5CsKrc^V1DIK#Cy3crEcUTH0U#1l*o34`lNG>D|Cbb4Z z{huABp&o~%NNtd5jB+rUv%GZ1N`Go0bRKF8Omb{wBv~^}FbYC#p02S{Sx-iJnYP$$ zmsta|O_rAQOL}EBGhS#$jHV`HYcMyp;3tPusYee$HqA}g0Yc~#!MPT*hs?aWlz>2x`+!( z#bzi(Cmo0KFv^Kvx=v@*le6)RjgD-VLUghj7#sCM9MTyZ%b85r_{XTG`AO)ANivj% z@=z|Sm5uoSrGt(cjYJF48rs-=g=i#NY20PA0ZL8jLURYwGF@R(!%xFqHcBx`(VnGY zbBU0~8UIu|(M8-b4%jboO`K3&Y(5scOZORf>>f#tSp>ZLui5IUk`it#|ULh}`RYGF@gG`FXBB5dYDTB0kI!i|k)FEsC_BbJWcVI@u*ALsSIX2%pOAaM%p8*p>swx*$Mm55wl3@L!R

G~fi1tug`ei(@G}JS(GFXpGXRP0*`=R4dKh1hIx*zIO z*%6&ldg|3_G@%}kMkO|?g+?eg7O|d|^|PT7i;Zh+WMh5i)4Y_1jTzJj)7v7-OV=1x zR1;Cb`f;Ly@kHlO`{5AHG^Vmqf!;|xr8tzEQOA00s*6fvGe*V}rDJ0Zqn+rax}Tmg z3RpU#fYr?85xPoej1G2}_M`>cKi!yS8MTZeRvMEX)yifTp;ALRX%0kX5#98ml>hxu zZBIX}Oxja^l4*zoO2wYO=?-0`ny6$(J>3uGX7^Ybbe~c%`9GZ(u^AIxXZP7$g^o#n zPdBC`c9-PJ#$GCs#$HOt_-1mTW0r$iA3G-gSPr6{bVTx|d#rYLOr;U!>>hDOJTXow zA00Cp;OBqsEz|}>+MyJ5kI9Igv3yiAU1e54M@&jjX<&DlG(xFa-jHNNw9qf53zb4L zrej70({QM6*50TVYAeJOJyB8lpGuv}YeCab~$}yk+k*MK|)e)j4bVlhI4@?^DI^%+*%hHEZ6V;4n#$V_4v78uGC_ zbeGBq&6C+V+q1NE{4_1oB|Cq5l~OYKKTRFVO;Ti%WOGI~Z>BjfJ+083leJrtG|7;? zx1+RlowZ21L-&|YSWQf`Ob<*?l#0q_Pi?fP(rEt5>ZBv4Tbk8U&1~jQGjN(^v)Q?? zA=lH{F3U~#sC=3Svr-sEWH+Hw*^^G_nTTncmCNX7+_7WIM>)w387FMd>Y+50lO#vE z=$Acv&>d!zq%n4nWY1EDN+Jo-G3{xi9PG-|ydle@bRqkqUuIoTudsOsyF>Tbh{BF& zTw&uKwRqO*shx+~KkFe_S*&DM7g5G+ol2umX#8QvAq!_^QqE8+);2=c+>rJEwIOy! zd08zX4q0hW@3JGdr z=`NK{d8nVDJ)N<6##6eeM9R-b1JV>pD5L>q32atC@?v~YijZxDYNDAueV$5{oj=t} z=+4u!pQaDVhtzsZsYLoR{V-Y6p2}x3raG9cLvm+a&>4N06uOUN#W#;0xT zzvI$Vo+u~f3Gv9zSb3xkxY@8o2cv;eMP)H=SZ;Qecw-yuRmd`^U6M^O z%VJhaHc9qE7DMu3a%PfZG!YNX_E?){?U?aMvpTAk_5PHS@xiIy_;a=verlR(JX+*;n3(v@2hA$WpBOcpYG5`|K*WC1HCV!5ub9<+fy3*X`Cmy zuuq6$k`{fKBuGM(hK=wcU6VGMM(CH``O`l)pB9%IsDnqcx|IwOi{X34b4q(d_jHqRiw**t*e9P|m< zDRE17Mfzal8 z`%-Qu$B^~Y6((cin(8923z8Dm#pqyT2qFBRJ zBFUAF7sNlwF4Pvto@fr{gEf&2kq2AN|MQSx?_LK_Q8iMM=OAk(AmD%UBd;3fp+6m=~M9%4>x4YkbL0W3X|M-32XA#W|oj1@tDuJC*!{;~S32w=@Aqsqp8_fR|@>gLh^Yd^^dvuK+%O4_A90Qkz~6i~{L{AvH^Q%+BKLlTZ8JRgcjNA1teZFwkK+6A$}Iv{0j%7$ z!rQPp=<0-6!aZeu;h{VTp5%kzuRH+Xf$&=HhwTMyy&L4;6`CN8bcBZ5!|V7t_#d}~ zHl78ajT^Y7-w2dTeqYpH$tvBjpkinv9ai!WD7Xk;?X$sO@I4;<0nwDdZ=ilJVl{sS zPxVhh^9NWbu>#u?_^p2cJ~n`hFOcd4e2Z^^|4eY;K`)|(24YeF^N?FFaL^yyV5}Ay zh4mtn5&iKUtT;Dlmf=fb8Q1^)u|KO&;@F3`Ige*W{YhGqiR3XDLE-50Tz zrX4T~nqkFKBx3AZf?BLp(qeUoJgCI45$n?I!7!{hYJoIuu~wij{QqCWDucx+Ya6uq z8>EyCtFuDJ!(?+n)%Tz@39D^%SX0#r@m;55eb6eb-uN7AF@MCmh70m5SZ93~(KHkA zeI$Q?RSq}hKjH3YSk*Ndt618|E%G{9yzGMPD@bGlv>65IQMB$WSUvd-q`nSZycv8Q z>r}cSnrtj6=YmooClDXF8Tc{qRbX9UJ|f?ZMr79ZfyP)NqzXuqN6N!`yc^OGjYl#KX>Nl}P(`j+&CbOCEW+5#UUHP8kcy%l@`D;VU6AbUXW!HT>0 zxl>#QXXe`?=ECdzIIIU9kNqHi4A$6n)h-4WJOn08wAflxj>+HN-F;>ey;x2Q? zxrJC!hnTutHCD0ghGYiIJIc!tpZYJX4LOCC;ftWlmhhM^KtFW~Hb!eMUd1X14|w|? ze2odT!FtJJ#ErdxwM3suE3xWsl++)qB;&wAK&%oo#fRc;@pr5{Js|GFTI5aQ7V$Ij zL#%+`Eq*WlCf*VA5c{;3v4}Z&9~#9jUEhJp!&kt@Kd34^5rG zdg9NdS%}Wt1uLcvQXN(hT@}xZKVntULGdf`Q}J{0ORO(iBrXt_h*QO(SY6peY$lq- zXwfNl61#}=#XrOxu^)KNl-ePhcPq3_72??5m%Ro{jOErrlii@jBm8|{#QNZFh=9{w zF&dZvO%>6=?~(Y2SdnxY>xIwqU-6&tOZjR12L4_CJ$^esiQfpVx8q%i$$SxOjGy81 zz~ct_068zei=gbKJVI-6kBE54ScBdISmG@)>J5>tmbnn+yH37}JI}R7JgAd=9$(G(N92y{>9Y)C-6C2dc_dbqRF zD_A3XT%0R*6Lq49NVboK+rkN9zwjYA8YA=;Itb4TI^iy&1Ml~b^^fpt{I`6Sz5~AP zzB#@LzHPpvK9A4mo9R#Sj}Ro`L$RrpEA0;~MlA7rvahkatcZJ`&%t``Ulgf8C4UR? zHGcxS(sAV`V9eZP4pdp(}dJpRr{ z<6i1+<}R;0Q&(CS>z?Lz)-SAI<5}ok?F;zJgs*^;)g9}9w{cGJmalvb(Q3X?pHRP` ziPF5KS)+Lm`&TtBfUaiH{G%>VA6D0>6VPr)nbC32UJ>+zojvtd)C{vjPKQ0_=P`^!_T^!zz9m-+)(B)t1)0S+l(QrD}K8 z`Ks{hKdP2ipQ_$kbG7!zx~28YJa72&{PE&9fxfa4+YOm(1cCs!~-&H?F z|2NQM?(4hh1zo!CO+-&nAbP@9pk~cMWTZpdZrX3PeYKsn5^ydjYW`5afoL<8%5Y@| z#Rz^CY}6&cig?l85j}nZMyC!KgBBy=#2J1x(4Br(7AxDRrm1?XyvjF}rxhFc#avVQ zjv%50BbLYyzJcCo&w2NY?u9@K`=@psP)W*aOtl}?OsrX4y|!vGa7mVxPc0o<`bDv~ zNLjMJ=zPh}lEPB4Vpf%f<~s} zh-lTEHc0_1bRYHw-!++;U3HytvbFl^8(baym85GV5n@8XstuIqDwYVj>` zp*S5;0!pR$z^jOuKAMkJ_E#+i7R~G0c8JRLt8S4lL2K0x(0Ei^m1f0G&MdDAz8Ua} zmxccRX+EFl#UB(MENGjr&Wp_g>m^)IMY!r4b(&_JZW546Ib&zzNaHl)F1YXKBfi74nh~nc6-&7RvJHVs;T>PJ zXM0`a8ckJv`SQ|_if0xU=5NIGetq_ptZte485=SVrC-hXEMr_&)0|B?m+}`Bxk~@7 ze68li`W*k*z)*gJYKLx)afh`_*b2u-;e7at$nlZgBa0$8hnIvealPq0?idjE*mlyo z+S1cpVj5&@WU%Oc+D)1%YQHjCVdj0Z%fZ=!ZQ@notUuC!!*|SA;ydrp6b2w_UxqBm z4Of1x&eF!~HyT@+cUq=fUqHkPm+iLom}R@!U@Fk-fM`EMnZykR*5L*J0>pglgxG1< z%MrJu=xD+Iykj|!vnnz^N>`+{O=*=}mS{{`lN6uSGvz>Pak?hwV*cpj=PI|uBG%!^(cZHap^_RYA3F^1^0$ZuSiT|2@K+y8}5Dota7fHc|gwf-aB zdhJh|z3TT=*A$)kdfBjGcS$RJ=Npdo`oGoxQU6(ek!POoRiSqvPj-}Vt?s1jW&F~z z*Ty?qJDa&uT!&mat{l#Nl`aNz@^yR1p;qx57*#9;kG^HD^=sE&}ZM?c4(CpQ! zwLox8MqK(Mvi#sn(gyLNe~|A-Z#&O*kJ{VSSLNR*?g&=O6P4d;YYiocs_~q&UHCr{ zE27p!J&alrxjtfG_!YZ}rkJl|QowZGc<;tSoh;ASAbIkgf z>Cx^;Yeca#J1l5>&ia9Au%U;pwPw0%jKT->wFzQ5c#*1qEsrc2Sx}i%l{qoJed?U# z$4NPfhQyB&xCGym6Hod)85Z9m@ln#lw8)Gjxo;P3u26df|L^<hkQr0E+ zNgSB))RVNIXyP=7)Gd^2_{^YLy6*d?ex7?} z?cCbNc>HeXO%`^@PAX^W>P%*P3)kzBhL~4leKE~r&qu!=^<3m^*Bj0$VUulI+cL{m z%RWTbS%~Nt9gS!7^RyqT4seqKKl!xo>s5}j&kDcF9gsCKqbO}tsw#DN%Jt+kDV>sU zC9O@|^yK^a!H;DL*AfFsBQo}7&M6eiyVqY1u2H^c8fG65xjwFG<1Wwq(YQ~eZn1MB zRykv={SBkE+msf?Y3?JeBcCdd;BAUoN)a)^ZYbjUDV$q&F>ptS@Hg|;)<5e>ufOM+ z=9?$R$#VE^8mIn}skyDY<5758MDNHQkryNOMZ6X0D9iH@I z;{3;HPj)@t_xPiPi}3@~MrZw4R9n5(`yOAXm26){X2xinI+}KFQru)>Y>%i2XFK!z zh6!qe@@0OW?EB!Az}PGFQ{X@Va@iz0`5X*(E$SJS+RcZTdjA^ zn~b08CaN#-z!Vl^z0SI-%6*6!@ls(~US)PlX3q>EePsHfG-cYQA_ z?=IUGd>&6FH-Qv2ic93?b1uFgHwM@a+koC-$LQDvm^e8Y<;MVJzn$U~aAxE5ZB0+i zIo5RhqOi-3fzG|o&8|k_ABPW#I3B({d{ub1>#FO#bG&1?-Dz!O8l#`7nW<iOt1US{;OkE?A6$Dae0lhqqhO8{yE$GMzvO{9L_D1FF_ozivdIMsO&|| zy1(N0C>{WtvXpnz8>qlSajUpf=pt?wCW-36DA{nueAP(ZK|{HTvyHQR95s%!&Ze%z zj@8aNj?Wz@oHpkeSBkUB*(>}HaMsJU+acLkSoFsJ+R2JGvK~UdJFX_K(p@Sq@f8j& z=u+?QReA(TS?~3mh6y*sy z%d;kA9nGjtotT{U_>+fk-amDJ*29hu+a|V2zMgfj&{=lQJ4HO9nr)79oyBd^TY<~cBFxgWrM`H}Gb2dP{Z|5mg01n) zk^&UNL40#n7tJwUW5WT%9^)sbG|M^Lr-;<@PFOJPZRZ!R!4cCV--*nOycqd&DY}|JA+IE7Th{Qb`5D90 zRwti%5_~xNe&M}A54Df@#N^~XS(U}_RSLor`6%5&`%K5Yn3rQtM!y?l3NLb&+O#Ib zCsGdNfWZPBrm507F(R;Bni%XWyAiCAEyG)^Nx|*`KF}Y}Rof+H@Lkz4`6i&(g{u!} zbM&v6PMF(Um)OqP&xY*_dk_}uh=g@5a#_N+L}(%`k>5r}MOIh%OR}bCj7=?1eC6@72Z{IIzIX6p z?c;XI=FE(IRh7)Una|TLw2h9aiEQ8Kv)Hd<{)}qtxNnWwMIU&_w z?w>8L5rO-McOIu?i-U=Q>cAslJ~jm!;)b9~{wB8%*lv3D4sA8?DL0yXTkcyI+sA}u zI0iZHIyO3gbQXr+kGK$dJ}NU(7nKuv7}3DYuBKrptO>@sx>D6QTyCI=(9Ltc?)~cY zieTB{;@66%7qrQLHs6!GCwFO%Dr<21cgf4*;~xL%*dLYm6#TjGALofqxla<-kd!E6S!H@n}G*{6Yq}XLaxw5#2diCGk6biS$tQl7bC>BQZ#UR{tW&u zdxgKRT&8|m7p_k>-ZvkxG_!B9U9!&(J856(=4>I-^P=c0`PF zjddKhzhyaU3`b8A#=jSQNqocims?%?52AZdDgC-Aws1iHuelp@$K>?QR%IE|7p9y` z>=3VsA0EFXp)mPQ+Pd6!CBIZR@(aPy8qSm!mf~t0F(9&o^E+n~+gVd9<21aNxG!&q zcPGdELxfS@8{T$)hi`zG?cXdkM+>BgRj)`Zf&&8z?qx-~a=hkGR|hO&$BmjoDEy=Tota5sEoWG`AlTn$VL${5ytRk&NBNx%ge?m+HzG(etYnQ zQ0C#>Rn^f|@0TZ(JSg(yugo8nC*}^w`#R_4>_M3?rj{jX69+u0OsGf*q@2#WmTN8F zS^dcW1$S1{(>m8S-t~_2uydGmhqbwFou1ddp>D}_vImkaU255iFK z54?-*i@$k36edWQq_e?lydj>Wn5q0iovG^yWa*8@-lk_QjjcWH@iqr$fz~jybEV@} zI2V35e0JoFh>lUZNHM(1WpdoGU$(SBL~);LFz*aD6PtV6xQ|r3Dn2f4UNo{`Ti&7E z1G%+1V{>O_3wR27G4VnBanee*rl=pl88SfPTO8-S)8|fFZ4{*Zfis!^x;+N8I!2z<* zxbOMF%H5hbwObIgb&T;(b64xPHnU?|Sg!Lg=kKoekr5GG^oYojk*cVy@Gl~~uF=kW z4$f}29x!#+AJQ0=!{w``-M&ulw`#PNdrR*YFE6OiJDWE=H!|<=4(n1)D(CtWX&|)3~$=Y!cMxL4gbNh&wd@z)gGyXs!{w^AZUH+ z8!oi*S-oHRAJz9vBiR67~T@YaEcWt+MspW5pKbhuW#y`?_I>=`h8b zVmoD@;+TN3t#kP2ct)NbnGim)loZ@0iLBBeY&+31Kk3!vuhM=@-ILoc-&z{uzFdEh z|3$q|eE`03!aMnV!KFNV z{+j%=u%x)`Md|xf7AO6c7@hJ`%F;}r#uvU{^;fM?K7jA2-)&!NEsuzGXzW|=dpwu)w1iQ~ucW-0 z`9p-&1EKTzA97QLbuOw+3+3W{(8-$5PkNF?Y!M=U+k<4 zbGnX(DIfh20C*t?iChgwHjHe39L0G`|33JZT()+k_?S_v-y zD*yMsR|FoLQkddb3x1%exA%VKG5ZAHS?P@gNUd zUbmh>ywEwW<`JIA?_##coQb_0_d{&Y*pAWih=VS#eTsFUsaE@*`Vc=^_M_C*zaM|+ zp0Da(&Xu+*ez#C6TvTwYpnHDX+@kC=8C-f)%JSsy$qQ1~WRA#LSd>*J*8J)JKG;e9 ztIlW~Z@Fn2X(=;~(r?q=z&|T{Wle)mq$26eFfjJB!8g$g;;;W?SI+*tLz`2t~JwT@kl5 zZh2gBOm37h>bmP+$2nVxagBbh>IkovwGrp|+Pk;a46R&Vo>)4fWL$}+cv|tiqSS&m z1%{jpnIEMVChbbfPy8SyD(ynnnt~SPh3+jv5tpL2>w8)5n2%eJm>V0$>P9JtD-Owi z4{Q;)p|8yJIfQrp)BPX%PxuVN0)G_R>_zWxZ;rRKcNZe0ca+q4oN6pIZ?FuuA9MWUGDXQ_vSW>nuEnj73&eaCJtk^oxZTyy{(&XYI9Pj9 z{SLoNb{)}S&ECm%Eo!<{9WFmuzM-^V>H8&ZibobbpT8ofAiW~RnRF|mJ|QYeO#K9t zkxk`wb;E^z+*)-X!xM9J%Q5Q%Q$OR^y7{X4ihyiq;JjoK)xsX(d7r_5-M7;#6OMVC z_=fnpeC(tcTg>pN!tiy@N%j%eVWyb|qqe_lG#@A17zh{s@wRmTSbe5) zUD=A#`^DkKuNS8jPA=Sz&I*7~0l?)rZ6?e|aht^}&6=xODj;#uc;*5jz}?OoxU zC1eE_%9irks$QC<`dNkrhM$eAjo%qF5YsZ=G{_t@kFj+MyY5^VaXuC)-+SyZJ+Gp&Ecpj}43;7Z9)WG6^Q~X4jD17Y?_$T;Z^`(0E zd)j%{*IVl^*QL9o>YZM>a9wOAo6ikWcF~N61uZciGZY%87#HLFrLlwY8FRdOzBK`3 zSo4T`k-n&RV`s*E67zXmew|{ zno)kfB)Mb-B69yxw zz?fv3XsFU|R>kqL@*+tTC;R97FXJ8kWxvti2YqRgN9!$i*SOCDQ0rRVmD;tQJH90G zh&)0uNOM)!#L&!i32`1C8b<1?bnA3`5KXG7v4?4sbzoSSt3y;$v@Y&uoF;B^YSE`!)4k`p4PL^A41Bs_I{VPWqU8MpLODW4vkl$++3DRy$YKi7O56 zlm-ir{9gkt`<(Y%Z(IEHmf+dqdEY(GEg}-kz}oVf;qE*2?+DR>iQFk=k|s~T*qCa3 z)ilC5-q73d9pb;4_3s+m7(X|UwZ0wpp0i{4vdB@9C!#JzwvPNJyo2jZ*!Ol!SgeBO zn)wDI+5xYZs|15GglNT;#j!{<6kbak+d&SxMd(shRik2NbWb`l7y0_*I^(n5tQ-o33A|KdXsVpWwgXjIs|fbDM>Kd{<*k z^Lyicy*wXzirw4m0(I}y+^FeTGrjhXdkx0v7lRtEy~?I-t+yd+QEQ{!uu@0A!b&Lt?9=0ef&ptZrI_9Qh>@l`9%crJ# zy`br)8o=ENP7*eFhr2J=d|ExBN>+8dvPaeAN=M}vWq%Ym%HNhfJx!5PkoY7qHRa{> zpK~`B&8VF3HVLC;w-w#h`?WhY4(+q*=atVWyutahTVhz?iMUu)NTY>cgfIPX_%vPv zTHW(Cma1o};;J;&8P$#5{k*%yo^r&_QF}E1>TV<6jK$E&ummyoKG1J8>@(Gxr(3OI zcO9|eLnFsUOpS~Sf7YdU9=Ff8X{=|>!!6q^e^`&>jcaem7(B&&Y(HwvHX8LmsP1xo zf?I_4-eh-8?c$m*YTmEb)!1wLSGTS{QW0M=wXkpYth7Ip4n9eEqD`2dG&6Hh?toHX zb->d(xSFp~pVM^E3F=(UPQ|CZUA_ru-HW88z)#}Vz)In+cnkCF4*t*U2iFg(-CGk? zqo}!B{ax+XbsN1Mh1S6-++gMJ>Swf#5LqzY(Adz$P^0_Q;5AG#PqfUl4!2*mw|7o< zo^Y*pCOD3U^{|h&53s3i@zy2Qt9YiK>=^DG>HOUxfR`TD7tAg77d6`ykL7dV9@|#9 z?A`7uboX?RsQaPzg__jrYn44Jc9)JWipt-Ubw6!+($$2)@vRcqCRe3T%KfnTnaca_ zE<#dpp>l}wZ_TraK>w3!km4RUUA8;e6uvwbNesLzW{XwAG=H4`C(pa}r)rC8=2kDS z8DH~no#-y}Ii+UuZpz-8zjb>Mky_CIh6r9?8TK2-8;%)0Caq#I5fv!W z+spf#XS*l1p09V*-K+VfvVD1F@!W!~ITh*OCf|%Ne?01O`-Evp(V63OpDW!|T~aSf zKsQy4Rer4Qs%oZ6R$z4Db_Pxae+gs;l)zDMEV~{o33QWw5a0AY=bh_bQ@g1)P&2)5 zUj2TrN$e@}a0k`PwNArw<7VS+V^2gMT&CZrpP`R6rWqp;>2y@suZ~Y$9#^9ChO<-H zBl`yHNc_vP7yq^XW*TNWW!-0=Wxp17B&?at7--#W-Dm~I&IwJp`d6c^-G0YOX=K7~Jx zM%d=>int6*@Sof7?yqY{RliqpsO+VZSw*(|u{oaySD?~Il`y-_=d0XWI&1(HbV~VM*b+YA&rLDP%>3!pT<7B*@?Pk7VQQ0bN=CD3^ ztG38K&i1RNhxupYw}x)|*YtkfQ2jW=2gc53(fqIZ2Xk|Cit!smp?1D{fa0S3aByTyx>Gq_1 z!jY8TStAO{%S&p@g#`I?>Zt~UsmQ$5w8VH$pQ9bA{ziF=TM3M1US278APPuspjYr; z;9D;c8exvMy8edup7&#)AZ(PJ@=MB3)JybTj9(j%nuDe&^P8r5Mw4MHV!uy&uD91Dz=oqQ~X|GXMc7hA6tJizi3W1 zE;Vd0*bUY{sqGm!~y?D ze^K|E?ttcbb+Ynz{x9xV`5M`8!BWHvl7LG8k))AYi1+-*edoL-o<*L4^?$gp)ZVC` zRXL-4LFv5WnFalGqBCAkem`OQli804Jei*OajKXTQ9P>pgm1oVq-Kesm35qLkUh&X z-Q3z(tLdysg4bfP{6|?77tIZYFH>jkJ9sJn9y~AY6vz2oUKyS<$NGNv!TT(*Pd;Ax zy!HouqH&mcqm@S*W%qr%+Mw8nVVkdC?jOtjMXxVQbu8+~KR~5pui{pNmpOUHM8L~q_8vhf%v|9qL zr5d5YZ^zp`x$lUtnb1pmEc=-suR5vK8?KusSz@jEmK5`T(=p8Q&gxEUH)x79HJVP^ za9tz)8vS*{4OsMi!&UIx2ocerQ;$}iQQlF8t4!(=^%hN-ZntiheuUnq`&;`JVn~0D z7zbMJXW6yj4fv~0hDT>#Ng4PWJ^X1PlFyO)ikuMb`^t0HJ*uupP3J1U!dP~$WODJ< z!sNU!a%N>bPJK1GPx9X6>Xg6Jle6X&4J_YO|DL!Jz983($Lykg3g&a$t!psz>7u#7 z@8#6+HNP3y9Pq$P_+#L7$H?D-Ut}%3V>=63zB9fpzAFDN;i9xJ=u-ToQsE7z(-ez$ zIC87Yl3`klD9qWKVoiy9i2Aa67h?0q>IUjt8r+5s;CqdJy>7F%pZb()3SvV|Q#_}5 zTM>iEjAfdhx^ud5`g{6n{SIh*2Yekq$1_S-K25et);{=oU<%%0F9nW&Q((zIl&VEU zJ@LQe>*yU_Us~6y_KWJ#Rhui4%JiirMLQ53c~#Ebtk=_yri@6oq%KHXnXxX%Qn0SP zOHGP@nQXrDW8F4mPuo=6i}vrV36>X33-vBUFEuHf@Ta+OAi)19Ys@W2+$K;FI9b+#}^T>Vvv>V8eY(CevWkr^c`Kz4UXndD=qFX-%o- z8BKTHNZmTU!O+?;!_WfJO&$8Tbak43nvUwvRXW7t=%@S_F%ZMmuc+6mGu2h<^@y2e z(_BDgl?;U!KEpchk~{*Qlc(Xy{t2+)FUm&Czn4|vpP@5|o{@s*tA!q=`%!IQL=V(d zE-6nd-Cy!UQCNM{SXot;UDCxtd~Cg<`kjU4AoS_%z|h@L$6-xgV#2?so)U30R~?4Wl?Gv={qG zZw1H7b%-%FK>fV#0AgvHOn(?NjLnTt44h%Seu_@3y`#CNd7#;>-JuU4q@ly53ZxmUP% z5Wk}Y=?dEE$tKO)%R@T0Bc*%;Q`vq6??&Ms} znwIfN`dg{?wArak(w1cQ&Hk~FFB?|V!`Cj*S!vX{O}%Vg!(PKI;BZ(6+d500Fd<)TAO1T|~Q+EsTo%CD}`DVn-S{6uaX2!ev(fTLa_q0tBi}yM0ExhM1OMOBQ9*7Mx{BZdIi2= zZSZgYJ8}i0%e*CD#O3q3inGdds_)gaH9u+YXlm6Js(N^tE?4yC8*^6q5_r1f*%`R` znX+_X>DMBbLoQl|N%~ocL)7D6>tArstxc}pS=FM_Tv1m#qvU$gm4X-ZW@fL-IFvRu z^-AiXwA1O9thTwGi?>vK=-w-A#sa0;I??nNo{zG_ezp&`36_`4olITz5!&w|iQR~9 zv|Y)=7vwg77@i@qyny&X0qGU-8^6lm%eTY3%6r;7-QQVk7hE8p!@sCns#&i)q^~mE zG*)9ibwKaY-PfMftkV=A>iOU5_cVUZVy&coUAJ8qsVmf~wOQ)RsyWI$-p;jwZ9gw7 z!a9qO5$ka+r&WBaOj2!7b7)Oj8l`reW}teOYN~Q6A}v*8^jQIK>K*VdEe~*Tl@&INdA&} zb5FD`G)F-a*=9|+NsG$l*brtGW>^DtJkaELfpzZ+(7ww=yx09 z!+j|GME)gL$N#RZP!(!E*FLNJM%Pa#g05Z|`?o1~DrWId;Bg(sHA3{cFnH4YF6>(XH zVYSLCcn_VzdVm*rCA?~R?oC;B;F6Reobk`}wSpJ(Xi&p`xK6bYEny!28f5;H zY!g|E1rv`V7!W~35kX0k6bTYkf(i->h=?Q+ zL9*l}Y|cBoGt>S3&i2*cEaLOL@Av0-T|Im4Om}tlsj9n9Rh_EtI=3gAH}6Nnfzlk7 z!6GFI=95~=IO_?sMdVUwKAN(#;N|>oc|CGp&$%I|d-ko_E3;~6&CD3fXi_aDeC4G} z9WS;&ciY*}=>umH&b@j5uA~)LZcbNow-@%dLY{Fkg{1~3`pdslzM%Yua*N9ToH(NN zPqE$n{f)PHH}9yjk-ZxA+50kBeTn@cA5$yywII838ZbW^9Da*;){}!h3SmGG&4}D* zbyxR!J~Y0G=@t7<+yZ7`^-8ZVH7#y&?5vp5ywf&XKj&TP`2f!5Rh|{zLa$+bVQewl z_|6!Y43AMwZ|>a=H|inSHd?ToXoFH#9mhN1b@lI!Uwj>6ss;uIG6N3;Jb^3zi@x8D zZo1|@p{d%d>QvfI8+HpV1JfzIpWjefV0~r23kTUG-i|z6&^`ZX?oBykv!`S!S(`E* z&zO|HFKt2U<&?cwCtbfZ{DZtdLMnaevk30aY|pNAM!Tg&CiwaP>xX7GtU~PltL$jkJR$i)sN^Q zqY|u@45a>3{#E`>zM8&PMh)HT^>`AsR_YTl!L+n4np@15&5l_1zeMimxo%Lz9*SA(=p#OtT`Hy-r;1RLU9|8oB@{|;Zs&<&rS=*`tKVRpG&nE|I$4%|0i zna`MgVaVAO=@VHKelv7q(W&6rf|Gfja=*xKlBH$p8GY0LOlz6;dTOuKqbaRZ!pV78 zJMwPO;wyu%th@R`^5K+?>8mrdb7ur=M`BqW939g*E;eCWVuiAe%SFm8DswL3rug6D zcE;@X-Kcl)^j7atR#?KwRf}DBkHT|wTKSGP|EYS5I*)y@r{U?jJ#sQUf~R>S+4D3u z^0E1ubyThDX`;_HI{1H&85~#}TNGOmyFE4#J2uz^CJ5iYQvWH zkmn6JuU_(0V~@*H?O83NPJw4=H^!*#gH%clMTTRBlyo zV@F;uwFkRE_rTcq4tsA-h1;^vXjZsoq__Ev)l_ANnrEJVm+ve8rkG`cP~bw~O5lw^ z!$7l`R@jaY7u%{?n8%ST%2kz4q>}VVb%h3-? z7kG}(@I35&ZwK9gmA*U{!bx_dZ1LT~>fA-R75jPzz$>{~tqeP-MNc{k{-iDN;@x2$ zh;)jK4G)4nu6AJsW_ttj%H_u9T+OPL^+e{Jj4m0w(wC**3T8>ET~p7ebW6!ij!o&9 zyd-62YTfh`nNMWjoWCbHIr5uQ-P^!_HfCa}xcC+c4=28zFgYPu`Wu+z+sD-PkJd+c z)3m#3algQqdM7(8y=s7cTpz3DVLS7~Mz)AukloBWksa8DOG3j!^FnjNF_9O+tB>mU zyzA{|Oz@@qkHxeNe2(?IiZOcuvx7K)H{VpFzA;JP>z(PX>}~3Oj2&gAys5CNt?~>7 zy$@hioJD&&2WMC%xZToW3!TcY$uHHxFt*nACVN-t{fvC0vv09)vTqpgZp<+L)DyvL zglDbxrE1V$Wzb*!Y>oq~-RuNRh)hMhDuj9$y*^r^e3fnjbFelw`}aQF-LrJjbC8Ud|QJto*hmzsqL!TIT)_*(0+x5Z-aV-nM{9c zuz7+xY+<-mxIRp;bHmdjkC}H{t&}V3c+WZS2;)N^Jby8t$2=cs5D3MLkNL?z$iK+9 z%&2Ab&_83x)ke=6SPSpzv_p0U9=Y&17g^&0wQ@AsaoT21XixG~!+GubbC5I(QB>9@S*M3~s- zhZ}`Qg`UTP=@qP3&@2Ceyu{qMa-PYK%N~|hEo)8YSDD>26Elxy^v}pn|2chMdb{-6 z8E4b~$T*WZ2}agO^OeHmMIV`Sl|7!j7`5sKTEvcw+XFx6GjT7(E(|;uvz=Ma!}?+` zBRX%AS7gUrV^}CxYx}jEVfX$;o2|`O2dO=k!PW|MCVY`?!jFVgLrQp8IK+D_O|3iG z+p|+^?_H|DYHWay_zZS(51uGq^8e`f`&arJ)8~{kKGBQ3N4(>_BWR!T%q&;2zi*JI zv&R6nYTA6YqPmGO|0*nAL)m#akX^}D*je0*app7a8Bas@$92>{W+eR;-s4Z0wJw3} zv@5Gkt+Y?o0(K~8vTu6|Y;-5+Wh=mJ**@}m_)zFj(eA<%u+pdJXXcr?cQHC^mop#8JejdJV`oOI%s(=>;e6np+z;|61$%_HM#z&^RiEQ~ zGbR`a$G#i4l6O$mxY4l@_6aTU-D&KmH~o$Gh{NnRmA-kVXA+umpQpPgPpbrf_`S+v za9as)=pfj4*M>HSbaqRQjNEG0wmwkaQ%`DbypQO0jTqldUlsqS{+a&q{)YaezL$Ls z87J>x9^Onp#vZVF@MV_rZuP8Yf9K<#9GEeyXm6-J)e~5tx4>>YpB>&C*=K(v_RMi* zl6ogI+WxR<-sm0eo#Z|1{m}b~w~;rA`%8JwF>}#q4UOql{#0hMx7)9L3ya?wv!OXS z@&&6J>v;M&E4ZRySN^HIGI`^3i*knL+?*4@6+}k%q3rJ2eY1134ra~HGBbB(j>_DX zd2?3fY%_aZPMf?(3hpY53zs%KE3>tc-km(-&W(AMr$@D8j|P4W+!d%C^S*z8udk7- z=TW0$ysxu&vkPpX&wD>-PwPofJ9h4!g{$!vrMcC`92Uugo9}k^&2|khXT*%-$YjsIJ!Vjs=t&QRl$0OcNZez%HbPBwTo^mtQ>r<;CO!d{MYhwb0_7# znEOC(z1;MiuW}aWbk6bS9LatnyGr)2S(!Z7>6ZOmPHIkK-p+ihpls2^(5A?@*3)W# z&sp!U`Z!-h|4z8fjhIrb-R3h?-*$A;_^PA55Q+h3ZyLSuku2kUdn?3B? z9zbthM;QQf@+5X)$D2u!zaok`+&pB?fv;@0(px>vyzxQL6;F5XF>kK7v%XwE2Tpa2 zSZc&)#PNHBk)+SmU(;*rmGt%Ox830x;5knZR$2Q$nxLhs`>=9%!^}Bfy@&nP8(^e7&(70T<}NsO z?+(uhZ7uSG+>L~nFL&(N#0dv>yAyQ61|KbFmpV3($!-c*fINIjJ<0QA2H_ z?bBjCuVI<4!=BK+&%&twm3NbOrMHS+R&Sv{rgzY9L#uPV-+G67t9oC>3i*J#htknnU)tJ*J!ArbPo1>Knz5CP*Y^fE@ z-d=(e_AP9yEvyjjRjScaTEH2(6mJ4vIIjoNE2V2SvBkEt@439UZt*Vew%%6W`tTL6 z@=Wv$@wCJ@!5{Q*T^OU7`hBi0 zP>w2vN-R98S6H7qhF6Lm?DTG_oQ9pTgLND}(EH7wBL9vwVWs{^_%-&be-)Y-8WO4! zI#^Ve$2I~i3D%h5vkiRf*LEhrL@p*mF!)E!5^2Zlc4}KmzU3h2F;LyZy zeK=h|fm?I4)dMEniEyZAtH)GJYl+P{irsNbyw&w>%;|2&VM|-%vYwPNlP z8M}(;iKlo@z|#E&+|El_gM1v@G@vEpb7DUXo0HLiYWO0k4zKYmaM5mpE&B#-v9=69 z2U(0YT^Moxh4pbAGT zeJHcGmS|u-R`H$&xf%HNn97`c0{rKFSwm@ymy|eV7j63!xHvn(7JS~^3Cn05^H5}Y zWJsh&q*CO^@cQti@G$oK*AE{Ky&Y;6iVLl0ZMt^RFNH4_RxUgr+#FmSoFB{!J{t@M zV+*SmKE?Z(8;TAT-ORhmHF*m;7@ixMMStGaJYn7jUuP_W^qTrXSnh&yvaOJjE{Iid5*tcwB%OUxaaW)^Ck?SzrpDI0bV_ZGCOa8uZRcD zXF+bI>9I25W-T(`$A`oivy0i>JO|J0AgsJDV&ODg&)I*!+#NWD*SagEnFt@B>c6!(;fLOQXWp>AI*i>Va=>3@z-`rP>%g zh%D4zWW-y66}Xlw{j}xU+vw*I+Ik#J+g|OGT8GwIO)EtUEvKDQ4eb+k75!X0SbJ{< zy*wp`nR}5^g*oO`TE`Et%f8Hb-BmNfRDN!T-dk3)_XTkMNgQGSl?a_ zUUgv{_M7J;b1&fB+H-<^#Yi4^q&BU&OTm}(oWyZ-1zHKBmnFuqtS2>CAo{iR6zP+$6KTONH$=Z*Hl>KImb%p0~ zoA6Td4j3Bn|Bf=pQ_f}PVf3&$JL@+V`!Jcu++qp-XI7f?&DqS%-Zd8yZ?L(*++`j$ zKPC1)GtZ2*Ug5kWamG+5o6K*Kf8OkcKadXe2;K3E)EaKpkH}G~b=tZEi=ZXXMtUoc z(JJ1cm;Qvct@ZRRD``Dz7}12cy|fy{%LgxLD=pBDSahR`+E%@h=SxlTsgtC1gYCJk znyBV0!+F;7h3QqW+>Dwia_7&&EI0B4n?$X7FCf6#U}M z#EZz=%wpDHQHY!^c*)v;rygO}y$Nf(FMVulu!^B(b&#rrwhh4wcm>~4{g|KDRG#9f z7a^VS7gdw*DlqomL9gAkn7?}z*Z;-+kI@r7gioWV@MAKXb+YmF;vcgPG!41$(emff z3w*(Nyc|pQ-~9Dg`Y_AtPTF1ZMb#JF?!`Y@TfB!=WVNgD2$5oZBRypM`sb8ixc2UzBs}#B1#|{|6GOeoE2Mxi{!mmv`P&ZjYF<^0Z~=XD__JP2hh77(WOey};;h>f&Mia<$^9Gg!rAyTvGv;)Saj9%2-H<{Y#J z(GnXezcXtHqIKzLT{=FBg4D3iuKptO5CaExX`Y=Fk}qM`EJu1bfOiF?#IV-wCxynG zJq7QwaQof?%8A6u1<4?#J&eRGJgt3)%nKkeAMM(PCM|}Kc{Rv<$hUBc?_>2^zNdr9 zO7gJ{mg?pcM_6(jQtyqxs0H!bppp0ETdo~i&;q-)Gv&CG>$R00ly4vyK1H}NduEYN z4NBCInz1O+x3r~g^cng5>gZe$pLQ2`+E$%so3)g^)O3*iHbEy-tjfx9!s{qUtY*|; zMV`y0v(|8)^}+zwiAugNAxZp*#Zwo#pmdaStw-`Pq)NG2hsLMTpS08;xqgb)n@L?< zr3Qayo*WOSd>Q2&I+O%DnY8$e^R#&equq-_B|<{DLwQgVXFb2GmIczQxKAnvO?3@jIDJ z3p+#GD75OJFNMXqERDA4j632V^D4c8)Rza8($R%{q@N|#4C?44>0JiDa@0l*ygv3t zBOjwrc$8;NgVEtGj2wBip3$tTjx#r73I3rB_e^HKFq`-`7iZOkqp_klHnnoG34 zL%XH)ldX5qsc~S_o-!@Mr`~kzzctJpeaMVwCc2KkBy2rNX_lgs6VTSDsOtmV)d#&A z47TT4=cq9a__gkp@-|vn#Z%$dIxg}b=OE~ zYqhc}s4eJG^UN^4!gGvQA5ovrQ7eh8=x3Qdm3BO_si_>ZdQb=2ud)znhT zTQ~CbdyU$feeJiXt(nt(V42D!e7b!^&+s|pYz20WY{o`DW>rmyGb;7b;f*;12^u3kLiC8Li ztgT8nR$xZZHrFaoU;~a-r}Hdrld_7wZMQNXtz1BVdM_)(kJESU6 zY8l?jPNT*2RhwCx%|FEZqVl%-0xkDX#>CZ(FG=)M4e9sN%vn4u+s10eUCJx8$5$Dx z-eLsG!Ok3|l(Oz)tb7dn>8QEV3M-dsvyFJ`=Oy~&Vf4BKI9o`s_aWBbMcz&O!D`J4 zMKap4*_uwTKUIB3ZO`a_)p{9gO~%o+*0b~(W3Yq1v+idFCxDbO_y{;^4PYc~PV0V) zGR>z4&c~jS`S**Yud{ZXMe96Ezf+1gJQgtm70@4dXIA$tBjhdEc@t=Tm6TT216Bw8 zQGTZMz|P#yQ`#?-FBrE5D|w6)Ypq{diJ47n8^{{KEsO^P>75Q+4Z&j+ed=X!HL!kH z(NjN(&9+e)%UTsaFtCR#+G;8K&(E0eZXvaO^!V>!cl<)%UIf0K(VO1XT_;L<2%XE~ z>On@~Kfz}XE&m91afs)f+wsTwBjfSyNNq&|mmd2ZwYia1h|R29uVm%_V@91^ z>u2t4tCUvvvWoN}{X}I-+m~6-BF46H^h3MpOZU)Qo}!QZ+WMS6xsV#J#;7=uHK0E9 z>kAmKeT*mH(WXD3%wKT*XT~9wQS@Hk->QxMQi{^-CdKENMJ;5kJIx4PLHPreU&n`N zUwm3Nww@>F-(ZbiW|VEjD6C>L?ZAd#i4`Oa6+5s(wos1U;I|A}qsdu+EbSSLG##kb zOtk1xEXh1-<0D4+mp~*Fz4?UkaxJ6xw~X5X?94%|Xgy25KSIkM0?QZqozJ{^IQI3o z*g@huHIw{xU}R}dTd#=K@)u)UeMZ}Ote|{skhWmklyUGceA!NB4)OyvD%Q>d#_cTn z%8QJPt4UA%-sX|(ZRB0*tT|S~UyM7q(b$MJ z1?%n%(mdD-Q_#HTj8MOl{_W_fpZ4=HGr+2h0Y%tWk5QIp)XKN`A{@?mI1GdrvliEl zUEXVu8b*pRzu$zWZN(374(+oWbD3|IRHZi0>RzGl<3N2Ev+|c{8#Or-J2e6Hcko`) zWNQBmxjxPtE-#>?BUoo>NAA+GRF-pn8U6ikG)2C*f%qU+9$qfi#LC2c8I4}Y-&^D| zo_4aHJZz_A3CaLMY8QJbK8u!J<^E3?!7qTVy#H2@Qq&{QSCE_w!U6h}0`&ekIA27< z7mQ0MLGcc3xMwND3+Uv%NbG_IB0ijBs6p}2EHjimq>G(;gw*F@-T#VKMLB0IA+etFBI3 zRmpQjG^rG+3hrgGqBfA%9rR1mH*G}zkF>Jce5a7oQd-65Sa?a;*fLvbSFD?D&@v7E zJ57BYrkv-os_Rm|`>-#YP@@OYz2VgH9kiio++CKo@+f^#E6N-IvmeR*MOOS8f{)Cw zx^tvs+5bX5Hqvjb0qx(p-j24~mzh{wW*03fLs@31HA&?~=C<$P$MbdN3H1Lo7Tjif zgCuaR%q*c3Gh1oL9{Sj1^z{@aDNoz&gf5Q6avp$OvFy)+-zoZyGU#nP^46HUvx=?V z@5sq9YQ@45t`15XdH#X+`aSx&pLE1ax%ksRPaHPnfqyCbsAlxEgJ`v)gO7tmb!s}5 z`Vrsd;`>}YvU^A&gIpyMnnb=l*j=@`w>7!z2&&>gx*W4DnL9~)YQgw)KNt)t?u(lf zGalU$wjc4AFFu>gaF#!uukI9%bS>CRw4cc+U4i8%SFX=R`EM6AA?b-2${#J_idnmoI7J^ z>C#`v6_+@Na-631B^Bo-mUMl^_zH>{lvx-L6f~p)xu{4w79m0KjRVO#)NNDh=T@*z zAf=1QKSOGTl*yi%S0p~}Xt32&>FLjtRvOa8Pj#51WI|-lbfx$>3FIy#-K5mE2YQ7Z zCNd9fi1yt?Da(-LGg|*zP-~a;c4zl zBA1tok zzg`W=r3e#VTj2r_FZ+@oo5M*w+ZPco zp7+J$zCmo68P_bnFAnVs6CP${5iJm@iTsNHdL0~uMIolROqIAxe7(n0Z&IrwlLw;3 zWmbqQ><&Rv5pDqS{VzvSdp5&GCOOI`Pr_j@yblqs2+}gUly6}{!Btjqnw*j+o7q9Q zGvwTZborJzQa{oXgb6?x%|++ss_=)%xOc^avPmA~U-)Gt+~y&Xm?9}sjD^&YXqe43BK0jfvU9D_ zUZm6#CORX1g|Ki)%}5Mk%CYOtMwg)C8n$cKmMCE&!REt}oJoz^GVO1vWBX$xEZ>qR z`InDhhonn;o`=N>y1JGA(=|0~szU}G)PoDey(8F`}8rS8In+N>NlmZCFGek27u zXHJR^r;i=WzHaM*NOwZ*V@X}|5&Z}Ui2MnUh#)GO>HLX)$QAoruGt~-ZR@>!+eZ@5 z{HB6s-sQE?WoeW6@3fy7Mg(>}wKYAD8@=FguQ;A+|$cqM+;y!M+xjinqOC33A|%|2Z~xjY!HzHYI)8S1bizfJ^avtJV%RO; zK9-nHT6PO48ON4qw{HoL$`xt$Ra>efP2xznd`s^kM|Lk_r(?@<`W!in#&_;@z&VS`vn7G(|ha3IE^a-)?RH zS-O*NM`|>+=sj{x{-Z6|zDwdp@3X_CojYOni0hm=zalGoEO*%-J1qJgJ#u11OB}u5 z$wO3T$@?V7a_;moj)d#)kES7Uq9rXEQ{p)Nr^G6ml5@YE;@_`1`G`t$;>)iSyJT4O zIC_utmeKH%M^5i5ah+qiE*7MdOZzA)&52u5rsPDzoqroS`^@=vB#L|qvp;g=lu|;X za-wH)^v|EDoPWCR)Q4OXxg~QfVfMM5KRb?mU;in2Uv%8LKHmQnB695ik`nE^>=0*^ z6uom|J5v9ttR?0AJ$^~Kj*MtnNi<4IiN?78o@l)5kDPkE_AUqG=x_AspUUHCMoE3L zCE9x7Tz68rR)&3cZJ2Ynq$^|MfB)F!5(!Rh39)s@KC<&)Qk$IgoL?u8a&Cv&d3Q=+ z=fJsA^30Ka?S0Of9rN1w|LI!E`YD-LxjTAI&ZMQ6Y?&qdJh?6($?LUu$=S6b#Sx3Y zN@`X#^uHZNk-9dPoq~X?D&^UM+e2pKIs(G^HM$ z|7aR^f9RAyDqD{1*@hiMj{o@+jVbT1*zx4ox!b literal 0 HcmV?d00001 diff --git a/__tests__/html/speech.customAudioConfig.html b/__tests__/html/speech.customAudioConfig.html new file mode 100644 index 0000000000..346f025893 --- /dev/null +++ b/__tests__/html/speech.customAudioConfig.html @@ -0,0 +1,74 @@ + + + + + + + + + +

+ + + diff --git a/packages/bundle/src/createCognitiveServicesSpeechServicesPonyfillFactory.ts b/packages/bundle/src/createCognitiveServicesSpeechServicesPonyfillFactory.ts index b170c4f284..ff72c4441c 100644 --- a/packages/bundle/src/createCognitiveServicesSpeechServicesPonyfillFactory.ts +++ b/packages/bundle/src/createCognitiveServicesSpeechServicesPonyfillFactory.ts @@ -5,7 +5,7 @@ import createPonyfill from 'web-speech-cognitive-services/lib/SpeechServices'; import CognitiveServicesAudioOutputFormat from './types/CognitiveServicesAudioOutputFormat'; import CognitiveServicesCredentials from './types/CognitiveServicesCredentials'; import CognitiveServicesTextNormalization from './types/CognitiveServicesTextNormalization'; -import createMicrophoneAudioConfig from './speech/createMicrophoneAudioConfig'; +import createMicrophoneAudioConfigAndAudioContext from './speech/createMicrophoneAudioConfigAndAudioContext'; export default function createCognitiveServicesSpeechServicesPonyfillFactory({ audioConfig, @@ -47,13 +47,11 @@ export default function createCognitiveServicesSpeechServicesPonyfillFactory({ 'botframework-webchat: "audioConfig" and "audioContext" cannot be set at the same time; ignoring "audioContext" for speech recognition.' ); } else { - const result = createMicrophoneAudioConfig({ + ({ audioConfig, audioContext } = createMicrophoneAudioConfigAndAudioContext({ audioContext, audioInputDeviceId, enableTelemetry - }); - - ({ audioConfig, audioContext } = result); + })); } return ({ referenceGrammarID } = {}) => { @@ -70,7 +68,7 @@ export default function createCognitiveServicesSpeechServicesPonyfillFactory({ }); return { - resumeAudioContext: () => audioContext.state === 'suspended' && audioContext.resume(), + resumeAudioContext: () => audioContext && audioContext.state === 'suspended' && audioContext.resume(), SpeechGrammarList, SpeechRecognition, speechSynthesis, diff --git a/packages/bundle/src/createDirectLineSpeechAdapters.ts b/packages/bundle/src/createDirectLineSpeechAdapters.ts index 7a75d274de..62efd16205 100644 --- a/packages/bundle/src/createDirectLineSpeechAdapters.ts +++ b/packages/bundle/src/createDirectLineSpeechAdapters.ts @@ -6,7 +6,7 @@ import { WebSpeechPonyfill } from 'botframework-webchat-api'; import CognitiveServicesAudioOutputFormat from './types/CognitiveServicesAudioOutputFormat'; import CognitiveServicesCredentials from './types/CognitiveServicesCredentials'; import CognitiveServicesTextNormalization from './types/CognitiveServicesTextNormalization'; -import createMicrophoneAudioConfig from './speech/createMicrophoneAudioConfig'; +import createMicrophoneAudioConfigAndAudioContext from './speech/createMicrophoneAudioConfigAndAudioContext'; const DEFAULT_LANGUAGE = 'en-US'; @@ -54,13 +54,11 @@ export default function createDirectLineSpeechAdapters({ 'botframework-webchat: "audioConfig" and "audioContext" cannot be set at the same time; ignoring "audioContext" for speech recognition.' ); } else { - const result = createMicrophoneAudioConfig({ + ({ audioConfig, audioContext } = createMicrophoneAudioConfigAndAudioContext({ audioContext, audioInputDeviceId, enableTelemetry - }); - - ({ audioConfig, audioContext } = result); + })); } return createAdapters({ diff --git a/packages/bundle/src/speech/CustomAudioInputStream.ts b/packages/bundle/src/speech/CustomAudioInputStream.ts index ee7caa2b91..9bb6a00a00 100644 --- a/packages/bundle/src/speech/CustomAudioInputStream.ts +++ b/packages/bundle/src/speech/CustomAudioInputStream.ts @@ -1,5 +1,3 @@ -// TODO: [P2] #XXX We should export this type of AudioInputStream to allow web developers to bring in their own microphone. -// For example, it should enable React Native devs to bring in their microphone implementation and use Cognitive Services Speech Services. import { AudioInputStream } from 'microsoft-cognitiveservices-speech-sdk'; import { @@ -82,7 +80,7 @@ const SYMBOL_OPTIONS = Symbol('options'); // They are: attach() and turnOff(). // Others are not used, including: blob(), close(), detach(), turnOn(). abstract class CustomAudioInputStream extends AudioInputStream { - constructor(options: Options) { + constructor(options: Options = {}) { super(); const normalizedOptions: NormalizedOptions = { diff --git a/packages/bundle/src/speech/MicrophoneAudioInputStream.ts b/packages/bundle/src/speech/MicrophoneAudioInputStream.ts deleted file mode 100644 index 5263ac82c5..0000000000 --- a/packages/bundle/src/speech/MicrophoneAudioInputStream.ts +++ /dev/null @@ -1,139 +0,0 @@ -import { ChunkedArrayBufferStream } from 'microsoft-cognitiveservices-speech-sdk/distrib/lib/src/common/Exports'; -import { PcmRecorder } from 'microsoft-cognitiveservices-speech-sdk/distrib/lib/src/common.browser/Exports'; - -import bytesPerSample from './bytesPerSample'; -import CustomAudioInputStream, { AudioStreamNode, DeviceInfo, Format } from './CustomAudioInputStream'; -import getUserMedia from './getUserMedia'; - -// This is how often we are flushing audio buffer to the network. Modify this value will affect latency. -const DEFAULT_BUFFER_DURATION_IN_MS = 100; - -// PcmRecorder always downscale to 16000 Hz. We cannot use the dynamic value from MediaConstraints or MediaTrackSettings. -const PCM_RECORDER_HARDCODED_SETTINGS: MediaTrackSettings = Object.freeze({ - channelCount: 1, - sampleRate: 16000, - sampleSize: 16 -}); - -const PCM_RECORDER_HARDCODED_FORMAT: Format = Object.freeze({ - bitsPerSample: PCM_RECORDER_HARDCODED_SETTINGS.sampleSize, - channels: PCM_RECORDER_HARDCODED_SETTINGS.channelCount, - samplesPerSec: PCM_RECORDER_HARDCODED_SETTINGS.sampleRate -}); - -type MicrophoneAudioInputStreamOptions = { - /** Specifies the constraints for selecting an audio device. */ - audioConstraints?: true | MediaTrackConstraints; - - /** Specifies the `AudioContext` to use. This object must be primed and ready to use. */ - audioContext: AudioContext; - - /** Specifies the buffering delay on how often to flush audio data to network. Increasing the value will increase audio latency. Default is 100 ms. */ - bufferDurationInMS?: number; - - /** Specifies if telemetry data should be sent. If not specified, telemetry data will NOT be sent. */ - enableTelemetry?: true; - - /** Specifies the `AudioWorklet` URL for `PcmRecorder`. If not specified, will use script processor on UI thread instead. */ - pcmRecorderWorkletUrl?: string; -}; - -const SYMBOL_AUDIO_CONSTRAINTS = Symbol('audioConstraints'); -const SYMBOL_AUDIO_CONTEXT = Symbol('audioContext'); -const SYMBOL_BUFFER_DURATION_IN_MS = Symbol('bufferDurationInMS'); -const SYMBOL_ENABLE_TELEMETRY = Symbol('enableTelemetry'); -const SYMBOL_OUTPUT_STREAM = Symbol('outputStream'); -const SYMBOL_PCM_RECORDER = Symbol('pcmRecorder'); - -export default class MicrophoneAudioInputStream extends CustomAudioInputStream { - constructor(options: MicrophoneAudioInputStreamOptions) { - super({ debug: true }); - - const { audioConstraints, audioContext, bufferDurationInMS, pcmRecorderWorkletUrl } = options; - - this[SYMBOL_AUDIO_CONSTRAINTS] = audioConstraints === 'boolean' || audioConstraints; - this[SYMBOL_AUDIO_CONTEXT] = audioContext; - this[SYMBOL_BUFFER_DURATION_IN_MS] = bufferDurationInMS || DEFAULT_BUFFER_DURATION_IN_MS; - - const pcmRecorder = (this[SYMBOL_PCM_RECORDER] = new PcmRecorder()); - - pcmRecorderWorkletUrl && pcmRecorder.setWorkletUrl(pcmRecorderWorkletUrl); - } - - [SYMBOL_AUDIO_CONSTRAINTS]: true | MediaTrackConstraints; - [SYMBOL_AUDIO_CONTEXT]: AudioContext; - [SYMBOL_BUFFER_DURATION_IN_MS]: number; - [SYMBOL_ENABLE_TELEMETRY]?: true; - [SYMBOL_OUTPUT_STREAM]?: ChunkedArrayBufferStream; - [SYMBOL_PCM_RECORDER]?: PcmRecorder; - - async performAttach( - audioNodeId: string - ): Promise<{ - audioStreamNode: AudioStreamNode; - deviceInfo: DeviceInfo; - format: Format; - }> { - const { - [SYMBOL_AUDIO_CONTEXT]: audioContext, - [SYMBOL_BUFFER_DURATION_IN_MS]: bufferDurationInMS, - [SYMBOL_PCM_RECORDER]: pcmRecorder - } = this; - - // We need to get new MediaStream on every attach(). - // This is because PcmRecorder.releaseMediaResources() disconnected/stopped them. - const mediaStream = await getUserMedia({ audio: this[SYMBOL_AUDIO_CONSTRAINTS], video: false }); - - const [firstAudioTrack] = mediaStream.getAudioTracks(); - - if (!firstAudioTrack) { - throw new Error('No audio device is found.'); - } - - const outputStream = (this[SYMBOL_OUTPUT_STREAM] = new ChunkedArrayBufferStream( - // Speech SDK quirks: PcmRecorder hardcoded sample rate of 16000 Hz. - // eslint-disable-next-line no-magic-numbers - bytesPerSample(PCM_RECORDER_HARDCODED_SETTINGS) * ((bufferDurationInMS || DEFAULT_BUFFER_DURATION_IN_MS) / 1000), - audioNodeId - )); - - pcmRecorder.record(audioContext, mediaStream, outputStream); - - return { - audioStreamNode: { - // Speech SDK quirks: In SDK's original MicAudioSource implementation, it call turnOff() during detach(). - // That means, it call turnOff(), then detach(), then turnOff() again. Seems redundant. - // When using with Direct Line Speech, turnOff() is never called. - detach: (): Promise => { - // Speech SDK quirks: In SDK, it call outputStream.close() in turnOff() before outputStream.readEnded() in detach(). - // I think it make sense to call readEnded() before close(). - this[SYMBOL_OUTPUT_STREAM].readEnded(); - this[SYMBOL_OUTPUT_STREAM].close(); - - // PcmRecorder.releaseMediaResources() will disconnect/stop the MediaStream. - // We cannot use MediaStream again after turned off. - this[SYMBOL_PCM_RECORDER].releaseMediaResources(this[SYMBOL_AUDIO_CONTEXT]); - - // MediaStream will become inactive after all tracks are removed. - mediaStream.getTracks().forEach(track => mediaStream.removeTrack(track)); - - // ESLint: "return" is required by TypeScript - // eslint-disable-next-line no-useless-return - return; - }, - id: () => audioNodeId, - read: () => outputStream.read() - }, - deviceInfo: { - manufacturer: 'Bot Framework Web Chat', - model: this[SYMBOL_ENABLE_TELEMETRY] ? firstAudioTrack.label : '', - type: this[SYMBOL_ENABLE_TELEMETRY] ? 'Microphones' : 'Unknown' - }, - // Speech SDK quirks: PcmRecorder hardcoded sample rate of 16000 Hz. - // We cannot obtain this number other than looking at their source code. - // I.e. no getter property. - // PcmRecorder always downscale to 16000 Hz. We cannot use the dynamic value from MediaConstraints or MediaTrackSettings. - format: PCM_RECORDER_HARDCODED_FORMAT - }; - } -} diff --git a/packages/bundle/src/speech/createAudioConfig.ts b/packages/bundle/src/speech/createAudioConfig.ts new file mode 100644 index 0000000000..211a404c32 --- /dev/null +++ b/packages/bundle/src/speech/createAudioConfig.ts @@ -0,0 +1,57 @@ +// TODO: [P2] #XXX We should export this to allow web developers to bring in their own microphone. +// For example, it should enable React Native devs to bring in their microphone implementation and use Cognitive Services Speech Services. + +import { AudioConfig } from 'microsoft-cognitiveservices-speech-sdk'; + +import CustomAudioInputStream, { AudioStreamNode, DeviceInfo, Format } from './CustomAudioInputStream'; + +type AttachFunction = ( + audioNodeId: string +) => Promise<{ + audioStreamNode: AudioStreamNode; + deviceInfo: DeviceInfo; + format: Format; +}>; + +type TurnOffFunction = () => Promise; + +const SYMBOL_ATTACH = Symbol('attach'); +const SYMBOL_TURN_OFF = Symbol('turnOff'); + +type CreateAudioConfigOptions = { + attach: AttachFunction; + debug?: true; + turnOff?: TurnOffFunction; +}; + +class CreateAudioConfigAudioInputStream extends CustomAudioInputStream { + constructor({ attach, debug, turnOff }: CreateAudioConfigOptions) { + super({ debug }); + + this[SYMBOL_ATTACH] = attach; + this[SYMBOL_TURN_OFF] = turnOff; + } + + [SYMBOL_ATTACH]: AttachFunction; + [SYMBOL_TURN_OFF]: TurnOffFunction; + + protected performAttach( + audioNodeId: string + ): Promise<{ + audioStreamNode: AudioStreamNode; + deviceInfo: DeviceInfo; + format: Format; + }> { + return this[SYMBOL_ATTACH](audioNodeId); + } + + protected performTurnOff(): Promise { + const { [SYMBOL_TURN_OFF]: turnOff } = this; + + return turnOff && turnOff(); + } +} + +export default function createAudioConfig(options: CreateAudioConfigOptions) { + return AudioConfig.fromStreamInput(new CreateAudioConfigAudioInputStream(options)); +} diff --git a/packages/bundle/src/speech/createMicrophoneAudioConfig.ts b/packages/bundle/src/speech/createMicrophoneAudioConfig.ts deleted file mode 100644 index 62e2176518..0000000000 --- a/packages/bundle/src/speech/createMicrophoneAudioConfig.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { AudioConfig } from 'microsoft-cognitiveservices-speech-sdk'; - -import createAudioContext from './createAudioContext'; -import MicrophoneAudioInputStream from './MicrophoneAudioInputStream'; - -export default function createMicrophoneAudioConfig({ - audioContext, - audioInputDeviceId, - enableTelemetry -}: { - audioContext?: AudioContext; - audioInputDeviceId?: string; - enableTelemetry?: true; -}) { - // Web Chat has an implementation of AudioConfig for microphone that would enable better support on Safari: - // - Maintain same instance of `AudioContext` across recognitions; - // - Resume suspended `AudioContext` on user gestures. - audioContext || (audioContext = createAudioContext()); - - return { - audioConfig: AudioConfig.fromStreamInput( - new MicrophoneAudioInputStream({ - audioConstraints: audioInputDeviceId ? { deviceId: audioInputDeviceId } : true, - audioContext, - enableTelemetry: enableTelemetry ? true : undefined - }) - ), - audioContext - }; -} diff --git a/packages/bundle/src/speech/createMicrophoneAudioConfigAndAudioContext.ts b/packages/bundle/src/speech/createMicrophoneAudioConfigAndAudioContext.ts new file mode 100644 index 0000000000..046fc6ba0a --- /dev/null +++ b/packages/bundle/src/speech/createMicrophoneAudioConfigAndAudioContext.ts @@ -0,0 +1,145 @@ +import { ChunkedArrayBufferStream } from 'microsoft-cognitiveservices-speech-sdk/distrib/lib/src/common/Exports'; +import { PcmRecorder } from 'microsoft-cognitiveservices-speech-sdk/distrib/lib/src/common.browser/Exports'; + +import { AudioStreamNode, DeviceInfo, Format } from './CustomAudioInputStream'; +import bytesPerSample from './bytesPerSample'; +import createAudioConfig from './createAudioConfig'; +import createAudioContext from './createAudioContext'; +import getUserMedia from './getUserMedia'; + +// This is how often we are flushing audio buffer to the network. Modify this value will affect latency. +const DEFAULT_BUFFER_DURATION_IN_MS = 100; + +// PcmRecorder always downscale to 16000 Hz. We cannot use the dynamic value from MediaConstraints or MediaTrackSettings. +const PCM_RECORDER_HARDCODED_SETTINGS: MediaTrackSettings = Object.freeze({ + channelCount: 1, + sampleRate: 16000, + sampleSize: 16 +}); + +const PCM_RECORDER_HARDCODED_FORMAT: Format = Object.freeze({ + bitsPerSample: PCM_RECORDER_HARDCODED_SETTINGS.sampleSize, + channels: PCM_RECORDER_HARDCODED_SETTINGS.channelCount, + samplesPerSec: PCM_RECORDER_HARDCODED_SETTINGS.sampleRate +}); + +type MicrophoneAudioInputStreamOptions = { + /** Specifies the constraints for selecting an audio device. */ + audioConstraints?: true | MediaTrackConstraints; + + /** Specifies the `AudioContext` to use. This object must be primed and ready to use. */ + audioContext: AudioContext; + + /** Specifies the buffering delay on how often to flush audio data to network. Increasing the value will increase audio latency. Default is 100 ms. */ + bufferDurationInMS?: number; + + /** Specifies whether to display diagnostic information. */ + debug?: true; + + /** Specifies if telemetry data should be sent. If not specified, telemetry data will NOT be sent. */ + enableTelemetry?: true; + + /** Specifies the `AudioWorklet` URL for `PcmRecorder`. If not specified, will use script processor on UI thread instead. */ + pcmRecorderWorkletUrl?: string; +}; + +function createMicrophoneAudioConfig(options: MicrophoneAudioInputStreamOptions) { + const { audioConstraints, audioContext, debug, enableTelemetry, pcmRecorderWorkletUrl } = options; + const bufferDurationInMS = options.bufferDurationInMS || DEFAULT_BUFFER_DURATION_IN_MS; + + const pcmRecorder = new PcmRecorder(); + + pcmRecorderWorkletUrl && pcmRecorder.setWorkletUrl(pcmRecorderWorkletUrl); + + return createAudioConfig({ + async attach( + audioNodeId: string + ): Promise<{ + audioStreamNode: AudioStreamNode; + deviceInfo: DeviceInfo; + format: Format; + }> { + // We need to get new MediaStream on every attach(). + // This is because PcmRecorder.releaseMediaResources() disconnected/stopped them. + const mediaStream = await getUserMedia({ audio: audioConstraints, video: false }); + + const [firstAudioTrack] = mediaStream.getAudioTracks(); + + if (!firstAudioTrack) { + throw new Error('No audio device is found.'); + } + + const outputStream = new ChunkedArrayBufferStream( + // Speech SDK quirks: PcmRecorder hardcoded sample rate of 16000 Hz. + bytesPerSample(PCM_RECORDER_HARDCODED_SETTINGS) * + // eslint-disable-next-line no-magic-numbers + ((bufferDurationInMS || DEFAULT_BUFFER_DURATION_IN_MS) / 1000), + audioNodeId + ); + + pcmRecorder.record(audioContext, mediaStream, outputStream); + + return { + audioStreamNode: { + // Speech SDK quirks: In SDK's original MicAudioSource implementation, it call turnOff() during detach(). + // That means, it call turnOff(), then detach(), then turnOff() again. Seems redundant. + // When using with Direct Line Speech, turnOff() is never called. + detach: (): Promise => { + // Speech SDK quirks: In SDK, it call outputStream.close() in turnOff() before outputStream.readEnded() in detach(). + // I think it make sense to call readEnded() before close(). + outputStream.readEnded(); + outputStream.close(); + + // PcmRecorder.releaseMediaResources() will disconnect/stop the MediaStream. + // We cannot use MediaStream again after turned off. + pcmRecorder.releaseMediaResources(audioContext); + + // MediaStream will become inactive after all tracks are removed. + mediaStream.getTracks().forEach(track => mediaStream.removeTrack(track)); + + // ESLint: "return" is required by TypeScript + // eslint-disable-next-line no-useless-return + return; + }, + id: () => audioNodeId, + read: () => outputStream.read() + }, + deviceInfo: { + manufacturer: 'Bot Framework Web Chat', + model: enableTelemetry ? firstAudioTrack.label : '', + type: enableTelemetry ? 'Microphones' : 'Unknown' + }, + // Speech SDK quirks: PcmRecorder hardcoded sample rate of 16000 Hz. + // We cannot obtain this number other than looking at their source code. + // I.e. no getter property. + // PcmRecorder always downscale to 16000 Hz. We cannot use the dynamic value from MediaConstraints or MediaTrackSettings. + format: PCM_RECORDER_HARDCODED_FORMAT + }; + }, + debug + }); +} + +export default function createMicrophoneAudioConfigAndAudioContext({ + audioContext, + audioInputDeviceId, + enableTelemetry +}: { + audioContext?: AudioContext; + audioInputDeviceId?: string; + enableTelemetry?: true; +}) { + // Web Chat has an implementation of AudioConfig for microphone that would enable better support on Safari: + // - Maintain same instance of `AudioContext` across recognitions; + // - Resume suspended `AudioContext` on user gestures. + audioContext || (audioContext = createAudioContext()); + + return { + audioConfig: createMicrophoneAudioConfig({ + audioConstraints: audioInputDeviceId ? { deviceId: audioInputDeviceId } : true, + audioContext, + enableTelemetry: enableTelemetry ? true : undefined + }), + audioContext + }; +} diff --git a/packages/bundle/src/speech/getUserMedia.ts b/packages/bundle/src/speech/getUserMedia.ts index 2ac9c6f7cd..abf3e9d5b8 100644 --- a/packages/bundle/src/speech/getUserMedia.ts +++ b/packages/bundle/src/speech/getUserMedia.ts @@ -5,7 +5,6 @@ export default function getUserMedia(constraints: MediaStreamConstraints): Promi return navigator.mediaDevices.getUserMedia(constraints); } - // TODO: Does it need vendor prefix? if (typeof navigator.getUserMedia !== 'undefined') { return new Promise((resolve, reject) => navigator.getUserMedia(constraints, resolve, reject)); } diff --git a/packages/test/page-object/package-lock.json b/packages/test/page-object/package-lock.json index 48c127b4df..9cfadaa29b 100644 --- a/packages/test/page-object/package-lock.json +++ b/packages/test/page-object/package-lock.json @@ -1271,9 +1271,9 @@ }, "dependencies": { "debug": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", - "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", + "integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==", "requires": { "ms": "2.1.2" } @@ -1433,9 +1433,9 @@ }, "dependencies": { "debug": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", - "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", + "integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==", "requires": { "ms": "2.1.2" } @@ -2929,9 +2929,9 @@ "integrity": "sha512-TMeqbNl2fMW0nMjTEPOwe3J/PRFP4vqeoNuQMG0HlMrtm5QxKqdvAkZ1pRBQ/ulIyDD5Yq0nJ7YbdD8ey0TO3g==" }, "debug": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", - "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", + "integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==", "requires": { "ms": "2.1.2" } @@ -3334,9 +3334,9 @@ } }, "microsoft-cognitiveservices-speech-sdk": { - "version": "1.16.0", - "resolved": "https://registry.npmjs.org/microsoft-cognitiveservices-speech-sdk/-/microsoft-cognitiveservices-speech-sdk-1.16.0.tgz", - "integrity": "sha512-97hrqDaM74ROVEZ4WLKs+jla/PoU5q1oxH4J1WNsmFgbNhyzGkBKlPyOVh1RKoLtWsrjmljPNSDh+Oi1BInIMg==", + "version": "1.17.0", + "resolved": "https://registry.npmjs.org/microsoft-cognitiveservices-speech-sdk/-/microsoft-cognitiveservices-speech-sdk-1.17.0.tgz", + "integrity": "sha512-RVUCpTeu1g+R4HB/PaLQmEfsdHzwEa6+2phgCiPA4lGIiR7ILEL7qZHHUWAG6W4zcjnWeiLnL7tVgMbyd5XGgA==", "requires": { "agent-base": "^6.0.1", "asn1.js-rfc2560": "^5.0.1", @@ -4612,9 +4612,9 @@ "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" }, "ws": { - "version": "7.4.6", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.4.6.tgz", - "integrity": "sha512-YmhHDO4MzaDLB+M9ym/mDA5z0naX8j7SIlT8f8z+I0VtzsRbekxEutHSme7NPS2qE8StCYQNUnfWdXta/Yu85A==" + "version": "7.5.2", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.2.tgz", + "integrity": "sha512-lkF7AWRicoB9mAgjeKbGqVUekLnSNO4VjKVnuPHpQeOxZOErX6BPXwJk70nFslRCEEA8EVW7ZjKwXaP9N+1sKQ==" }, "xmlhttprequest-ts": { "version": "1.0.1", diff --git a/packages/test/page-object/package.json b/packages/test/page-object/package.json index cd2ec86d98..68f1b37886 100644 --- a/packages/test/page-object/package.json +++ b/packages/test/page-object/package.json @@ -25,7 +25,7 @@ "core-js": "3.11.2", "event-iterator": "2.0.0", "event-target-shim": "6.0.2", - "microsoft-cognitiveservices-speech-sdk": "1.16.0", + "microsoft-cognitiveservices-speech-sdk": "1.17.0", "p-defer": "4.0.0", "simple-update-in": "2.2.0" }, diff --git a/packages/test/page-object/src/globals/testHelpers/index.js b/packages/test/page-object/src/globals/testHelpers/index.js index a7c733377e..6d9ff64f22 100644 --- a/packages/test/page-object/src/globals/testHelpers/index.js +++ b/packages/test/page-object/src/globals/testHelpers/index.js @@ -4,6 +4,7 @@ import * as speech from './speech/index'; import * as token from './token/index'; import * as transcriptNavigation from './transcriptNavigation'; import arrayBufferToBase64 from './arrayBufferToBase64'; +import createAudioInputStreamFromRiffWavArrayBuffer from './speech/audioConfig/createAudioInputStreamFromRiffWavArrayBuffer'; import createDirectLineWithTranscript from './createDirectLineWithTranscript'; import createRunHookActivityMiddleware from './createRunHookActivityMiddleware'; import createStore from './createStore'; @@ -18,6 +19,7 @@ export { accessibility, activityGrouping, arrayBufferToBase64, + createAudioInputStreamFromRiffWavArrayBuffer, createDirectLineWithTranscript, createRunHookActivityMiddleware, createStore, diff --git a/packages/test/page-object/src/globals/testHelpers/speech/audioConfig/createAudioInputStreamFromRiffWavArrayBuffer.js b/packages/test/page-object/src/globals/testHelpers/speech/audioConfig/createAudioInputStreamFromRiffWavArrayBuffer.js new file mode 100644 index 0000000000..539249bc9a --- /dev/null +++ b/packages/test/page-object/src/globals/testHelpers/speech/audioConfig/createAudioInputStreamFromRiffWavArrayBuffer.js @@ -0,0 +1,71 @@ +// eslint no-magic-numbers: "off" + +// Importing from "bundle" as we do not expose this function. +import createAudioConfig from '../../../../../../../bundle/src/speech/createAudioConfig'; + +const QUORUM_SIZE = 9600; + +/** Creates a Speech SDK AudioConfig for audio input based from an ArrayBuffer of RIFF WAV. */ +export default function fromRiffWavArrayBuffer(arrayBuffer) { + const channels = new Uint16Array(arrayBuffer.slice(22, 24))[0]; + const samplesPerSec = new Uint32Array(arrayBuffer.slice(24, 28))[0]; + const bitsPerSample = new Uint16Array(arrayBuffer.slice(34, 36))[0]; + + // Search the offset of "data" marker. + // "data" === 64-61-74-61. + + let dataOffset; + + for (let index = 36; index < 200; index++) { + const bytes = new Uint8Array(arrayBuffer.slice(index, index + 4)); + + if (bytes[0] === 0x64 && bytes[1] === 0x61 && bytes[2] === 0x74 && bytes[3] === 0x61) { + dataOffset = index + 4; + break; + } + } + + if (!dataOffset) { + throw new Error('Cannot find "data" section marker in the RIFF WAV ArrayBuffer.'); + } + + const bytesPerSecond = samplesPerSec * channels * (bitsPerSample >> 3); + const now = Date.now(); + + return createAudioConfig({ + attach(audioNodeId) { + let offset = dataOffset; + + return Promise.resolve({ + audioStreamNode: { + detach: () => {}, + id: () => audioNodeId, + read: () => { + if (offset >= arrayBuffer.byteLength) { + return { isEnd: true }; + } + + const buffer = arrayBuffer.slice(offset, offset + QUORUM_SIZE); + + offset += QUORUM_SIZE; + + return { + isEnd: false, + buffer, + timeReceived: now + ~~((offset - dataOffset) / bytesPerSecond) + }; + } + }, + deviceInfo: { + model: 'RIFF WAV ArrayBuffer', + type: 'Stream' + }, + format: { + bitsPerSample, + channels, + samplesPerSec + } + }); + } + }); +} From 837a9ec5f7e3ced530ea24056e852386bb6b1435 Mon Sep 17 00:00:00 2001 From: William Wong Date: Tue, 6 Jul 2021 09:45:32 -0700 Subject: [PATCH 10/19] Update comment --- .../audioConfig/createAudioInputStreamFromRiffWavArrayBuffer.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/test/page-object/src/globals/testHelpers/speech/audioConfig/createAudioInputStreamFromRiffWavArrayBuffer.js b/packages/test/page-object/src/globals/testHelpers/speech/audioConfig/createAudioInputStreamFromRiffWavArrayBuffer.js index 539249bc9a..e1a7fd9797 100644 --- a/packages/test/page-object/src/globals/testHelpers/speech/audioConfig/createAudioInputStreamFromRiffWavArrayBuffer.js +++ b/packages/test/page-object/src/globals/testHelpers/speech/audioConfig/createAudioInputStreamFromRiffWavArrayBuffer.js @@ -11,7 +11,7 @@ export default function fromRiffWavArrayBuffer(arrayBuffer) { const samplesPerSec = new Uint32Array(arrayBuffer.slice(24, 28))[0]; const bitsPerSample = new Uint16Array(arrayBuffer.slice(34, 36))[0]; - // Search the offset of "data" marker. + // Search the offset of "data" marker, earliest possible position is 36. // "data" === 64-61-74-61. let dataOffset; From 529424efbd91c6f65b12166a2fb709cc7a7f6106 Mon Sep 17 00:00:00 2001 From: William Wong Date: Tue, 6 Jul 2021 10:44:23 -0700 Subject: [PATCH 11/19] Add comments --- .../src/speech/CustomAudioInputStream.ts | 22 +++++++++++++++---- .../bundle/src/speech/createAudioConfig.ts | 9 ++++++++ .../bundle/src/speech/createAudioContext.ts | 2 ++ ...ateMicrophoneAudioConfigAndAudioContext.ts | 3 +++ packages/bundle/src/speech/getUserMedia.ts | 1 + 5 files changed, 33 insertions(+), 4 deletions(-) diff --git a/packages/bundle/src/speech/CustomAudioInputStream.ts b/packages/bundle/src/speech/CustomAudioInputStream.ts index 9bb6a00a00..9c67365ad0 100644 --- a/packages/bundle/src/speech/CustomAudioInputStream.ts +++ b/packages/bundle/src/speech/CustomAudioInputStream.ts @@ -99,6 +99,7 @@ abstract class CustomAudioInputStream extends AudioInputStream { [SYMBOL_FORMAT_DEFERRED]: DeferredPromise; [SYMBOL_OPTIONS]: NormalizedOptions; + /** Gets the event source for listening to events. */ // ESLint: This code will only works in browsers other than IE11. Only works in ES5 is okay. // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore Accessors are only available when targeting ECMAScript 5 and higher.ts(1056) @@ -106,10 +107,10 @@ abstract class CustomAudioInputStream extends AudioInputStream { return this[SYMBOL_EVENTS]; } + /** Gets the format of the audio stream. */ // Speech SDK quirks: AudioStreamFormatImpl is internal implementation while AudioStreamFormat is public. // It is weird to expose AudioStreamFormatImpl instead of AudioStreamFormat. // Speech SDK quirks: It is weird to return a Promise in a property. - // Especially this is audio format. Setup options should be initialized synchronously. // Speech SDK quirks: In normal speech recognition, getter of "format" is called only after "attach". // But in Direct Line Speech, it is called before "attach". // ESLint: This code will only works in browsers other than IE11. Only works in ES5 is okay. @@ -121,10 +122,12 @@ abstract class CustomAudioInputStream extends AudioInputStream { return this[SYMBOL_FORMAT_DEFERRED].promise; } + /** Gets the ID of this audio stream. */ id(): string { return this[SYMBOL_OPTIONS].id; } + /** Emits an event. */ // Speech SDK quirks: In JavaScript, onXxx means "listen to event XXX". // Instead, in Speech SDK, it means "emit event XXX". protected onEvent(event: AudioSourceEvent): void { @@ -132,16 +135,19 @@ abstract class CustomAudioInputStream extends AudioInputStream { Events.instance.onEvent(event); } + /** Emits an `AudioSourceInitializingEvent`. */ protected emitInitializing(): void { this.debug('Emitting "AudioSourceInitializingEvent".'); this.onEvent(new AudioSourceInitializingEvent(this.id())); } + /** Emits an `AudioSourceReadyEvent`. */ protected emitReady(): void { this.debug('Emitting "AudioSourceReadyEvent".'); this.onEvent(new AudioSourceReadyEvent(this.id())); } + /** Emits an `AudioSourceErrorEvent`. */ // Speech SDK quirks: Since "turnOn" is never called and "turnOff" does not work in Direct Line Speech, the "source error" event is not emitted at all. // Instead, we only emit "node error" event. protected emitError(error: Error): void { @@ -151,16 +157,19 @@ abstract class CustomAudioInputStream extends AudioInputStream { this.onEvent(new AudioSourceErrorEvent(this.id(), error.message)); } + /** Emits an `AudioStreamNodeAttachingEvent`. */ protected emitNodeAttaching(audioNodeId: string): void { this.debug(`Emitting "AudioStreamNodeAttachingEvent" for node "${audioNodeId}".`); this.onEvent(new AudioStreamNodeAttachingEvent(this.id(), audioNodeId)); } + /** Emits an `AudioStreamNodeAttachedEvent`. */ protected emitNodeAttached(audioNodeId: string): void { this.debug(`Emitting "AudioStreamNodeAttachedEvent" for node "${audioNodeId}".`); this.onEvent(new AudioStreamNodeAttachedEvent(this.id(), audioNodeId)); } + /** Emits an `AudioStreamNodeErrorEvent`. */ protected emitNodeError(audioNodeId: string, error: Error): void { this.debug(`Emitting "AudioStreamNodeErrorEvent" for node "${audioNodeId}".`, { error }); @@ -168,18 +177,19 @@ abstract class CustomAudioInputStream extends AudioInputStream { this.onEvent(new AudioStreamNodeErrorEvent(this.id(), audioNodeId, error.message)); } + /** Emits an `AudioStreamNodeDetachedEvent`. */ protected emitNodeDetached(audioNodeId: string): void { this.debug('Emitting "AudioStreamNodeDetachedEvent".'); this.onEvent(new AudioStreamNodeDetachedEvent(this.id(), audioNodeId)); } + /** Emits an `AudioSourceOffEvent`. */ protected emitOff(): void { this.debug('Emitting "AudioSourceOffEvent".'); this.onEvent(new AudioSourceOffEvent(this.id())); } // Speech SDK quirks: Although "close" is marked as abstract, it is never called in our observations. - // ESLint: Speech SDK requires this function, but we are not implementing it. // eslint-disable-next-line class-methods-use-this close(): void { @@ -188,20 +198,21 @@ abstract class CustomAudioInputStream extends AudioInputStream { throw new Error('Not implemented'); } - // Speech SDK quirks: Although "turnOn" is implemented in XxxAudioInputStream, it is never called in our observations. + // Speech SDK quirks: Although "turnOn" is implemented in Speech SDK Push/PullAudioInputStream, it is never called in our observations. turnOn(): void { this.debug('Callback for "turnOn".'); throw new Error('Not implemented'); } - // Speech SDK quirks: Although "detach" is implemented in XxxAudioInputStream, it is never called in our observations. + // Speech SDK quirks: Although "detach" is implemented in Speech SDK Push/PullAudioInputStream, it is never called in our observations. detach(): void { this.debug('Callback for "detach".'); throw new Error('Not implemented'); } + /** Log the message to console if `debug` is set to `true`. */ private debug(message, ...args) { // ESLint: For debugging, will only log when "debug" is set to "true". // eslint-disable-next-line no-console @@ -217,6 +228,7 @@ abstract class CustomAudioInputStream extends AudioInputStream { format: Format; }>; + /** Attaches the device by returning an audio node. */ attach(audioNodeId: string): Promise { this.debug(`Callback for "attach" with "${audioNodeId}".`); @@ -277,6 +289,7 @@ abstract class CustomAudioInputStream extends AudioInputStream { return; } + /** Turn off the audio device. This is called before detaching from the graph. */ // Speech SDK quirks: It is confused to have both "turnOff" and "detach". "turnOff" is called before "detach". // Why don't we put all logics at "detach"? // Speech SDK quirks: Direct Line Speech never call "turnOff". "Source off" event need to be emitted during "detach" instead. @@ -287,6 +300,7 @@ abstract class CustomAudioInputStream extends AudioInputStream { await this.performTurnOff(); } + /** Gets the device information. */ // ESLint: This code will only works in browsers other than IE11. Only works in ES5 is okay. // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore Accessors are only available when targeting ECMAScript 5 and higher.ts(1056) diff --git a/packages/bundle/src/speech/createAudioConfig.ts b/packages/bundle/src/speech/createAudioConfig.ts index 211a404c32..1d73009333 100644 --- a/packages/bundle/src/speech/createAudioConfig.ts +++ b/packages/bundle/src/speech/createAudioConfig.ts @@ -19,8 +19,17 @@ const SYMBOL_ATTACH = Symbol('attach'); const SYMBOL_TURN_OFF = Symbol('turnOff'); type CreateAudioConfigOptions = { + /** Callback function for attaching the device by returning an audio node. */ attach: AttachFunction; + + /** `true` to enable diagnostic information, otherwise, `false`. */ debug?: true; + + /** + * Callback function for turning off the device before detaching its node from an audio graph. + * + * Note: this is not called for Direct Line Speech. + */ turnOff?: TurnOffFunction; }; diff --git a/packages/bundle/src/speech/createAudioContext.ts b/packages/bundle/src/speech/createAudioContext.ts index 4e0574f627..4fa3ff32c9 100644 --- a/packages/bundle/src/speech/createAudioContext.ts +++ b/packages/bundle/src/speech/createAudioContext.ts @@ -1,3 +1,4 @@ +/** Creates an AudioContext object. */ export default function createAudioContext(): AudioContext { if (typeof window.AudioContext !== 'undefined') { return new window.AudioContext(); @@ -5,6 +6,7 @@ export default function createAudioContext(): AudioContext { // Required by TypeScript. // eslint-disable-next-line dot-notation } else if (typeof window['webkitAudioContext'] !== 'undefined') { + // This is for Safari as Web Audio API is still under vendor-prefixed. // eslint-disable-next-line dot-notation return new window['webkitAudioContext'](); } diff --git a/packages/bundle/src/speech/createMicrophoneAudioConfigAndAudioContext.ts b/packages/bundle/src/speech/createMicrophoneAudioConfigAndAudioContext.ts index 046fc6ba0a..6c81ff091f 100644 --- a/packages/bundle/src/speech/createMicrophoneAudioConfigAndAudioContext.ts +++ b/packages/bundle/src/speech/createMicrophoneAudioConfigAndAudioContext.ts @@ -10,6 +10,9 @@ import getUserMedia from './getUserMedia'; // This is how often we are flushing audio buffer to the network. Modify this value will affect latency. const DEFAULT_BUFFER_DURATION_IN_MS = 100; +// TODO: [P2] #XXX We should consider building our own PcmRecorder: +// - Use Audio Worklet via blob URL +// - Not hardcoding the sample rate or other values // PcmRecorder always downscale to 16000 Hz. We cannot use the dynamic value from MediaConstraints or MediaTrackSettings. const PCM_RECORDER_HARDCODED_SETTINGS: MediaTrackSettings = Object.freeze({ channelCount: 1, diff --git a/packages/bundle/src/speech/getUserMedia.ts b/packages/bundle/src/speech/getUserMedia.ts index abf3e9d5b8..ade90c6375 100644 --- a/packages/bundle/src/speech/getUserMedia.ts +++ b/packages/bundle/src/speech/getUserMedia.ts @@ -5,6 +5,7 @@ export default function getUserMedia(constraints: MediaStreamConstraints): Promi return navigator.mediaDevices.getUserMedia(constraints); } + // Although getUserMedia has vendor prefix, they are only used in very old version of browsers. if (typeof navigator.getUserMedia !== 'undefined') { return new Promise((resolve, reject) => navigator.getUserMedia(constraints, resolve, reject)); } From 135e47ea9155a6dfdf69f99baa4c21f072cbd17f Mon Sep 17 00:00:00 2001 From: William Wong Date: Tue, 6 Jul 2021 11:02:44 -0700 Subject: [PATCH 12/19] Add entries --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 170045bde9..915c8430a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,7 +40,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - `"any"` will show when there are any offscreen messages; - `false` will always hide the button. - Added new [`scrollToEndButtonMiddleware`](https://github.com/microsoft/BotFramework-WebChat/blob/main/packages/api/src/types/scrollToEndButtonMiddleware.ts) to customize the appearance of the scroll to end button. -- Resolves [#3752](https://github.com/microsoft/BotFramework-WebChat/issues/3752). Added typings (`*.d.ts`) for all public interfaces, by [@compulim](https://github.com), in PR [#3931](https://github.com/microsoft/BotFramework-WebChat/pull/3931) and [#3946](https://github.com/microsoft/BotFramework-WebChat/pull/3946) +- Resolves [#3752](https://github.com/microsoft/BotFramework-WebChat/issues/3752). Added typings (`*.d.ts`) for all public interfaces, by [@compulim](https://github.com/compulim), in PR [#3931](https://github.com/microsoft/BotFramework-WebChat/pull/3931) and [#3946](https://github.com/microsoft/BotFramework-WebChat/pull/3946) +- Resolves [#2316](https://github.com/microsoft/BotFramework-WebChat/issues/2316). Added blessing/priming of `AudioContext` when clicking on microphone button, by [@compulim](https://github.com/compulim), in PR [#3974](https://github.com/microsoft/BotFramework-WebChat/pull/3974) ### Fixed @@ -57,6 +58,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Fixes [#3856](https://github.com/microsoft/BotFramework-WebChat/issues/3856). Fix missing typings, by [@compulim](https://github.com/compulim) and [@corinagum](https://github.com/corinagum), in PR [#3931](https://github.com/microsoft/BotFramework-WebChat/pull/3931) - Fixes [#3943](https://github.com/microsoft/BotFramework-WebChat/issues/3943). Auto-scroll should skip invisible activities, such as post back or event activity, by [@compulim](https://github.com/compulim), in PR [#3945](https://github.com/microsoft/BotFramework-WebChat/pull/3945) - Fixes [#3947](https://github.com/microsoft/BotFramework-WebChat/issues/3947). Adaptive Cards: all action sets (which has `role="menubar"`) must have at least 1 or more `role="menuitem"`, by [@compulim](https://github.com/compulim), in PR [#3950](https://github.com/microsoft/BotFramework-WebChat/pull/3950) +- Fixes [#3823](https://github.com/microsoft/BotFramework-WebChat/issues/3823) and [#3899](https://github.com/microsoft/BotFramework-WebChat/issues/3899). Fix speech recognition and synthesis on Safari, in PR [#3974](https://github.com/microsoft/BotFramework-WebChat/pull/3974) ### Changed From 43d3706c44e46bfd0f29e7c672481f58b8416a18 Mon Sep 17 00:00:00 2001 From: William Wong Date: Tue, 6 Jul 2021 11:34:36 -0700 Subject: [PATCH 13/19] Clean up --- packages/bundle/src/speech/CustomAudioInputStream.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/bundle/src/speech/CustomAudioInputStream.ts b/packages/bundle/src/speech/CustomAudioInputStream.ts index 9c67365ad0..35027116e8 100644 --- a/packages/bundle/src/speech/CustomAudioInputStream.ts +++ b/packages/bundle/src/speech/CustomAudioInputStream.ts @@ -90,8 +90,8 @@ abstract class CustomAudioInputStream extends AudioInputStream { this[SYMBOL_DEVICE_INFO_DEFERRED] = createDeferred(); this[SYMBOL_EVENTS] = new EventSource(); - this[SYMBOL_OPTIONS] = normalizedOptions; this[SYMBOL_FORMAT_DEFERRED] = createDeferred(); + this[SYMBOL_OPTIONS] = normalizedOptions; } [SYMBOL_DEVICE_INFO_DEFERRED]: DeferredPromise; @@ -108,9 +108,9 @@ abstract class CustomAudioInputStream extends AudioInputStream { } /** Gets the format of the audio stream. */ - // Speech SDK quirks: AudioStreamFormatImpl is internal implementation while AudioStreamFormat is public. - // It is weird to expose AudioStreamFormatImpl instead of AudioStreamFormat. - // Speech SDK quirks: It is weird to return a Promise in a property. + // Speech SDK quirks: `AudioStreamFormatImpl` is internal implementation while `AudioStreamFormat` is public. + // It is weird to expose `AudioStreamFormatImpl` instead of `AudioStreamFormat`. + // Speech SDK quirks: It is weird to return a `Promise` in a property. // Speech SDK quirks: In normal speech recognition, getter of "format" is called only after "attach". // But in Direct Line Speech, it is called before "attach". // ESLint: This code will only works in browsers other than IE11. Only works in ES5 is okay. From f0c2b3a71074b5c169f3be0b4a1a6236dfb73c93 Mon Sep 17 00:00:00 2001 From: William Wong Date: Tue, 6 Jul 2021 11:44:35 -0700 Subject: [PATCH 14/19] Update comment --- packages/bundle/src/speech/createAudioConfig.ts | 2 +- .../src/speech/createMicrophoneAudioConfigAndAudioContext.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/bundle/src/speech/createAudioConfig.ts b/packages/bundle/src/speech/createAudioConfig.ts index 1d73009333..ead95723fa 100644 --- a/packages/bundle/src/speech/createAudioConfig.ts +++ b/packages/bundle/src/speech/createAudioConfig.ts @@ -1,4 +1,4 @@ -// TODO: [P2] #XXX We should export this to allow web developers to bring in their own microphone. +// TODO: [P2] #3976 We should export this to allow web developers to bring in their own microphone. // For example, it should enable React Native devs to bring in their microphone implementation and use Cognitive Services Speech Services. import { AudioConfig } from 'microsoft-cognitiveservices-speech-sdk'; diff --git a/packages/bundle/src/speech/createMicrophoneAudioConfigAndAudioContext.ts b/packages/bundle/src/speech/createMicrophoneAudioConfigAndAudioContext.ts index 6c81ff091f..524921bebf 100644 --- a/packages/bundle/src/speech/createMicrophoneAudioConfigAndAudioContext.ts +++ b/packages/bundle/src/speech/createMicrophoneAudioConfigAndAudioContext.ts @@ -10,7 +10,7 @@ import getUserMedia from './getUserMedia'; // This is how often we are flushing audio buffer to the network. Modify this value will affect latency. const DEFAULT_BUFFER_DURATION_IN_MS = 100; -// TODO: [P2] #XXX We should consider building our own PcmRecorder: +// TODO: [P2] #3975 We should consider building our own PcmRecorder: // - Use Audio Worklet via blob URL // - Not hardcoding the sample rate or other values // PcmRecorder always downscale to 16000 Hz. We cannot use the dynamic value from MediaConstraints or MediaTrackSettings. From e3fbd6c5441f30810bc5351c5140afbebd9c8193 Mon Sep 17 00:00:00 2001 From: William Wong Date: Tue, 6 Jul 2021 12:55:55 -0700 Subject: [PATCH 15/19] Fix tests --- ...vicesSpeechServicesPonyfillFactory.spec.js | 36 +++++++++++++++++-- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/packages/bundle/src/createCognitiveServicesSpeechServicesPonyfillFactory.spec.js b/packages/bundle/src/createCognitiveServicesSpeechServicesPonyfillFactory.spec.js index 162771ed43..056fc67388 100644 --- a/packages/bundle/src/createCognitiveServicesSpeechServicesPonyfillFactory.spec.js +++ b/packages/bundle/src/createCognitiveServicesSpeechServicesPonyfillFactory.spec.js @@ -21,7 +21,25 @@ beforeEach(() => { createCognitiveServicesSpeechServicesPonyfillFactory = require('./createCognitiveServicesSpeechServicesPonyfillFactory') .default; - window.navigator.mediaDevices = {}; + window.AudioContext = class MockAudioContext { + // eslint-disable-next-line class-methods-use-this + createMediaStreamSource() { + // eslint-disable-next-line @typescript-eslint/no-empty-function + return { connect: () => {} }; + } + + // eslint-disable-next-line class-methods-use-this + createScriptProcessor() { + // eslint-disable-next-line @typescript-eslint/no-empty-function + return { connect: () => {} }; + } + }; + + window.navigator.mediaDevices = { + getUserMedia: jest.fn(() => ({ + getAudioTracks: () => ['mock-media-stream-track'] + })) + }; }); afterEach(() => { @@ -60,7 +78,8 @@ test('not providing reference grammar ID', () => { expect(referenceGrammars).toEqual([]); }); -test('supplying audioInputDeviceId', () => { +test('supplying audioInputDeviceId', async () => { + // GIVEN: Set up Web Speech with "audioInputDeviceId" of "audio-input-device-1". const ponyfillFactory = createCognitiveServicesSpeechServicesPonyfillFactory({ audioInputDeviceId: 'audio-input-device-1', credentials: { @@ -69,9 +88,20 @@ test('supplying audioInputDeviceId', () => { } }); + // WHEN: Polyfill is created. ponyfillFactory({}); - expect(createPonyfill.mock.calls[0][0]).toHaveProperty('audioConfig.privSource.deviceId', 'audio-input-device-1'); + // WHEN: Audio source is attached and audio device is opened. + await createPonyfill.mock.calls[0][0].audioConfig.privSource.attach(); + + // THEN: It should call getUserMedia() with "audio" constraints of { deviceId: 'audio-input-device-1' }. + expect(window.navigator.mediaDevices.getUserMedia.mock.calls[0][0]).toHaveProperty( + 'audio.deviceId', + 'audio-input-device-1' + ); + + // THEN: It should call getUserMedia() with "video" constraint of false. + expect(window.navigator.mediaDevices.getUserMedia.mock.calls[0][0]).toHaveProperty('video', false); }); test('supplying both audioConfig and audioInputDeviceId', () => { From 2976141e7e4af8ba50b97b0ab1e5f534c4774d96 Mon Sep 17 00:00:00 2001 From: William Wong Date: Tue, 6 Jul 2021 12:58:16 -0700 Subject: [PATCH 16/19] Apply PR suggestions --- packages/bundle/src/speech/createAudioConfig.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/bundle/src/speech/createAudioConfig.ts b/packages/bundle/src/speech/createAudioConfig.ts index ead95723fa..ca8cae0c1b 100644 --- a/packages/bundle/src/speech/createAudioConfig.ts +++ b/packages/bundle/src/speech/createAudioConfig.ts @@ -35,6 +35,14 @@ type CreateAudioConfigOptions = { class CreateAudioConfigAudioInputStream extends CustomAudioInputStream { constructor({ attach, debug, turnOff }: CreateAudioConfigOptions) { + if (typeof attach !== 'function') { + throw new Error('"attach" must be a function.'); + } + + if (typeof turnOff !== 'function') { + throw new Error('"turnOff" must be a function.'); + } + super({ debug }); this[SYMBOL_ATTACH] = attach; @@ -55,9 +63,7 @@ class CreateAudioConfigAudioInputStream extends CustomAudioInputStream { } protected performTurnOff(): Promise { - const { [SYMBOL_TURN_OFF]: turnOff } = this; - - return turnOff && turnOff(); + return this[SYMBOL_TURN_OFF](); } } From 203bcdd545c8731f266bdf3ced21c96227780470 Mon Sep 17 00:00:00 2001 From: William Wong Date: Tue, 6 Jul 2021 16:01:12 -0700 Subject: [PATCH 17/19] Fix type check --- packages/bundle/src/speech/createAudioConfig.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/bundle/src/speech/createAudioConfig.ts b/packages/bundle/src/speech/createAudioConfig.ts index ca8cae0c1b..d31329cdcf 100644 --- a/packages/bundle/src/speech/createAudioConfig.ts +++ b/packages/bundle/src/speech/createAudioConfig.ts @@ -35,12 +35,12 @@ type CreateAudioConfigOptions = { class CreateAudioConfigAudioInputStream extends CustomAudioInputStream { constructor({ attach, debug, turnOff }: CreateAudioConfigOptions) { - if (typeof attach !== 'function') { + if (!attach || typeof attach !== 'function') { throw new Error('"attach" must be a function.'); } - if (typeof turnOff !== 'function') { - throw new Error('"turnOff" must be a function.'); + if (turnOff && typeof turnOff !== 'function') { + throw new Error('"turnOff", if defined, must be a function.'); } super({ debug }); From 0ae7c237c1ac3e444df66dd1d4466be48be448ed Mon Sep 17 00:00:00 2001 From: William Wong Date: Tue, 6 Jul 2021 16:42:39 -0700 Subject: [PATCH 18/19] Add test --- .../src/speech/createAudioConfig.spec.js | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 packages/bundle/src/speech/createAudioConfig.spec.js diff --git a/packages/bundle/src/speech/createAudioConfig.spec.js b/packages/bundle/src/speech/createAudioConfig.spec.js new file mode 100644 index 0000000000..fba818e89f --- /dev/null +++ b/packages/bundle/src/speech/createAudioConfig.spec.js @@ -0,0 +1,23 @@ +/* eslint @typescript-eslint/no-empty-function: "off" */ + +import createAudioConfig from './createAudioConfig'; + +describe('Verify arguments', () => { + test('should not throw if only "attach" is supplied', () => { + expect(() => createAudioConfig({ attach: () => {} })).not.toThrow(); + }); + + test('should not throw if both "attach" and "turnOff" are supplied', () => { + expect(() => createAudioConfig({ attach: () => {}, turnOff: () => {} })).not.toThrow(); + }); + + test('should throw if "attach" is not supplied', () => { + expect(() => createAudioConfig({})).toThrow('"attach" must be a function.'); + }); + + test('should throw if "turnOff" is not a function', () => { + expect(() => createAudioConfig({ attach: () => {}, turnOff: '123' })).toThrow( + '"turnOff", if defined, must be a function.' + ); + }); +}); From 928052ba0297221f5bc0708d26bac95907d22032 Mon Sep 17 00:00:00 2001 From: William Wong Date: Tue, 6 Jul 2021 22:56:20 -0700 Subject: [PATCH 19/19] Fix type check --- packages/bundle/src/speech/createAudioConfig.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/bundle/src/speech/createAudioConfig.ts b/packages/bundle/src/speech/createAudioConfig.ts index d31329cdcf..59ef2e098e 100644 --- a/packages/bundle/src/speech/createAudioConfig.ts +++ b/packages/bundle/src/speech/createAudioConfig.ts @@ -63,7 +63,9 @@ class CreateAudioConfigAudioInputStream extends CustomAudioInputStream { } protected performTurnOff(): Promise { - return this[SYMBOL_TURN_OFF](); + const { [SYMBOL_TURN_OFF]: turnOff } = this; + + return turnOff && turnOff(); } }