From 9dc8e1f4827eb47b6e6d31769ad9365ed067c73c Mon Sep 17 00:00:00 2001 From: Jesse Date: Mon, 11 Apr 2022 10:54:55 -0400 Subject: [PATCH] Add voicingVisibleProperty for https://github.com/phetsims/scenery/issues/1300 --- js/accessibility/voicing/Voicing.ts | 172 +++++++++++++++++++++++++++- js/display/Display.ts | 3 +- js/display/Instance.js | 29 ++++- js/nodes/Node.ts | 34 ++++++ 4 files changed, 226 insertions(+), 12 deletions(-) diff --git a/js/accessibility/voicing/Voicing.ts b/js/accessibility/voicing/Voicing.ts index 07b309243..287400e51 100644 --- a/js/accessibility/voicing/Voicing.ts +++ b/js/accessibility/voicing/Voicing.ts @@ -29,11 +29,12 @@ import ResponsePacket, { ResolvedResponse, ResponsePacketOptions, VoicingRespons import ResponsePatternCollection from '../../../../utterance-queue/js/ResponsePatternCollection.js'; import Utterance, { IAlertable } from '../../../../utterance-queue/js/Utterance.js'; import UtteranceQueue from '../../../../utterance-queue/js/UtteranceQueue.js'; -import { InteractiveHighlighting, InteractiveHighlightingOptions, Node, scenery, SceneryListenerFunction, voicingUtteranceQueue } from '../../imports.js'; +import { Instance, InteractiveHighlighting, InteractiveHighlightingOptions, Node, scenery, SceneryListenerFunction, voicingUtteranceQueue } from '../../imports.js'; import optionize from '../../../../phet-core/js/optionize.js'; import Constructor from '../../../../phet-core/js/types/Constructor.js'; import IntentionalAny from '../../../../phet-core/js/types/IntentionalAny.js'; import responseCollector from '../../../../utterance-queue/js/responseCollector.js'; +import TinyProperty from '../../../../axon/js/TinyProperty.js'; // options that are supported by Voicing.js. Added to mutator keys so that Voicing properties can be set with mutate. const VOICING_OPTION_KEYS = [ @@ -93,6 +94,8 @@ export type SpeakingOptions = { ResponsePacketOptions[PropertyName]; } +type InstanceListener = ( instance: Instance ) => void; + /** * @param Type * @param optionsArgPosition - zero-indexed number that the options argument is provided at @@ -120,6 +123,22 @@ const Voicing = ( Type: SuperType, optionsArgPosi // Called when this node is focused. _voicingFocusListener!: SceneryListenerFunction | null; + // Indicates whether this Node can speak. A Node can speak if self and all of its ancestors are visible and + // voicingVisible. + _voicingCanSpeakProperty!: TinyProperty; + + // A counter that keeps track of visible and voicingVisible Instances of this Node. + // As long as this value is greater than zero, this Node can speak. See onInstanceVisibilityChange + // and onInstanceVoicingVisibilityChange for more implementation details. + _voicingCanSpeakCount!: number; + + // Called when `visible` or `voicingVisible` change for an Instance. + _boundInstanceVisibilityChangeListener!: InstanceListener; + _boundInstanceVoicingVisibilityChangeListener!: InstanceListener; + + // Called when instances of this Node change. + _boundInstancesChangedListener!: ( instance: Instance, added: boolean ) => void; + // Input listener that speaks content on focus. This is the only input listener added // by Voicing, but it is the one that is consistent for all Voicing nodes. On focus, speak the name, object // response, and interaction hint. @@ -148,9 +167,28 @@ const Voicing = ( Type: SuperType, optionsArgPosi // @ts-ignore super.initialize && super.initialize( args ); + // Indicates whether this Node can speak. A Node can speak if self and all of its ancestors are visible and + // voicingVisible. + this._voicingCanSpeakProperty = new TinyProperty( true ); + this._voicingResponsePacket = new ResponsePacket(); this._voicingFocusListener = this.defaultFocusListener; - this._voicingUtterance = new Utterance(); + + // Sets the default voicingUtterance and makes this.canSpeakProperty a dependency on its ability to announce. + this.setVoicingUtterance( new Utterance() ); + + // A counter that keeps track of visible and voicingVisible Instances of this Node. As long as this value is + // greater than zero, this Node can speak. See onInstanceVisibilityChange and onInstanceVoicingVisibilityChange + // for more details. + this._voicingCanSpeakCount = 0; + + this._boundInstanceVisibilityChangeListener = this.onInstanceVisibilityChange.bind( this ); + this._boundInstanceVoicingVisibilityChangeListener = this.onInstanceVoicingVisibilityChange.bind( this ); + + // Whenever an Instance of this Node is added or removed, add/remove listeners that will update the + // canSpeakProperty. + this._boundInstancesChangedListener = this.addOrRemoveInstanceListeners.bind( this ); + ( this as unknown as Node ).changedInstanceEmitter.addListener( this._boundInstancesChangedListener ); this._speakContentOnFocusListener = { focus: event => { @@ -449,8 +487,20 @@ const Voicing = ( Type: SuperType, optionsArgPosi * Sets the utterance through which voicing associated with this Node will be spoken. By default on initialize, * one will be created, but a custom one can optionally be provided. */ - setVoicingUtterance( utterance: Utterance ) { - this._voicingUtterance = utterance; + public setVoicingUtterance( utterance: Utterance ) { + if ( this._voicingUtterance !== utterance ) { + + // First remove the _voicingCanSpeakProperty from the old Utterance since this Node's visible and voicingVisible + // no longer dictate whether the Utterance can be announced. + if ( this._voicingUtterance ) { + this._voicingUtterance.canAnnounceProperties = _.without( this._voicingUtterance.canAnnounceProperties, this._voicingCanSpeakProperty ); + } + + const previousCanAnnounceProperties = utterance.canAnnounceProperties; + utterance.canAnnounceProperties = [ this._voicingCanSpeakProperty, ...previousCanAnnounceProperties ]; + + this._voicingUtterance = utterance; + } } set voicingUtterance( utterance: Utterance ) { this.setVoicingUtterance( utterance ); } @@ -484,6 +534,16 @@ const Voicing = ( Type: SuperType, optionsArgPosi get voicingUtteranceQueue(): UtteranceQueue | null { return this.getVoicingUtteranceQueue(); } + /** + * Get the Property indicating that this Voicing Node can speak. True when this Voicing Node and all of its + * ancestors are visible and voicingVisible. + */ + getVoicingCanSpeakProperty() { + return this._voicingCanSpeakProperty; + } + + get voicingCanSpeakProperty() { return this.getVoicingCanSpeakProperty(); } + /** * Called whenever this Node is focused. */ @@ -524,16 +584,120 @@ const Voicing = ( Type: SuperType, optionsArgPosi */ override dispose() { ( this as unknown as Node ).removeInputListener( this._speakContentOnFocusListener ); + ( this as unknown as Node ).changedInstanceEmitter.removeListener( this._boundInstancesChangedListener ); super.dispose(); } clean() { ( this as unknown as Node ).removeInputListener( this._speakContentOnFocusListener ); + ( this as unknown as Node ).changedInstanceEmitter.removeListener( this._boundInstancesChangedListener ); // @ts-ignore super.clean && super.clean(); } + + /***********************************************************************************************************/ + // PRIVATE METHODS + /***********************************************************************************************************/ + + /** + * When visibility changes on an Instance update the canSpeakProperty. A counting variable keeps track + * of the instances attached to the display that are both globally visible and voicingVisible. If any + * Instance is voicingVisible and visible, this Node can speak. + * + * @private + */ + onInstanceVisibilityChange( instance: Instance ) { + + // Since this is called on the change and `visible` is a boolean wasVisible is the not of the current value. + // From the change we can determine if the count should be incremented or decremented. + const wasVisible = !instance.visible && instance.voicingVisible; + const isVisible = instance.visible && instance.voicingVisible; + + if ( wasVisible && !isVisible ) { + this._voicingCanSpeakCount--; + } + else if ( !wasVisible && isVisible ) { + this._voicingCanSpeakCount++; + } + + this._voicingCanSpeakProperty.value = this._voicingCanSpeakCount > 0; + } + + /** + * When voicingVisible changes on an Instance, update the canSpeakProperty. A counting variable keeps track of + * the instances attached to the display that are both globally visible and voicingVisible. If any Instance + * is voicingVisible and visible this Node can speak. + * + * @private + */ + onInstanceVoicingVisibilityChange( instance: Instance ) { + + // Since this is called on the change and `visible` is a boolean wasVisible is the not of the current value. + // From the change we can determine if the count should be incremented or decremented. + const wasVoicingVisible = !instance.voicingVisible && instance.visible; + const isVoicingVisible = instance.voicingVisible && instance.visible; + + if ( wasVoicingVisible && !isVoicingVisible ) { + this._voicingCanSpeakCount--; + } + else if ( !wasVoicingVisible && isVoicingVisible ) { + this._voicingCanSpeakCount++; + } + + this._voicingCanSpeakProperty.value = this._voicingCanSpeakCount > 0; + } + + /** + * Update the canSpeakProperty and counting variable in response to an Instance of this Node being added or + * removed. + * + * @private + */ + handleInstancesChanged( instance: Instance, added: boolean ) { + const isVisible = instance.visible && instance.voicingVisible; + if ( isVisible ) { + + // If the added Instance was visible and voicingVisible it should increment the counter. If the removed + // instance is NOT visible/voicingVisible it would not have contributed to the counter so we should not + // decrement in that case. + this._voicingCanSpeakCount = added ? this._voicingCanSpeakCount + 1 : this._voicingCanSpeakCount - 1; + } + + this._voicingCanSpeakProperty.value = this._voicingCanSpeakCount > 0; + } + + /** + * Add or remove listeners on an Instance watching for changes to visible or voicingVisible that will modify + * the voicingCanSpeakCount. See documentation for voicingCanSpeakCount for details about how this controls the + * voicingCanSpeakProperty. + * + * @private + */ + addOrRemoveInstanceListeners( instance: Instance, added: boolean ) { + assert && assert( instance.voicingVisibleEmitter, 'Instance must be initialized.' ); + assert && assert( instance.visibleEmitter, 'Instance must be initialized.' ); + + if ( added ) { + + // @ts-ignore - Emitters in Instance need typing + instance.voicingVisibleEmitter!.addListener( this._boundInstanceVoicingVisibilityChangeListener ); + + // @ts-ignore - Emitters in Instance need typing + instance.visibleEmitter!.addListener( this._boundInstanceVisibilityChangeListener ); + } + else { + // @ts-ignore - Emitters in Instance need typing + instance.voicingVisibleEmitter!.removeListener( this._boundInstanceVoicingVisibilityChangeListener ); + + // @ts-ignore - Emitters in Instance need typing + instance.visibleEmitter!.removeListener( this._boundInstanceVisibilityChangeListener ); + } + + // eagerly update the canSpeakProperty and counting variables in addition to adding change listeners + this.handleInstancesChanged( instance, added ); + } }; /** diff --git a/js/display/Display.ts b/js/display/Display.ts index d989a460f..a7a2d08be 100644 --- a/js/display/Display.ts +++ b/js/display/Display.ts @@ -579,8 +579,7 @@ export default class Display { // pre-repaint phase: update relative transform information for listeners (notification) and precomputation where desired this.updateDirtyTransformRoots(); // pre-repaint phase update visibility information on instances - this._baseInstance!.updateVisibility( true, true, false ); - + this._baseInstance!.updateVisibility( true, true, true, false ); if ( assertSlow ) { this._baseInstance!.auditVisibility( true ); } if ( assertSlow ) { this._baseInstance!.audit( this._frameId, true ); } diff --git a/js/display/Instance.js b/js/display/Instance.js index 378488f00..fddbe93b7 100644 --- a/js/display/Instance.js +++ b/js/display/Instance.js @@ -24,7 +24,7 @@ import TinyEmitter from '../../../axon/js/TinyEmitter.js'; import arrayRemove from '../../../phet-core/js/arrayRemove.js'; import cleanArray from '../../../phet-core/js/cleanArray.js'; import Poolable from '../../../phet-core/js/Poolable.js'; -import { scenery, Trail, Utils, Renderer, RelativeTransform, Drawable, ChangeInterval, Fittability, BackboneDrawable, CanvasBlock, InlineCanvasCacheDrawable, SharedCanvasCacheDrawable } from '../imports.js'; +import { BackboneDrawable, CanvasBlock, ChangeInterval, Drawable, Fittability, InlineCanvasCacheDrawable, RelativeTransform, Renderer, scenery, SharedCanvasCacheDrawable, Trail, Utils } from '../imports.js'; let globalIdCounter = 1; @@ -92,6 +92,7 @@ class Instance { this.selfVisible = true; // like relative visibility, but is always true if we are a visibility root this.visibilityDirty = true; // entire subtree of visibility will need to be updated this.childVisibilityDirty = true; // an ancestor needs its visibility updated + this.voicingVisible = true; // whether this instance is "visible" for Voicing and allows speech with that feature // @private {Object.} - Maps another instance's `instance.id` {number} => branch index // {number} (first index where the two trails are different). This effectively operates as a cache (since it's more @@ -138,6 +139,7 @@ class Instance { this.visibleEmitter = new TinyEmitter(); this.relativeVisibleEmitter = new TinyEmitter(); this.selfVisibleEmitter = new TinyEmitter(); + this.voicingVisibleEmitter = new TinyEmitter(); this.cleanInstance( display, trail ); @@ -1515,10 +1517,11 @@ class Instance { * @public * * @param {boolean} parentGloballyVisible - Whether our parent (if any) is globally visible + * @param {boolean} parentGloballyVoicingVisible - Whether our parent (if any) is globally voicingVisible. * @param {boolean} parentRelativelyVisible - Whether our parent (if any) is relatively visible * @param {boolean} updateFullSubtree - If true, we will visit the entire subtree to ensure visibility is correct. */ - updateVisibility( parentGloballyVisible, parentRelativelyVisible, updateFullSubtree ) { + updateVisibility( parentGloballyVisible, parentGloballyVoicingVisible, parentRelativelyVisible, updateFullSubtree ) { // If our visibility flag for ourself is dirty, we need to update our entire subtree if ( this.visibilityDirty ) { updateFullSubtree = true; @@ -1529,7 +1532,10 @@ class Instance { const wasVisible = this.visible; const wasRelativeVisible = this.relativeVisible; const wasSelfVisible = this.selfVisible; + const nodeVoicingVisible = this.node.voicingVisibleProperty.value; + const wasVoicingVisible = this.voicingVisible; this.visible = parentGloballyVisible && nodeVisible; + this.voicingVisible = parentGloballyVoicingVisible && nodeVoicingVisible; this.relativeVisible = parentRelativelyVisible && nodeVisible; this.selfVisible = this.isVisibilityApplied ? true : this.relativeVisible; @@ -1539,7 +1545,7 @@ class Instance { if ( updateFullSubtree || child.visibilityDirty || child.childVisibilityDirty ) { // if we are a visibility root (isVisibilityApplied===true), disregard ancestor visibility - child.updateVisibility( this.visible, this.isVisibilityApplied ? true : this.relativeVisible, updateFullSubtree ); + child.updateVisibility( this.visible, this.voicingVisible, this.isVisibilityApplied ? true : this.relativeVisible, updateFullSubtree ); } } @@ -1548,13 +1554,16 @@ class Instance { // trigger changes after we do the full visibility update if ( this.visible !== wasVisible ) { - this.visibleEmitter.emit(); + this.visibleEmitter.emit( this ); } if ( this.relativeVisible !== wasRelativeVisible ) { - this.relativeVisibleEmitter.emit(); + this.relativeVisibleEmitter.emit( this ); } if ( this.selfVisible !== wasSelfVisible ) { - this.selfVisibleEmitter.emit(); + this.selfVisibleEmitter.emit( this ); + } + if ( this.voicingVisible !== wasVoicingVisible ) { + this.voicingVisibleEmitter.emit( this ); } } @@ -1670,6 +1679,9 @@ class Instance { this.node.childrenReorderedEmitter.addListener( this.childrenReorderedListener ); this.node.visibleProperty.lazyLink( this.visibilityListener ); + // Marks all visibility dirty when voicingVisible changes to cause necessary updates for voicingVisible + this.node.voicingVisibleProperty.lazyLink( this.visibilityListener ); + this.node.filterChangeEmitter.addListener( this.markRenderStateDirtyListener ); this.node.clipAreaProperty.lazyLink( this.markRenderStateDirtyListener ); this.node.instanceRefreshEmitter.addListener( this.markRenderStateDirtyListener ); @@ -1687,6 +1699,7 @@ class Instance { this.node.childRemovedEmitter.removeListener( this.childRemovedListener ); this.node.childrenReorderedEmitter.removeListener( this.childrenReorderedListener ); this.node.visibleProperty.unlink( this.visibilityListener ); + this.node.voicingVisibleProperty.unlink( this.visibilityListener ); this.node.filterChangeEmitter.removeListener( this.markRenderStateDirtyListener ); this.node.clipAreaProperty.unlink( this.markRenderStateDirtyListener ); @@ -1802,6 +1815,7 @@ class Instance { this.visibleEmitter.removeAllListeners(); this.relativeVisibleEmitter.removeAllListeners(); this.selfVisibleEmitter.removeAllListeners(); + this.voicingVisibleEmitter.removeAllListeners(); this.freeToPool(); @@ -1867,6 +1881,9 @@ class Instance { assertSlow( visible === trailVisible, 'Trail visibility failure' ); assertSlow( visible === this.visible, 'Visible flag failure' ); + assertSlow( this.voicingVisible === _.reduce( this.trail.nodes, ( value, node ) => value && node.voicingVisibleProperty.value, true ), + 'When this Instance is voicingVisible: true, all Trail Nodes must also be voicingVisible: true' ); + // validate the subtree for ( let i = 0; i < this.children.length; i++ ) { const childInstance = this.children[ i ]; diff --git a/js/nodes/Node.ts b/js/nodes/Node.ts index 23484d008..163db167c 100644 --- a/js/nodes/Node.ts +++ b/js/nodes/Node.ts @@ -399,6 +399,13 @@ class Node extends ParallelDOM { // NOTE: This is fired synchronously when the clipArea of the Node is toggled clipAreaProperty: TinyProperty; + // Whether this Node and its subtree can announce content with Voicing and SpeechSynthesis. Though + // related to Voicing it exists in Node because it is useful to set voicingVisible on a subtree where the + // root does not compose Voicing. This is not ideal but the entirety of Voicing cannot be composed into every + // Node because it would produce incorrect behaviors and have a massive memory footprint. See setVoicingVisible() + // and Voicing.ts for more information about Voicing. + voicingVisibleProperty: TinyProperty; + // Areas for hit intersection. If set on a Node, no descendants can handle events. _mouseArea: Shape | Bounds2 | null; // for mouse position in the local coordinate frame _touchArea: Shape | Bounds2 | null; // for touch and pen position in the local coordinate frame @@ -652,6 +659,7 @@ class Node extends ParallelDOM { this._inputEnabledProperty = new TinyForwardingProperty( DEFAULT_OPTIONS.inputEnabled, DEFAULT_OPTIONS.phetioInputEnabledPropertyInstrumented ); this.clipAreaProperty = new TinyProperty( DEFAULT_OPTIONS.clipArea ); + this.voicingVisibleProperty = new TinyProperty( true ); this._mouseArea = DEFAULT_OPTIONS.mouseArea; this._touchArea = DEFAULT_OPTIONS.touchArea; this._cursor = DEFAULT_OPTIONS.cursor; @@ -6025,6 +6033,32 @@ class Node extends ParallelDOM { } } + /** + * Set the visibility of this Node with respect to the Voicing feature. Totally separate from graphical display. + * When visible, this Node and all of its ancestors will be able to speak with Voicing. When voicingVisible + * is false, all Voicing under this Node will be muted. `voicingVisible` properties exist in Node.ts because + * it is useful to set `voicingVisible` on a root that is composed with Voicing.ts. We cannot put all of the + * Voicing.ts implementation in Node because that would have a massive memory impact. See Voicing.ts for more + * information. + */ + public setVoicingVisible( visible: boolean ): void { + if ( this.voicingVisibleProperty.value !== visible ) { + this.voicingVisibleProperty.value = visible; + } + } + + set voicingVisible( visible ) { this.setVoicingVisible( visible ); } + + /** + * Returns whether this Node is voicingVisible. When true Utterances for this Node can be announced with the + * Voicing feature, see Voicing.ts for more information. + */ + public isVoicingVisible(): boolean { + return this.voicingVisibleProperty.value; + } + + get voicingVisible() { return this.isVoicingVisible(); } + /** * Override for extra information in the debugging output (from Display.getDebugHTML()). (scenery-internal) */