From 2036a88b52efcf7084669eb7d9c3550834f62c92 Mon Sep 17 00:00:00 2001 From: Tony Jin Date: Sat, 7 Jul 2018 18:38:05 -0700 Subject: [PATCH] New: Auto-generated captions Note: This depends on video intelligence being enabled for your enterprise. See https://www.box.com/skills for details. This patch detects if Box Skills transcript cards are loaded and if so, converts them to Shaka Text Cues to be used for captioning. This patch doesn't support switching between native subtitles and auto-generated subtitles. If there are subtitles already within the video, we do not attempt to load auto-generated captions. --- src/i18n/en-US.properties | 4 + src/lib/viewers/media/DashViewer.js | 63 ++++++++++++- .../media/__tests__/DashViewer-test.js | 88 ++++++++++++++++++- 3 files changed, 150 insertions(+), 5 deletions(-) diff --git a/src/i18n/en-US.properties b/src/i18n/en-US.properties index c01979c45..dc123301c 100644 --- a/src/i18n/en-US.properties +++ b/src/i18n/en-US.properties @@ -108,6 +108,10 @@ off=Off media_audio=Audio # Label for alternate audio tracks in media player track=Track +# Label for auto-generated captions +media_auto_generated_captions=Auto-Generated Captions +# Label for auto-generated language choice +auto_generated=Auto-Generated # 3D Preview # Button tooltip for showing/hiding the list of animation clips diff --git a/src/lib/viewers/media/DashViewer.js b/src/lib/viewers/media/DashViewer.js index 90de4230c..68b671357 100644 --- a/src/lib/viewers/media/DashViewer.js +++ b/src/lib/viewers/media/DashViewer.js @@ -1,7 +1,7 @@ import VideoBaseViewer from './VideoBaseViewer'; import PreviewError from '../../PreviewError'; import fullscreen from '../../Fullscreen'; -import { appendQueryParams, get } from '../../util'; +import { appendQueryParams, get, getProp } from '../../util'; import { getRepresentation } from '../../file'; import { MEDIA_STATIC_ASSETS_VERSION } from '../../constants'; import getLanguageName from '../../lang'; @@ -19,6 +19,9 @@ const DEFAULT_VIDEO_HEIGHT_PX = 480; const SHAKA_CODE_ERROR_RECOVERABLE = 1; class DashViewer extends VideoBaseViewer { + /** @property {Object} - shakaExtern.TextDisplayer that displays auto-generated captions, if available */ + autoCaptionDisplayer; + /** * @inheritdoc */ @@ -83,6 +86,7 @@ class DashViewer extends VideoBaseViewer { if (this.mediaControls) { this.mediaControls.removeListener('qualitychange', this.handleQuality); this.mediaControls.removeListener('subtitlechange', this.handleSubtitle); + this.mediaControls.removeListener('audiochange', this.handleAudioTrack); } this.removeStats(); super.destroy(); @@ -279,12 +283,26 @@ class DashViewer extends VideoBaseViewer { */ handleSubtitle() { const subtitleIdx = parseInt(this.cache.get('media-subtitles'), 10); - if (this.textTracks[subtitleIdx] !== undefined) { + + // Auto-generated index 0 ==> turn auto-generated text track on + if (this.autoCaptionDisplayer && subtitleIdx === 0) { + // Manually set text visibility with the custom Shaka Text Displayer + this.autoCaptionDisplayer.setTextVisibility(true); + this.emit('subtitlechange', __('auto_generated')); + + // Valid non-auto-generated index ==> turn specified text track on + } else if (this.textTracks[subtitleIdx] !== undefined) { const track = this.textTracks[subtitleIdx]; this.player.selectTextTrack(track); this.player.setTextTrackVisibility(true); this.emit('subtitlechange', track.language); + + // Index -1 ==> turn subtitles/captions off } else { + if (this.autoCaptionDisplayer) { + this.autoCaptionDisplayer.setTextVisibility(false); + } + this.player.setTextTrackVisibility(false); this.emit('subtitlechange', null); } @@ -426,12 +444,53 @@ class DashViewer extends VideoBaseViewer { * @return {void} */ loadSubtitles() { + // Load subtitles from video, if available this.textTracks = this.player.getTextTracks().sort((track1, track2) => track1.id - track2.id); if (this.textTracks.length > 0) { this.mediaControls.initSubtitles( this.textTracks.map((track) => getLanguageName(track.language) || track.language), getLanguageName(this.options.location.locale.substring(0, 2)) ); + return; + } + + // Attempt to load auto-generated captions + // @TODO 07-07-18: Support both auto-generated captions and subtitles from videos + this.loadAutoGeneratedCaptions(); + } + + /** + * Loads auto-generated captions + * + * @return {void} + */ + loadAutoGeneratedCaptions() { + const textCues = []; + + // Convert Box Skill transcript cards to Shaka Text Cues + const skillsCards = getProp(this.options, 'file.metadata.global.boxSkillsCards.cards'); + if (skillsCards) { + const transcriptCard = skillsCards.find((card) => card.skill_card_type === 'transcript'); + const entries = transcriptCard.entries || []; + entries.forEach((entry) => { + // Set defaults if transcript data is malformed (start/end: 0s, text: '') + const { appears = [{}], text = '' } = entry; + const { start = 0, end = 0 } = Array.isArray(appears) && appears.length > 0 ? appears[0] : {}; + textCues.push(new shaka.text.Cue(start, end, text)); + }); + } + + if (textCues.length > 0) { + this.autoCaptionDisplayer = new shaka.text.SimpleTextDisplayer(this.mediaEl); + this.autoCaptionDisplayer.append(textCues); + this.player.configure({ textDisplayFactory: this.autoCaptionDisplayer }); + this.mediaControls.initSubtitles( + [__('auto_generated')], + getLanguageName(this.options.location.locale.substring(0, 2)) + ); + + // Update the subtitles/caption button to reflect auto-translation + this.mediaControls.setLabel(this.mediaControls.subtitlesButtonEl, __('media_auto_generated_captions')); } } diff --git a/src/lib/viewers/media/__tests__/DashViewer-test.js b/src/lib/viewers/media/__tests__/DashViewer-test.js index 603daf241..d53813179 100644 --- a/src/lib/viewers/media/__tests__/DashViewer-test.js +++ b/src/lib/viewers/media/__tests__/DashViewer-test.js @@ -73,7 +73,12 @@ describe('lib/viewers/media/DashViewer', () => { selectAudioLanguage: () => {}, setTextTrackVisibility: () => {} }; + dash.autoCaptionDisplayer = { + append: () => {}, + setTextVisibility: () => {} + }; stubs.mockPlayer = sandbox.mock(dash.player); + stubs.mockDisplayer = sandbox.mock(dash.autoCaptionDisplayer); dash.mediaControls = { addListener: () => {}, @@ -84,7 +89,8 @@ describe('lib/viewers/media/DashViewer', () => { initAlternateAudio: () => {}, removeAllListeners: () => {}, removeListener: () => {}, - show: sandbox.stub() + show: sandbox.stub(), + setLabel: () => {} }; stubs.mockControls = sandbox.mock(dash.mediaControls); @@ -616,7 +622,7 @@ describe('lib/viewers/media/DashViewer', () => { expect(dash.showMedia).to.not.be.called; }); - it('should load the meta data for the media element, show the media/play button, load subs, check for autoplay, and set focus', () => { + it('should load the metadata for the media element, show the media/play button, load subs, check for autoplay, and set focus', () => { sandbox.stub(dash, 'isDestroyed').returns(false); sandbox.stub(dash, 'showMedia'); sandbox.stub(dash, 'isAutoplayEnabled').returns(true); @@ -789,12 +795,72 @@ describe('lib/viewers/media/DashViewer', () => { expect(dash.textTracks).to.deep.equal([russian, foo, und, empty, doesntmatter, zero]); }); - it('should do nothing if there are no available subtitles', () => { + it('should attempt to load auto-generated subtitles if no there are no subtitles from the video', () => { + sandbox.stub(dash, 'loadAutoGeneratedCaptions'); + const subs = []; stubs.mockPlayer.expects('getTextTracks').returns(subs); stubs.mockControls.expects('initSubtitles').never(); dash.loadSubtitles(); + + expect(dash.loadAutoGeneratedCaptions).to.be.called; + }); + }); + + describe('loadAutoGeneratedCaptions()', () => { + it('should convert to Shaka Cues, initialize subtitles, set button label & return true if there are auto-generated subtitles', () => { + dash.options = { + file: { + metadata: { + global: { + boxSkillsCards: { + cards: [ + { + skill_card_type: 'transcript', + entries: [ + { + appears: [ + { + start: 0, + end: 1 + } + ], + text: 'sometext' + } + ] + } + ] + } + } + } + }, + location: { + locale: 'en-US' + } + }; + const appendStub = sandbox.stub(); + sandbox.stub(shaka.text, 'SimpleTextDisplayer').returns({ + append: appendStub + }); + + stubs.mockPlayer.expects('configure').withArgs({ + textDisplayFactory: sandbox.match.any + }); + stubs.mockControls.expects('initSubtitles').withArgs(['Auto-Generated'], 'English'); + stubs.mockControls.expects('setLabel').withArgs(sandbox.match.any, 'Auto-Generated Captions'); + + dash.loadAutoGeneratedCaptions(); + + expect(appendStub).to.be.called; + }); + + it('should not set a custom text displayer if there are no transcript cards', () => { + stubs.mockPlayer.expects('configure').never(); + stubs.mockControls.expects('initSubtitles').never(); + stubs.mockControls.expects('setLabel').never(); + + dash.loadAutoGeneratedCaptions(); }); }); @@ -846,7 +912,18 @@ describe('lib/viewers/media/DashViewer', () => { }); describe('handleSubtitle()', () => { + it('should select auto-generated track if auto-caption displayer exists', () => { + stubs.mockDisplayer.expects('setTextVisibility').withArgs(true); + sandbox.stub(dash.cache, 'get').returns('0'); + + dash.handleSubtitle(); + + expect(stubs.emit).to.be.calledWith('subtitlechange', 'Auto-Generated'); + }); + it('should select track from front of text track list', () => { + dash.autoCaptionDisplayer = undefined; + const english = { language: 'eng', id: 3 }; const russian = { language: 'rus', id: 4 }; const french = { language: 'fra', id: 5 }; @@ -862,6 +939,8 @@ describe('lib/viewers/media/DashViewer', () => { }); it('should select track from end of text track list', () => { + dash.autoCaptionDisplayer = undefined; + const english = { language: 'eng', id: 3 }; const russian = { language: 'rus', id: 4 }; const french = { language: 'fre', id: 5 }; @@ -877,6 +956,8 @@ describe('lib/viewers/media/DashViewer', () => { }); it('should select track from middle of text track list', () => { + dash.autoCaptionDisplayer = undefined; + const english = { language: 'eng', id: 3 }; const russian = { language: 'rus', id: 4 }; const french = { language: 'fre', id: 5 }; @@ -900,6 +981,7 @@ describe('lib/viewers/media/DashViewer', () => { sandbox.stub(dash.cache, 'get').returns('-1'); stubs.mockPlayer.expects('selectTextTrack').never(); stubs.mockPlayer.expects('setTextTrackVisibility').withArgs(false); + stubs.mockDisplayer.expects('setTextVisibility').withArgs(false); dash.handleSubtitle();