Skip to content

Commit

Permalink
New: Auto-generated captions
Browse files Browse the repository at this point in the history 
Note: This depends on video intelligence being enabled for your enterprise. See https://www.box.com/skills for details.

This patch detects if Box Skills transcript cards are loaded and if so, converts them to Shaka Text Cues to be used for captioning. This patch doesn't support switching between native subtitles and auto-generated subtitles. If there are subtitles already within the video, we do not attempt to load auto-generated captions.
  • Loading branch information
Tony Jin committed Jul 9, 2018
1 parent e1e4b57 commit 2036a88
Show file tree
Hide file tree
Showing 3 changed files with 150 additions and 5 deletions.
4 changes: 4 additions & 0 deletions src/i18n/en-US.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,10 @@ off=Off
media_audio=Audio
# Label for alternate audio tracks in media player
track=Track
# Label for auto-generated captions
media_auto_generated_captions=Auto-Generated Captions
# Label for auto-generated language choice
auto_generated=Auto-Generated

# 3D Preview
# Button tooltip for showing/hiding the list of animation clips
Expand Down
63 changes: 61 additions & 2 deletions src/lib/viewers/media/DashViewer.js
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import VideoBaseViewer from './VideoBaseViewer';
import PreviewError from '../../PreviewError';
import fullscreen from '../../Fullscreen';
import { appendQueryParams, get } from '../../util';
import { appendQueryParams, get, getProp } from '../../util';
import { getRepresentation } from '../../file';
import { MEDIA_STATIC_ASSETS_VERSION } from '../../constants';
import getLanguageName from '../../lang';
Expand All @@ -19,6 +19,9 @@ const DEFAULT_VIDEO_HEIGHT_PX = 480;
const SHAKA_CODE_ERROR_RECOVERABLE = 1;

class DashViewer extends VideoBaseViewer {
/** @property {Object} - shakaExtern.TextDisplayer that displays auto-generated captions, if available */
autoCaptionDisplayer;

/**
* @inheritdoc
*/
Expand Down Expand Up @@ -83,6 +86,7 @@ class DashViewer extends VideoBaseViewer {
if (this.mediaControls) {
this.mediaControls.removeListener('qualitychange', this.handleQuality);
this.mediaControls.removeListener('subtitlechange', this.handleSubtitle);
this.mediaControls.removeListener('audiochange', this.handleAudioTrack);
}
this.removeStats();
super.destroy();
Expand Down Expand Up @@ -279,12 +283,26 @@ class DashViewer extends VideoBaseViewer {
*/
handleSubtitle() {
const subtitleIdx = parseInt(this.cache.get('media-subtitles'), 10);
if (this.textTracks[subtitleIdx] !== undefined) {

// Auto-generated index 0 ==> turn auto-generated text track on
if (this.autoCaptionDisplayer && subtitleIdx === 0) {
// Manually set text visibility with the custom Shaka Text Displayer
this.autoCaptionDisplayer.setTextVisibility(true);
this.emit('subtitlechange', __('auto_generated'));

// Valid non-auto-generated index ==> turn specified text track on
} else if (this.textTracks[subtitleIdx] !== undefined) {
const track = this.textTracks[subtitleIdx];
this.player.selectTextTrack(track);
this.player.setTextTrackVisibility(true);
this.emit('subtitlechange', track.language);

// Index -1 ==> turn subtitles/captions off
} else {
if (this.autoCaptionDisplayer) {
this.autoCaptionDisplayer.setTextVisibility(false);
}

this.player.setTextTrackVisibility(false);
this.emit('subtitlechange', null);
}
Expand Down Expand Up @@ -426,12 +444,53 @@ class DashViewer extends VideoBaseViewer {
* @return {void}
*/
loadSubtitles() {
// Load subtitles from video, if available
this.textTracks = this.player.getTextTracks().sort((track1, track2) => track1.id - track2.id);
if (this.textTracks.length > 0) {
this.mediaControls.initSubtitles(
this.textTracks.map((track) => getLanguageName(track.language) || track.language),
getLanguageName(this.options.location.locale.substring(0, 2))
);
return;
}

// Attempt to load auto-generated captions
// @TODO 07-07-18: Support both auto-generated captions and subtitles from videos
this.loadAutoGeneratedCaptions();
}

/**
* Loads auto-generated captions
*
* @return {void}
*/
loadAutoGeneratedCaptions() {
const textCues = [];

// Convert Box Skill transcript cards to Shaka Text Cues
const skillsCards = getProp(this.options, 'file.metadata.global.boxSkillsCards.cards');
if (skillsCards) {
const transcriptCard = skillsCards.find((card) => card.skill_card_type === 'transcript');
const entries = transcriptCard.entries || [];
entries.forEach((entry) => {
// Set defaults if transcript data is malformed (start/end: 0s, text: '')
const { appears = [{}], text = '' } = entry;
const { start = 0, end = 0 } = Array.isArray(appears) && appears.length > 0 ? appears[0] : {};
textCues.push(new shaka.text.Cue(start, end, text));
});
}

if (textCues.length > 0) {
this.autoCaptionDisplayer = new shaka.text.SimpleTextDisplayer(this.mediaEl);
this.autoCaptionDisplayer.append(textCues);
this.player.configure({ textDisplayFactory: this.autoCaptionDisplayer });
this.mediaControls.initSubtitles(
[__('auto_generated')],
getLanguageName(this.options.location.locale.substring(0, 2))
);

// Update the subtitles/caption button to reflect auto-translation
this.mediaControls.setLabel(this.mediaControls.subtitlesButtonEl, __('media_auto_generated_captions'));
}
}

Expand Down
88 changes: 85 additions & 3 deletions src/lib/viewers/media/__tests__/DashViewer-test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,12 @@ describe('lib/viewers/media/DashViewer', () => {
selectAudioLanguage: () => {},
setTextTrackVisibility: () => {}
};
dash.autoCaptionDisplayer = {
append: () => {},
setTextVisibility: () => {}
};
stubs.mockPlayer = sandbox.mock(dash.player);
stubs.mockDisplayer = sandbox.mock(dash.autoCaptionDisplayer);

dash.mediaControls = {
addListener: () => {},
Expand All @@ -84,7 +89,8 @@ describe('lib/viewers/media/DashViewer', () => {
initAlternateAudio: () => {},
removeAllListeners: () => {},
removeListener: () => {},
show: sandbox.stub()
show: sandbox.stub(),
setLabel: () => {}
};
stubs.mockControls = sandbox.mock(dash.mediaControls);

Expand Down Expand Up @@ -616,7 +622,7 @@ describe('lib/viewers/media/DashViewer', () => {
expect(dash.showMedia).to.not.be.called;
});

it('should load the meta data for the media element, show the media/play button, load subs, check for autoplay, and set focus', () => {
it('should load the metadata for the media element, show the media/play button, load subs, check for autoplay, and set focus', () => {
sandbox.stub(dash, 'isDestroyed').returns(false);
sandbox.stub(dash, 'showMedia');
sandbox.stub(dash, 'isAutoplayEnabled').returns(true);
Expand Down Expand Up @@ -789,12 +795,72 @@ describe('lib/viewers/media/DashViewer', () => {
expect(dash.textTracks).to.deep.equal([russian, foo, und, empty, doesntmatter, zero]);
});

it('should do nothing if there are no available subtitles', () => {
it('should attempt to load auto-generated subtitles if no there are no subtitles from the video', () => {
sandbox.stub(dash, 'loadAutoGeneratedCaptions');

const subs = [];
stubs.mockPlayer.expects('getTextTracks').returns(subs);
stubs.mockControls.expects('initSubtitles').never();

dash.loadSubtitles();

expect(dash.loadAutoGeneratedCaptions).to.be.called;
});
});

describe('loadAutoGeneratedCaptions()', () => {
it('should convert to Shaka Cues, initialize subtitles, set button label & return true if there are auto-generated subtitles', () => {
dash.options = {
file: {
metadata: {
global: {
boxSkillsCards: {
cards: [
{
skill_card_type: 'transcript',
entries: [
{
appears: [
{
start: 0,
end: 1
}
],
text: 'sometext'
}
]
}
]
}
}
}
},
location: {
locale: 'en-US'
}
};
const appendStub = sandbox.stub();
sandbox.stub(shaka.text, 'SimpleTextDisplayer').returns({
append: appendStub
});

stubs.mockPlayer.expects('configure').withArgs({
textDisplayFactory: sandbox.match.any
});
stubs.mockControls.expects('initSubtitles').withArgs(['Auto-Generated'], 'English');
stubs.mockControls.expects('setLabel').withArgs(sandbox.match.any, 'Auto-Generated Captions');

dash.loadAutoGeneratedCaptions();

expect(appendStub).to.be.called;
});

it('should not set a custom text displayer if there are no transcript cards', () => {
stubs.mockPlayer.expects('configure').never();
stubs.mockControls.expects('initSubtitles').never();
stubs.mockControls.expects('setLabel').never();

dash.loadAutoGeneratedCaptions();
});
});

Expand Down Expand Up @@ -846,7 +912,18 @@ describe('lib/viewers/media/DashViewer', () => {
});

describe('handleSubtitle()', () => {
it('should select auto-generated track if auto-caption displayer exists', () => {
stubs.mockDisplayer.expects('setTextVisibility').withArgs(true);
sandbox.stub(dash.cache, 'get').returns('0');

dash.handleSubtitle();

expect(stubs.emit).to.be.calledWith('subtitlechange', 'Auto-Generated');
});

it('should select track from front of text track list', () => {
dash.autoCaptionDisplayer = undefined;

const english = { language: 'eng', id: 3 };
const russian = { language: 'rus', id: 4 };
const french = { language: 'fra', id: 5 };
Expand All @@ -862,6 +939,8 @@ describe('lib/viewers/media/DashViewer', () => {
});

it('should select track from end of text track list', () => {
dash.autoCaptionDisplayer = undefined;

const english = { language: 'eng', id: 3 };
const russian = { language: 'rus', id: 4 };
const french = { language: 'fre', id: 5 };
Expand All @@ -877,6 +956,8 @@ describe('lib/viewers/media/DashViewer', () => {
});

it('should select track from middle of text track list', () => {
dash.autoCaptionDisplayer = undefined;

const english = { language: 'eng', id: 3 };
const russian = { language: 'rus', id: 4 };
const french = { language: 'fre', id: 5 };
Expand All @@ -900,6 +981,7 @@ describe('lib/viewers/media/DashViewer', () => {
sandbox.stub(dash.cache, 'get').returns('-1');
stubs.mockPlayer.expects('selectTextTrack').never();
stubs.mockPlayer.expects('setTextTrackVisibility').withArgs(false);
stubs.mockDisplayer.expects('setTextVisibility').withArgs(false);

dash.handleSubtitle();

Expand Down

0 comments on commit 2036a88

Please sign in to comment.