Merge pull request #5631 from bjester/interactive-video-transcript

Interactive video transcript

integration_testing/features/learner/learner-engage-video-transcript.feature

@@ -0,0 +1,58 @@
+Feature: Learner engages with content of the video kind using transcript
+  Leaner can engage with video content using the accompanying transcript feature
+
+  Background:
+    Given I am signed in as a learner user
+      And there are one or more channels imported on the device with video content containing captions
+      And I am on the *Channels* page for a channel with captioned video content
+
+    Scenario: Browse and find captioned video content
+      When I am on the *Channels* page for <channel>
+      Then I see the *Channels > '<channel>'* breadcrumb
+        And I see all the topics for the channel <channel>
+      When I click the topic <topic>
+      Then I see the *Channels > '<channel'> > '<topic>' breadcrumb
+        And I see all the subtopics and resources of the topic <topic>
+      When I click the subtopic <subtopic>
+      Then I see the *Channels > '<channel'> > '<topic>' > '<subtopic>' breadcrumb
+        And I see all the subtopics and resources of the subtopic <subtopic>
+        And I recognize <resource> resource as a video by the content type icon in the upper left corner
+
+    Scenario: Open video
+      Given that <resource> resource is a video
+        When I click the <resource> resource
+        Then I see the *Channels > '<channel>' > '<topic>' > '<subtopic>' > '<resource>'* breadcrumb
+          And I see the <resource> content
+          And I see a *CC* button to control captions
+          And I see a *Globe* button to control caption language
+
+    Scenario: Engage with the captioned video content controls
+      When I click the *CC* button
+      Then I see an option for *Subtitles*
+        And I see an option for *Transcript*
+      When I click *Subtitles*
+      Then I see captions overlaid as subtitles on the video
+      When I click *Transcript*
+      Then I see a transcript of the captions alongside the video
+      When I click the *Globe* button
+      Then I see the <language_option> option
+      When I select <language_option>
+      Then I see subtitles in <language>
+        And I see the transcript in <language>
+
+    Scenario: Engaging with the transcript
+      When the transcript is enabled
+        And the video is playing
+      Then I see transcript cues highlighted within the transcript
+        And I see the transcript automatically scroll to the next cue
+      When I hover the mouse over the transcript
+      Then the transcript stops scrolling automatically
+      When I click a transcript cue
+      Then the video seeks to the time of the transcript cue
+      When I move the mouse outside of the transcript
+      Then the transcript resumes automatically scrolling
+
+
+Examples:
+  | channel              | topic | subtopic         | resource        | language_option            | language |
+  | Touchable Earth (en) | India | Culture in India | Girl's Clothing | Français, langue française | French   |

kolibri/core/assets/src/utils/i18n.js

@@ -52,13 +52,13 @@ export let currentLanguage = defaultLocale;
 // Default to ltr
 export let languageDirection = languageDirections.LTR;
 
-export const getLangDir = id => {
+export function getLangDir(id) {
   return (availableLanguages[id] || {}).lang_direction || languageDirections.LTR;
-};
+}
 
-export const isRtl = id => {
+export function isRtl(id) {
   return getLangDir(id) === languageDirections.RTL;
-};
+}
 
 export const languageDensities = {
   englishLike: 'english_like',
@@ -93,7 +93,7 @@ const languageDensityMapping = {
   zh: languageDensities.dense,
 };
 
-function languageIdToCode(id) {
+export function languageIdToCode(id) {
   return id.split('-')[0].toLowerCase();
 }
 

kolibri/core/assets/src/views/ContentRenderer/filePresetStrings.js

@@ -14,7 +14,7 @@ const filePresetStrings = {
   vector_video: 'Vectorized ({fileSize})',
   // Same 'thumbnail' string is used for video, audio, document, exercise, and topic
   thumbnail: 'Thumbnail ({fileSize})',
-  subtitle: 'Subtitles - {langCode} ({fileSize})',
+  video_subtitle: 'Subtitles - {langCode} ({fileSize})',
   audio: 'Audio ({fileSize})',
   document: 'Document ({fileSize})',
   exercise: 'Exercise ({fileSize})',
@@ -41,7 +41,7 @@ export function getFilePresetString(file) {
     return filePresetTranslator.$tr('thumbnail', params);
   }
   if (filePresetStrings[preset]) {
-    return filePresetTranslator.$tr(preset, { fileSize: bytesForHumans(file_size) });
+    return filePresetTranslator.$tr(preset, params);
   }
   logging.error(`Download translation string not defined for '${preset}'`);
   return preset;

kolibri/plugins/media_player/assets/src/mixins/videojsButtonMixin.js

@@ -0,0 +1,60 @@
+import videojs from 'video.js';
+
+/**
+ * @param {String} videojsComponent A string of the videojs component to extend
+ */
+export default function videojsButtonMixin(videojsComponent) {
+  return class extends videojs.getComponent(videojsComponent) {
+    /**
+     * @param player
+     * @param options
+     * @param ready
+     */
+    constructor(player, options, ready) {
+      super(player, options, ready);
+
+      // Add missing hide handler since we're not using video.js hover CSS to show it
+      this.on(this.menuButton_.el().parentElement, 'mouseleave', () => {
+        this.menu.hide();
+      });
+    }
+
+    /**
+     * Should build and return an instance of a Video.js Menu
+     * @return {Menu}
+     */
+    buildMenu() {
+      throw new Error('Not implemented');
+    }
+
+    /**
+     * @override
+     * @return {Menu}
+     */
+    createMenu() {
+      if (this.items) {
+        this.items.forEach(item => item.dispose());
+        this.items = [];
+      }
+
+      const menu = this.buildMenu();
+      this.items = this.createItems();
+      this.items.forEach(item => {
+        menu.addItem(item);
+        item.on('hide', () => this.unpressButton());
+      });
+
+      return menu;
+    }
+
+    /**
+     * Removes class that adds specific functionality we don't want
+     *
+     * @param {String} classNames
+     * @return {String}
+     */
+    removePopupClass(classNames) {
+      return classNames.replace(/\bvjs-menu-button-popup\b/, ' ');
+    }
+  };
+}

kolibri/plugins/media_player/assets/src/mixins/videojsMenuItemVueMixin.js

@@ -0,0 +1,39 @@
+import videojsVueMixin from './videojsVueMixin';
+
+/**
+ * @param {Object} vueComponent A compiled vue component object
+ */
+export default function videojsMenuItemVueMixin(vueComponent) {
+  return class extends videojsVueMixin('MenuItem', vueComponent) {
+    createVueComponent(options = {}) {
+      const component = super.createVueComponent(options);
+      component.$on('hide', () => this.trigger('hide'));
+      return component;
+    }
+
+    /**
+     * Pass responsibility to focus down to Vue component
+     */
+    focus() {
+      this.getVueComponent().focus();
+    }
+
+    /**
+     * @override
+     */
+    selected() {
+      return this.getVueComponent().selected;
+    }
+
+    /**
+     * We don't need to handle clicks
+     * @override
+     */
+    handleClick() {}
+
+    /**
+     * Remove Video.js tap event handling so it doesn't mess with menu on mobile
+     */
+    emitTapEvents() {}
+  };
+}

kolibri/plugins/media_player/assets/src/mixins/videojsMenuVueMixin.js

@@ -0,0 +1,126 @@
+import videojsVueMixin from './videojsVueMixin';
+
+/**
+ * @param {Object} vueComponent A compiled vue component object
+ */
+export default function videojsMenuVueMixin(vueComponent) {
+  return class extends videojsVueMixin('Menu', vueComponent) {
+    /**
+     * @param player
+     * @param options
+     */
+    constructor(player, options) {
+      super(player, options);
+
+      this.isLocked = false;
+      this.focusedChild_ = 0;
+    }
+
+    /**
+     * `contentEl` is used when `addItem` is called, so this allows the addition of the text track
+     * options (the languages) in the right spot
+     *
+     * @override
+     * @return {*|Element}
+     */
+    contentEl() {
+      return this.getVueComponent().contentEl();
+    }
+
+    /**
+     * Override parent's method, which adds event handlers we don't want
+     *
+     * @override
+     * @param {Component|String} item The name or instance of the item to add
+     */
+    addItem(item) {
+      this.addChild(item);
+    }
+
+    /**
+     * Triggered by mouseenter of button container
+     *
+     * @override
+     */
+    show() {
+      this.doShow();
+    }
+
+    /**
+     * Triggered by mouseleave of button container
+     *
+     * @override
+     */
+    hide() {
+      this.doHide();
+    }
+
+    /**
+     * Triggered on click in ancestor
+     *
+     * @override
+     */
+    lockShowing() {
+      this.doShow(true);
+    }
+
+    /**
+     * Triggered on blur in ancestor
+     *
+     * @override
+     */
+    unlockShowing() {
+      this.doHide(true);
+    }
+
+    /**
+     * @param {Boolean} lock Whether or not to lock it open
+     */
+    doShow(lock = false) {
+      const component = this.getVueComponent();
+      this.isLocked = this.isLocked || lock;
+
+      if (!component || component.showing()) {
+        return;
+      }
+
+      component.show();
+    }
+
+    /**
+     * @param {Boolean} unlock Whether or not to unlock it if it's locked open
+     */
+    doHide(unlock = false) {
+      const component = this.getVueComponent();
+
+      if (!component || !component.showing() || (!unlock && this.isLocked)) {
+        return;
+      }
+
+      this.isLocked = false;
+      component.hide(unlock);
+    }
+
+    /**
+     * Called by Video.js key event handlers
+     */
+    focus(index) {
+      const children = this.children();
+
+      if (!children) {
+        return;
+      }
+
+      if (!index && index !== 0) {
+        index = this.focusedChild_;
+      } else if (index >= children.length) {
+        index = 0;
+      } else if (index < 0) {
+        index = children.length - 1;
+      }
+
+      this.focusedChild_ = index;
+      children[index].focus();
+    }
+  };
+}

kolibri/plugins/media_player/assets/src/mixins/videojsVueMixin.js

@@ -0,0 +1,59 @@
+import Vue from 'kolibri.lib.vue';
+import store from 'kolibri.coreVue.vuex.store';
+import videojs from 'video.js';
+
+/**
+ * @param {String} videojsComponent A string of the videojs component to extend
+ * @param {Object} vueComponent A compiled vue component object
+ */
+export default function videojsVueMixin(videojsComponent, vueComponent) {
+  const VideojsComponent = videojs.getComponent(videojsComponent);
+  const VueComponent = Vue.extend(vueComponent);
+
+  return class extends VideojsComponent {
+    /**
+     * This is called by video.js code that usually constructs an element, but here we'll leverage
+     * vue by calling it manually.
+     *
+     * @return {Element}
+     */
+    createEl() {
+      return this.createVueComponent().$el;
+    }
+
+    /**
+     * @param {Object} [options]
+     * @return {VueComponent}
+     */
+    createVueComponent(options) {
+      this.clearVueComponent();
+      this._vueComponent = new VueComponent(Object.assign({ store }, options)).$mount();
+      return this.getVueComponent();
+    }
+
+    /**
+     * @return {VueComponent}
+     */
+    getVueComponent() {
+      return this._vueComponent;
+    }
+
+    /**
+     * Clears held Vue component instance, destroying it first
+     */
+    clearVueComponent() {
+      if (this._vueComponent) {
+        this._vueComponent.$destroy();
+        this._vueComponent = null;
+      }
+    }
+
+    /**
+     * video.js hook to dispose this video.js component, so be sure to `clearComponent`
+     */
+    dispose() {
+      this.clearVueComponent();
+      super.dispose();
+    }
+  };
+}