feat(MediaCap): Add preferredDecodingAttributes config

We'll allow users to configure the decoding attributes they prefer when
choosing codecs through the configuration. The attributes include
'smooth', 'powerEfficient' and 'bandwidth'.

For example, if the user configures the field as ['smooth',
'powerEfficient'], we'll filter the variants and keep the smooth ones
first, and if we have more than one available variants, we'll filter and
keep the power efficient variants.
After that, we choose the codecs with lowest bandwidth if we have
multiple codecs available.

Issue #1391

Change-Id: Ief3f6d8ff98fabff5ec99bb0365cdc6a36d2ab2d

externs/shaka/player.js

@@ -981,6 +981,7 @@ shaka.extern.OfflineConfiguration;
  *   preferredVariantRole: string,
  *   preferredTextRole: string,
  *   preferredAudioChannelCount: number,
+ *   preferredDecodingAttributes: !Array.<string>,
  *   preferForcedSubs: boolean,
  *   restrictions: shaka.extern.Restrictions,
  *   playRangeStart: number,
@@ -1016,6 +1017,9 @@ shaka.extern.OfflineConfiguration;
  *   The preferred role to use for text tracks.
  * @property {number} preferredAudioChannelCount
  *   The preferred number of audio channels.
+ * @property {!Array.<string>} preferredDecodingAttributes
+ *   The list of preferred attributes of decodingInfo, in the order of their
+ *   priorities.
  * @property {boolean} preferForcedSubs
  *   If true, a forced text track is preferred.  Defaults to false.
  *   If the content has no forced captions and the value is true,

lib/offline/storage.js

@@ -501,8 +501,10 @@ shaka.offline.Storage = class {
 
     // Choose the codec that has the lowest average bandwidth.
     const preferredAudioChannelCount = config.preferredAudioChannelCount;
+    const preferredDecodingAttributes = config.preferredDecodingAttributes;
+
     StreamUtils.chooseCodecsAndFilterManifest(
-        manifest, preferredAudioChannelCount);
+        manifest, preferredAudioChannelCount, preferredDecodingAttributes);
 
     for (const variant of manifest.variants) {
       goog.asserts.assert(

lib/player.js

@@ -1849,7 +1849,8 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
     // If the content is multi-codec and the browser can play more than one of
     // them, choose codecs now before we initialize streaming.
     shaka.util.StreamUtils.chooseCodecsAndFilterManifest(
-        this.manifest_, this.config_.preferredAudioChannelCount);
+        this.manifest_, this.config_.preferredAudioChannelCount,
+        this.config_.preferredDecodingAttributes);
 
     this.streamingEngine_ = this.createStreamingEngine();
     this.streamingEngine_.configure(this.config_.streaming);

lib/util/multi_map.js

@@ -102,4 +102,12 @@ shaka.util.MultiMap = class {
   size() {
     return Object.keys(this.map_).length;
   }
+
+  /**
+   * Get a list of all the keys.
+   * @return {!Array.<string>}
+   */
+  keys() {
+    return Object.keys(this.map_);
+  }
 };

lib/util/player_configuration.js

@@ -243,6 +243,7 @@ shaka.util.PlayerConfiguration = class {
       preferredTextRole: '',
       preferredAudioChannelCount: 2,
       preferForcedSubs: false,
+      preferredDecodingAttributes: [],
       restrictions: {
         minWidth: 0,
         maxWidth: Infinity,

lib/util/stream_utils.js

@@ -29,8 +29,10 @@ shaka.util.StreamUtils = class {
    * Also filters out variants that have too many audio channels.
    * @param {!shaka.extern.Manifest} manifest
    * @param {number} preferredAudioChannelCount
+   * @param {!Array.<string>} preferredDecodingAttributes
    */
-  static chooseCodecsAndFilterManifest(manifest, preferredAudioChannelCount) {
+  static chooseCodecsAndFilterManifest(manifest, preferredAudioChannelCount,
+      preferredDecodingAttributes) {
     const StreamUtils = shaka.util.StreamUtils;
 
     // To start, consider a subset of variants based on audio channel
@@ -46,12 +48,13 @@ shaka.util.StreamUtils = class {
     let variantsByCodecs = StreamUtils.getVariantsByCodecs_(variants);
     variantsByCodecs = StreamUtils.filterVariantsByDensity_(variantsByCodecs);
 
-    const bestCodecs = StreamUtils.findBestCodecs_(variantsByCodecs);
+    const bestCodecs = StreamUtils.chooseCodecs_(variantsByCodecs,
+        preferredDecodingAttributes);
 
     // Filter out any variants that don't match, forcing AbrManager to choose
-    // from the most efficient variants possible.
+    // from a single video codec and a single audio codec possible.
     manifest.variants = manifest.variants.filter((variant) => {
-      const codecs = StreamUtils.getGroupVariantCodecs_(variant);
+      const codecs = StreamUtils.getVariantCodecs_(variant);
       if (codecs == bestCodecs) {
         return true;
       }
@@ -71,15 +74,17 @@ shaka.util.StreamUtils = class {
   static getVariantsByCodecs_(variants) {
     const variantsByCodecs = new shaka.util.MultiMap();
     for (const variant of variants) {
-      const group = shaka.util.StreamUtils.getGroupVariantCodecs_(variant);
-      variantsByCodecs.push(group, variant);
+      const variantCodecs = shaka.util.StreamUtils.getVariantCodecs_(variant);
+      variantsByCodecs.push(variantCodecs, variant);
     }
 
     return variantsByCodecs;
   }
 
   /**
   * Filters variants by density.
+  * Get variants by codecs map with the max density where all codecs are
+  * present.
   *
   * @param {!shaka.util.MultiMap.<shaka.extern.Variant>} variantsByCodecs
   * @return {!shaka.util.MultiMap.<shaka.extern.Variant>}
@@ -120,6 +125,74 @@ shaka.util.StreamUtils = class {
     return maxDensity ? codecGroupsByDensity.get(maxDensity) : variantsByCodecs;
   }
 
+  /**
+   * Choose the codecs by configured preferred decoding attributes.
+   *
+   * @param {!shaka.util.MultiMap.<shaka.extern.Variant>} variantsByCodecs
+   * @param {!Array.<string>} attributes
+   * @return {string}
+   * @private
+   */
+  static chooseCodecs_(variantsByCodecs, attributes) {
+    const StreamUtils = shaka.util.StreamUtils;
+
+    for (const attribute of attributes) {
+      if (attribute == StreamUtils.DecodingAttributes.SMOOTH ||
+          attribute == StreamUtils.DecodingAttributes.POWER) {
+        variantsByCodecs = StreamUtils.chooseCodecsByMediaCapabilitiesInfo_(
+            variantsByCodecs, attribute);
+        // If we only have one smooth or powerEfficient codecs, choose it as the
+        // best codecs.
+        if (variantsByCodecs.size() == 1) {
+          return variantsByCodecs.keys()[0];
+        }
+      } else if (attribute == StreamUtils.DecodingAttributes.BANDWIDTH) {
+        return StreamUtils.findCodecsByLowestBandwidth_(variantsByCodecs);
+      }
+    }
+    // If there's no configured decoding preferences, or we have multiple codecs
+    // that meets the configured decoding preferences, choose the one with
+    // the lowest bandwidth.
+    return StreamUtils.findCodecsByLowestBandwidth_(variantsByCodecs);
+  }
+
+  /**
+   * Choose the best codecs by configured preferred MediaCapabilitiesInfo
+   * attributes.
+   *
+   * @param {!shaka.util.MultiMap.<shaka.extern.Variant>} variantsByCodecs
+   * @param {string} attribute
+   * @return {!shaka.util.MultiMap.<shaka.extern.Variant>}
+   * @private
+   */
+  static chooseCodecsByMediaCapabilitiesInfo_(variantsByCodecs, attribute) {
+    let highestScore = 0;
+    const bestVariantsByCodecs = new shaka.util.MultiMap();
+    variantsByCodecs.forEach((codecs, variants) => {
+      let sum = 0;
+      let num = 0;
+
+      for (const variant of variants) {
+        if (variant.decodingInfos.length) {
+          sum += variant.decodingInfos[0][attribute] ? 1 : 0;
+          num++;
+        }
+      }
+
+      const averageScore = sum / num;
+      shaka.log.info('codecs', codecs, 'avg', attribute, averageScore);
+
+      if (averageScore > highestScore) {
+        bestVariantsByCodecs.clear();
+        bestVariantsByCodecs.push(codecs, variants);
+        highestScore = averageScore;
+      } else if (averageScore == highestScore) {
+        bestVariantsByCodecs.push(codecs, variants);
+      }
+    });
+    return bestVariantsByCodecs;
+  }
+
   /**
    * Find the lowest-bandwidth (best) codecs.
    * Compute the average bandwidth for each group of variants.
@@ -128,7 +201,7 @@ shaka.util.StreamUtils = class {
    * @return {string}
    * @private
    */
-  static findBestCodecs_(variantsByCodecs) {
+  static findCodecsByLowestBandwidth_(variantsByCodecs) {
     let bestCodecs = '';
     let lowestAverageBandwidth = Infinity;
 
@@ -163,7 +236,7 @@ shaka.util.StreamUtils = class {
    * @return {string}
    * @private
    */
-  static getGroupVariantCodecs_(variant) {
+  static getVariantCodecs_(variant) {
     // Only consider the base of the codec string.  For example, these should
     // both be considered the same codec: avc1.42c01e, avc1.4d401f
     let baseVideoCodec = '';
@@ -1332,3 +1405,12 @@ shaka.util.StreamUtils = class {
 
 /** @private {number} */
 shaka.util.StreamUtils.nextTrackId_ = 0;
+
+/**
+ * @enum {string}
+ */
+shaka.util.StreamUtils.DecodingAttributes = {
+  SMOOTH: 'smooth',
+  POWER: 'powerEfficient',
+  BANDWIDTH: 'bandwidth',
+};

test/util/stream_utils_unit.js

@@ -707,7 +707,7 @@ describe('StreamUtils', () => {
         addVariant2160Vp9(manifest);
       });
 
-      shaka.util.StreamUtils.chooseCodecsAndFilterManifest(manifest, 2);
+      shaka.util.StreamUtils.chooseCodecsAndFilterManifest(manifest, 2, []);
 
       expect(manifest.variants.length).toBe(2);
       expect(manifest.variants[0].video.codecs).toBe(vp09Codecs);
@@ -720,10 +720,51 @@ describe('StreamUtils', () => {
         addVariant1080Vp9(manifest);
       });
 
-      shaka.util.StreamUtils.chooseCodecsAndFilterManifest(manifest, 2);
+      shaka.util.StreamUtils.chooseCodecsAndFilterManifest(manifest, 2, []);
 
       expect(manifest.variants.length).toBe(1);
       expect(manifest.variants[0].video.codecs).toBe(vp09Codecs);
     });
+
+    it('chooses variants by decoding attributes', async () => {
+      manifest = shaka.test.ManifestGenerator.generate((manifest) => {
+        manifest.addVariant(0, (variant) => {
+          variant.bandwidth = 4058558;
+          variant.addVideo(1, (stream) => {
+            stream.mime('video', 'notsmooth');
+          });
+        });
+        manifest.addVariant(1, (variant) => {
+          variant.bandwidth = 4781002;
+          variant.addVideo(2, (stream) => {
+            stream.mime('video', 'smooth');
+          });
+        });
+        manifest.addVariant(3, (variant) => {
+          variant.addVideo(4, (stream) => {
+            variant.bandwidth = 5058558;
+            stream.mime('video', 'smooth-2');
+          });
+        });
+      });
+      navigator.mediaCapabilities.decodingInfo =
+          shaka.test.Util.spyFunc(decodingInfoSpy);
+      decodingInfoSpy.and.callFake((config) => {
+        const res = config.video.contentType.includes('notsmooth') ?
+           {supported: true, smooth: false} :
+           {supported: true, smooth: true};
+        return Promise.resolve(res);
+      });
+
+      await StreamUtils.getDecodingInfosForVariants(manifest.variants,
+          /* usePersistentLicenses= */false);
+
+      shaka.util.StreamUtils.chooseCodecsAndFilterManifest(manifest, 2,
+          [shaka.util.StreamUtils.DecodingAttributes.SMOOTH]);
+      // 2 video codecs are smooth. Choose the one with the lowest bandwidth.
+      expect(manifest.variants.length).toBe(1);
+      expect(manifest.variants[0].id).toBe(1);
+      expect(manifest.variants[0].video.id).toBe(2);
+    });
   });
 });