feat: Automatic ABR quality restrictions based on size (#4404)

Closes #2333

demo/common/message_ids.js

@@ -188,6 +188,7 @@ shakaDemo.MessageIds = {
   IGNORE_DASH_DRM: 'DEMO_IGNORE_DASH_DRM',
   IGNORE_DASH_MAX_SEGMENT_DURATION: 'DEMO_IGNORE_DASH_MAX_SEGMENT_DURATION',
   IGNORE_DASH_SUGGESTED_PRESENTATION_DELAY: 'DEMO_IGNORE_DASH_SUGGESTED_PRESENTATION_DELAY',
+  IGNORE_DEVICE_PIXEL_RATIO: 'DEMO_IGNORE_DEVICE_PIXEL_RATIO',
   IGNORE_HLS_IMAGE_FAILURES: 'DEMO_IGNORE_HLS_IMAGE_FAILURES',
   IGNORE_HLS_TEXT_FAILURES: 'DEMO_IGNORE_HLS_TEXT_FAILURES',
   IGNORE_MANIFEST_PROGRAM_DATE_TIME: 'DEMO_IGNORE_MANIFEST_PROGRAM_DATE_TIME',
@@ -229,6 +230,7 @@ shakaDemo.MessageIds = {
   PREFER_FORCED_SUBS: 'DEMO_PREFER_FORCED_SUBS',
   PREFER_NATIVE_HLS: 'DEMO_PREFER_NATIVE_HLS',
   REBUFFERING_GOAL: 'DEMO_REBUFFERING_GOAL',
+  RESTRICT_TO_ELEMENT_SIZE: 'DEMO_RESTRICT_TO_ELEMENT_SIZE',
   RESTRICTIONS_SECTION_HEADER: 'DEMO_RESTRICTIONS_SECTION_HEADER',
   SAFE_SEEK_OFFSET: 'DEMO_SAFE_SEEK_OFFSET',
   SAFE_SKIP_DISTANCE: 'DEMO_SAFE_SKIP_DISTANCE',

demo/config.js

@@ -271,7 +271,11 @@ shakaDemo.Config = class {
             /* canBeDecimal= */ true)
         .addNumberInput_(MessageIds.SLOW_HALF_LIFE,
             'abr.advanced.slowHalfLife',
-            /* canBeDecimal= */ true);
+            /* canBeDecimal= */ true)
+        .addBoolInput_(MessageIds.RESTRICT_TO_ELEMENT_SIZE,
+            'abr.restrictToElementSize')
+        .addBoolInput_(MessageIds.IGNORE_DEVICE_PIXEL_RATIO,
+            'abr.ignoreDevicePixelRatio');
     this.addRetrictionsSection_('abr',
         MessageIds.ADAPTATION_RESTRICTIONS_SECTION_HEADER);
   }

demo/locales/en.json

@@ -95,6 +95,7 @@
   "DEMO_IGNORE_DASH_EMPTY_ADAPTATION_SET": "Ignore empty DASH AdaptationSets",
   "DEMO_IGNORE_DASH_MAX_SEGMENT_DURATION": "Ignore DASH maxSegmentDuration",
   "DEMO_IGNORE_DASH_SUGGESTED_PRESENTATION_DELAY": "Ignore DASH suggestedPresentationDelay",
+  "DEMO_IGNORE_DEVICE_PIXEL_RATIO": "Ignore device pixel ratio",
   "DEMO_IGNORE_HLS_IMAGE_FAILURES": "Ignore HLS Image Stream Failures",
   "DEMO_IGNORE_HLS_TEXT_FAILURES": "Ignore HLS Text Stream Failures",
   "DEMO_IMA_ASSET_KEY": "Asset key (for LIVE DAI Content)",
@@ -177,6 +178,7 @@
   "DEMO_PROMPT_YES": "Yes",
   "DEMO_REBUFFERING_GOAL": "Rebuffering Goal",
   "DEMO_REPORT_BUG": "REPORT BUG",
+  "DEMO_RESTRICT_TO_ELEMENT_SIZE": "Restrict to element size",
   "DEMO_RESTRICTIONS_SECTION_HEADER": "Restrictions",
   "DEMO_SAFE_SEEK_OFFSET": "Safe Seek Offset",
   "DEMO_SAFE_SKIP_DISTANCE": "Safe Skip Distance",

demo/locales/source.json

@@ -383,6 +383,10 @@
     "description": "The name of a configuration value.",
     "message": "Ignore [PROPER_NAME:DASH] [JARGON:suggestedPresentationDelay]"
   },
+  "DEMO_IGNORE_DEVICE_PIXEL_RATIO": {
+    "description": "The name of a configuration value.",
+    "message": "Ignore device pixel ratio"
+  },
   "DEMO_IGNORE_HLS_TEXT_FAILURES": {
     "description": "The name of a configuration value.",
     "message": "Ignore [PROPER_NAME:HLS] Text Stream Failures"
@@ -711,6 +715,10 @@
     "description": "A link in the header, that files a bug report.",
     "message": "REPORT BUG"
   },
+  "DEMO_RESTRICT_TO_ELEMENT_SIZE": {
+    "description": "The name of a configuration value.",
+    "message": "Restrict to element size"
+  },
   "DEMO_RESTRICTIONS_SECTION_HEADER": {
     "description": "The header for a section of configuration values.",
     "message": "Restrictions"

externs/shaka/abr_manager.js

@@ -103,6 +103,14 @@ shaka.extern.AbrManager = class {
    */
   playbackRateChanged(rate) {}
 
+  /**
+   * Set media element.
+   *
+   * @param {HTMLMediaElement} mediaElement
+   * @exportDoc
+   */
+  setMediaElement(mediaElement) {}
+
   /**
    * Sets the ABR configuration.
    *

externs/shaka/player.js

@@ -1000,7 +1000,9 @@ shaka.extern.StreamingConfiguration;
  *   switchInterval: number,
  *   bandwidthUpgradeTarget: number,
  *   bandwidthDowngradeTarget: number,
- *   advanced: shaka.extern.AdvancedAbrConfiguration
+ *   advanced: shaka.extern.AdvancedAbrConfiguration,
+ *   restrictToElementSize: boolean,
+ *   ignoreDevicePixelRatio: boolean
  * }}
  *
  * @property {boolean} enabled
@@ -1029,7 +1031,16 @@ shaka.extern.StreamingConfiguration;
  *   The largest fraction of the estimated bandwidth we should use. We should
  *   downgrade to avoid this.
  * @property {shaka.extern.AdvancedAbrConfiguration} advanced
- *   Advanced ABR configuration.
+ *   Advanced ABR configuration
+ * @property {boolean} restrictToElementSize
+ *   If true, restrict the quality to media element size.
+ *   Note: The use of ResizeObserver is required for it to work properly. If
+ *   true without ResizeObserver, it behaves as false.
+ *   Defaults false.
+ * @property {boolean} ignoreDevicePixelRatio
+ *   If true,device pixel ratio is ignored when restricting the quality to
+ *   media element size.
+ *   Defaults false.
  * @exportDoc
  */
 shaka.extern.AbrConfiguration;

lib/abr/simple_abr_manager.js

@@ -10,6 +10,7 @@ goog.require('goog.asserts');
 goog.require('shaka.abr.EwmaBandwidthEstimator');
 goog.require('shaka.log');
 goog.require('shaka.util.StreamUtils');
+goog.require('shaka.util.Timer');
 
 
 /**
@@ -85,6 +86,22 @@ shaka.abr.SimpleAbrManager = class {
 
     /** @private {?shaka.extern.AbrConfiguration} */
     this.config_ = null;
+
+    /** @private {HTMLMediaElement} */
+    this.mediaElement_ = null;
+
+    /** @private {ResizeObserver} */
+    this.resizeObserver_ = null;
+
+    /** @private {shaka.util.Timer} */
+    this.resizeObserverTimer_ = new shaka.util.Timer(() => {
+      if (this.config_.restrictToElementSize) {
+        const chosenVariant = this.chooseVariant();
+        if (chosenVariant) {
+          this.switch_(chosenVariant);
+        }
+      }
+    });
   }
 
 
@@ -98,6 +115,14 @@ shaka.abr.SimpleAbrManager = class {
     this.variants_ = [];
     this.playbackRate_ = 1;
     this.lastTimeChosenMs_ = null;
+    this.mediaElement_ = null;
+
+    if (this.resizeObserver_) {
+      this.resizeObserver_.disconnect();
+      this.resizeObserver_ = null;
+    }
+
+    this.resizeObserverTimer_.stop();
 
     // Don't reset |startupComplete_|: if we've left the startup interval, we
     // can start using bandwidth estimates right away after init() is called.
@@ -120,9 +145,19 @@ shaka.abr.SimpleAbrManager = class {
   chooseVariant() {
     const SimpleAbrManager = shaka.abr.SimpleAbrManager;
 
+    let maxHeight = Infinity;
+    let maxWidth = Infinity;
+
+    if (this.resizeObserver_ && this.config_.restrictToElementSize) {
+      const devicePixelRatio =
+          this.config_.ignoreDevicePixelRatio ? 1 : window.devicePixelRatio;
+      maxHeight = this.mediaElement_.clientWidth * devicePixelRatio;
+      maxWidth = this.mediaElement_.clientHeight * devicePixelRatio;
+    }
+
     // Get sorted Variants.
     let sortedVariants = SimpleAbrManager.filterAndSortVariants_(
-        this.config_.restrictions, this.variants_);
+        this.config_.restrictions, this.variants_, maxHeight, maxWidth);
 
     const defaultBandwidthEstimate = this.getDefaultBandwidth_();
     const currentBandwidth = this.bandwidthEstimator_.getBandwidthEstimate(
@@ -137,7 +172,8 @@ shaka.abr.SimpleAbrManager = class {
       shaka.log.warning('No variants met the ABR restrictions. ' +
                         'Choosing a variant by lowest bandwidth.');
       sortedVariants = SimpleAbrManager.filterAndSortVariants_(
-          /* restrictions= */ null, this.variants_);
+          /* restrictions= */ null, this.variants_,
+          /* maxHeight= */ Infinity, /* maxWidth= */ Infinity);
       sortedVariants = [sortedVariants[0]];
     }
 
@@ -243,6 +279,28 @@ shaka.abr.SimpleAbrManager = class {
   }
 
 
+  /**
+   * @override
+   * @export
+   */
+  setMediaElement(mediaElement) {
+    this.mediaElement_ = mediaElement;
+    if (this.resizeObserver_) {
+      this.resizeObserver_.disconnect();
+      this.resizeObserver_ = null;
+    }
+    if (this.mediaElement_ && 'ResizeObserver' in window) {
+      this.resizeObserver_ = new ResizeObserver(() => {
+        const SimpleAbrManager = shaka.abr.SimpleAbrManager;
+        // Batch up resize changes before checking them.
+        this.resizeObserverTimer_.tickAfter(
+            /* seconds= */ SimpleAbrManager.RESIZE_OBSERVER_BATCH_TIME);
+      });
+      this.resizeObserver_.observe(this.mediaElement_);
+    }
+  }
+
+
   /**
    * @override
    * @export
@@ -320,11 +378,13 @@ shaka.abr.SimpleAbrManager = class {
   /**
    * @param {?shaka.extern.Restrictions} restrictions
    * @param {!Array.<shaka.extern.Variant>} variants
+   * @param {!number} maxHeight
+   * @param {!number} maxWidth
    * @return {!Array.<shaka.extern.Variant>} variants filtered according to
    *   |restrictions| and sorted in ascending order of bandwidth.
    * @private
    */
-  static filterAndSortVariants_(restrictions, variants) {
+  static filterAndSortVariants_(restrictions, variants, maxHeight, maxWidth) {
     if (restrictions) {
       variants = variants.filter((variant) => {
         // This was already checked in another scope, but the compiler doesn't
@@ -333,7 +393,7 @@ shaka.abr.SimpleAbrManager = class {
 
         return shaka.util.StreamUtils.meetsRestrictions(
             variant, restrictions,
-            /* maxHwRes= */ {width: Infinity, height: Infinity});
+            /* maxHwRes= */ {width: maxWidth, height: maxHeight});
       });
     }
 
@@ -342,3 +402,11 @@ shaka.abr.SimpleAbrManager = class {
     });
   }
 };
+
+
+/**
+ * The amount of time, in seconds, we wait to batch up rapid resize changes.
+ * This allows us to avoid multiple resize events in most cases.
+ * @type {number}
+ */
+shaka.abr.SimpleAbrManager.RESIZE_OBSERVER_BATCH_TIME = 1;

lib/player.js

@@ -1948,6 +1948,12 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
     if (!this.abrManager_ || this.abrManagerFactory_ != abrFactory) {
       this.abrManagerFactory_ = abrFactory;
       this.abrManager_ = abrFactory();
+      if (typeof this.abrManager_.setMediaElement != 'function') {
+        shaka.Deprecate.deprecateFeature(5,
+            'AbrManager',
+            'Please use an AbrManager with setMediaElement function.');
+        this.abrManager_.setMediaElement = () => {};
+      }
       this.abrManager_.configure(this.config_.abr);
     }
 
@@ -1970,6 +1976,7 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
     this.abrManager_.init((variant, clearBuffer, safeMargin) => {
       return this.switch_(variant, clearBuffer, safeMargin);
     });
+    this.abrManager_.setMediaElement(mediaElement);
 
     this.playhead_ = this.createPlayhead(has.startTime);
     this.playheadObservers_ = this.createPlayheadObserversForMSE_();

lib/util/player_configuration.js

@@ -244,6 +244,8 @@ shaka.util.PlayerConfiguration = class {
         fastHalfLife: 2,
         slowHalfLife: 5,
       },
+      restrictToElementSize: false,
+      ignoreDevicePixelRatio: false,
     };
 
     const cmcd = {