diff --git a/externs/shaka/player.js b/externs/shaka/player.js index 1371753c65..27895a32ef 100644 --- a/externs/shaka/player.js +++ b/externs/shaka/player.js @@ -972,6 +972,7 @@ shaka.extern.OfflineConfiguration; * preferredVariantRole: string, * preferredTextRole: string, * preferredAudioChannelCount: number, + * preferredDecodingAttributes: !Array., * preferForcedSubs: boolean, * restrictions: shaka.extern.Restrictions, * playRangeStart: number, @@ -1007,6 +1008,9 @@ shaka.extern.OfflineConfiguration; * The preferred role to use for text tracks. * @property {number} preferredAudioChannelCount * The preferred number of audio channels. + * @property {!Array.} preferredDecodingAttributes + * The list of preferred attributes of decodingInfo, in the order of their + * priorities. * @property {boolean} preferForcedSubs * If true, a forced text track is preferred. Defaults to false. * If the content has no forced captions and the value is true, diff --git a/lib/offline/storage.js b/lib/offline/storage.js index 5b813134c6..15c91e01ff 100644 --- a/lib/offline/storage.js +++ b/lib/offline/storage.js @@ -501,8 +501,10 @@ shaka.offline.Storage = class { // Choose the codec that has the lowest average bandwidth. const preferredAudioChannelCount = config.preferredAudioChannelCount; + const preferredDecodingAttributes = config.preferredDecodingAttributes; + StreamUtils.chooseCodecsAndFilterManifest( - manifest, preferredAudioChannelCount); + manifest, preferredAudioChannelCount, preferredDecodingAttributes); for (const variant of manifest.variants) { goog.asserts.assert( diff --git a/lib/player.js b/lib/player.js index fae5d1ba69..eeec462a28 100644 --- a/lib/player.js +++ b/lib/player.js @@ -1849,7 +1849,8 @@ shaka.Player = class extends shaka.util.FakeEventTarget { // If the content is multi-codec and the browser can play more than one of // them, choose codecs now before we initialize streaming. shaka.util.StreamUtils.chooseCodecsAndFilterManifest( - this.manifest_, this.config_.preferredAudioChannelCount); + this.manifest_, this.config_.preferredAudioChannelCount, + this.config_.preferredDecodingAttributes); this.streamingEngine_ = this.createStreamingEngine(); this.streamingEngine_.configure(this.config_.streaming); diff --git a/lib/util/multi_map.js b/lib/util/multi_map.js index a26c033c7d..96363dc3e8 100644 --- a/lib/util/multi_map.js +++ b/lib/util/multi_map.js @@ -102,4 +102,12 @@ shaka.util.MultiMap = class { size() { return Object.keys(this.map_).length; } + + /** + * Get a list of all the keys. + * @return {!Array.} + */ + keys() { + return Object.keys(this.map_); + } }; diff --git a/lib/util/player_configuration.js b/lib/util/player_configuration.js index 926ba77862..01951159cb 100644 --- a/lib/util/player_configuration.js +++ b/lib/util/player_configuration.js @@ -242,6 +242,7 @@ shaka.util.PlayerConfiguration = class { preferredTextRole: '', preferredAudioChannelCount: 2, preferForcedSubs: false, + preferredDecodingAttributes: [], restrictions: { minWidth: 0, maxWidth: Infinity, diff --git a/lib/util/stream_utils.js b/lib/util/stream_utils.js index 3c54229902..3333d155fc 100644 --- a/lib/util/stream_utils.js +++ b/lib/util/stream_utils.js @@ -29,8 +29,10 @@ shaka.util.StreamUtils = class { * Also filters out variants that have too many audio channels. * @param {!shaka.extern.Manifest} manifest * @param {number} preferredAudioChannelCount + * @param {!Array.} preferredDecodingAttributes */ - static chooseCodecsAndFilterManifest(manifest, preferredAudioChannelCount) { + static chooseCodecsAndFilterManifest(manifest, preferredAudioChannelCount, + preferredDecodingAttributes) { const StreamUtils = shaka.util.StreamUtils; // To start, consider a subset of variants based on audio channel @@ -46,12 +48,13 @@ shaka.util.StreamUtils = class { let variantsByCodecs = StreamUtils.getVariantsByCodecs_(variants); variantsByCodecs = StreamUtils.filterVariantsByDensity_(variantsByCodecs); - const bestCodecs = StreamUtils.findBestCodecs_(variantsByCodecs); + const bestCodecs = StreamUtils.chooseCodecs_(variantsByCodecs, + preferredDecodingAttributes); // Filter out any variants that don't match, forcing AbrManager to choose - // from the most efficient variants possible. + // from a single video codec and a single audio codec possible. manifest.variants = manifest.variants.filter((variant) => { - const codecs = StreamUtils.getGroupVariantCodecs_(variant); + const codecs = StreamUtils.getVariantCodecs_(variant); if (codecs == bestCodecs) { return true; } @@ -71,8 +74,8 @@ shaka.util.StreamUtils = class { static getVariantsByCodecs_(variants) { const variantsByCodecs = new shaka.util.MultiMap(); for (const variant of variants) { - const group = shaka.util.StreamUtils.getGroupVariantCodecs_(variant); - variantsByCodecs.push(group, variant); + const variantCodecs = shaka.util.StreamUtils.getVariantCodecs_(variant); + variantsByCodecs.push(variantCodecs, variant); } return variantsByCodecs; @@ -80,6 +83,8 @@ shaka.util.StreamUtils = class { /** * Filters variants by density. + * Get variants by codecs map with the max density where all codecs are + * present. * * @param {!shaka.util.MultiMap.} variantsByCodecs * @return {!shaka.util.MultiMap.} @@ -120,6 +125,74 @@ shaka.util.StreamUtils = class { return maxDensity ? codecGroupsByDensity.get(maxDensity) : variantsByCodecs; } + /** + * Choose the codecs by configured preferred decoding attributes. + * + * @param {!shaka.util.MultiMap.} variantsByCodecs + * @param {!Array.} attributes + * @return {string} + * @private + */ + static chooseCodecs_(variantsByCodecs, attributes) { + const StreamUtils = shaka.util.StreamUtils; + + for (const attribute of attributes) { + if (attribute == StreamUtils.DecodingAttributes.SMOOTH || + attribute == StreamUtils.DecodingAttributes.POWER) { + variantsByCodecs = StreamUtils.chooseCodecsByMediaCapabilitiesInfo_( + variantsByCodecs, attribute); + // If we only have one smooth or powerEfficient codecs, choose it as the + // best codecs. + if (variantsByCodecs.size() == 1) { + return variantsByCodecs.keys()[0]; + } + } else if (attribute == StreamUtils.DecodingAttributes.BANDWIDTH) { + return StreamUtils.findCodecsByLowestBandwidth_(variantsByCodecs); + } + } + // If there's no configured decoding preferences, or we have multiple codecs + // that meets the configured decoding preferences, choose the one with + // the lowest bandwidth. + return StreamUtils.findCodecsByLowestBandwidth_(variantsByCodecs); + } + + /** + * Choose the best codecs by configured preferred MediaCapabilitiesInfo + * attributes. + * + * @param {!shaka.util.MultiMap.} variantsByCodecs + * @param {string} attribute + * @return {!shaka.util.MultiMap.} + * @private + */ + static chooseCodecsByMediaCapabilitiesInfo_(variantsByCodecs, attribute) { + let highestScore = 0; + const bestVariantsByCodecs = new shaka.util.MultiMap(); + variantsByCodecs.forEach((codecs, variants) => { + let sum = 0; + let num = 0; + + for (const variant of variants) { + if (variant.decodingInfos.length) { + sum += variant.decodingInfos[0][attribute] ? 1 : 0; + num++; + } + } + + const averageScore = sum / num; + shaka.log.info('codecs', codecs, 'avg', attribute, averageScore); + + if (averageScore > highestScore) { + bestVariantsByCodecs.clear(); + bestVariantsByCodecs.push(codecs, variants); + highestScore = averageScore; + } else if (averageScore == highestScore) { + bestVariantsByCodecs.push(codecs, variants); + } + }); + return bestVariantsByCodecs; + } + /** * Find the lowest-bandwidth (best) codecs. * Compute the average bandwidth for each group of variants. @@ -128,7 +201,7 @@ shaka.util.StreamUtils = class { * @return {string} * @private */ - static findBestCodecs_(variantsByCodecs) { + static findCodecsByLowestBandwidth_(variantsByCodecs) { let bestCodecs = ''; let lowestAverageBandwidth = Infinity; @@ -163,7 +236,7 @@ shaka.util.StreamUtils = class { * @return {string} * @private */ - static getGroupVariantCodecs_(variant) { + static getVariantCodecs_(variant) { // Only consider the base of the codec string. For example, these should // both be considered the same codec: avc1.42c01e, avc1.4d401f let baseVideoCodec = ''; @@ -1332,3 +1405,12 @@ shaka.util.StreamUtils = class { /** @private {number} */ shaka.util.StreamUtils.nextTrackId_ = 0; + +/** + * @enum {string} + */ +shaka.util.StreamUtils.DecodingAttributes = { + SMOOTH: 'smooth', + POWER: 'powerEfficient', + BANDWIDTH: 'bandwidth', +}; diff --git a/test/util/stream_utils_unit.js b/test/util/stream_utils_unit.js index da7534722d..1bbec0edc5 100644 --- a/test/util/stream_utils_unit.js +++ b/test/util/stream_utils_unit.js @@ -707,7 +707,7 @@ describe('StreamUtils', () => { addVariant2160Vp9(manifest); }); - shaka.util.StreamUtils.chooseCodecsAndFilterManifest(manifest, 2); + shaka.util.StreamUtils.chooseCodecsAndFilterManifest(manifest, 2, []); expect(manifest.variants.length).toBe(2); expect(manifest.variants[0].video.codecs).toBe(vp09Codecs); @@ -720,10 +720,51 @@ describe('StreamUtils', () => { addVariant1080Vp9(manifest); }); - shaka.util.StreamUtils.chooseCodecsAndFilterManifest(manifest, 2); + shaka.util.StreamUtils.chooseCodecsAndFilterManifest(manifest, 2, []); expect(manifest.variants.length).toBe(1); expect(manifest.variants[0].video.codecs).toBe(vp09Codecs); }); + + it('chooses variants by decoding attributes', async () => { + manifest = shaka.test.ManifestGenerator.generate((manifest) => { + manifest.addVariant(0, (variant) => { + variant.bandwidth = 4058558; + variant.addVideo(1, (stream) => { + stream.mime('video', 'notsmooth'); + }); + }); + manifest.addVariant(1, (variant) => { + variant.bandwidth = 4781002; + variant.addVideo(2, (stream) => { + stream.mime('video', 'smooth'); + }); + }); + manifest.addVariant(3, (variant) => { + variant.addVideo(4, (stream) => { + variant.bandwidth = 5058558; + stream.mime('video', 'smooth-2'); + }); + }); + }); + navigator.mediaCapabilities.decodingInfo = + shaka.test.Util.spyFunc(decodingInfoSpy); + decodingInfoSpy.and.callFake((config) => { + const res = config.video.contentType.includes('notsmooth') ? + {supported: true, smooth: false} : + {supported: true, smooth: true}; + return Promise.resolve(res); + }); + + await StreamUtils.getDecodingInfosForVariants(manifest.variants, + /* usePersistentLicenses= */false); + + shaka.util.StreamUtils.chooseCodecsAndFilterManifest(manifest, 2, + [shaka.util.StreamUtils.DecodingAttributes.SMOOTH]); + // 2 video codecs are smooth. Choose the one with the lowest bandwidth. + expect(manifest.variants.length).toBe(1); + expect(manifest.variants[0].id).toBe(1); + expect(manifest.variants[0].video.id).toBe(2); + }); }); });