From 2af80122a4e6f5cc36c5c51fbba140333a5f0be1 Mon Sep 17 00:00:00 2001 From: Rob Walch Date: Tue, 21 Feb 2023 20:31:54 -0800 Subject: [PATCH 1/8] Handle fragments and parts with a GAP tag/attribute --- README.md | 2 +- api-extractor/report/hls.js.api.md | 4 +++ docs/API.md | 10 +++++--- src/controller/audio-stream-controller.ts | 9 ++++++- src/controller/base-stream-controller.ts | 8 +++++- src/controller/eme-controller.ts | 2 +- src/controller/error-controller.ts | 5 +++- src/controller/fragment-tracker.ts | 17 ++++++++++--- src/controller/gap-controller.ts | 10 ++++++-- src/controller/latency-controller.ts | 8 +++--- src/controller/stream-controller.ts | 1 + src/errors.ts | 2 ++ src/loader/fragment-loader.ts | 31 +++++++++++++++++++++++ src/loader/fragment.ts | 2 ++ src/loader/m3u8-parser.ts | 1 + tests/unit/loader/playlist-loader.ts | 5 +++- 16 files changed, 99 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 9910533ca4e..ce64ca58067 100644 --- a/README.md +++ b/README.md @@ -110,11 +110,11 @@ The following properties are added to their respective variants' attribute list - `#EXT-X-RENDITION-REPORT:` - `#EXT-X-DATERANGE:` Metadata - `#EXT-X-DEFINE:` Variable Import and Substitution (`NAME,VALUE,IMPORT,QUERYPARAM` attributes) +- `#EXT-X-GAP` (Skips loading GAP segments and parts. Skips playback of unbuffered program containing only GAP content and no suitable alternates. See [#2940](https://github.com/video-dev/hls.js/issues/2940)) The following tags are added to their respective fragment's attribute list but are not implemented in streaming and playback. - `#EXT-X-BITRATE` (Not used in ABR controller) -- `#EXT-X-GAP` (Not implemented. See [#2940](https://github.com/video-dev/hls.js/issues/2940)) Parsed but missing feature support diff --git a/api-extractor/report/hls.js.api.md b/api-extractor/report/hls.js.api.md index b65644a17ee..cd83daff5cf 100644 --- a/api-extractor/report/hls.js.api.md +++ b/api-extractor/report/hls.js.api.md @@ -991,6 +991,8 @@ export enum ErrorDetails { // (undocumented) FRAG_DECRYPT_ERROR = "fragDecryptError", // (undocumented) + FRAG_GAP = "fragGap", + // (undocumented) FRAG_LOAD_ERROR = "fragLoadError", // (undocumented) FRAG_LOAD_TIMEOUT = "fragLoadTimeOut", @@ -1343,6 +1345,8 @@ export class Fragment extends BaseSegment { // (undocumented) endPTS?: number; // (undocumented) + gap?: boolean; + // (undocumented) initSegment: Fragment | null; // Warning: (ae-forgotten-export) The symbol "KeyLoaderContext" needs to be exported by the entry point hls.d.ts // diff --git a/docs/API.md b/docs/API.md index 84daff4e501..047470539cb 100644 --- a/docs/API.md +++ b/docs/API.md @@ -1726,14 +1726,14 @@ Full list of errors is described below: - data: { type : `NETWORK_ERROR`, details : `Hls.ErrorDetails.MANIFEST_LOAD_TIMEOUT`, fatal : `true`, url : manifest URL, loader : URL loader } - `Hls.ErrorDetails.MANIFEST_PARSING_ERROR` - raised when manifest parsing failed to find proper content - data: { type : `NETWORK_ERROR`, details : `Hls.ErrorDetails.MANIFEST_PARSING_ERROR`, fatal : `true`, url : manifest URL, reason : parsing error reason } -- `Hls.ErrorDetails.LEVEL_EMPTY_ERROR` - raised when loaded level contains no fragments - - data: { type : `NETWORK_ERROR`, details : `Hls.ErrorDetails.LEVEL_EMPTY_ERROR`, url: playlist URL, reason: error reason, level: index of the bad level } +- `Hls.ErrorDetails.LEVEL_EMPTY_ERROR` - raised when loaded level contains no fragments (applies to levels and audio and subtitle tracks) + - data: { type : `NETWORK_ERROR`, details : `Hls.ErrorDetails.LEVEL_EMPTY_ERROR`, url: playlist URL, reason: error reason, level: index of the bad level or undefined, parent: PlaylistLevelType } - `Hls.ErrorDetails.LEVEL_LOAD_ERROR` - raised when level loading fails because of a network error - data: { type : `NETWORK_ERROR`, details : `Hls.ErrorDetails.LEVEL_LOAD_ERROR`, fatal : `true`, url : level URL, response : { code: error code, text: error text }, loader : URL loader } - `Hls.ErrorDetails.LEVEL_LOAD_TIMEOUT` - raised when level loading fails because of a timeout - data: { type : `NETWORK_ERROR`, details : `Hls.ErrorDetails.LEVEL_LOAD_TIMEOUT`, fatal : `false`, url : level URL, loader : URL loader } -- `Hls.ErrorDetails.LEVEL_PARSING_ERROR` - raised when level parsing failed or found invalid content - - data: { type : `NETWORK_ERROR`, details : `Hls.ErrorDetails.LEVEL_PARSING_ERROR`, fatal : `false`, url : level URL, error: Error } +- `Hls.ErrorDetails.LEVEL_PARSING_ERROR` - raised when playlist parsing failed or found invalid content (applies to levels and audio and subtitle tracks) + - data: { type : `NETWORK_ERROR`, details : `Hls.ErrorDetails.LEVEL_PARSING_ERROR`, fatal : `false`, url : level URL, error: Error, parent: PlaylistLevelType } - `Hls.ErrorDetails.AUDIO_TRACK_LOAD_ERROR` - raised when audio playlist loading fails because of a network error - data: { type : `NETWORK_ERROR`, details : `Hls.ErrorDetails.AUDIO_TRACK_LOAD_ERROR`, fatal : `false`, url : audio URL, response : { code: error code, text: error text }, loader : URL loader } - `Hls.ErrorDetails.AUDIO_TRACK_LOAD_TIMEOUT` - raised when audio playlist loading fails because of a timeout @@ -1759,6 +1759,8 @@ Full list of errors is described below: - data: { type : `MEDIA_ERROR`, details : `Hls.ErrorDetails.FRAG_DECRYPT_ERROR`, fatal : `true`, reason : failure reason } - `Hls.ErrorDetails.FRAG_PARSING_ERROR` - raised when fragment parsing fails - data: { type : `MEDIA_ERROR`, details : `Hls.ErrorDetails.FRAG_PARSING_ERROR`, fatal : `true` or `false`, reason : failure reason } +- `Hls.ErrorDetails.FRAG_GAP` - raised when segment loading is skipped because a fragment with a GAP tag or part with GAP=YES attribute was encountered + - data: { type : `MEDIA_ERROR`, details : `Hls.ErrorDetails.FRAG_GAP`, fatal : `false`, frag : fragment object, part? : part object (if any) } - `Hls.ErrorDetails.BUFFER_ADD_CODEC_ERROR` - raised when MediaSource fails to add new sourceBuffer - data: { type : `MEDIA_ERROR`, details : `Hls.ErrorDetails.BUFFER_ADD_CODEC_ERROR`, fatal : `false`, error : error raised by MediaSource, mimeType: mimeType on which the failure happened } - `Hls.ErrorDetails.BUFFER_INCOMPATIBLE_CODECS_ERROR` - raised when no MediaSource(s) could be created based on track codec(s) diff --git a/src/controller/audio-stream-controller.ts b/src/controller/audio-stream-controller.ts index 2e13cf43ec2..26b4f3ce1ab 100644 --- a/src/controller/audio-stream-controller.ts +++ b/src/controller/audio-stream-controller.ts @@ -356,7 +356,13 @@ class AudioStreamController } // wait for main buffer after buffing some audio if (!mainBufferInfo?.len && bufferInfo.len) { - return; + if ( + !mainBufferInfo?.nextStart || + mainBufferInfo.nextStart > + bufferInfo.end + trackDetails.targetduration * 2 + ) { + return; + } } const frag = this.getNextFragment(targetBufferTime, trackDetails); @@ -643,6 +649,7 @@ class AudioStreamController return; } switch (data.details) { + case ErrorDetails.FRAG_GAP: case ErrorDetails.FRAG_PARSING_ERROR: case ErrorDetails.FRAG_DECRYPT_ERROR: case ErrorDetails.FRAG_LOAD_ERROR: diff --git a/src/controller/base-stream-controller.ts b/src/controller/base-stream-controller.ts index c1a69251094..61bcaa95818 100644 --- a/src/controller/base-stream-controller.ts +++ b/src/controller/base-stream-controller.ts @@ -1161,7 +1161,10 @@ export default class BaseStreamController const curSNIdx = frag.sn - levelDetails.startSN; // Move fragPrevious forward to support forcing the next fragment to load // when the buffer catches up to a previously buffered range. - if (this.fragmentTracker.getState(frag) === FragmentState.OK) { + if ( + this.fragmentTracker.getState(frag) === FragmentState.OK || + (frag.gap && frag.stats.aborted) + ) { fragPrevious = frag; } if (fragPrevious && frag.sn === fragPrevious.sn && !loadingParts) { @@ -1372,6 +1375,9 @@ export default class BaseStreamController ); return; } + if (data.details === ErrorDetails.FRAG_GAP) { + this.fragmentTracker.fragBuffered(frag, true); + } // keep retrying until the limit will be reached const errorAction = data.errorAction; const { action, retryCount = 0, retryConfig } = errorAction || {}; diff --git a/src/controller/eme-controller.ts b/src/controller/eme-controller.ts index 92c8299424c..d1c013ab926 100644 --- a/src/controller/eme-controller.ts +++ b/src/controller/eme-controller.ts @@ -24,7 +24,7 @@ import { base64Decode } from '../utils/numeric-encoding-utils'; import { DecryptData, LevelKey } from '../loader/level-key'; import Hex from '../utils/hex'; import { bin2str, parsePssh, parseSinf } from '../utils/mp4-tools'; -import EventEmitter from 'eventemitter3'; +import { EventEmitter } from 'eventemitter3'; import type Hls from '../hls'; import type { ComponentAPI } from '../types/component-api'; import type { diff --git a/src/controller/error-controller.ts b/src/controller/error-controller.ts index b9159e95a61..a8d698a74a3 100644 --- a/src/controller/error-controller.ts +++ b/src/controller/error-controller.ts @@ -96,6 +96,7 @@ export default class ErrorController implements NetworkComponentAPI { case ErrorDetails.KEY_LOAD_TIMEOUT: data.errorAction = this.getFragRetryOrSwitchAction(data); return; + case ErrorDetails.FRAG_GAP: case ErrorDetails.FRAG_PARSING_ERROR: case ErrorDetails.FRAG_DECRYPT_ERROR: { // Switch level if possible, otherwise allow retry count to reach max error retries @@ -264,7 +265,9 @@ export default class ErrorController implements NetworkComponentAPI { ); // Switch levels when out of retried or level index out of bounds if (level) { - level.fragmentError++; + if (data.details !== ErrorDetails.FRAG_GAP) { + level.fragmentError++; + } const httpStatus = data.response?.code; const retry = shouldRetry( retryConfig, diff --git a/src/controller/fragment-tracker.ts b/src/controller/fragment-tracker.ts index 133d9f74f03..b3d80757228 100644 --- a/src/controller/fragment-tracker.ts +++ b/src/controller/fragment-tracker.ts @@ -220,9 +220,18 @@ export class FragmentTracker implements ComponentAPI { } } - public fragBuffered(frag: Fragment) { + public fragBuffered(frag: Fragment, force?: boolean) { const fragKey = getFragmentKey(frag); - const fragmentEntity = this.fragments[fragKey]; + let fragmentEntity = this.fragments[fragKey]; + if (!fragmentEntity && force) { + fragmentEntity = this.fragments[fragKey] = { + body: frag, + appendedPTS: null, + loaded: null, + buffered: false, + range: Object.create(null), + }; + } if (fragmentEntity) { fragmentEntity.loaded = null; fragmentEntity.buffered = true; @@ -482,7 +491,9 @@ export class FragmentTracker implements ComponentAPI { function isPartial(fragmentEntity: FragmentEntity): boolean { return ( fragmentEntity.buffered && - (fragmentEntity.range.video?.partial || fragmentEntity.range.audio?.partial) + (fragmentEntity.body.gap || + fragmentEntity.range.video?.partial || + fragmentEntity.range.audio?.partial) ); } diff --git a/src/controller/gap-controller.ts b/src/controller/gap-controller.ts index 2591c77b983..ff3bf04efe5 100644 --- a/src/controller/gap-controller.ts +++ b/src/controller/gap-controller.ts @@ -131,7 +131,11 @@ export default class GapController { const maxStartGapJump = isLive ? level!.details!.targetduration * 2 : MAX_START_GAP_JUMP; - if (startJump > 0 && startJump <= maxStartGapJump) { + if ( + startJump > 0 && + (startJump <= maxStartGapJump || + this.fragmentTracker.getPartialFragment(0)) + ) { this._trySkipBufferHole(null); return; } @@ -194,7 +198,9 @@ export default class GapController { // needs to cross some sort of threshold covering all source-buffers content // to start playing properly. if ( - bufferInfo.len > config.maxBufferHole && + (bufferInfo.len > config.maxBufferHole || + (bufferInfo.nextStart && + bufferInfo.nextStart - currentTime < config.maxBufferHole)) && stalledDurationMs > config.highBufferWatchdogPeriod * 1000 ) { logger.warn('Trying to nudge playhead over buffer-hole'); diff --git a/src/controller/latency-controller.ts b/src/controller/latency-controller.ts index bf60895bb9e..707079f4b66 100644 --- a/src/controller/latency-controller.ts +++ b/src/controller/latency-controller.ts @@ -184,9 +184,11 @@ export default class LatencyController implements ComponentAPI { return; } this.stallCount++; - logger.warn( - '[playback-rate-controller]: Stall detected, adjusting target latency' - ); + if (this.levelDetails?.live) { + logger.warn( + '[playback-rate-controller]: Stall detected, adjusting target latency' + ); + } } private timeupdate() { diff --git a/src/controller/stream-controller.ts b/src/controller/stream-controller.ts index 6dc0c754aff..90f7411d627 100644 --- a/src/controller/stream-controller.ts +++ b/src/controller/stream-controller.ts @@ -855,6 +855,7 @@ export default class StreamController return; } switch (data.details) { + case ErrorDetails.FRAG_GAP: case ErrorDetails.FRAG_PARSING_ERROR: case ErrorDetails.FRAG_DECRYPT_ERROR: case ErrorDetails.FRAG_LOAD_ERROR: diff --git a/src/errors.ts b/src/errors.ts index 5687c34e77e..2de40defbcf 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -57,6 +57,8 @@ export enum ErrorDetails { // Identifier for a fragment parsing error event - data: { id : demuxer Id, reason : parsing error description } // will be renamed DEMUX_PARSING_ERROR and switched to MUX_ERROR in the next major release FRAG_PARSING_ERROR = 'fragParsingError', + // Identifier for a fragment or part load skipped because of a GAP tag or attribute + FRAG_GAP = 'fragGap', // Identifier for a remux alloc error event - data: { id : demuxer Id, frag : fragment object, bytes : nb of bytes on which allocation failed , reason : error text } REMUX_ALLOC_ERROR = 'remuxAllocError', // Identifier for decrypt key load error - data: { frag : fragment object, response : { code: error code, text: error text }} diff --git a/src/loader/fragment-loader.ts b/src/loader/fragment-loader.ts index 136d7ed4abb..27884939923 100644 --- a/src/loader/fragment-loader.ts +++ b/src/loader/fragment-loader.ts @@ -68,6 +68,21 @@ export default class FragmentLoader { if (this.loader) { this.loader.destroy(); } + if (frag.gap) { + frag.stats.aborted = true; + frag.stats.retry++; + reject( + new LoadError({ + type: ErrorTypes.MEDIA_ERROR, + details: ErrorDetails.FRAG_GAP, + fatal: false, + frag, + error: new Error('GAP tag found'), + networkDetails: null, + }) + ); + return; + } const loader = (this.loader = frag.loader = @@ -175,6 +190,22 @@ export default class FragmentLoader { if (this.loader) { this.loader.destroy(); } + if (frag.gap || part.gap) { + frag.stats.aborted = true; + frag.stats.retry++; + reject( + new LoadError({ + type: ErrorTypes.MEDIA_ERROR, + details: ErrorDetails.FRAG_GAP, + fatal: false, + frag, + part, + error: new Error(`GAP ${frag.gap ? 'tag' : 'attribute'} found`), + networkDetails: null, + }) + ); + return; + } const loader = (this.loader = frag.loader = diff --git a/src/loader/fragment.ts b/src/loader/fragment.ts index 892c60c89ea..a6087544042 100644 --- a/src/loader/fragment.ts +++ b/src/loader/fragment.ts @@ -147,6 +147,8 @@ export class Fragment extends BaseSegment { public initSegment: Fragment | null = null; // Fragment is the last fragment in the media playlist public endList?: boolean; + // Fragment is marked by an EXT-X-GAP tag indicating that it does not contain media data and should not be loaded + public gap?: boolean; constructor(type: PlaylistLevelType, baseurl: string) { super(baseurl); diff --git a/src/loader/m3u8-parser.ts b/src/loader/m3u8-parser.ts index a7ac6529743..2cc2a998d33 100644 --- a/src/loader/m3u8-parser.ts +++ b/src/loader/m3u8-parser.ts @@ -513,6 +513,7 @@ export default class M3U8Parser { frag.tagList.push(['DIS']); break; case 'GAP': + frag.gap = true; frag.tagList.push([tag]); break; case 'BITRATE': diff --git a/tests/unit/loader/playlist-loader.ts b/tests/unit/loader/playlist-loader.ts index 471e2e36ab5..5bc9d544622 100644 --- a/tests/unit/loader/playlist-loader.ts +++ b/tests/unit/loader/playlist-loader.ts @@ -1582,7 +1582,7 @@ fileSequence2.ts ]); }); - it('adds GAP to fragment.tagList', function () { + it('adds GAP to fragment.tagList and sets fragment.gap', function () { const playlist = `#EXTM3U #EXT-X-TARGETDURATION:5 #EXT-X-VERSION:3 @@ -1613,6 +1613,9 @@ fileSequence2.ts ['GAP'], ]); expectWithJSONMessage(fragments[2].tagList).to.deep.equal([['INF', '5']]); + expect(fragments[0].gap).to.equal(undefined); + expect(fragments[1].gap).to.equal(true); + expect(fragments[2].gap).to.equal(undefined); }); it('adds unhandled tags (DATERANGE) and comments to fragment.tagList', function () { From b8244479ab8e92162672896bd86f1d4719471013 Mon Sep 17 00:00:00 2001 From: Rob Walch Date: Tue, 28 Feb 2023 22:40:29 -0800 Subject: [PATCH 2/8] Improve GAP fragment picking, post gap buffering, and level switching error resolution --- src/controller/audio-stream-controller.ts | 33 ++------- src/controller/base-stream-controller.ts | 57 ++++++++++++--- src/controller/error-controller.ts | 22 +++++- src/controller/stream-controller.ts | 84 +++++++++++------------ 4 files changed, 114 insertions(+), 82 deletions(-) diff --git a/src/controller/audio-stream-controller.ts b/src/controller/audio-stream-controller.ts index 26b4f3ce1ab..e68f7e75e48 100644 --- a/src/controller/audio-stream-controller.ts +++ b/src/controller/audio-stream-controller.ts @@ -671,33 +671,12 @@ class AudioStreamController } break; case ErrorDetails.BUFFER_FULL_ERROR: - // if in appending state - if ( - data.parent === 'audio' && - (this.state === State.PARSING || this.state === State.PARSED) - ) { - let flushBuffer = true; - const bufferedInfo = this.getFwdBufferInfo( - this.mediaBuffer, - PlaylistLevelType.AUDIO - ); - // 0.5 : tolerance needed as some browsers stalls playback before reaching buffered end - // reduce max buf len if current position is buffered - if (bufferedInfo && bufferedInfo.len > 0.5) { - flushBuffer = !this.reduceMaxBufferLength(bufferedInfo.len); - } - if (flushBuffer) { - // current position is not buffered, but browser is still complaining about buffer full error - // this happens on IE/Edge, refer to https://github.com/video-dev/hls.js/pull/708 - // in that case flush the whole audio buffer to recover - this.warn( - 'Buffer full error also media.currentTime is not buffered, flush audio buffer' - ); - this.fragCurrent = null; - this.bufferedTrack = null; - super.flushMainBuffer(0, Number.POSITIVE_INFINITY, 'audio'); - } - this.resetLoadingState(); + if (!data.parent || data.parent !== 'audio') { + return; + } + if (this.reduceLengthAndFlushBuffer(data)) { + this.bufferedTrack = null; + super.flushMainBuffer(0, Number.POSITIVE_INFINITY, 'audio'); } break; case ErrorDetails.INTERNAL_EXCEPTION: diff --git a/src/controller/base-stream-controller.ts b/src/controller/base-stream-controller.ts index 61bcaa95818..082f481beab 100644 --- a/src/controller/base-stream-controller.ts +++ b/src/controller/base-stream-controller.ts @@ -900,16 +900,22 @@ export default class BaseStreamController bufferable: Bufferable | null, type: PlaylistLevelType ): BufferInfo | null { - const { config } = this; const pos = this.getLoadPosition(); if (!Number.isFinite(pos)) { return null; } - const bufferInfo = BufferHelper.bufferInfo( - bufferable, - pos, - config.maxBufferHole - ); + return this.getFwdBufferInfoAtPos(bufferable, pos, type); + } + + protected getFwdBufferInfoAtPos( + bufferable: Bufferable | null, + pos: number, + type: PlaylistLevelType + ): BufferInfo | null { + const { + config: { maxBufferHole }, + } = this; + const bufferInfo = BufferHelper.bufferInfo(bufferable, pos, maxBufferHole); // Workaround flaw in getting forward buffer when maxBufferHole is smaller than gap at current pos if (bufferInfo.len === 0 && bufferInfo.nextStart !== undefined) { const bufferedFragAtPos = this.fragmentTracker.getBufferedFrag(pos, type); @@ -917,7 +923,7 @@ export default class BaseStreamController return BufferHelper.bufferInfo( bufferable, pos, - Math.max(bufferInfo.nextStart, config.maxBufferHole) + Math.max(bufferInfo.nextStart, maxBufferHole) ); } } @@ -938,7 +944,7 @@ export default class BaseStreamController return Math.min(maxBufLen, config.maxMaxBufferLength); } - protected reduceMaxBufferLength(threshold?: number) { + protected reduceMaxBufferLength(threshold: number) { const config = this.config; const minLength = threshold || config.maxBufferLength; if (config.maxMaxBufferLength >= minLength) { @@ -1178,7 +1184,7 @@ export default class BaseStreamController this.fragmentTracker.getState(nextFrag) !== FragmentState.OK ) { this.log( - `SN ${frag.sn} just loaded, load next one: ${nextFrag.sn}` + `Skipping loaded ${frag.type} SN ${frag.sn} at buffer end` ); frag = nextFrag; } else { @@ -1414,6 +1420,39 @@ export default class BaseStreamController } else { this.state = State.ERROR; } + // Perform next async tick sooner to speed up error action resolution + this.tickImmediate(); + } + + protected reduceLengthAndFlushBuffer(data: ErrorData): boolean { + // if in appending state + if (this.state === State.PARSING || this.state === State.PARSED) { + const playlistType = data.parent as PlaylistLevelType; + const bufferedInfo = this.getFwdBufferInfo( + this.mediaBuffer, + playlistType + ); + // 0.5 : tolerance needed as some browsers stalls playback before reaching buffered end + // reduce max buf len if current position is buffered + let flushBuffer = true; + if (bufferedInfo && bufferedInfo.len > 0.5) { + flushBuffer = !this.reduceMaxBufferLength(bufferedInfo.len); + } + if (flushBuffer) { + // current position is not buffered, but browser is still complaining about buffer full error + // this happens on IE/Edge, refer to https://github.com/video-dev/hls.js/pull/708 + // in that case flush the whole audio buffer to recover + this.warn( + `Buffer full error while media.currentTime is not buffered, flush ${playlistType} buffer` + ); + } + if (data.frag) { + this.nextLoadPosition = data.frag.start; + } + this.resetLoadingState(); + return flushBuffer; + } + return false; } protected resetFragmentErrors(filterType: PlaylistLevelType) { diff --git a/src/controller/error-controller.ts b/src/controller/error-controller.ts index a8d698a74a3..f709fe9f3d1 100644 --- a/src/controller/error-controller.ts +++ b/src/controller/error-controller.ts @@ -6,6 +6,7 @@ import { isTimeoutError, shouldRetry, } from '../utils/error-helper'; +import { findFragmentByPTS } from './fragment-finders'; import { HdcpLevel, HdcpLevels } from '../types/level'; import { logger } from '../utils/logger'; import type Hls from '../hls'; @@ -305,7 +306,10 @@ export default class ErrorController implements NetworkComponentAPI { } const level = this.hls.levels[levelIndex]; if (level) { - level.loadError++; + // No penalty for GAP tags so that player can switch back when GAPs are found in other levels + if (data.details !== ErrorDetails.FRAG_GAP) { + level.loadError++; + } const redundantLevels = level.url.length; // Try redundant fail-over until level.loadError reaches redundantLevels if (redundantLevels > 1 && level.loadError < redundantLevels) { @@ -320,6 +324,20 @@ export default class ErrorController implements NetworkComponentAPI { candidate !== hls.loadLevel && levels[candidate].loadError === 0 ) { + // Skip level switch if GAP tag is found in next level + if (data.details === ErrorDetails.FRAG_GAP && data.frag) { + const levelDetails = hls.levels[candidate].details; + if (levelDetails) { + const fragCandidate = findFragmentByPTS( + data.frag, + levelDetails.fragments, + data.frag.start + ); + if (fragCandidate?.gap) { + continue; + } + } + } nextLevel = candidate; break; } @@ -405,7 +423,7 @@ export default class ErrorController implements NetworkComponentAPI { private switchLevel(data: ErrorData, levelIndex: number | undefined) { if (levelIndex !== undefined && data.errorAction) { - this.warn(`${data.details}: switching to level ${levelIndex}`); + this.warn(`switching to level ${levelIndex} after ${data.details}`); this.hls.nextAutoLevel = levelIndex; data.errorAction.resolved = true; // Stream controller is responsible for this but won't switch on false start diff --git a/src/controller/stream-controller.ts b/src/controller/stream-controller.ts index 90f7411d627..7e1137f006c 100644 --- a/src/controller/stream-controller.ts +++ b/src/controller/stream-controller.ts @@ -302,25 +302,42 @@ export default class StreamController } else if (this.backtrackFragment && bufferInfo.len) { this.backtrackFragment = null; } - // Avoid loop loading by using nextLoadPosition set for backtracking - if ( - frag && - this.fragmentTracker.getState(frag) === FragmentState.OK && - this.nextLoadPosition > targetBufferTime - ) { - // Cleanup the fragment tracker before trying to find the next unbuffered fragment - const type = - this.audioOnly && !this.altAudio - ? ElementaryStreamTypes.AUDIO - : ElementaryStreamTypes.VIDEO; - const mediaBuffer = - (type === ElementaryStreamTypes.VIDEO - ? this.videoBuffer - : this.mediaBuffer) || this.media; - if (mediaBuffer) { - this.afterBufferFlushed(mediaBuffer, type, PlaylistLevelType.MAIN); + if (frag) { + // Avoid loop loading by using nextLoadPosition set for backtracking and skipping consecutive GAP tags + const trackerState = this.fragmentTracker.getState(frag); + if ( + (trackerState === FragmentState.OK || + (trackerState === FragmentState.PARTIAL && frag.gap)) && + this.nextLoadPosition > targetBufferTime + ) { + const gapStart = frag.gap; + if (!gapStart) { + // Cleanup the fragment tracker before trying to find the next unbuffered fragment + const type = + this.audioOnly && !this.altAudio + ? ElementaryStreamTypes.AUDIO + : ElementaryStreamTypes.VIDEO; + const mediaBuffer = + (type === ElementaryStreamTypes.VIDEO + ? this.videoBuffer + : this.mediaBuffer) || this.media; + if (mediaBuffer) { + this.afterBufferFlushed(mediaBuffer, type, PlaylistLevelType.MAIN); + } + } + frag = this.getNextFragment(this.nextLoadPosition, levelDetails); + if (gapStart && frag && !frag.gap && bufferInfo.nextStart) { + // Make sure this doesn't make the next buffer timerange exceed forward buffer length after a gap + const nextbufferInfo = this.getFwdBufferInfoAtPos( + this.mediaBuffer ? this.mediaBuffer : this.media, + bufferInfo.nextStart, + PlaylistLevelType.MAIN + ); + if (nextbufferInfo !== null && nextbufferInfo.len > maxBufLen) { + return; + } + } } - frag = this.getNextFragment(this.nextLoadPosition, levelDetails); } if (!frag) { return; @@ -877,32 +894,11 @@ export default class StreamController } break; case ErrorDetails.BUFFER_FULL_ERROR: - // if in appending state - if ( - data.parent === 'main' && - (this.state === State.PARSING || this.state === State.PARSED) - ) { - let flushBuffer = true; - const bufferedInfo = this.getFwdBufferInfo( - this.media, - PlaylistLevelType.MAIN - ); - // 0.5 : tolerance needed as some browsers stalls playback before reaching buffered end - // reduce max buf len if current position is buffered - if (bufferedInfo && bufferedInfo.len > 0.5) { - flushBuffer = !this.reduceMaxBufferLength(bufferedInfo.len); - } - if (flushBuffer) { - // current position is not buffered, but browser is still complaining about buffer full error - // this happens on IE/Edge, refer to https://github.com/video-dev/hls.js/pull/708 - // in that case flush the whole buffer to recover - this.warn( - 'buffer full error also media.currentTime is not buffered, flush main' - ); - // flush main buffer - this.immediateLevelSwitch(); - } - this.resetLoadingState(); + if (!data.parent || data.parent !== 'main') { + return; + } + if (this.reduceLengthAndFlushBuffer(data)) { + this.flushMainBuffer(0, Number.POSITIVE_INFINITY); } break; case ErrorDetails.INTERNAL_EXCEPTION: From 35f9c78a891f68937675dd0205c3024e24abb3f2 Mon Sep 17 00:00:00 2001 From: Rob Walch Date: Thu, 2 Mar 2023 11:56:11 -0800 Subject: [PATCH 3/8] Cleanup FRAG_GAP error creation and inlcude current buffer length in forward gap buffer limit --- api-extractor/report/hls.js.api.md | 6 +++- src/controller/base-stream-controller.ts | 5 +-- src/controller/error-controller.ts | 2 +- src/controller/stream-controller.ts | 7 ++-- src/loader/fragment-loader.ts | 43 ++++++++++-------------- 5 files changed, 32 insertions(+), 31 deletions(-) diff --git a/api-extractor/report/hls.js.api.md b/api-extractor/report/hls.js.api.md index cd83daff5cf..19dfe549e86 100644 --- a/api-extractor/report/hls.js.api.md +++ b/api-extractor/report/hls.js.api.md @@ -330,6 +330,8 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP // (undocumented) protected getFwdBufferInfo(bufferable: Bufferable | null, type: PlaylistLevelType): BufferInfo | null; // (undocumented) + protected getFwdBufferInfoAtPos(bufferable: Bufferable | null, pos: number, type: PlaylistLevelType): BufferInfo | null; + // (undocumented) protected getInitialLiveFragment(levelDetails: LevelDetails, fragments: Array): Fragment | null; // (undocumented) protected getLevelDetails(): LevelDetails | undefined; @@ -406,7 +408,9 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP // (undocumented) protected recoverWorkerError(data: ErrorData): void; // (undocumented) - protected reduceMaxBufferLength(threshold?: number): boolean; + protected reduceLengthAndFlushBuffer(data: ErrorData): boolean; + // (undocumented) + protected reduceMaxBufferLength(threshold: number): boolean; // (undocumented) protected resetFragmentErrors(filterType: PlaylistLevelType): void; // (undocumented) diff --git a/src/controller/base-stream-controller.ts b/src/controller/base-stream-controller.ts index 082f481beab..2ef823fe17b 100644 --- a/src/controller/base-stream-controller.ts +++ b/src/controller/base-stream-controller.ts @@ -1167,9 +1167,10 @@ export default class BaseStreamController const curSNIdx = frag.sn - levelDetails.startSN; // Move fragPrevious forward to support forcing the next fragment to load // when the buffer catches up to a previously buffered range. + const fragState = this.fragmentTracker.getState(frag); if ( - this.fragmentTracker.getState(frag) === FragmentState.OK || - (frag.gap && frag.stats.aborted) + fragState === FragmentState.OK || + (fragState === FragmentState.PARTIAL && frag.gap) ) { fragPrevious = frag; } diff --git a/src/controller/error-controller.ts b/src/controller/error-controller.ts index f709fe9f3d1..95a7366698b 100644 --- a/src/controller/error-controller.ts +++ b/src/controller/error-controller.ts @@ -324,7 +324,7 @@ export default class ErrorController implements NetworkComponentAPI { candidate !== hls.loadLevel && levels[candidate].loadError === 0 ) { - // Skip level switch if GAP tag is found in next level + // Skip level switch if GAP tag is found in next level at same position if (data.details === ErrorDetails.FRAG_GAP && data.frag) { const levelDetails = hls.levels[candidate].details; if (levelDetails) { diff --git a/src/controller/stream-controller.ts b/src/controller/stream-controller.ts index 7e1137f006c..77598f87ae4 100644 --- a/src/controller/stream-controller.ts +++ b/src/controller/stream-controller.ts @@ -327,13 +327,16 @@ export default class StreamController } frag = this.getNextFragment(this.nextLoadPosition, levelDetails); if (gapStart && frag && !frag.gap && bufferInfo.nextStart) { - // Make sure this doesn't make the next buffer timerange exceed forward buffer length after a gap + // Media buffered after GAP tags should not make the next buffer timerange exceed forward buffer length const nextbufferInfo = this.getFwdBufferInfoAtPos( this.mediaBuffer ? this.mediaBuffer : this.media, bufferInfo.nextStart, PlaylistLevelType.MAIN ); - if (nextbufferInfo !== null && nextbufferInfo.len > maxBufLen) { + if ( + nextbufferInfo !== null && + bufferLen + nextbufferInfo.len >= maxBufLen + ) { return; } } diff --git a/src/loader/fragment-loader.ts b/src/loader/fragment-loader.ts index 27884939923..cd697543db8 100644 --- a/src/loader/fragment-loader.ts +++ b/src/loader/fragment-loader.ts @@ -69,18 +69,7 @@ export default class FragmentLoader { this.loader.destroy(); } if (frag.gap) { - frag.stats.aborted = true; - frag.stats.retry++; - reject( - new LoadError({ - type: ErrorTypes.MEDIA_ERROR, - details: ErrorDetails.FRAG_GAP, - fatal: false, - frag, - error: new Error('GAP tag found'), - networkDetails: null, - }) - ); + reject(createGapLoadError(frag)); return; } const loader = @@ -191,19 +180,7 @@ export default class FragmentLoader { this.loader.destroy(); } if (frag.gap || part.gap) { - frag.stats.aborted = true; - frag.stats.retry++; - reject( - new LoadError({ - type: ErrorTypes.MEDIA_ERROR, - details: ErrorDetails.FRAG_GAP, - fatal: false, - frag, - part, - error: new Error(`GAP ${frag.gap ? 'tag' : 'attribute'} found`), - networkDetails: null, - }) - ); + reject(createGapLoadError(frag, part)); return; } const loader = @@ -373,6 +350,22 @@ function createLoaderContext( return loaderContext; } +function createGapLoadError(frag: Fragment, part?: Part): LoadError { + const error = new Error(`GAP ${frag.gap ? 'tag' : 'attribute'} found`); + const errorData: FragLoadFailResult = { + type: ErrorTypes.MEDIA_ERROR, + details: ErrorDetails.FRAG_GAP, + fatal: false, + frag, + error, + networkDetails: null, + }; + if (part) { + errorData.part = part; + } + return new LoadError(errorData); +} + export class LoadError extends Error { public readonly data: FragLoadFailResult; constructor(data: FragLoadFailResult) { From b7bb26ba3abe6a7da582ab07f771291e7b95dd00 Mon Sep 17 00:00:00 2001 From: Rob Walch Date: Fri, 3 Mar 2023 08:43:11 -0800 Subject: [PATCH 4/8] Fix gap jump over waiting for data stall when buffer length is larger than maxBufferHole but less than one second --- src/controller/gap-controller.ts | 35 +++++++++++++++----------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/src/controller/gap-controller.ts b/src/controller/gap-controller.ts index ff3bf04efe5..ea45e092524 100644 --- a/src/controller/gap-controller.ts +++ b/src/controller/gap-controller.ts @@ -131,12 +131,9 @@ export default class GapController { const maxStartGapJump = isLive ? level!.details!.targetduration * 2 : MAX_START_GAP_JUMP; - if ( - startJump > 0 && - (startJump <= maxStartGapJump || - this.fragmentTracker.getPartialFragment(0)) - ) { - this._trySkipBufferHole(null); + const partialOrGap = this.fragmentTracker.getPartialFragment(currentTime); + if (startJump > 0 && (startJump <= maxStartGapJump || partialOrGap)) { + this._trySkipBufferHole(partialOrGap); return; } } @@ -187,7 +184,7 @@ export default class GapController { // This method isn't limited by the size of the gap between buffered ranges const targetTime = this._trySkipBufferHole(partial); // we return here in this case, meaning - // the branch below only executes when we don't handle a partial fragment + // the branch below only executes when we haven't seeked to a new position if (targetTime || !this.media) { return; } @@ -248,19 +245,20 @@ export default class GapController { if (media === null) { return 0; } - const currentTime = media.currentTime; - let lastEndTime = 0; + // Check if currentTime is between unbuffered regions of partial fragments - const buffered = BufferHelper.getBuffered(media); - for (let i = 0; i < buffered.length; i++) { - const startTime = buffered.start(i); - if ( - currentTime + config.maxBufferHole >= lastEndTime && - currentTime < startTime - ) { + const currentTime = media.currentTime; + const bufferInfo = BufferHelper.bufferInfo(media, currentTime, 0); + const startTime = + currentTime < bufferInfo.start ? bufferInfo.start : bufferInfo.nextStart; + if (startTime) { + const bufferStarved = bufferInfo.len <= config.maxBufferHole; + const waiting = + bufferInfo.len > 0 && bufferInfo.len < 1 && media.readyState < 3; + if (currentTime < startTime && (bufferStarved || waiting)) { const targetTime = Math.max( startTime + SKIP_BUFFER_RANGE_START, - media.currentTime + SKIP_BUFFER_HOLE_STEP_SECONDS + currentTime + SKIP_BUFFER_HOLE_STEP_SECONDS ); logger.warn( `skipping hole, adjusting currentTime from ${currentTime} to ${targetTime}` @@ -268,7 +266,7 @@ export default class GapController { this.moved = true; this.stalled = null; media.currentTime = targetTime; - if (partial) { + if (partial && !partial.gap) { const error = new Error( `fragment loaded with buffer holes, seeking from ${currentTime} to ${targetTime}` ); @@ -283,7 +281,6 @@ export default class GapController { } return targetTime; } - lastEndTime = buffered.end(i); } return 0; } From 9c495ded8e84c9d15edfcfd9ea4689c3e682eb65 Mon Sep 17 00:00:00 2001 From: Rob Walch Date: Wed, 8 Mar 2023 14:20:56 -0800 Subject: [PATCH 5/8] Warn when observing "seeked" event with empty buffer Related to #5274 --- src/controller/gap-controller.ts | 1 + src/controller/stream-controller.ts | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/src/controller/gap-controller.ts b/src/controller/gap-controller.ts index ea45e092524..8d7e160a40b 100644 --- a/src/controller/gap-controller.ts +++ b/src/controller/gap-controller.ts @@ -77,6 +77,7 @@ export default class GapController { // Clear stalled state when beginning or finishing seeking so that we don't report stalls coming out of a seek if (beginSeek || seeked) { this.stalled = null; + return; } // The playhead should not be moving diff --git a/src/controller/stream-controller.ts b/src/controller/stream-controller.ts index 77598f87ae4..48fc46f6b12 100644 --- a/src/controller/stream-controller.ts +++ b/src/controller/stream-controller.ts @@ -557,6 +557,17 @@ export default class StreamController this.log(`Media seeked to ${(currentTime as number).toFixed(3)}`); } + // If seeked was issued before buffer was appended do not tick immediately + const bufferInfo = this.getMainFwdBufferInfo(); + if (bufferInfo === null || bufferInfo.len === 0) { + this.warn( + `Main forward buffer length on "seeked" event ${ + bufferInfo ? bufferInfo.len : 'empty' + })` + ); + return; + } + // tick to speed up FRAG_CHANGED triggering this.tick(); } From 45582def1a6dada7aed1bf40f8f3b538abd333d1 Mon Sep 17 00:00:00 2001 From: Rob Walch Date: Mon, 20 Mar 2023 18:44:18 -0700 Subject: [PATCH 6/8] Handle forward buffering past GAP tags in alt audio Playlists --- api-extractor/report/hls.js.api.md | 4 ++ src/controller/audio-stream-controller.ts | 54 +++++++++++++++------ src/controller/base-stream-controller.ts | 49 +++++++++++++++++-- src/controller/stream-controller.ts | 59 +++++++++-------------- src/loader/fragment-loader.ts | 1 + 5 files changed, 111 insertions(+), 56 deletions(-) diff --git a/api-extractor/report/hls.js.api.md b/api-extractor/report/hls.js.api.md index 19dfe549e86..bca6891e17b 100644 --- a/api-extractor/report/hls.js.api.md +++ b/api-extractor/report/hls.js.api.md @@ -342,6 +342,8 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP // (undocumented) protected getNextFragment(pos: number, levelDetails: LevelDetails): Fragment | null; // (undocumented) + protected getNextFragmentLoopLoading(frag: Fragment, levelDetails: LevelDetails, bufferInfo: BufferInfo, playlistType: PlaylistLevelType, maxBufLen: number): Fragment | null; + // (undocumented) getNextPart(partList: Part[], frag: Fragment, targetBufferTime: number): number; // Warning: (ae-forgotten-export) The symbol "PartsLoadedData" needs to be exported by the entry point hls.d.ts // @@ -358,6 +360,8 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP // (undocumented) protected initPTS: RationalTimestamp[]; // (undocumented) + protected isLoopLoading(frag: Fragment, targetBufferTime: number): boolean; + // (undocumented) protected keyLoader: KeyLoader; // (undocumented) protected lastCurrentTime: number; diff --git a/src/controller/audio-stream-controller.ts b/src/controller/audio-stream-controller.ts index e68f7e75e48..19adf6aaf43 100644 --- a/src/controller/audio-stream-controller.ts +++ b/src/controller/audio-stream-controller.ts @@ -347,30 +347,52 @@ class AudioStreamController } } - // buffer audio up to one target duration ahead of main buffer - if ( - mainBufferInfo && - targetBufferTime > mainBufferInfo.end + trackDetails.targetduration - ) { + let frag = this.getNextFragment(targetBufferTime, trackDetails); + let atGap = false; + // Avoid loop loading by using nextLoadPosition set for backtracking and skipping consecutive GAP tags + if (frag && this.isLoopLoading(frag, targetBufferTime)) { + atGap = !!frag.gap; + frag = this.getNextFragmentLoopLoading( + frag, + trackDetails, + bufferInfo, + PlaylistLevelType.MAIN, + maxBufLen + ); + } + if (!frag) { + this.bufferFlushed = true; return; } - // wait for main buffer after buffing some audio - if (!mainBufferInfo?.len && bufferInfo.len) { + + // Buffer audio up to one target duration ahead of main buffer + const atBufferSyncLimit = + mainBufferInfo && + frag.start > mainBufferInfo.end + trackDetails.targetduration; + if ( + atBufferSyncLimit || + // Or wait for main buffer after buffing some audio + (!mainBufferInfo?.len && bufferInfo.len) + ) { + // Check fragment-tracker for main fragments since GAP segments do not show up in bufferInfo + const mainFrag = this.fragmentTracker.getBufferedFrag( + frag.start, + PlaylistLevelType.MAIN + ); + if (mainFrag === null) { + return; + } + // Bridge gaps in main buffer + atGap ||= + !!mainFrag.gap || (!!atBufferSyncLimit && mainBufferInfo.len === 0); if ( - !mainBufferInfo?.nextStart || - mainBufferInfo.nextStart > - bufferInfo.end + trackDetails.targetduration * 2 + (atBufferSyncLimit && !atGap) || + (atGap && bufferInfo.nextStart && bufferInfo.nextStart < mainFrag.end) ) { return; } } - const frag = this.getNextFragment(targetBufferTime, trackDetails); - if (!frag) { - this.bufferFlushed = true; - return; - } - this.loadFragment(frag, levelInfo, targetBufferTime); } diff --git a/src/controller/base-stream-controller.ts b/src/controller/base-stream-controller.ts index 2ef823fe17b..f8cf8371df3 100644 --- a/src/controller/base-stream-controller.ts +++ b/src/controller/base-stream-controller.ts @@ -1010,6 +1010,52 @@ export default class BaseStreamController return this.mapToInitFragWhenRequired(frag); } + protected isLoopLoading(frag: Fragment, targetBufferTime: number): boolean { + const trackerState = this.fragmentTracker.getState(frag); + return ( + (trackerState === FragmentState.OK || + (trackerState === FragmentState.PARTIAL && !!frag.gap)) && + this.nextLoadPosition > targetBufferTime + ); + } + + protected getNextFragmentLoopLoading( + frag: Fragment, + levelDetails: LevelDetails, + bufferInfo: BufferInfo, + playlistType: PlaylistLevelType, + maxBufLen: number + ): Fragment | null { + const gapStart = frag.gap; + const nextFragment = this.getNextFragment( + this.nextLoadPosition, + levelDetails + ); + if (nextFragment === null) { + return nextFragment; + } + frag = nextFragment; + if (gapStart && frag && !frag.gap && bufferInfo.nextStart) { + // Media buffered after GAP tags should not make the next buffer timerange exceed forward buffer length + const nextbufferInfo = this.getFwdBufferInfoAtPos( + this.mediaBuffer ? this.mediaBuffer : this.media, + bufferInfo.nextStart, + playlistType + ); + if ( + nextbufferInfo !== null && + bufferInfo.len + nextbufferInfo.len >= maxBufLen + ) { + // Returning here might result in not finding an audio and video candiate to skip to + this.log( + `buffer full after gaps in "${playlistType}" playlist starting at sn: ${frag.sn}` + ); + return null; + } + } + return frag; + } + mapToInitFragWhenRequired(frag: Fragment | null): typeof frag { // If an initSegment is present, it must be buffered first if (frag?.initSegment && !frag?.initSegment.data && !this.bitrateTest) { @@ -1184,9 +1230,6 @@ export default class BaseStreamController frag.sn < endSN && this.fragmentTracker.getState(nextFrag) !== FragmentState.OK ) { - this.log( - `Skipping loaded ${frag.type} SN ${frag.sn} at buffer end` - ); frag = nextFrag; } else { frag = null; diff --git a/src/controller/stream-controller.ts b/src/controller/stream-controller.ts index 48fc46f6b12..acf21fc1454 100644 --- a/src/controller/stream-controller.ts +++ b/src/controller/stream-controller.ts @@ -302,45 +302,30 @@ export default class StreamController } else if (this.backtrackFragment && bufferInfo.len) { this.backtrackFragment = null; } - if (frag) { - // Avoid loop loading by using nextLoadPosition set for backtracking and skipping consecutive GAP tags - const trackerState = this.fragmentTracker.getState(frag); - if ( - (trackerState === FragmentState.OK || - (trackerState === FragmentState.PARTIAL && frag.gap)) && - this.nextLoadPosition > targetBufferTime - ) { - const gapStart = frag.gap; - if (!gapStart) { - // Cleanup the fragment tracker before trying to find the next unbuffered fragment - const type = - this.audioOnly && !this.altAudio - ? ElementaryStreamTypes.AUDIO - : ElementaryStreamTypes.VIDEO; - const mediaBuffer = - (type === ElementaryStreamTypes.VIDEO - ? this.videoBuffer - : this.mediaBuffer) || this.media; - if (mediaBuffer) { - this.afterBufferFlushed(mediaBuffer, type, PlaylistLevelType.MAIN); - } - } - frag = this.getNextFragment(this.nextLoadPosition, levelDetails); - if (gapStart && frag && !frag.gap && bufferInfo.nextStart) { - // Media buffered after GAP tags should not make the next buffer timerange exceed forward buffer length - const nextbufferInfo = this.getFwdBufferInfoAtPos( - this.mediaBuffer ? this.mediaBuffer : this.media, - bufferInfo.nextStart, - PlaylistLevelType.MAIN - ); - if ( - nextbufferInfo !== null && - bufferLen + nextbufferInfo.len >= maxBufLen - ) { - return; - } + // Avoid loop loading by using nextLoadPosition set for backtracking and skipping consecutive GAP tags + if (frag && this.isLoopLoading(frag, targetBufferTime)) { + const gapStart = frag.gap; + if (!gapStart) { + // Cleanup the fragment tracker before trying to find the next unbuffered fragment + const type = + this.audioOnly && !this.altAudio + ? ElementaryStreamTypes.AUDIO + : ElementaryStreamTypes.VIDEO; + const mediaBuffer = + (type === ElementaryStreamTypes.VIDEO + ? this.videoBuffer + : this.mediaBuffer) || this.media; + if (mediaBuffer) { + this.afterBufferFlushed(mediaBuffer, type, PlaylistLevelType.MAIN); } } + frag = this.getNextFragmentLoopLoading( + frag, + levelDetails, + bufferInfo, + PlaylistLevelType.MAIN, + maxBufLen + ); } if (!frag) { return; diff --git a/src/loader/fragment-loader.ts b/src/loader/fragment-loader.ts index cd697543db8..a92f50ecf25 100644 --- a/src/loader/fragment-loader.ts +++ b/src/loader/fragment-loader.ts @@ -363,6 +363,7 @@ function createGapLoadError(frag: Fragment, part?: Part): LoadError { if (part) { errorData.part = part; } + (part ? part : frag).stats.aborted = true; return new LoadError(errorData); } From ad6c4354fdd7ba9830253924daeabfb68d83eac3 Mon Sep 17 00:00:00 2001 From: Rob Walch Date: Tue, 21 Mar 2023 11:48:34 -0700 Subject: [PATCH 7/8] Fix GAP tag handling with Redundant Streams --- api-extractor/report/hls.js.api.md | 4 ++ src/controller/base-stream-controller.ts | 7 +- src/controller/error-controller.ts | 70 +++++++++++++------ src/controller/level-controller.ts | 2 +- src/types/level.ts | 8 +++ .../unit/controller/audio-track-controller.ts | 4 +- tests/unit/controller/error-controller.ts | 4 +- 7 files changed, 71 insertions(+), 28 deletions(-) diff --git a/api-extractor/report/hls.js.api.md b/api-extractor/report/hls.js.api.md index bca6891e17b..96a1393ab94 100644 --- a/api-extractor/report/hls.js.api.md +++ b/api-extractor/report/hls.js.api.md @@ -1949,6 +1949,8 @@ export class Level { // (undocumented) readonly audioCodec: string | undefined; // (undocumented) + get audioGroupId(): string | undefined; + // (undocumented) audioGroupIds?: (string | undefined)[]; // (undocumented) readonly bitrate: number; @@ -1978,6 +1980,8 @@ export class Level { // (undocumented) realBitrate: number; // (undocumented) + get textGroupId(): string | undefined; + // (undocumented) textGroupIds?: (string | undefined)[]; // (undocumented) readonly unknownCodecs: string[] | undefined; diff --git a/src/controller/base-stream-controller.ts b/src/controller/base-stream-controller.ts index f8cf8371df3..63a56f5aecc 100644 --- a/src/controller/base-stream-controller.ts +++ b/src/controller/base-stream-controller.ts @@ -1425,7 +1425,8 @@ export default class BaseStreamController ); return; } - if (data.details === ErrorDetails.FRAG_GAP) { + const gapTagEncountered = data.details === ErrorDetails.FRAG_GAP; + if (gapTagEncountered) { this.fragmentTracker.fragBuffered(frag, true); } // keep retrying until the limit will be reached @@ -1455,7 +1456,9 @@ export default class BaseStreamController this.resetFragmentErrors(filterType); if (retryCount < retryConfig.maxNumRetry) { // Network retry is skipped when level switch is preferred - errorAction.resolved = true; + if (!gapTagEncountered) { + errorAction.resolved = true; + } } else { logger.warn( `${data.details} reached or exceeded max retry (${retryCount})` diff --git a/src/controller/error-controller.ts b/src/controller/error-controller.ts index 95a7366698b..bd11073d000 100644 --- a/src/controller/error-controller.ts +++ b/src/controller/error-controller.ts @@ -44,6 +44,7 @@ export type IErrorAction = { export default class ErrorController implements NetworkComponentAPI { private readonly hls: Hls; private playlistError: number = 0; + private failoverError?: ErrorData; private log: (msg: any) => void; private warn: (msg: any) => void; private error: (msg: any) => void; @@ -146,11 +147,9 @@ export default class ErrorController implements NetworkComponentAPI { if ( level && ((context.type === PlaylistContextType.AUDIO_TRACK && - level.audioGroupIds && - context.groupId === level.audioGroupIds[level.urlId]) || + context.groupId === level.audioGroupId) || (context.type === PlaylistContextType.SUBTITLE_TRACK && - level.textGroupIds && - context.groupId === level.textGroupIds[level.urlId])) + context.groupId === level.textGroupId)) ) { // Perform Pathway switch or Redundant failover if possible for fastest recovery // otherwise allow playlist retry count to reach max error retries @@ -310,23 +309,23 @@ export default class ErrorController implements NetworkComponentAPI { if (data.details !== ErrorDetails.FRAG_GAP) { level.loadError++; } - const redundantLevels = level.url.length; - // Try redundant fail-over until level.loadError reaches redundantLevels - if (redundantLevels > 1 && level.loadError < redundantLevels) { - data.levelRetry = true; - } else if (hls.autoLevelEnabled) { + if (hls.autoLevelEnabled) { // Search for next level to retry let nextLevel = -1; const levels = hls.levels; + const fragErrorType = data.frag?.type; + const { type: playlistErrorType, groupId: playlistErrorGroupId } = + data.context ?? {}; for (let i = levels.length; i--; ) { const candidate = (i + hls.loadLevel) % levels.length; if ( candidate !== hls.loadLevel && levels[candidate].loadError === 0 ) { + const levelCandidate = levels[candidate]; // Skip level switch if GAP tag is found in next level at same position if (data.details === ErrorDetails.FRAG_GAP && data.frag) { - const levelDetails = hls.levels[candidate].details; + const levelDetails = levels[candidate].details; if (levelDetails) { const fragCandidate = findFragmentByPTS( data.frag, @@ -337,6 +336,22 @@ export default class ErrorController implements NetworkComponentAPI { continue; } } + } else if ( + (playlistErrorType === PlaylistContextType.AUDIO_TRACK && + playlistErrorGroupId === levelCandidate.audioGroupId) || + (playlistErrorType === PlaylistContextType.SUBTITLE_TRACK && + playlistErrorGroupId === levelCandidate.textGroupId) + ) { + // For audio/subs playlist errors find another group ID or fallthrough to redundant fail-over + continue; + } else if ( + (fragErrorType === PlaylistLevelType.AUDIO && + level.audioGroupId === levelCandidate.audioGroupId) || + (fragErrorType === PlaylistLevelType.SUBTITLE && + level.textGroupId === levelCandidate.textGroupId) + ) { + // For audio/subs frag errors find another group ID or fallthrough to redundant fail-over + continue; } nextLevel = candidate; break; @@ -366,7 +381,10 @@ export default class ErrorController implements NetworkComponentAPI { break; case NetworkErrorAction.SendAlternateToPenaltyBox: this.sendAlternateToPenaltyBox(data); - if (!data.errorAction.resolved) { + if ( + !data.errorAction.resolved && + data.details !== ErrorDetails.FRAG_GAP + ) { data.fatal = true; } break; @@ -395,13 +413,9 @@ export default class ErrorController implements NetworkComponentAPI { break; case ErrorActionFlags.MoveAllAlternatesMatchingHost: { - const levelIndex = - data.parent === PlaylistLevelType.MAIN - ? (data.level as number) - : hls.loadLevel; - // Handle Redundant Levels here. Patway switching is handled by content-steering-controller + // Handle Redundant Levels here. Pathway switching is handled by content-steering-controller if (!errorAction.resolved) { - errorAction.resolved = this.redundantFailover(levelIndex); + errorAction.resolved = this.redundantFailover(data); } } break; @@ -431,13 +445,27 @@ export default class ErrorController implements NetworkComponentAPI { } } - private redundantFailover(levelIndex: number): boolean { - const hls = this.hls; + private redundantFailover(data: ErrorData): boolean { + const { hls, failoverError } = this; + const levelIndex: number = + data.parent === PlaylistLevelType.MAIN + ? (data.level as number) + : hls.loadLevel; const level = hls.levels[levelIndex]; const redundantLevels = level.url.length; - if (redundantLevels > 1) { + const newUrlId = (level.urlId + 1) % redundantLevels; + if ( + redundantLevels > 1 && + // FIXME: Don't loop back to first redundant renditions unless the last gap is more than 60 seconds ago + // TODO: We throw out information about the other redundant renditions but should keep track of gap ranges to avoid looping back to the same problem + (newUrlId !== 0 || + !failoverError || + (failoverError.frag && + data.frag && + Math.abs(data.frag.start - failoverError.frag.start) > 60)) + ) { + this.failoverError = data; // Update the url id of all levels so that we stay on the same set of variants when level switching - const newUrlId = (level.urlId + 1) % redundantLevels; this.log( `Switching to Redundant Stream ${newUrlId + 1}/${redundantLevels}: "${ level.url[newUrlId] diff --git a/src/controller/level-controller.ts b/src/controller/level-controller.ts index 37bbceacf1f..7ce9496766d 100644 --- a/src/controller/level-controller.ts +++ b/src/controller/level-controller.ts @@ -479,7 +479,7 @@ export default class LevelController extends BasePlaylistController { const audioGroupId = this.hls.audioTracks[data.id].groupId; if ( currentLevel.audioGroupIds && - currentLevel.audioGroupIds[currentLevel.urlId] !== audioGroupId + currentLevel.audioGroupId !== audioGroupId ) { let urlId = -1; for (let i = 0; i < currentLevel.audioGroupIds.length; i++) { diff --git a/src/types/level.ts b/src/types/level.ts index c62541e7c58..49b21a63ff6 100644 --- a/src/types/level.ts +++ b/src/types/level.ts @@ -155,6 +155,14 @@ export class Level { } } + get audioGroupId(): string | undefined { + return this.audioGroupIds?.[this.urlId]; + } + + get textGroupId(): string | undefined { + return this.textGroupIds?.[this.urlId]; + } + addFallback(data: LevelParsed) { this.url.push(data.url); this._attrs.push(data.attrs); diff --git a/tests/unit/controller/audio-track-controller.ts b/tests/unit/controller/audio-track-controller.ts index 45a5e887d41..38daa9f8a3b 100644 --- a/tests/unit/controller/audio-track-controller.ts +++ b/tests/unit/controller/audio-track-controller.ts @@ -208,7 +208,7 @@ describe('AudioTrackController', function () { }); const newLevelInfo = hls.levels[0]; - const newGroupId = newLevelInfo.audioGroupIds?.[newLevelInfo.urlId]; + const newGroupId = newLevelInfo.audioGroupId; audioTrackController.tracks = tracks; // Update the level to set audioGroupId @@ -325,7 +325,7 @@ describe('AudioTrackController', function () { }; const newLevelInfo = hls.levels[levelLoadedEvent.level]; - const newGroupId = newLevelInfo.audioGroupIds?.[newLevelInfo.urlId]; + const newGroupId = newLevelInfo.audioGroupId; audioTrackController.tracks = tracks; audioTrackController.onLevelLoading(Events.LEVEL_LOADING, { diff --git a/tests/unit/controller/error-controller.ts b/tests/unit/controller/error-controller.ts index 58aa4a31beb..f681ec5a7cc 100644 --- a/tests/unit/controller/error-controller.ts +++ b/tests/unit/controller/error-controller.ts @@ -737,7 +737,7 @@ segment.mp4 expect( errors.length, 'fragment errors after yeilding to second error event' - ).to.equal(4); + ).to.equal(6); expect(hls.levels[0].uri).to.equal('http://www.baz.com/tier6.m3u8'); return new Promise((resolve, reject) => { hls.on(Events.FRAG_LOADED, (event, data) => { @@ -847,7 +847,7 @@ segment.mp4 expect( errors.length, 'fragment errors after yeilding to second error event' - ).to.equal(4); + ).to.equal(6); expect(hls.levels[0].uri).to.equal('http://www.baz.com/tier6.m3u8'); return new Promise((resolve, reject) => { hls.on(Events.FRAG_LOADED, (event, data) => { From 02a8b475df3a000573f5cab8a3b514c232084629 Mon Sep 17 00:00:00 2001 From: Rob Walch Date: Tue, 21 Mar 2023 16:47:29 -0700 Subject: [PATCH 8/8] Implement penalty box for Redundant Streams --- src/controller/error-controller.ts | 136 +++++++++++++++---- src/controller/subtitle-stream-controller.ts | 6 +- 2 files changed, 113 insertions(+), 29 deletions(-) diff --git a/src/controller/error-controller.ts b/src/controller/error-controller.ts index bd11073d000..f7008fb78cd 100644 --- a/src/controller/error-controller.ts +++ b/src/controller/error-controller.ts @@ -7,13 +7,16 @@ import { shouldRetry, } from '../utils/error-helper'; import { findFragmentByPTS } from './fragment-finders'; -import { HdcpLevel, HdcpLevels } from '../types/level'; +import { HdcpLevel, HdcpLevels, type Level } from '../types/level'; import { logger } from '../utils/logger'; import type Hls from '../hls'; import type { RetryConfig } from '../config'; import type { NetworkComponentAPI } from '../types/component-api'; import type { ErrorData } from '../types/events'; import type { Fragment } from '../loader/fragment'; +import type { LevelDetails } from '../hls'; + +const RENDITION_PENALTY_DURATION_MS = 300000; export const enum NetworkErrorAction { DoNothing = 0, @@ -41,10 +44,18 @@ export type IErrorAction = { resolved?: boolean; }; +type PenalizedRendition = { + lastErrorPerfMs: number; + errors: ErrorData[]; + details?: LevelDetails; +}; + +type PenalizedRenditions = { [key: number]: PenalizedRendition }; + export default class ErrorController implements NetworkComponentAPI { private readonly hls: Hls; private playlistError: number = 0; - private failoverError?: ErrorData; + private penalizedRenditions: PenalizedRenditions = {}; private log: (msg: any) => void; private warn: (msg: any) => void; private error: (msg: any) => void; @@ -58,12 +69,19 @@ export default class ErrorController implements NetworkComponentAPI { } private registerListeners() { - this.hls.on(Events.ERROR, this.onError, this); + const hls = this.hls; + hls.on(Events.ERROR, this.onError, this); + hls.on(Events.MANIFEST_LOADING, this.onManifestLoading, this); } private unregisterListeners() { - this.hls.off(Events.ERROR, this.onError, this); - this.hls.off(Events.ERROR, this.onErrorOut, this); + const hls = this.hls; + if (!hls) { + return; + } + hls.off(Events.ERROR, this.onError, this); + hls.off(Events.ERROR, this.onErrorOut, this); + hls.off(Events.MANIFEST_LOADING, this.onManifestLoading, this); } destroy() { @@ -84,6 +102,11 @@ export default class ErrorController implements NetworkComponentAPI { : this.hls.loadLevel; } + private onManifestLoading() { + this.playlistError = 0; + this.penalizedRenditions = {}; + } + private onError(event: Events.ERROR, data: ErrorData) { if (data.fatal) { return; @@ -446,38 +469,95 @@ export default class ErrorController implements NetworkComponentAPI { } private redundantFailover(data: ErrorData): boolean { - const { hls, failoverError } = this; + const { hls, penalizedRenditions } = this; const levelIndex: number = data.parent === PlaylistLevelType.MAIN ? (data.level as number) : hls.loadLevel; const level = hls.levels[levelIndex]; const redundantLevels = level.url.length; - const newUrlId = (level.urlId + 1) % redundantLevels; + this.penalizeRendition(level, data); + for (let i = 1; i < redundantLevels; i++) { + const newUrlId = (level.urlId + i) % redundantLevels; + const penalizedRendition = penalizedRenditions[newUrlId]; + // Check if rendition is penalized and skip if it is a bad fit for failover + if ( + !penalizedRendition || + checkExpired(penalizedRendition, data, penalizedRenditions[level.urlId]) + ) { + // delete penalizedRenditions[newUrlId]; + // Update the url id of all levels so that we stay on the same set of variants when level switching + this.warn( + `Switching to Redundant Stream ${newUrlId + 1}/${redundantLevels}: "${ + level.url[newUrlId] + }" after ${data.details}` + ); + this.playlistError = 0; + hls.levels.forEach((lv) => { + lv.urlId = newUrlId; + }); + hls.nextLoadLevel = levelIndex; + return true; + } + } + return false; + } + + private penalizeRendition(level: Level, data: ErrorData) { + const { penalizedRenditions } = this; + const penalizedRendition = penalizedRenditions[level.urlId] || { + lastErrorPerfMs: 0, + errors: [], + details: undefined, + }; + penalizedRendition.lastErrorPerfMs = performance.now(); + penalizedRendition.errors.push(data); + penalizedRendition.details = level.details; + penalizedRenditions[level.urlId] = penalizedRendition; + } +} + +function checkExpired( + penalizedRendition: PenalizedRendition, + data: ErrorData, + currentPenaltyState: PenalizedRendition | undefined +): boolean { + // Expire penalty for switching back to rendition after RENDITION_PENALTY_DURATION_MS + if ( + performance.now() - penalizedRendition.lastErrorPerfMs > + RENDITION_PENALTY_DURATION_MS + ) { + return true; + } + // Expire penalty on GAP tag error if rendition has no GAP at position (does not cover media tracks) + const lastErrorDetails = penalizedRendition.details; + if (data.details === ErrorDetails.FRAG_GAP && lastErrorDetails && data.frag) { + const position = data.frag.start; + const candidateFrag = findFragmentByPTS( + null, + lastErrorDetails.fragments, + position + ); + if (candidateFrag && !candidateFrag.gap) { + return true; + } + } + // Expire penalty if there are more errors in currentLevel than in penalizedRendition + if ( + currentPenaltyState && + penalizedRendition.errors.length < currentPenaltyState.errors.length + ) { + const lastCandidateError = + penalizedRendition.errors[penalizedRendition.errors.length - 1]; if ( - redundantLevels > 1 && - // FIXME: Don't loop back to first redundant renditions unless the last gap is more than 60 seconds ago - // TODO: We throw out information about the other redundant renditions but should keep track of gap ranges to avoid looping back to the same problem - (newUrlId !== 0 || - !failoverError || - (failoverError.frag && - data.frag && - Math.abs(data.frag.start - failoverError.frag.start) > 60)) + lastErrorDetails && + lastCandidateError.frag && + data.frag && + Math.abs(lastCandidateError.frag.start - data.frag.start) > + lastErrorDetails.targetduration * 3 ) { - this.failoverError = data; - // Update the url id of all levels so that we stay on the same set of variants when level switching - this.log( - `Switching to Redundant Stream ${newUrlId + 1}/${redundantLevels}: "${ - level.url[newUrlId] - }"` - ); - this.playlistError = 0; - hls.levels.forEach((lv) => { - lv.urlId = newUrlId; - }); - hls.nextLoadLevel = levelIndex; return true; } - return false; } + return false; } diff --git a/src/controller/subtitle-stream-controller.ts b/src/controller/subtitle-stream-controller.ts index c5434dd34c2..73e250d2c1c 100644 --- a/src/controller/subtitle-stream-controller.ts +++ b/src/controller/subtitle-stream-controller.ts @@ -220,7 +220,11 @@ export class SubtitleStreamController this.levels = subtitleTracks.map( (mediaPlaylist) => new Level(mediaPlaylist) ); - this.fragmentTracker.removeAllFragments(); + this.fragmentTracker.removeFragmentsInRange( + 0, + Number.POSITIVE_INFINITY, + PlaylistLevelType.SUBTITLE + ); this.fragPrevious = null; this.levels.forEach((level: Level) => { this.tracksBuffered[level.id] = [];