GAP Tag/Attribute Support with FRAG_GAP Error (#5257)

* Handle fragments and parts with a GAP tag/attribute * Improve GAP fragment picking, post gap buffering, and level switching error resolution * Cleanup FRAG_GAP error creation and inlcude current buffer length in forward gap buffer limit * Fix gap jump over waiting for data stall when buffer length is larger than maxBufferHole but less than one second * Warn when observing "seeked" event with empty buffer (Related to #5274) * Handle forward buffering past GAP tags in alt audio Playlists * Fix GAP tag handling with Redundant Streams * Implement penalty box for Redundant Streams
video-dev · Mar 23, 2023 · 3e959c2 · 3e959c2
1 parent 945e4dc
commit 3e959c2
Show file tree

Hide file tree

Showing 21 changed files with 476 additions and 171 deletions.
diff --git a/README.md b/README.md
@@ -110,11 +110,11 @@ The following properties are added to their respective variants' attribute list
 - `#EXT-X-RENDITION-REPORT:<attribute-list>`
 - `#EXT-X-DATERANGE:<attribute-list>` Metadata
 - `#EXT-X-DEFINE:<attribute-list>` Variable Import and Substitution (`NAME,VALUE,IMPORT,QUERYPARAM` attributes)
+- `#EXT-X-GAP` (Skips loading GAP segments and parts. Skips playback of unbuffered program containing only GAP content and no suitable alternates. See [#2940](https://github.com/video-dev/hls.js/issues/2940))
 
 The following tags are added to their respective fragment's attribute list but are not implemented in streaming and playback.
 
 - `#EXT-X-BITRATE` (Not used in ABR controller)
-- `#EXT-X-GAP` (Not implemented. See [#2940](https://github.com/video-dev/hls.js/issues/2940))
 
 Parsed but missing feature support
 

diff --git a/api-extractor/report/hls.js.api.md b/api-extractor/report/hls.js.api.md
@@ -330,6 +330,8 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP
     // (undocumented)
     protected getFwdBufferInfo(bufferable: Bufferable | null, type: PlaylistLevelType): BufferInfo | null;
     // (undocumented)
+    protected getFwdBufferInfoAtPos(bufferable: Bufferable | null, pos: number, type: PlaylistLevelType): BufferInfo | null;
+    // (undocumented)
     protected getInitialLiveFragment(levelDetails: LevelDetails, fragments: Array<Fragment>): Fragment | null;
     // (undocumented)
     protected getLevelDetails(): LevelDetails | undefined;
@@ -340,6 +342,8 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP
     // (undocumented)
     protected getNextFragment(pos: number, levelDetails: LevelDetails): Fragment | null;
     // (undocumented)
+    protected getNextFragmentLoopLoading(frag: Fragment, levelDetails: LevelDetails, bufferInfo: BufferInfo, playlistType: PlaylistLevelType, maxBufLen: number): Fragment | null;
+    // (undocumented)
     getNextPart(partList: Part[], frag: Fragment, targetBufferTime: number): number;
     // Warning: (ae-forgotten-export) The symbol "PartsLoadedData" needs to be exported by the entry point hls.d.ts
     //
@@ -356,6 +360,8 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP
     // (undocumented)
     protected initPTS: RationalTimestamp[];
     // (undocumented)
+    protected isLoopLoading(frag: Fragment, targetBufferTime: number): boolean;
+    // (undocumented)
     protected keyLoader: KeyLoader;
     // (undocumented)
     protected lastCurrentTime: number;
@@ -406,7 +412,9 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP
     // (undocumented)
     protected recoverWorkerError(data: ErrorData): void;
     // (undocumented)
-    protected reduceMaxBufferLength(threshold?: number): boolean;
+    protected reduceLengthAndFlushBuffer(data: ErrorData): boolean;
+    // (undocumented)
+    protected reduceMaxBufferLength(threshold: number): boolean;
     // (undocumented)
     protected resetFragmentErrors(filterType: PlaylistLevelType): void;
     // (undocumented)
@@ -991,6 +999,8 @@ export enum ErrorDetails {
     // (undocumented)
     FRAG_DECRYPT_ERROR = "fragDecryptError",
     // (undocumented)
+    FRAG_GAP = "fragGap",
+    // (undocumented)
     FRAG_LOAD_ERROR = "fragLoadError",
     // (undocumented)
     FRAG_LOAD_TIMEOUT = "fragLoadTimeOut",
@@ -1343,6 +1353,8 @@ export class Fragment extends BaseSegment {
     // (undocumented)
     endPTS?: number;
     // (undocumented)
+    gap?: boolean;
+    // (undocumented)
     initSegment: Fragment | null;
     // Warning: (ae-forgotten-export) The symbol "KeyLoaderContext" needs to be exported by the entry point hls.d.ts
     //
@@ -1937,6 +1949,8 @@ export class Level {
     // (undocumented)
     readonly audioCodec: string | undefined;
     // (undocumented)
+    get audioGroupId(): string | undefined;
+    // (undocumented)
     audioGroupIds?: (string | undefined)[];
     // (undocumented)
     readonly bitrate: number;
@@ -1966,6 +1980,8 @@ export class Level {
     // (undocumented)
     realBitrate: number;
     // (undocumented)
+    get textGroupId(): string | undefined;
+    // (undocumented)
     textGroupIds?: (string | undefined)[];
     // (undocumented)
     readonly unknownCodecs: string[] | undefined;

diff --git a/docs/API.md b/docs/API.md
@@ -1726,14 +1726,14 @@ Full list of errors is described below:
   - data: { type : `NETWORK_ERROR`, details : `Hls.ErrorDetails.MANIFEST_LOAD_TIMEOUT`, fatal : `true`, url : manifest URL, loader : URL loader }
 - `Hls.ErrorDetails.MANIFEST_PARSING_ERROR` - raised when manifest parsing failed to find proper content
   - data: { type : `NETWORK_ERROR`, details : `Hls.ErrorDetails.MANIFEST_PARSING_ERROR`, fatal : `true`, url : manifest URL, reason : parsing error reason }
-- `Hls.ErrorDetails.LEVEL_EMPTY_ERROR` - raised when loaded level contains no fragments
-  - data: { type : `NETWORK_ERROR`, details : `Hls.ErrorDetails.LEVEL_EMPTY_ERROR`, url: playlist URL, reason: error reason, level: index of the bad level }
+- `Hls.ErrorDetails.LEVEL_EMPTY_ERROR` - raised when loaded level contains no fragments (applies to levels and audio and subtitle tracks)
+  - data: { type : `NETWORK_ERROR`, details : `Hls.ErrorDetails.LEVEL_EMPTY_ERROR`, url: playlist URL, reason: error reason, level: index of the bad level or undefined, parent: PlaylistLevelType }
 - `Hls.ErrorDetails.LEVEL_LOAD_ERROR` - raised when level loading fails because of a network error
   - data: { type : `NETWORK_ERROR`, details : `Hls.ErrorDetails.LEVEL_LOAD_ERROR`, fatal : `true`, url : level URL, response : { code: error code, text: error text }, loader : URL loader }
 - `Hls.ErrorDetails.LEVEL_LOAD_TIMEOUT` - raised when level loading fails because of a timeout
   - data: { type : `NETWORK_ERROR`, details : `Hls.ErrorDetails.LEVEL_LOAD_TIMEOUT`, fatal : `false`, url : level URL, loader : URL loader }
-- `Hls.ErrorDetails.LEVEL_PARSING_ERROR` - raised when level parsing failed or found invalid content
-  - data: { type : `NETWORK_ERROR`, details : `Hls.ErrorDetails.LEVEL_PARSING_ERROR`, fatal : `false`, url : level URL, error: Error }
+- `Hls.ErrorDetails.LEVEL_PARSING_ERROR` - raised when playlist parsing failed or found invalid content (applies to levels and audio and subtitle tracks)
+  - data: { type : `NETWORK_ERROR`, details : `Hls.ErrorDetails.LEVEL_PARSING_ERROR`, fatal : `false`, url : level URL, error: Error, parent: PlaylistLevelType }
 - `Hls.ErrorDetails.AUDIO_TRACK_LOAD_ERROR` - raised when audio playlist loading fails because of a network error
   - data: { type : `NETWORK_ERROR`, details : `Hls.ErrorDetails.AUDIO_TRACK_LOAD_ERROR`, fatal : `false`, url : audio URL, response : { code: error code, text: error text }, loader : URL loader }
 - `Hls.ErrorDetails.AUDIO_TRACK_LOAD_TIMEOUT` - raised when audio playlist loading fails because of a timeout
@@ -1759,6 +1759,8 @@ Full list of errors is described below:
   - data: { type : `MEDIA_ERROR`, details : `Hls.ErrorDetails.FRAG_DECRYPT_ERROR`, fatal : `true`, reason : failure reason }
 - `Hls.ErrorDetails.FRAG_PARSING_ERROR` - raised when fragment parsing fails
   - data: { type : `MEDIA_ERROR`, details : `Hls.ErrorDetails.FRAG_PARSING_ERROR`, fatal : `true` or `false`, reason : failure reason }
+- `Hls.ErrorDetails.FRAG_GAP` - raised when segment loading is skipped because a fragment with a GAP tag or part with GAP=YES attribute was encountered
+  - data: { type : `MEDIA_ERROR`, details : `Hls.ErrorDetails.FRAG_GAP`, fatal : `false`, frag : fragment object, part? : part object (if any) }
 - `Hls.ErrorDetails.BUFFER_ADD_CODEC_ERROR` - raised when MediaSource fails to add new sourceBuffer
   - data: { type : `MEDIA_ERROR`, details : `Hls.ErrorDetails.BUFFER_ADD_CODEC_ERROR`, fatal : `false`, error : error raised by MediaSource, mimeType: mimeType on which the failure happened }
 - `Hls.ErrorDetails.BUFFER_INCOMPATIBLE_CODECS_ERROR` - raised when no MediaSource(s) could be created based on track codec(s)

diff --git a/src/controller/audio-stream-controller.ts b/src/controller/audio-stream-controller.ts
@@ -347,24 +347,52 @@ class AudioStreamController
       }
     }
 
-    // buffer audio up to one target duration ahead of main buffer
-    if (
-      mainBufferInfo &&
-      targetBufferTime > mainBufferInfo.end + trackDetails.targetduration
-    ) {
-      return;
-    }
-    // wait for main buffer after buffing some audio
-    if (!mainBufferInfo?.len && bufferInfo.len) {
-      return;
+    let frag = this.getNextFragment(targetBufferTime, trackDetails);
+    let atGap = false;
+    // Avoid loop loading by using nextLoadPosition set for backtracking and skipping consecutive GAP tags
+    if (frag && this.isLoopLoading(frag, targetBufferTime)) {
+      atGap = !!frag.gap;
+      frag = this.getNextFragmentLoopLoading(
+        frag,
+        trackDetails,
+        bufferInfo,
+        PlaylistLevelType.MAIN,
+        maxBufLen
+      );
     }
-
-    const frag = this.getNextFragment(targetBufferTime, trackDetails);
     if (!frag) {
       this.bufferFlushed = true;
       return;
     }
 
+    // Buffer audio up to one target duration ahead of main buffer
+    const atBufferSyncLimit =
+      mainBufferInfo &&
+      frag.start > mainBufferInfo.end + trackDetails.targetduration;
+    if (
+      atBufferSyncLimit ||
+      // Or wait for main buffer after buffing some audio
+      (!mainBufferInfo?.len && bufferInfo.len)
+    ) {
+      // Check fragment-tracker for main fragments since GAP segments do not show up in bufferInfo
+      const mainFrag = this.fragmentTracker.getBufferedFrag(
+        frag.start,
+        PlaylistLevelType.MAIN
+      );
+      if (mainFrag === null) {
+        return;
+      }
+      // Bridge gaps in main buffer
+      atGap ||=
+        !!mainFrag.gap || (!!atBufferSyncLimit && mainBufferInfo.len === 0);
+      if (
+        (atBufferSyncLimit && !atGap) ||
+        (atGap && bufferInfo.nextStart && bufferInfo.nextStart < mainFrag.end)
+      ) {
+        return;
+      }
+    }
+
     this.loadFragment(frag, levelInfo, targetBufferTime);
   }
 
@@ -643,6 +671,7 @@ class AudioStreamController
       return;
     }
     switch (data.details) {
+      case ErrorDetails.FRAG_GAP:
       case ErrorDetails.FRAG_PARSING_ERROR:
       case ErrorDetails.FRAG_DECRYPT_ERROR:
       case ErrorDetails.FRAG_LOAD_ERROR:
@@ -664,33 +693,12 @@ class AudioStreamController
         }
         break;
       case ErrorDetails.BUFFER_FULL_ERROR:
-        // if in appending state
-        if (
-          data.parent === 'audio' &&
-          (this.state === State.PARSING || this.state === State.PARSED)
-        ) {
-          let flushBuffer = true;
-          const bufferedInfo = this.getFwdBufferInfo(
-            this.mediaBuffer,
-            PlaylistLevelType.AUDIO
-          );
-          // 0.5 : tolerance needed as some browsers stalls playback before reaching buffered end
-          // reduce max buf len if current position is buffered
-          if (bufferedInfo && bufferedInfo.len > 0.5) {
-            flushBuffer = !this.reduceMaxBufferLength(bufferedInfo.len);
-          }
-          if (flushBuffer) {
-            // current position is not buffered, but browser is still complaining about buffer full error
-            // this happens on IE/Edge, refer to https://github.com/video-dev/hls.js/pull/708
-            // in that case flush the whole audio buffer to recover
-            this.warn(
-              'Buffer full error also media.currentTime is not buffered, flush audio buffer'
-            );
-            this.fragCurrent = null;
-            this.bufferedTrack = null;
-            super.flushMainBuffer(0, Number.POSITIVE_INFINITY, 'audio');
-          }
-          this.resetLoadingState();
+        if (!data.parent || data.parent !== 'audio') {
+          return;
+        }
+        if (this.reduceLengthAndFlushBuffer(data)) {
+          this.bufferedTrack = null;
+          super.flushMainBuffer(0, Number.POSITIVE_INFINITY, 'audio');
         }
         break;
       case ErrorDetails.INTERNAL_EXCEPTION: