From 9013f669562169c648f84fcc38b105a98eca87be Mon Sep 17 00:00:00 2001 From: Adam Waldron Date: Thu, 23 Mar 2023 18:14:49 -0700 Subject: [PATCH 1/2] fix: allow audio fmp4 emsg probe --- src/media-segment-request.js | 49 ++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/src/media-segment-request.js b/src/media-segment-request.js index 504fece5a..086f0fe48 100644 --- a/src/media-segment-request.js +++ b/src/media-segment-request.js @@ -497,38 +497,37 @@ const handleSegmentBytes = ({ if (trackInfo.hasVideo) { timingInfoFn(segment, 'video', 'start', startTime); } - - // Run through the CaptionParser in case there are captions. - // Initialize CaptionParser if it hasn't been yet - if (!tracks.video || !data.byteLength || !segment.transmuxer) { - finishLoading(); - return; - } - workerCallback({ - action: 'pushMp4Captions', - endAction: 'mp4Captions', - transmuxer: segment.transmuxer, + action: 'probeEmsgID3', data: bytesAsUint8Array, - timescales: segment.map.timescales, - trackIds: [tracks.video.id], - callback: (message) => { + transmuxer: segment.transmuxer, + offset: startTime, + callback: ({emsgData, id3Frames}) => { // transfer bytes back to us - bytes = message.data.buffer; - segment.bytes = bytesAsUint8Array = message.data; - message.logs.forEach(function(log) { - onTransmuxerLog(merge(log, {stream: 'mp4CaptionParser'})); - }); + bytes = emsgData.buffer; + segment.bytes = bytesAsUint8Array = emsgData; + + // Run through the CaptionParser in case there are captions. + // Initialize CaptionParser if it hasn't been yet + if (!tracks.video || !data.byteLength || !segment.transmuxer) { + finishLoading(undefined, id3Frames); + return; + } workerCallback({ - action: 'probeEmsgID3', - data: bytesAsUint8Array, + action: 'pushMp4Captions', + endAction: 'mp4Captions', transmuxer: segment.transmuxer, - offset: startTime, - callback: ({emsgData, id3Frames}) => { + data: bytesAsUint8Array, + timescales: segment.map.timescales, + trackIds: [tracks.video.id], + callback: (message) => { // transfer bytes back to us - bytes = emsgData.buffer; - segment.bytes = bytesAsUint8Array = emsgData; + bytes = message.data.buffer; + segment.bytes = bytesAsUint8Array = message.data; + message.logs.forEach(function(log) { + onTransmuxerLog(merge(log, {stream: 'mp4CaptionParser'})); + }); finishLoading(message.captions, id3Frames); } }); From 594710bd0a384827e65a1b34b79904ffab12afa0 Mon Sep 17 00:00:00 2001 From: Adam Waldron Date: Thu, 30 Mar 2023 17:16:49 -0700 Subject: [PATCH 2/2] add audio only test --- test/media-segment-request.test.js | 134 ++++++++++++++++++++++++++++- 1 file changed, 133 insertions(+), 1 deletion(-) diff --git a/test/media-segment-request.test.js b/test/media-segment-request.test.js index 80aec92de..bfb89843e 100644 --- a/test/media-segment-request.test.js +++ b/test/media-segment-request.test.js @@ -15,6 +15,8 @@ import { ac3WithoutId3 as ac3WithoutId3Segment, video as videoSegment, audio as audioSegment, + mp4Audio, + mp4AudioInit, mp4Video, mp4VideoInit, muxed as muxedSegment, @@ -1576,7 +1578,7 @@ QUnit.test('non-TS segment will get parsed for captions on next segment request this.standardXHRResponse(initReq, mp4VideoInit()); }); -QUnit.test('can get emsg ID3 frames from fmp4 segment', function(assert) { +QUnit.test('can get emsg ID3 frames from fmp4 video segment', function(assert) { const done = assert.async(); let gotEmsgId3 = 0; let gotData = 0; @@ -1705,3 +1707,133 @@ QUnit.test('can get emsg ID3 frames from fmp4 segment', function(assert) { // Simulate receiving the init segment after the media this.standardXHRResponse(initReq, mp4VideoInit()); }); + +QUnit.test('can get emsg ID3 frames from fmp4 audio segment', function(assert) { + const done = assert.async(); + let gotEmsgId3 = 0; + let gotData = 0; + // expected frame data + const id3Frames = [{ + cueTime: 1, + duration: 0, + frames: [{ + id: 'TXXX', + description: 'foo bar', + data: { key: 'value' } + }, + { + id: 'PRIV', + owner: 'priv-owner@foo.bar', + // 'foo' + data: new Uint8Array([0x66, 0x6F, 0x6F]) + }] + }, + { + cueTime: 3, + duration: 0, + frames: [{ + id: 'PRIV', + owner: 'priv-owner@foo.bar', + // 'bar' + data: new Uint8Array([0x62, 0x61, 0x72]) + }, + { + id: 'TXXX', + description: 'bar foo', + data: { key: 'value' } + }] + }]; + const transmuxer = new videojs.EventTarget(); + + transmuxer.postMessage = (event) => { + if (event.action === 'pushMp4Captions') { + transmuxer.trigger({ + type: 'message', + data: { + action: 'mp4Captions', + data: event.data, + captions: 'foo bar', + logs: [] + } + }); + } + + if (event.action === 'probeMp4StartTime') { + transmuxer.trigger({ + type: 'message', + data: { + action: 'probeMp4StartTime', + data: event.data, + timingInfo: {} + } + }); + } + + if (event.action === 'probeMp4Tracks') { + transmuxer.trigger({ + type: 'message', + data: { + action: 'probeMp4Tracks', + data: event.data, + tracks: [{type: 'audio', codec: 'mp4a.40.2'}] + } + }); + } + + if (event.action === 'probeEmsgID3') { + transmuxer.trigger({ + type: 'message', + data: { + action: 'probeEmsgID3', + emsgData: event.data, + id3Frames + } + }); + } + }; + + mediaSegmentRequest({ + xhr: this.xhr, + xhrOptions: this.xhrOptions, + decryptionWorker: this.mockDecrypter, + segment: { + transmuxer, + resolvedUri: 'mp4Audio.mp4', + map: { + resolvedUri: 'mp4AudioInit.mp4' + } + }, + progressFn: this.noop, + trackInfoFn: this.noop, + timingInfoFn: this.noop, + id3Fn: (segment, _id3Frames) => { + gotEmsgId3++; + assert.deepEqual(_id3Frames, id3Frames, 'got expected emsg id3 data.'); + }, + captionsFn: this.noop, + dataFn: (segment, segmentData) => { + gotData++; + assert.ok(segmentData, 'init segment bytes in map'); + assert.ok(segment.map.tracks, 'added tracks'); + assert.ok(segment.map.tracks.audio, 'added audio track'); + }, + doneFn: () => { + assert.equal(gotEmsgId3, 1, 'received emsg ID3 event'); + assert.equal(gotData, 1, 'received data event'); + transmuxer.off(); + done(); + } + }); + assert.equal(this.requests.length, 2, 'there are two requests'); + + const initReq = this.requests.shift(); + const segmentReq = this.requests.shift(); + + assert.equal(initReq.uri, 'mp4AudioInit.mp4', 'the first request is for the init segment'); + assert.equal(segmentReq.uri, 'mp4Audio.mp4', 'the second request is for a segment'); + + // Simulate receiving the media first + this.standardXHRResponse(segmentReq, mp4Audio()); + // Simulate receiving the init segment after the media + this.standardXHRResponse(initReq, mp4AudioInit()); +});