Skip to content

Commit

Permalink
feat: add fmp4 emsg ID3 support (#1370)
Browse files Browse the repository at this point in the history
* feat: add fmp4 emsg ID3 support

* fix worker function

* add id3fn to finishLoading

* fix captions tests

* add media segment request test
  • Loading branch information
adrums86 authored Mar 6, 2023
1 parent e5b4bf6 commit 906f29e
Show file tree
Hide file tree
Showing 3 changed files with 187 additions and 2 deletions.
19 changes: 17 additions & 2 deletions src/media-segment-request.js
Original file line number Diff line number Diff line change
Expand Up @@ -462,7 +462,7 @@ const handleSegmentBytes = ({
// Note that the start time returned by the probe reflects the baseMediaDecodeTime, as
// that is the true start of the segment (where the playback engine should begin
// decoding).
const finishLoading = (captions) => {
const finishLoading = (captions, id3Frames) => {
// if the track still has audio at this point it is only possible
// for it to be audio only. See `tracks.video && tracks.audio` if statement
// above.
Expand All @@ -471,6 +471,9 @@ const handleSegmentBytes = ({
data: bytesAsUint8Array,
type: trackInfo.hasAudio && !trackInfo.isMuxed ? 'audio' : 'video'
});
if (id3Frames && id3Frames.length) {
id3Fn(segment, id3Frames);
}
if (captions && captions.length) {
captionsFn(segment, captions);
}
Expand Down Expand Up @@ -516,7 +519,19 @@ const handleSegmentBytes = ({
message.logs.forEach(function(log) {
onTransmuxerLog(merge(log, {stream: 'mp4CaptionParser'}));
});
finishLoading(message.captions);

workerCallback({
action: 'probeEmsgID3',
data: bytesAsUint8Array,
transmuxer: segment.transmuxer,
offset: startTime,
callback: ({emsgData, id3Frames}) => {
// transfer bytes back to us
bytes = emsgData.buffer;
segment.bytes = bytesAsUint8Array = emsgData;
finishLoading(message.captions, id3Frames);
}
});
}
});
}
Expand Down
18 changes: 18 additions & 0 deletions src/transmuxer-worker.js
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,24 @@ class MessageHandlers {
}, [data.buffer]);
}

/**
* Probes an mp4 segment for EMSG boxes containing ID3 data.
* https://aomediacodec.github.io/id3-emsg/
*
* @param {Uint8Array} data segment data
* @param {number} offset segment start time
* @return {Object[]} an array of ID3 frames
*/
probeEmsgID3({data, offset}) {
const id3Frames = mp4probe.getEmsgID3(data, offset);

this.self.postMessage({
action: 'probeEmsgID3',
id3Frames,
emsgData: data
}, [data.buffer]);
}

/**
* Probe an mpeg2-ts segment to determine the start time of the segment in it's
* internal "media time," as well as whether it contains video and/or audio.
Expand Down
152 changes: 152 additions & 0 deletions test/media-segment-request.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -1357,6 +1357,17 @@ QUnit.test('non-TS segment will get parsed for captions', function(assert) {
}
});
}

if (event.action === 'probeEmsgID3') {
transmuxer.trigger({
type: 'message',
data: {
action: 'probeEmsgID3',
emsgData: event.data,
id3Frames: []
}
});
}
};

mediaSegmentRequest({
Expand Down Expand Up @@ -1498,6 +1509,17 @@ QUnit.test('non-TS segment will get parsed for captions on next segment request
}
});
}

if (event.action === 'probeEmsgID3') {
transmuxer.trigger({
type: 'message',
data: {
action: 'probeEmsgID3',
emsgData: event.data,
id3Frames: []
}
});
}
};

mediaSegmentRequest({
Expand Down Expand Up @@ -1553,3 +1575,133 @@ QUnit.test('non-TS segment will get parsed for captions on next segment request
// Simulate receiving the init segment after the media
this.standardXHRResponse(initReq, mp4VideoInit());
});

QUnit.test('can get emsg ID3 frames from fmp4 segment', function(assert) {
const done = assert.async();
let gotEmsgId3 = 0;
let gotData = 0;
// expected frame data
const id3Frames = [{
cueTime: 1,
duration: 0,
frames: [{
id: 'TXXX',
description: 'foo bar',
data: { key: 'value' }
},
{
id: 'PRIV',
owner: '[email protected]',
// 'foo'
data: new Uint8Array([0x66, 0x6F, 0x6F])
}]
},
{
cueTime: 3,
duration: 0,
frames: [{
id: 'PRIV',
owner: '[email protected]',
// 'bar'
data: new Uint8Array([0x62, 0x61, 0x72])
},
{
id: 'TXXX',
description: 'bar foo',
data: { key: 'value' }
}]
}];
const transmuxer = new videojs.EventTarget();

transmuxer.postMessage = (event) => {
if (event.action === 'pushMp4Captions') {
transmuxer.trigger({
type: 'message',
data: {
action: 'mp4Captions',
data: event.data,
captions: 'foo bar',
logs: []
}
});
}

if (event.action === 'probeMp4StartTime') {
transmuxer.trigger({
type: 'message',
data: {
action: 'probeMp4StartTime',
data: event.data,
timingInfo: {}
}
});
}

if (event.action === 'probeMp4Tracks') {
transmuxer.trigger({
type: 'message',
data: {
action: 'probeMp4Tracks',
data: event.data,
tracks: [{type: 'video', codec: 'avc1.4d400d'}]
}
});
}

if (event.action === 'probeEmsgID3') {
transmuxer.trigger({
type: 'message',
data: {
action: 'probeEmsgID3',
emsgData: event.data,
id3Frames
}
});
}
};

mediaSegmentRequest({
xhr: this.xhr,
xhrOptions: this.xhrOptions,
decryptionWorker: this.mockDecrypter,
segment: {
transmuxer,
resolvedUri: 'mp4Video.mp4',
map: {
resolvedUri: 'mp4VideoInit.mp4'
}
},
progressFn: this.noop,
trackInfoFn: this.noop,
timingInfoFn: this.noop,
id3Fn: (segment, _id3Frames) => {
gotEmsgId3++;
assert.deepEqual(_id3Frames, id3Frames, 'got expected emsg id3 data.');
},
captionsFn: this.noop,
dataFn: (segment, segmentData) => {
gotData++;
assert.ok(segmentData, 'init segment bytes in map');
assert.ok(segment.map.tracks, 'added tracks');
assert.ok(segment.map.tracks.video, 'added video track');
},
doneFn: () => {
assert.equal(gotEmsgId3, 1, 'received emsg ID3 event');
assert.equal(gotData, 1, 'received data event');
transmuxer.off();
done();
}
});
assert.equal(this.requests.length, 2, 'there are two requests');

const initReq = this.requests.shift();
const segmentReq = this.requests.shift();

assert.equal(initReq.uri, 'mp4VideoInit.mp4', 'the first request is for the init segment');
assert.equal(segmentReq.uri, 'mp4Video.mp4', 'the second request is for a segment');

// Simulate receiving the media first
this.standardXHRResponse(segmentReq, mp4Video());
// Simulate receiving the init segment after the media
this.standardXHRResponse(initReq, mp4VideoInit());
});

0 comments on commit 906f29e

Please sign in to comment.