Skip to content

Commit

Permalink
refactor: offload mp4/ts probe to the web worker (#1117)
Browse files Browse the repository at this point in the history
  • Loading branch information
@brandonocasey
brandonocasey authored Apr 22, 2021
1 parent 82ff4f5 commit 3c9f721
Show file tree
Hide file tree
Showing 8 changed files with 375 additions and 214 deletions.
6 changes: 3 additions & 3 deletions package-lock.json
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@
"lodash-compat": "^3.10.0",
"nomnoml": "^0.3.0",
"rollup": "^2.36.1",
"rollup-plugin-worker-factory": "0.5.4",
"rollup-plugin-worker-factory": "0.5.5",
"shelljs": "^0.8.4",
"sinon": "^8.1.1",
"url-toolkit": "^2.2.1",
Expand Down
194 changes: 108 additions & 86 deletions src/media-segment-request.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,8 @@ import videojs from 'video.js';
import { createTransferableMessage } from './bin-utils';
import { stringToArrayBuffer } from './util/string-to-array-buffer';
import { transmux } from './segment-transmuxer';
import { probeTsSegment } from './util/segment';
import mp4probe from 'mux.js/lib/mp4/probe';
import { segmentXhrHeaders } from './xhr';
import {workerCallback} from './util/worker-callback.js';
import {
detectContainerForBytes,
isLikelyFmp4MediaSegment
Expand Down Expand Up @@ -182,25 +181,34 @@ const handleInitSegmentResponse =
}, segment);
}

const tracks = mp4probe.tracks(segment.map.bytes);
workerCallback({
action: 'probeMp4Tracks',
data: segment.map.bytes,
transmuxer: segment.transmuxer,
callback: ({tracks, data}) => {
// transfer bytes back to us
segment.map.bytes = data;

tracks.forEach(function(track) {
segment.map.tracks = segment.map.tracks || {};
tracks.forEach(function(track) {
segment.map.tracks = segment.map.tracks || {};

// only support one track of each type for now
if (segment.map.tracks[track.type]) {
return;
}
// only support one track of each type for now
if (segment.map.tracks[track.type]) {
return;
}

segment.map.tracks[track.type] = track;
segment.map.tracks[track.type] = track;

if (typeof track.id === 'number' && track.timescale) {
segment.map.timescales = segment.map.timescales || {};
segment.map.timescales[track.id] = track.timescale;
}

});

if (typeof track.id === 'number' && track.timescale) {
segment.map.timescales = segment.map.timescales || {};
segment.map.timescales[track.id] = track.timescale;
return finishProcessingFn(null, segment);
}
});

return finishProcessingFn(null, segment);
};

/**
Expand Down Expand Up @@ -282,34 +290,7 @@ const transmuxAndNotify = ({
let videoStartFn = timingInfoFn.bind(null, segment, 'video', 'start');
const videoEndFn = timingInfoFn.bind(null, segment, 'video', 'end');

// Check to see if we are appending a full segment.
if (!isPartial && !segment.lastReachedChar) {
// In the full segment transmuxer, we don't yet have the ability to extract a "proper"
// start time. Meaning cached frame data may corrupt our notion of where this segment
// really starts. To get around this, full segment appends should probe for the info
// needed.
const probeResult = probeTsSegment(bytes, segment.baseStartTime);

if (probeResult) {
trackInfoFn(segment, {
hasAudio: probeResult.hasAudio,
hasVideo: probeResult.hasVideo,
isMuxed
});
trackInfoFn = null;

if (probeResult.hasAudio && !isMuxed) {
audioStartFn(probeResult.audioStart);
}
if (probeResult.hasVideo) {
videoStartFn(probeResult.videoStart);
}
audioStartFn = null;
videoStartFn = null;
}
}

transmux({
const finish = () => transmux({
bytes,
transmuxer: segment.transmuxer,
audioAppendStart: segment.audioAppendStart,
Expand Down Expand Up @@ -379,6 +360,47 @@ const transmuxAndNotify = ({
doneFn(null, segment, result);
}
});

// Check to see if we are appending a full segment.
if (!isPartial && !segment.lastReachedChar) {
// In the full segment transmuxer, we don't yet have the ability to extract a "proper"
// start time. Meaning cached frame data may corrupt our notion of where this segment
// really starts. To get around this, full segment appends should probe for the info
// needed.
workerCallback({
action: 'probeTs',
transmuxer: segment.transmuxer,
data: bytes,
baseStartTime: segment.baseStartTime,
callback: (data) => {
segment.bytes = bytes = data.data;

const probeResult = data.result;

if (probeResult) {
trackInfoFn(segment, {
hasAudio: probeResult.hasAudio,
hasVideo: probeResult.hasVideo,
isMuxed
});
trackInfoFn = null;

if (probeResult.hasAudio && !isMuxed) {
audioStartFn(probeResult.audioStart);
}
if (probeResult.hasVideo) {
videoStartFn(probeResult.videoStart);
}
audioStartFn = null;
videoStartFn = null;
}

finish();
}
});
} else {
finish();
}
};

const handleSegmentBytes = ({
Expand All @@ -396,7 +418,7 @@ const handleSegmentBytes = ({
dataFn,
doneFn
}) => {
const bytesAsUint8Array = new Uint8Array(bytes);
let bytesAsUint8Array = new Uint8Array(bytes);

// TODO:
// We should have a handler that fetches the number of bytes required
Expand Down Expand Up @@ -438,62 +460,62 @@ const handleSegmentBytes = ({
// Note that the start time returned by the probe reflects the baseMediaDecodeTime, as
// that is the true start of the segment (where the playback engine should begin
// decoding).
const timingInfo = mp4probe.startTime(segment.map.timescales, bytesAsUint8Array);

if (trackInfo.hasAudio && !trackInfo.isMuxed) {
timingInfoFn(segment, 'audio', 'start', timingInfo);
}

if (trackInfo.hasVideo) {
timingInfoFn(segment, 'video', 'start', timingInfo);
}

const finishLoading = (captions) => {
// if the track still has audio at this point it is only possible
// for it to be audio only. See `tracks.video && tracks.audio` if statement
// above.
// we make sure to use segment.bytes here as that
dataFn(segment, {data: bytes, type: trackInfo.hasAudio && !trackInfo.isMuxed ? 'audio' : 'video'});
dataFn(segment, {
data: bytesAsUint8Array,
type: trackInfo.hasAudio && !trackInfo.isMuxed ? 'audio' : 'video'
});
if (captions && captions.length) {
captionsFn(segment, captions);
}
doneFn(null, segment, {});
};

// Run through the CaptionParser in case there are captions.
// Initialize CaptionParser if it hasn't been yet
if (!tracks.video || !bytes.byteLength || !segment.transmuxer) {
finishLoading();
return;
}

const buffer = bytes instanceof ArrayBuffer ? bytes : bytes.buffer;
const byteOffset = bytes instanceof ArrayBuffer ? 0 : bytes.byteOffset;
const listenForCaptions = (event) => {
if (event.data.action !== 'mp4Captions') {
return;
}
segment.transmuxer.removeEventListener('message', listenForCaptions);

const data = event.data.data;

// transfer ownership of bytes back to us.
segment.bytes = bytes = new Uint8Array(data, data.byteOffset || 0, data.byteLength);
workerCallback({
action: 'probeMp4StartTime',
timescales: segment.map.timescales,
data: bytesAsUint8Array,
transmuxer: segment.transmuxer,
callback: ({data, startTime}) => {
// transfer bytes back to us
bytes = data.buffer;
segment.bytes = bytesAsUint8Array = data;

if (trackInfo.hasAudio && !trackInfo.isMuxed) {
timingInfoFn(segment, 'audio', 'start', startTime);
}

finishLoading(event.data.captions);
};
if (trackInfo.hasVideo) {
timingInfoFn(segment, 'video', 'start', startTime);
}

segment.transmuxer.addEventListener('message', listenForCaptions);
// Run through the CaptionParser in case there are captions.
// Initialize CaptionParser if it hasn't been yet
if (!tracks.video || !data.byteLength || !segment.transmuxer) {
finishLoading();
return;
}

// transfer ownership of bytes to worker.
segment.transmuxer.postMessage({
action: 'pushMp4Captions',
timescales: segment.map.timescales,
trackIds: [tracks.video.id],
data: buffer,
byteOffset,
byteLength: bytes.byteLength
}, [ buffer ]);
workerCallback({
action: 'pushMp4Captions',
endAction: 'mp4Captions',
transmuxer: segment.transmuxer,
data: bytesAsUint8Array,
timescales: segment.map.timescales,
trackIds: [tracks.video.id],
callback: (message) => {
// transfer bytes back to us
bytes = message.data.buffer;
segment.bytes = bytesAsUint8Array = message.data;
finishLoading(message.captions);
}
});
}
});
return;
}

Expand Down
65 changes: 65 additions & 0 deletions src/transmuxer-worker.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,10 @@
import {Transmuxer as FullMux} from 'mux.js/lib/mp4/transmuxer';
import PartialMux from 'mux.js/lib/partial/transmuxer';
import CaptionParser from 'mux.js/lib/mp4/caption-parser';
import mp4probe from 'mux.js/lib/mp4/probe';
import tsInspector from 'mux.js/lib/tools/ts-inspector.js';
import {
ONE_SECOND_IN_TS,
secondsToVideoTs,
videoTsToSeconds
} from 'mux.js/lib/utils/clock';
Expand Down Expand Up @@ -335,6 +338,68 @@ class MessageHandlers {
}, [segment.buffer]);
}

probeMp4StartTime({timescales, data}) {
const startTime = mp4probe.startTime(timescales, data);

this.self.postMessage({
action: 'probeMp4StartTime',
startTime,
data
}, [data.buffer]);
}

probeMp4Tracks({data}) {
const tracks = mp4probe.tracks(data);

this.self.postMessage({
action: 'probeMp4Tracks',
tracks,
data
}, [data.buffer]);
}

/**
* Probe an mpeg2-ts segment to determine the start time of the segment in it's
* internal "media time," as well as whether it contains video and/or audio.
*
* @private
* @param {Uint8Array} bytes - segment bytes
* @param {number} baseStartTime
* Relative reference timestamp used when adjusting frame timestamps for rollover.
* This value should be in seconds, as it's converted to a 90khz clock within the
* function body.
* @return {Object} The start time of the current segment in "media time" as well as
* whether it contains video and/or audio
*/
probeTs({data, baseStartTime}) {
const tsStartTime = (typeof baseStartTime === 'number' && !isNaN(baseStartTime)) ?
(baseStartTime * ONE_SECOND_IN_TS) :
void 0;
const timeInfo = tsInspector.inspect(data, tsStartTime);
let result = null;

if (timeInfo) {
result = {
// each type's time info comes back as an array of 2 times, start and end
hasVideo: timeInfo.video && timeInfo.video.length === 2 || false,
hasAudio: timeInfo.audio && timeInfo.audio.length === 2 || false
};

if (result.hasVideo) {
result.videoStart = timeInfo.video[0].ptsTime;
}
if (result.hasAudio) {
result.audioStart = timeInfo.audio[0].ptsTime;
}
}

this.self.postMessage({
action: 'probeTs',
result,
data
}, [data.buffer]);
}

clearAllMp4Captions() {
if (this.captionParser) {
this.captionParser.clearAllCaptions();
Expand Down
Loading

0 comments on commit 3c9f721

Please sign in to comment.