From 9352feb8cc7401f43557a96fe2671888e92b5920 Mon Sep 17 00:00:00 2001 From: fpv-wtf <> Date: Fri, 14 Jul 2023 11:10:54 +0000 Subject: [PATCH] Update from https://github.com/fpv-wtf/wtfos-configurator/commit/15e8e5dc9a140bbfeadc95cedf72e1c6bafc0045 --- src/osd-overlay/mp4/parsers.ts | 27 ++++++++++++++++ src/osd-overlay/mp4/types.ts | 8 +++++ src/osd-overlay/processor.ts | 43 ++++++++++++++++++++++++-- src/translations/el/osdOverlay.json | 2 +- src/translations/en/osdOverlay.json | 2 +- src/translations/nl/osdOverlay.json | 2 +- src/translations/sk/osdOverlay.json | 2 +- src/translations/sv/osdOverlay.json | 2 +- src/translations/uk/osdOverlay.json | 2 +- src/translations/vi/osdOverlay.json | 2 +- src/translations/zh-TW/osdOverlay.json | 2 +- 11 files changed, 83 insertions(+), 11 deletions(-) diff --git a/src/osd-overlay/mp4/parsers.ts b/src/osd-overlay/mp4/parsers.ts index bf07415..f6b3527 100644 --- a/src/osd-overlay/mp4/parsers.ts +++ b/src/osd-overlay/mp4/parsers.ts @@ -34,6 +34,7 @@ import { UrlBox, UrnBox, VmhdBox, + CttsBox, } from "./types"; export async function parseBox(stream: FileStreamReader): Promise { @@ -88,6 +89,7 @@ export async function parseBox(stream: FileStreamReader): Promise { trak: TrakBoxParser, udta: UdtaBoxParser, vmhd: VmhdBoxParser, + ctts: CttsBoxParser, }; let parser: BoxParser; @@ -457,6 +459,7 @@ class StblBoxParser extends SimpleBoxParser { stss: childBoxes.stss![0] as StssBox, stsz: childBoxes.stsz![0] as StszBox, stts: childBoxes.stts![0] as SttsBox, + ctts: childBoxes.ctts ? childBoxes.ctts![0] as CttsBox : undefined }; } } @@ -762,6 +765,30 @@ class VmhdBoxParser extends FullBoxParser { } } +class CttsBoxParser extends FullBoxParser { + async parseBox( + header: BoxHeader, + fullBoxHeader: FullBoxHeader + ): Promise { + const sampleCount = await this.stream.getNextUint32(); + const sampleCounts = []; + const sampleOffsets = []; + + for (let i = 0; i < sampleCount; i++) { + sampleCounts.push(await this.stream.getNextUint32()); + sampleOffsets.push(await this.stream.getNextUint32()); + } + + return { + header, + fullBoxHeader, + type: "ctts", + sampleCounts, + sampleOffsets, + }; + } +} + class DrefBoxParser extends FullBoxParser { async parseBox( header: BoxHeader, diff --git a/src/osd-overlay/mp4/types.ts b/src/osd-overlay/mp4/types.ts index f198e46..11dc44c 100644 --- a/src/osd-overlay/mp4/types.ts +++ b/src/osd-overlay/mp4/types.ts @@ -33,6 +33,7 @@ export type BoxType = | "url " | "urn " | "vmhd" + | "ctts" | UnknownBoxType; export type ContainerBox = @@ -68,6 +69,7 @@ export type Box = | UrlBox | UrnBox | VmhdBox + | CttsBox | UnknownBox; export interface BoxHeader { @@ -124,6 +126,7 @@ export interface StblBox extends BaseBox<"stbl"> { stss: StssBox; stsz: StszBox; stts: SttsBox; + ctts?: CttsBox; } export interface UdtaBox extends BaseBox<"udta"> {} @@ -263,6 +266,11 @@ export interface VmhdBox extends BaseFullBox<"vmhd"> { opColor: number[]; } +export interface CttsBox extends BaseFullBox<"ctts"> { + sampleCounts: number[]; + sampleOffsets: number[]; +} + export interface UrlBox extends BaseFullBox<"url "> { location: string; } diff --git a/src/osd-overlay/processor.ts b/src/osd-overlay/processor.ts index 6b891d0..ab327ac 100644 --- a/src/osd-overlay/processor.ts +++ b/src/osd-overlay/processor.ts @@ -173,7 +173,7 @@ export class Processor { this.outMp4?.setFramerate(60); - this.expectedFrames = this.inMp4!.moov!.trak[0].mdia.mdhd.duration; + this.expectedFrames = this.inMp4!.moov!.trak[0].mdia.minf.stbl.stsz.sampleCount; this.decodedFrames = {}; this.progressInit({ @@ -258,16 +258,19 @@ export class Processor { }); this.decoder!.decode(encodedChunk); - lastSampleIndex = chunk.index + 1; this.queuedForDecode++; } // Wait for all samples to be decoded. await this.decoder!.flush(); + // DJI recordings straight from the goggles have all frames in sequence. + // Processed files may need reordering of the frames described in the ctts box. + const orderedFrames = this.reorderFrames(lastSampleIndex); + // Modify and enque frames for encoding. this.encodedFrames = []; - for (const [index, entry] of Object.values(this.decodedFrames).entries()) { + for (const [index, entry] of orderedFrames.entries()) { if (!entry.image) { console.error(`Frame ${entry.index} was never decoded!`); this.framesDecodedMissing++; @@ -300,6 +303,8 @@ export class Processor { for (const frame of this.encodedFrames) { this.outMp4!.writeSample(frame.data, frame.sync); } + + lastSampleIndex += sampleChunks.length } await this.outMp4!.close(); @@ -307,6 +312,38 @@ export class Processor { this.processResolve!(); } + private reorderFrames(lastSampleIndex: number) { + const orderedFrames = []; + const ctts = this.inMp4!.moov!.trak[0].mdia.minf.stbl.ctts; + + if (!ctts) { + // No ctts box found: no reordering needed + for (let i = 0; i < Object.keys(this.decodedFrames).length; i++) { + orderedFrames.push(this.decodedFrames[lastSampleIndex + i]); + } + } else { + // Reorder frames according to ctts table + const sampleDelta = this.inMp4!.moov!.trak[0].mdia.minf.stbl.stts.entries[0].sampleDelta; + const initialOffset = ctts.sampleOffsets[0] / sampleDelta; + + for (let i = 0; i < Object.keys(this.decodedFrames).length; i++) { + const frameNumber = lastSampleIndex + i; + + let j = 0; + let frame = 0; + while (frameNumber >= frame) { + j++; + frame = ctts.sampleCounts.slice(0, j).reduce((acc, e) => acc + e, 0); + } + + const newPosition = i + ctts.sampleOffsets[j - 1] / sampleDelta - initialOffset; + orderedFrames[newPosition] = Object.assign({}, this.decodedFrames[lastSampleIndex + i], {index: lastSampleIndex + newPosition}); + } + } + + return orderedFrames; + } + private async handleDecodedFrame(frame: VideoFrame) { this.framesDecoded++; this.decodedFrames[frame.timestamp!].image = await createImageBitmap(frame); diff --git a/src/translations/el/osdOverlay.json b/src/translations/el/osdOverlay.json index c2c260c..d2f77de 100644 --- a/src/translations/el/osdOverlay.json +++ b/src/translations/el/osdOverlay.json @@ -22,7 +22,7 @@ "fileDropVideo": "Video", "noteConfigLink": "Configure this in the Package Manager.", "noteHeader": "OSD recording is an opt-in feature on the goggle side.", - "noteWarning": "Any video files used must come directly from the goggles, with no modifications.", + "noteWarning": "The video file must either come directly from the goggles, or have only modifications that don't change the length of the video. Altering the aspect ratio, stabilization, etc are supported.", "processing": "Processing...", "start": "Start" } diff --git a/src/translations/en/osdOverlay.json b/src/translations/en/osdOverlay.json index c2c260c..d2f77de 100644 --- a/src/translations/en/osdOverlay.json +++ b/src/translations/en/osdOverlay.json @@ -22,7 +22,7 @@ "fileDropVideo": "Video", "noteConfigLink": "Configure this in the Package Manager.", "noteHeader": "OSD recording is an opt-in feature on the goggle side.", - "noteWarning": "Any video files used must come directly from the goggles, with no modifications.", + "noteWarning": "The video file must either come directly from the goggles, or have only modifications that don't change the length of the video. Altering the aspect ratio, stabilization, etc are supported.", "processing": "Processing...", "start": "Start" } diff --git a/src/translations/nl/osdOverlay.json b/src/translations/nl/osdOverlay.json index c2c260c..d2f77de 100644 --- a/src/translations/nl/osdOverlay.json +++ b/src/translations/nl/osdOverlay.json @@ -22,7 +22,7 @@ "fileDropVideo": "Video", "noteConfigLink": "Configure this in the Package Manager.", "noteHeader": "OSD recording is an opt-in feature on the goggle side.", - "noteWarning": "Any video files used must come directly from the goggles, with no modifications.", + "noteWarning": "The video file must either come directly from the goggles, or have only modifications that don't change the length of the video. Altering the aspect ratio, stabilization, etc are supported.", "processing": "Processing...", "start": "Start" } diff --git a/src/translations/sk/osdOverlay.json b/src/translations/sk/osdOverlay.json index c2c260c..d2f77de 100644 --- a/src/translations/sk/osdOverlay.json +++ b/src/translations/sk/osdOverlay.json @@ -22,7 +22,7 @@ "fileDropVideo": "Video", "noteConfigLink": "Configure this in the Package Manager.", "noteHeader": "OSD recording is an opt-in feature on the goggle side.", - "noteWarning": "Any video files used must come directly from the goggles, with no modifications.", + "noteWarning": "The video file must either come directly from the goggles, or have only modifications that don't change the length of the video. Altering the aspect ratio, stabilization, etc are supported.", "processing": "Processing...", "start": "Start" } diff --git a/src/translations/sv/osdOverlay.json b/src/translations/sv/osdOverlay.json index c2c260c..d2f77de 100644 --- a/src/translations/sv/osdOverlay.json +++ b/src/translations/sv/osdOverlay.json @@ -22,7 +22,7 @@ "fileDropVideo": "Video", "noteConfigLink": "Configure this in the Package Manager.", "noteHeader": "OSD recording is an opt-in feature on the goggle side.", - "noteWarning": "Any video files used must come directly from the goggles, with no modifications.", + "noteWarning": "The video file must either come directly from the goggles, or have only modifications that don't change the length of the video. Altering the aspect ratio, stabilization, etc are supported.", "processing": "Processing...", "start": "Start" } diff --git a/src/translations/uk/osdOverlay.json b/src/translations/uk/osdOverlay.json index c2c260c..d2f77de 100644 --- a/src/translations/uk/osdOverlay.json +++ b/src/translations/uk/osdOverlay.json @@ -22,7 +22,7 @@ "fileDropVideo": "Video", "noteConfigLink": "Configure this in the Package Manager.", "noteHeader": "OSD recording is an opt-in feature on the goggle side.", - "noteWarning": "Any video files used must come directly from the goggles, with no modifications.", + "noteWarning": "The video file must either come directly from the goggles, or have only modifications that don't change the length of the video. Altering the aspect ratio, stabilization, etc are supported.", "processing": "Processing...", "start": "Start" } diff --git a/src/translations/vi/osdOverlay.json b/src/translations/vi/osdOverlay.json index c2c260c..d2f77de 100644 --- a/src/translations/vi/osdOverlay.json +++ b/src/translations/vi/osdOverlay.json @@ -22,7 +22,7 @@ "fileDropVideo": "Video", "noteConfigLink": "Configure this in the Package Manager.", "noteHeader": "OSD recording is an opt-in feature on the goggle side.", - "noteWarning": "Any video files used must come directly from the goggles, with no modifications.", + "noteWarning": "The video file must either come directly from the goggles, or have only modifications that don't change the length of the video. Altering the aspect ratio, stabilization, etc are supported.", "processing": "Processing...", "start": "Start" } diff --git a/src/translations/zh-TW/osdOverlay.json b/src/translations/zh-TW/osdOverlay.json index c2c260c..d2f77de 100644 --- a/src/translations/zh-TW/osdOverlay.json +++ b/src/translations/zh-TW/osdOverlay.json @@ -22,7 +22,7 @@ "fileDropVideo": "Video", "noteConfigLink": "Configure this in the Package Manager.", "noteHeader": "OSD recording is an opt-in feature on the goggle side.", - "noteWarning": "Any video files used must come directly from the goggles, with no modifications.", + "noteWarning": "The video file must either come directly from the goggles, or have only modifications that don't change the length of the video. Altering the aspect ratio, stabilization, etc are supported.", "processing": "Processing...", "start": "Start" }