fix: reading mp4v file. close #1

src/processors/frameGrabber.ts

@@ -1,5 +1,5 @@
 import { MP4ArrayBuffer } from 'mp4box';
-import { makePromise, noop } from 'yon-utils';
+import { debouncePromise, makePromise, noop } from 'yon-utils';
 import { reportError } from '../report';
 import type { FFmpeg } from '@ffmpeg/ffmpeg';
 
@@ -224,7 +224,7 @@ export async function grabFramesWithFFMpeg({ fileContent, fileExtname, resizeWid
 
   const start = frameTimestamps[0]
   const end = frameTimestamps[frameTimestamps.length - 1]
-  const fps = frameTimestamps.length / (end - start) // TODO: rewrite this with ffmpeg "select" filter
+  const frameDuration = (end - start) / frameTimestamps.length
 
   // const mountPoint = "/mounted/"
   // await ffmpeg.mount('WORKERFS' as any, { files: [file] }, mountPoint) // see https://emscripten.org/docs/api_reference/Filesystem-API.html#workerfs
@@ -241,43 +241,46 @@ export async function grabFramesWithFFMpeg({ fileContent, fileExtname, resizeWid
   tempCanvas.height = resizeHeight
   const tempCtx = tempCanvas.getContext('2d')!
 
-  const poll = async () => {
+  const frameDataSize = resizeWidth * resizeHeight * 4
+  const poll = debouncePromise(async () => {
     while (!aborted) {
       const frameFilename = `out${outputFrames.length + 1}.rgba`;
       const data = await ffmpeg.readFile(frameFilename).catch(noop)
-      if (!data?.length) break;
+      if (data?.length !== frameDataSize) break;
 
       tempCtx.clearRect(0, 0, tempCanvas.width, tempCanvas.height)
       tempCtx.putImageData(new ImageData(new Uint8ClampedArray(data.slice() as Uint8Array), resizeWidth, resizeHeight), 0, 0)
-      await ffmpeg.deleteFile(`out${outputFrames.length}.rgba`).catch(noop)
+      setTimeout(() => ffmpeg.deleteFile(frameFilename).catch(e => { console.error('cannot delete', frameFilename, e) }), 100)
       outputFrames.push({
         image: tempCanvas,
         time: frameTimestamps[outputFrames.length]
       })
 
-      aborted = reportProgress(outputFrames.length, outputFrames.at(-1)!)
+      aborted = !reportProgress(outputFrames.length, outputFrames.at(-1)!)
       if (aborted) abortController.abort()
     }
-  }
+  })
 
   try {
     const vf: string[] = [
-      'scale=' + resizeWidth + ':' + resizeHeight
+      `select='isnan(prev_selected_t)*gte(t\\,${start})+gte(t-prev_selected_t\\,${frameDuration})'`,
+      `scale=${resizeWidth}:${resizeHeight}`
     ]
 
     ffmpeg.on('progress', poll)
     await ffmpeg.exec([
       '-i', inputFileName, //mountPoint + file.name,
       '-ss', String(start),
-      '-r', String(fps),
+      '-to', String(end + 0.5),
       '-vframes', String(frameTimestamps.length),
-      ...vf.length ? ['-vf', vf.join(',')] : [],
+      '-vf', vf.join(','),
       '-c:v', 'rawvideo',
       '-pix_fmt', 'rgba',
       '-f', 'image2',
       'out%d.rgba'
     ], undefined, { signal: abortController.signal })
     await poll()
+    await poll()
     await ffmpeg.deleteFile(`out${outputFrames.length}.rgba`).catch(noop)
 
   } finally {
@@ -295,8 +298,8 @@ export async function grabFrames(options: GrabFrameOptions) {
     grabFramesWithMP4Box(options)
       .catch(err => {
         reportError('grabFramesWithMP4Box error', err);
-        if (confirm('MP4Box+WebCodec cannot decode this file. Try with video tag?')) return grabFramesWithVideoTag(options);
-        else error = err;
+        /*if (confirm('MP4Box+WebCodec cannot decode this file. Try with video tag?'))*/ return grabFramesWithVideoTag(options);
+        // else error = err;
       })
       .catch(err => {
         reportError('grabFramesWithVideoTag error', err);

src/processors/readFileInfo.ts

@@ -0,0 +1,56 @@
+import { MP4ArrayBuffer } from "mp4box"
+
+export interface VideoFileInfo {
+  width: number,
+  height: number,
+  duration: number,
+}
+
+export async function readVideoFileInfo({ fileContent, fileURL }: { fileContent: ArrayBufferLike, fileURL: string }) {
+  const outInfo = await new Promise<VideoFileInfo>(resolve => {
+    const video = document.createElement('video');
+    setTimeout(() => resolve({
+      width: 0,
+      height: 0,
+      duration: 0,
+    }), 1200)
+
+    video.onloadedmetadata = () => resolve({
+      width: video.videoWidth,
+      height: video.videoHeight,
+      duration: video.duration,
+    })
+    video.src = fileURL;
+    video.muted = true;
+  })
+
+  if (!(outInfo.width && outInfo.height && outInfo.duration)) {
+    // use mp4box to get info of malformed mp4 file (eg. mpeg4 is not supported by chrome )
+
+    const MP4Box = await import('mp4box')
+    const mp4boxInputFile = MP4Box.createFile();
+    const info = await new Promise<any>((resolve, reject) => {
+      const buffer = new Uint8Array(fileContent).buffer.slice(0) as MP4ArrayBuffer
+      buffer.fileStart = 0
+
+      mp4boxInputFile.onReady = resolve
+      mp4boxInputFile.onError = reject
+      mp4boxInputFile.appendBuffer(buffer)
+    })
+
+    const track = info.videoTracks[0] || info.otherTracks.find((x: any) => /video/i.test(x.name));
+    if (track) {
+      if (!outInfo.width || !outInfo.height) {
+        const [a, c, e, b, d, f] = Array.from(track.matrix.slice(0, 6), (x: number) => x / 65536) // Fixed-float number
+        outInfo.width = Math.abs(a * track.track_width + c * track.track_height)
+        outInfo.height = Math.abs(b * track.track_width + d * track.track_height)
+      }
+    }
+
+    if (!outInfo.duration) {
+      outInfo.duration = info.duration / 1000 // seconds
+    }
+  }
+
+  return outInfo
+}

src/store.tsx

@@ -3,6 +3,7 @@ import { FileData } from '@ffmpeg/ffmpeg/dist/esm/types';
 import { createEffect, createMemo, createRoot, on, onCleanup } from 'solid-js';
 import { createStore } from 'solid-js/store';
 import { debounce } from 'lodash-es';
+import { readVideoFileInfo } from './processors/readFileInfo';
 
 export enum watermarkLocation {
   top = 'top',
@@ -118,26 +119,22 @@ createRoot(() => {
       extname: file.name.split('.').pop() ?? '',
     })
 
-    file.arrayBuffer().then(data => {
-      if (file !== store.file) return
-      updateStore('fileContent', new Uint8Array(data))
-    })
-
     const url = URL.createObjectURL(file);
-    const video = document.createElement('video');
-    video.onloadedmetadata = () => {
-      if (file !== store.file) return;
-
-      updateStore('fileInfo', {
-        url,
-        width: video.videoWidth,
-        height: video.videoHeight,
-        duration: video.duration,
+    updateStore('fileInfo', { url })
+
+    file.arrayBuffer()
+      .then(data => {
+        if (file !== store.file) return
+
+        const fileContent = new Uint8Array(data);
+        updateStore('fileContent', fileContent)
+        return readVideoFileInfo({ fileContent, fileURL: url })
+      })
+      .then(videoInfo => {
+        if (!videoInfo || file !== store.file) return
+        updateStore('fileInfo', videoInfo)
+        updateStore('options', { ...defaultOptions, end: videoInfo.duration })
       })
-      updateStore('options', { ...defaultOptions, end: video.duration })
-    }
-    video.src = url;
-    video.muted = true;
 
     onCleanup(() => {
       URL.revokeObjectURL(url)