Skip to content

Commit

Permalink
fix(📹): Better support for rotated videos (#2461)
Browse files Browse the repository at this point in the history
  • Loading branch information
wcandillon authored Jun 6, 2024
1 parent 8bd1ffb commit bcab375
Show file tree
Hide file tree
Showing 23 changed files with 296 additions and 50 deletions.
48 changes: 47 additions & 1 deletion docs/docs/video.md
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,53 @@ export const useVideoFromAsset = (

## Returned Values

The `useVideo` hook returns `currentFrame` which contains the current video frame, as well as `currentTime`, and `rotationInDegrees`.
The `useVideo` hook returns `currentFrame` which contains the current video frame, as well as `currentTime`, `rotation`, and `size`.

## Rotated Video

`rotation` can either be `0`, `90`, `180`, or `270`.
We provide a `fitbox` function that can help rotating and scaling the video.

```tsx twoslash
import React from "react";
import {
Canvas,
Image,
useVideo,
fitbox,
rect
} from "@shopify/react-native-skia";
import { Pressable, useWindowDimensions } from "react-native";
import { useSharedValue } from "react-native-reanimated";

interface VideoExampleProps {
localVideoFile: string;
}

// The URL needs to be a local path; we usually use expo-asset for that.
export const VideoExample = ({ localVideoFile }: VideoExampleProps) => {
const paused = useSharedValue(false);
const { width, height } = useWindowDimensions();
const { currentFrame, rotation, size } = useVideo(require(localVideoFile));
const src = rect(0, 0, size.width, size.height);
const dst = rect(0, 0, width, height)
const transform = fitbox("cover", src, dst, rotation);
return (
<Canvas style={{ flex: 1 }}>
<Image
image={currentFrame}
x={0}
y={0}
width={width}
height={height}
fit="none"
transform={transform}
/>
</Canvas>
);
};
```


## Playback Options

Expand Down
26 changes: 26 additions & 0 deletions package/android/cpp/rnskia-android/RNSkAndroidVideo.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
#pragma clang diagnostic ignored "-Wdocumentation"

#include "include/core/SkImage.h"
#include "include/core/SkSize.h"

#pragma clang diagnostic pop

Expand Down Expand Up @@ -102,4 +103,29 @@ float RNSkAndroidVideo::getRotationInDegrees() {
return static_cast<float>(rotation);
}

SkISize RNSkAndroidVideo::getSize() {
JNIEnv *env = facebook::jni::Environment::current();
jclass cls = env->GetObjectClass(_jniVideo.get());
jmethodID mid =
env->GetMethodID(cls, "getSize", "()Landroid/graphics/Point;");
if (!mid) {
RNSkLogger::logToConsole("getSize method not found");
return SkISize::Make(0, 0);
}
jobject jPoint = env->CallObjectMethod(_jniVideo.get(), mid);
jclass pointCls = env->GetObjectClass(jPoint);

jfieldID xFid = env->GetFieldID(pointCls, "x", "I");
jfieldID yFid = env->GetFieldID(pointCls, "y", "I");
if (!xFid || !yFid) {
RNSkLogger::logToConsole("Point class fields not found");
return SkISize::Make(0, 0);
}

jint width = env->GetIntField(jPoint, xFid);
jint height = env->GetIntField(jPoint, yFid);

return SkISize::Make(width, height);
}

} // namespace RNSkia
1 change: 1 addition & 0 deletions package/android/cpp/rnskia-android/RNSkAndroidVideo.h
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ class RNSkAndroidVideo : public RNSkVideo {
double framerate() override;
void seek(double timestamp) override;
float getRotationInDegrees() override;
SkISize getSize() override;
};

} // namespace RNSkia
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import android.net.Uri;
import android.os.Build;
import android.view.Surface;
import android.graphics.Point;

import androidx.annotation.RequiresApi;

Expand All @@ -30,6 +31,8 @@ public class RNSkVideo {
private double durationMs;
private double frameRate;
private int rotationDegrees = 0;
private int width = 0;
private int height = 0;

RNSkVideo(Context context, String localUri) {
this.uri = Uri.parse(localUri);
Expand Down Expand Up @@ -57,8 +60,8 @@ private void initializeReader() {
if (format.containsKey(MediaFormat.KEY_ROTATION)) {
rotationDegrees = format.getInteger(MediaFormat.KEY_ROTATION);
}
int width = format.getInteger(MediaFormat.KEY_WIDTH);
int height = format.getInteger(MediaFormat.KEY_HEIGHT);
width = format.getInteger(MediaFormat.KEY_WIDTH);
height = format.getInteger(MediaFormat.KEY_HEIGHT);
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
imageReader = ImageReader.newInstance(
width,
Expand Down Expand Up @@ -125,6 +128,11 @@ public void seek(long timestamp) {
}
}

@DoNotStrip
public Point getSize() {
return new Point(width, height);
}

private int selectVideoTrack(MediaExtractor extractor) {
int numTracks = extractor.getTrackCount();
for (int i = 0; i < numTracks; i++) {
Expand Down
14 changes: 12 additions & 2 deletions package/cpp/api/JsiVideo.h
Original file line number Diff line number Diff line change
Expand Up @@ -53,17 +53,27 @@ class JsiVideo : public JsiSkWrappingSharedPtrHostObject<RNSkVideo> {
return jsi::Value::undefined();
}

JSI_HOST_FUNCTION(getRotationInDegrees) {
JSI_HOST_FUNCTION(rotation) {
auto context = getContext();
auto rot = getObject()->getRotationInDegrees();
return jsi::Value(static_cast<double>(rot));
}

JSI_HOST_FUNCTION(size) {
auto context = getContext();
auto size = getObject()->getSize();
auto result = jsi::Object(runtime);
result.setProperty(runtime, "width", static_cast<double>(size.width()));
result.setProperty(runtime, "height", static_cast<double>(size.height()));
return result;
}

JSI_EXPORT_FUNCTIONS(JSI_EXPORT_FUNC(JsiVideo, nextImage),
JSI_EXPORT_FUNC(JsiVideo, duration),
JSI_EXPORT_FUNC(JsiVideo, framerate),
JSI_EXPORT_FUNC(JsiVideo, seek),
JSI_EXPORT_FUNC(JsiVideo, getRotationInDegrees),
JSI_EXPORT_FUNC(JsiVideo, rotation),
JSI_EXPORT_FUNC(JsiVideo, size),
JSI_EXPORT_FUNC(JsiVideo, dispose))

JsiVideo(std::shared_ptr<RNSkPlatformContext> context,
Expand Down
1 change: 1 addition & 0 deletions package/cpp/rnskia/RNSkVideo.h
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ class RNSkVideo {
virtual double framerate() = 0;
virtual void seek(double timestamp) = 0;
virtual float getRotationInDegrees() = 0;
virtual SkISize getSize() = 0;
};

} // namespace RNSkia
4 changes: 4 additions & 0 deletions package/ios/RNSkia-iOS/RNSkiOSVideo.h
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
#pragma clang diagnostic ignored "-Wdocumentation"

#include "include/core/SkImage.h"
#include "include/core/SkSize.h"

#pragma clang diagnostic pop

Expand All @@ -25,6 +26,8 @@ class RNSkiOSVideo : public RNSkVideo {
RNSkPlatformContext *_context;
double _duration = 0;
double _framerate = 0;
float _videoWidth = 0;
float _videoHeight = 0;
void setupReader(CMTimeRange timeRange);
NSDictionary *getOutputSettings();
CGAffineTransform _preferredTransform;
Expand All @@ -37,6 +40,7 @@ class RNSkiOSVideo : public RNSkVideo {
double framerate() override;
void seek(double timestamp) override;
float getRotationInDegrees() override;
SkISize getSize() override;
};

} // namespace RNSkia
20 changes: 11 additions & 9 deletions package/ios/RNSkia-iOS/RNSkiOSVideo.mm
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,9 @@
[[asset tracksWithMediaType:AVMediaTypeVideo] firstObject];
_framerate = videoTrack.nominalFrameRate;
_preferredTransform = videoTrack.preferredTransform;

CGSize videoSize = videoTrack.naturalSize;
_videoWidth = videoSize.width;
_videoHeight = videoSize.height;
NSDictionary *outputSettings = getOutputSettings();
AVAssetReaderTrackOutput *trackOutput =
[[AVAssetReaderTrackOutput alloc] initWithTrack:videoTrack
Expand Down Expand Up @@ -104,19 +106,15 @@
// Determine the rotation angle in radians
if (transform.a == 0 && transform.b == 1 && transform.c == -1 &&
transform.d == 0) {
rotationAngle = M_PI_2; // 90 degrees
rotationAngle = 90;
} else if (transform.a == 0 && transform.b == -1 && transform.c == 1 &&
transform.d == 0) {
rotationAngle = -M_PI_2; // -90 degrees
rotationAngle = 270;
} else if (transform.a == -1 && transform.b == 0 && transform.c == 0 &&
transform.d == -1) {
rotationAngle = M_PI; // 180 degrees
} else if (transform.a == 1 && transform.b == 0 && transform.c == 0 &&
transform.d == 1) {
rotationAngle = 0.0; // 0 degrees
rotationAngle = 180;
}
// Convert the rotation angle from radians to degrees
return rotationAngle * 180 / M_PI;
return rotationAngle;
}

void RNSkiOSVideo::seek(double timeInMilliseconds) {
Expand All @@ -136,4 +134,8 @@

double RNSkiOSVideo::framerate() { return _framerate; }

SkISize RNSkiOSVideo::getSize() {
return SkISize::Make(_videoWidth, _videoHeight);
}

} // namespace RNSkia
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
49 changes: 28 additions & 21 deletions package/src/dom/nodes/datatypes/Fitting.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,10 @@ export interface Size {
height: number;
}

export const size = (width = 0, height = 0) => ({ width, height });
export const size = (width = 0, height = 0) => {
"worklet";
return { width, height };
};

export const rect2rect = (
src: SkRect,
Expand All @@ -18,37 +21,19 @@ export const rect2rect = (
{ scaleX: number },
{ scaleY: number }
] => {
"worklet";
const scaleX = dst.width / src.width;
const scaleY = dst.height / src.height;
const translateX = dst.x - src.x * scaleX;
const translateY = dst.y - src.y * scaleY;
return [{ translateX }, { translateY }, { scaleX }, { scaleY }];
};

export const fitRects = (
fit: Fit,
rect: SkRect,
{ x, y, width, height }: SkRect
) => {
const sizes = applyBoxFit(
fit,
{ width: rect.width, height: rect.height },
{ width, height }
);
const src = inscribe(sizes.src, rect);
const dst = inscribe(sizes.dst, {
x,
y,
width,
height,
});
return { src, dst };
};

const inscribe = (
{ width, height }: Size,
rect: { x: number; y: number; width: number; height: number }
) => {
"worklet";
const halfWidthDelta = (rect.width - width) / 2.0;
const halfHeightDelta = (rect.height - height) / 2.0;
return {
Expand All @@ -60,6 +45,7 @@ const inscribe = (
};

const applyBoxFit = (fit: Fit, input: Size, output: Size) => {
"worklet";
let src = size(),
dst = size();
if (
Expand Down Expand Up @@ -122,3 +108,24 @@ const applyBoxFit = (fit: Fit, input: Size, output: Size) => {
}
return { src, dst };
};

export const fitRects = (
fit: Fit,
rect: SkRect,
{ x, y, width, height }: SkRect
) => {
"worklet";
const sizes = applyBoxFit(
fit,
{ width: rect.width, height: rect.height },
{ width, height }
);
const src = inscribe(sizes.src, rect);
const dst = inscribe(sizes.dst, {
x,
y,
width,
height,
});
return { src, dst };
};
15 changes: 10 additions & 5 deletions package/src/external/reanimated/useVideo.ts
Original file line number Diff line number Diff line change
Expand Up @@ -49,10 +49,8 @@ export const useVideo = (
const lastTimestamp = Rea.useSharedValue(-1);
const duration = useMemo(() => video?.duration() ?? 0, [video]);
const framerate = useMemo(() => video?.framerate() ?? 0, [video]);
const rotationInDegrees = useMemo(
() => video?.getRotationInDegrees() ?? 0,
[video]
);
const size = useMemo(() => video?.size() ?? { width: 0, height: 0 }, [video]);
const rotation = useMemo(() => video?.rotation() ?? 0, [video]);
Rea.useFrameCallback((frameInfo: FrameInfo) => {
processVideoState(
video,
Expand All @@ -78,5 +76,12 @@ export const useVideo = (
};
}, [video]);

return { currentFrame, currentTime, duration, framerate, rotationInDegrees };
return {
currentFrame,
currentTime,
duration,
framerate,
rotation,
size,
};
};
Loading

0 comments on commit bcab375

Please sign in to comment.