Skip to content

Commit

Permalink
Parse metadata from srfr atom in SMTA.
Browse files Browse the repository at this point in the history
PiperOrigin-RevId: 573829216
  • Loading branch information
Samrobbo authored and copybara-github committed Oct 16, 2023
1 parent f53e1bc commit fe11444
Show file tree
Hide file tree
Showing 6 changed files with 204 additions and 35 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import android.net.Uri;
import androidx.media3.common.C;
import androidx.media3.common.MediaItem;
import androidx.media3.common.Metadata;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.util.Util;
import androidx.media3.container.MdtaMetadataEntry;
Expand All @@ -36,6 +37,7 @@
import androidx.media3.test.utils.FakeClock;
import androidx.test.core.app.ApplicationProvider;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.common.collect.ImmutableList;
import com.google.common.util.concurrent.ListenableFuture;
import java.util.ArrayList;
import java.util.List;
Expand Down Expand Up @@ -151,7 +153,7 @@ public void retrieveMetadata_heicStillPhoto_outputsEmptyMetadata() throws Except
}

@Test
public void retrieveMetadata_sefSlowMotion_outputsExpectedMetadata() throws Exception {
public void retrieveMetadata_sefSlowMotionAvc_outputsExpectedMetadata() throws Exception {
MediaItem mediaItem =
MediaItem.fromUri(Uri.parse("asset://android_asset/media/mp4/sample_sef_slow_motion.mp4"));
MdtaMetadataEntry expectedAndroidVersionMetadata =
Expand Down Expand Up @@ -216,6 +218,60 @@ public void retrieveMetadata_sefSlowMotion_outputsExpectedMetadata() throws Exce
assertThat(trackGroups.get(1).getFormat(0).metadata.get(5)).isEqualTo(expectedMp4TimestampData);
}

@Test
public void retrieveMetadata_sefSlowMotionHevc_outputsExpectedMetadata() throws Exception {
MediaItem mediaItem =
MediaItem.fromUri(
Uri.parse("asset://android_asset/media/mp4/sample_sef_slow_motion_hevc.mp4"));
MdtaMetadataEntry expectedAndroidVersionMetadata =
new MdtaMetadataEntry(
/* key= */ "com.android.version",
/* value= */ Util.getUtf8Bytes("13"),
/* localeIndicator= */ 0,
MdtaMetadataEntry.TYPE_INDICATOR_STRING);
SmtaMetadataEntry expectedSmtaEntry =
new SmtaMetadataEntry(/* captureFrameRate= */ 240, /* svcTemporalLayerCount= */ 4);
SlowMotionData expectedSlowMotionData =
new SlowMotionData(
ImmutableList.of(
new SlowMotionData.Segment(
/* startTimeMs= */ 2128, /* endTimeMs= */ 9856, /* speedDivisor= */ 8)));
MdtaMetadataEntry expectedCaptureFpsMdtaEntry =
new MdtaMetadataEntry(
KEY_ANDROID_CAPTURE_FPS,
/* value= */ new byte[] {67, 112, 0, 0},
/* localeIndicator= */ 0,
/* typeIndicator= */ 23);

ListenableFuture<TrackGroupArray> trackGroupsFuture =
retrieveMetadata(context, mediaItem, clock);
ShadowLooper.idleMainLooper();
TrackGroupArray trackGroups = trackGroupsFuture.get(TEST_TIMEOUT_SEC, TimeUnit.SECONDS);

assertThat(trackGroups.length).isEqualTo(2); // Video and audio

// Video
Metadata videoFormatMetadata = trackGroups.get(0).getFormat(0).metadata;
List<Metadata.Entry> videoMetadataEntries = new ArrayList<>();
for (int i = 0; i < videoFormatMetadata.length(); i++) {
videoMetadataEntries.add(videoFormatMetadata.get(i));
}
assertThat(videoMetadataEntries).contains(expectedAndroidVersionMetadata);
assertThat(videoMetadataEntries).contains(expectedSlowMotionData);
assertThat(videoMetadataEntries).contains(expectedSmtaEntry);
assertThat(videoMetadataEntries).contains(expectedCaptureFpsMdtaEntry);

// Audio
Metadata audioFormatMetadata = trackGroups.get(1).getFormat(0).metadata;
List<Metadata.Entry> audioMetadataEntries = new ArrayList<>();
for (int i = 0; i < audioFormatMetadata.length(); i++) {
audioMetadataEntries.add(audioFormatMetadata.get(i));
}
assertThat(audioMetadataEntries).contains(expectedAndroidVersionMetadata);
assertThat(audioMetadataEntries).contains(expectedSlowMotionData);
assertThat(audioMetadataEntries).contains(expectedSmtaEntry);
}

@Test
public void retrieveMetadata_invalidMediaItem_throwsError() {
MediaItem mediaItem =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
/**
* Stores metadata from the Samsung smta box.
*
* <p>See [Internal: b/150138465#comment76].
* <p>See [Internal: b/150138465#comment76], [Internal: b/301273734#comment17].
*/
@UnstableApi
public final class SmtaMetadataEntry implements Metadata.Entry {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -359,6 +359,9 @@
@SuppressWarnings("ConstantCaseForConstants")
public static final int TYPE_saut = 0x73617574;

@SuppressWarnings("ConstantCaseForConstants")
public static final int TYPE_srfr = 0x73726672;

@SuppressWarnings("ConstantCaseForConstants")
public static final int TYPE_keys = 0x6b657973;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@
import androidx.media3.extractor.HevcConfig;
import androidx.media3.extractor.OpusUtil;
import androidx.media3.extractor.VorbisUtil;
import androidx.media3.extractor.metadata.mp4.SmtaMetadataEntry;
import androidx.media3.extractor.mp4.Atom.LeafAtom;
import com.google.common.base.Function;
import com.google.common.collect.ImmutableList;
Expand Down Expand Up @@ -177,7 +176,8 @@ public static Metadata parseUdta(Atom.LeafAtom udtaAtom) {
} else if (atomType == Atom.TYPE_smta) {
udtaData.setPosition(atomPosition);
metadata =
metadata.copyWithAppendedEntriesFrom(parseSmta(udtaData, atomPosition + atomSize));
metadata.copyWithAppendedEntriesFrom(
SmtaAtomUtil.parseSmta(udtaData, atomPosition + atomSize));
} else if (atomType == Atom.TYPE_xyz) {
metadata = metadata.copyWithAppendedEntriesFrom(parseXyz(udtaData));
}
Expand Down Expand Up @@ -825,37 +825,6 @@ private static Metadata parseXyz(ParsableByteArray xyzBox) {
}
}

/**
* Parses metadata from a Samsung smta atom.
*
* <p>See [Internal: b/150138465#comment76].
*/
@Nullable
private static Metadata parseSmta(ParsableByteArray smta, int limit) {
smta.skipBytes(Atom.FULL_HEADER_SIZE);
while (smta.getPosition() < limit) {
int atomPosition = smta.getPosition();
int atomSize = smta.readInt();
int atomType = smta.readInt();
if (atomType == Atom.TYPE_saut) {
if (atomSize < 14) {
return null;
}
smta.skipBytes(5); // author (4), reserved = 0 (1).
int recordingMode = smta.readUnsignedByte();
if (recordingMode != 12 && recordingMode != 13) {
return null;
}
float captureFrameRate = recordingMode == 12 ? 240 : 120;
smta.skipBytes(1); // reserved = 1 (1).
int svcTemporalLayerCount = smta.readUnsignedByte();
return new Metadata(new SmtaMetadataEntry(captureFrameRate, svcTemporalLayerCount));
}
smta.setPosition(atomPosition + atomSize);
}
return null;
}

/**
* Parses a tkhd atom (defined in ISO/IEC 14496-12).
*
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
/*
* Copyright 2023 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.extractor.mp4;

import static java.lang.annotation.ElementType.TYPE_USE;

import androidx.annotation.IntDef;
import androidx.annotation.Nullable;
import androidx.media3.common.C;
import androidx.media3.common.Metadata;
import androidx.media3.common.util.ParsableByteArray;
import androidx.media3.common.util.UnstableApi;
import androidx.media3.extractor.metadata.mp4.SmtaMetadataEntry;
import java.lang.annotation.Documented;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;

/**
* Utility methods for handling SMTA atoms.
*
* <p>See [Internal: b/150138465#comment76], [Internal: b/301273734#comment17].
*/
@UnstableApi
public final class SmtaAtomUtil {

@Documented
@Retention(RetentionPolicy.SOURCE)
@Target(TYPE_USE)
@IntDef(
open = true,
value = {
NO_VALUE,
CAMCORDER_NORMAL,
CAMCORDER_SINGLE_SUPERSLOW_MOTION,
CAMCORDER_FRC_SUPERSLOW_MOTION,
CAMCORDER_SLOW_MOTION_V2,
CAMCORDER_SLOW_MOTION_V2_120,
CAMCORDER_SLOW_MOTION_V2_HEVC,
CAMCORDER_FRC_SUPERSLOW_MOTION_HEVC,
CAMCORDER_QFRC_SUPERSLOW_MOTION,
})
private @interface RecordingMode {}

private static final int NO_VALUE = -1;
private static final int CAMCORDER_NORMAL = 0;
private static final int CAMCORDER_SINGLE_SUPERSLOW_MOTION = 7;
private static final int CAMCORDER_FRC_SUPERSLOW_MOTION = 9;
private static final int CAMCORDER_SLOW_MOTION_V2 = 12;
private static final int CAMCORDER_SLOW_MOTION_V2_120 = 13;
private static final int CAMCORDER_SLOW_MOTION_V2_HEVC = 21;
private static final int CAMCORDER_FRC_SUPERSLOW_MOTION_HEVC = 22;
private static final int CAMCORDER_QFRC_SUPERSLOW_MOTION = 23;

private SmtaAtomUtil() {}

/** Parses metadata from a Samsung smta atom. */
@Nullable
public static Metadata parseSmta(ParsableByteArray smta, int limit) {
smta.skipBytes(Atom.FULL_HEADER_SIZE);
while (smta.getPosition() < limit) {
int atomPosition = smta.getPosition();
int atomSize = smta.readInt();
int atomType = smta.readInt();
if (atomType == Atom.TYPE_saut) {
// Size (4), Type (4), Author (4), Recording mode (2), SVC layer count (2).
if (atomSize < 16) {
return null;
}
smta.skipBytes(4); // Author (4)

// Each field is stored as a key (1 byte) value (1 byte) pairs.
// The order of the fields is not guaranteed.
@RecordingMode int recordingMode = NO_VALUE;
int svcTemporalLayerCount = 0;
for (int i = 0; i < 2; i++) {
int key = smta.readUnsignedByte();
int value = smta.readUnsignedByte();
if (key == 0x00) { // recordingMode key
recordingMode = value;
} else if (key == 0x01) { // svcTemporalLayerCount key
svcTemporalLayerCount = value;
}
}

int captureFrameRate = getCaptureFrameRate(recordingMode, smta, limit);
if (captureFrameRate == C.RATE_UNSET_INT) {
return null;
}

return new Metadata(new SmtaMetadataEntry(captureFrameRate, svcTemporalLayerCount));
}
smta.setPosition(atomPosition + atomSize);
}
return null;
}

/**
* Returns the capture frame rate for the given recording mode, if supported.
*
* <p>For {@link #CAMCORDER_SLOW_MOTION_V2_HEVC}, this is done by parsing the Samsung 'srfr' atom.
*
* @return The capture frame rate value, or {@link C#RATE_UNSET_INT} if unavailable.
*/
private static int getCaptureFrameRate(
@RecordingMode int recordingMode, ParsableByteArray smta, int limit) {
// V2 and V2_120 have fixed capture frame rates.
if (recordingMode == CAMCORDER_SLOW_MOTION_V2) {
return 240;
} else if (recordingMode == CAMCORDER_SLOW_MOTION_V2_120) {
return 120;
} else if (recordingMode != CAMCORDER_SLOW_MOTION_V2_HEVC) {
return C.RATE_UNSET_INT;
}

if (smta.bytesLeft() < Atom.HEADER_SIZE || smta.getPosition() + Atom.HEADER_SIZE > limit) {
return C.RATE_UNSET_INT;
}

int atomSize = smta.readInt();
int atomType = smta.readInt();
if (atomSize < 12 || atomType != Atom.TYPE_srfr) {
return C.RATE_UNSET_INT;
}
// Capture frame rate is in Q16 format.
return smta.readUnsignedFixedPoint1616();
}
}
Binary file not shown.

0 comments on commit fe11444

Please sign in to comment.