From d1317b60fc6ecd909c73da256f2b83200b690f67 Mon Sep 17 00:00:00 2001 From: Manisha Jajoo Date: Fri, 4 Feb 2022 01:08:09 +0530 Subject: [PATCH 1/2] Add support for RTSP AMR-NB/WB Added AMR-NB/WB RTP packet reader and added support for AMR-NB/WB playback through RTSP Change-Id: I0a975fa1e1aa8450bda1c828599a523ba796bc48 --- .../exoplayer/rtsp/RtpPayloadFormat.java | 8 + .../media3/exoplayer/rtsp/RtspMediaTrack.java | 12 ++ .../DefaultRtpPayloadReaderFactory.java | 3 + .../exoplayer/rtsp/reader/RtpAmrReader.java | 193 ++++++++++++++++++ 4 files changed, 216 insertions(+) create mode 100644 libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpAmrReader.java diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtpPayloadFormat.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtpPayloadFormat.java index 297353167b9..9d7c354b113 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtpPayloadFormat.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtpPayloadFormat.java @@ -37,6 +37,8 @@ public final class RtpPayloadFormat { private static final String RTP_MEDIA_AC3 = "AC3"; + private static final String RTP_MEDIA_AMR = "AMR"; + private static final String RTP_MEDIA_AMR_WB = "AMR-WB"; private static final String RTP_MEDIA_MPEG4_GENERIC = "MPEG4-GENERIC"; private static final String RTP_MEDIA_H264 = "H264"; private static final String RTP_MEDIA_H265 = "H265"; @@ -45,6 +47,8 @@ public final class RtpPayloadFormat { public static boolean isFormatSupported(MediaDescription mediaDescription) { switch (Ascii.toUpperCase(mediaDescription.rtpMapAttribute.mediaEncoding)) { case RTP_MEDIA_AC3: + case RTP_MEDIA_AMR: + case RTP_MEDIA_AMR_WB: case RTP_MEDIA_H264: case RTP_MEDIA_H265: case RTP_MEDIA_MPEG4_GENERIC: @@ -71,6 +75,10 @@ public static String getMimeTypeFromRtpMediaType(String mediaType) { return MimeTypes.VIDEO_H265; case RTP_MEDIA_MPEG4_GENERIC: return MimeTypes.AUDIO_AAC; + case RTP_MEDIA_AMR: + return MimeTypes.AUDIO_AMR_NB; + case RTP_MEDIA_AMR_WB: + return MimeTypes.AUDIO_AMR_WB; default: throw new IllegalArgumentException(mediaType); } diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java index 7547f1ea188..236c921fca8 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java @@ -48,6 +48,8 @@ private static final String PARAMETER_H265_SPROP_PPS = "sprop-pps"; private static final String PARAMETER_H265_SPROP_VPS = "sprop-vps"; private static final String PARAMETER_H265_SPROP_MAX_DON_DIFF = "sprop-max-don-diff"; + private static final String PARAMETER_AMR_OCTET_ALIGN = "octet-align"; + private static final String PARAMETER_AMR_INTERLEAVING = "interleaving"; /** Prefix for the RFC6381 codecs string for AAC formats. */ private static final String AAC_CODECS_PREFIX = "mp4a.40."; @@ -121,6 +123,16 @@ public int hashCode() { checkArgument(!fmtpParameters.isEmpty()); processAacFmtpAttribute(formatBuilder, fmtpParameters, channelCount, clockRate); break; + case MimeTypes.AUDIO_AMR_NB: + case MimeTypes.AUDIO_AMR_WB: + checkArgument(channelCount == 1, "multi channel is not supported currently"); + checkArgument(!fmtpParameters.isEmpty()); + checkArgument( + fmtpParameters.containsKey(PARAMETER_AMR_OCTET_ALIGN), "mode not supported currently"); + checkArgument( + !fmtpParameters.containsKey(PARAMETER_AMR_INTERLEAVING), + "mode not supported currently"); + break; case MimeTypes.VIDEO_H264: checkArgument(!fmtpParameters.isEmpty()); processH264FmtpAttribute(formatBuilder, fmtpParameters); diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/DefaultRtpPayloadReaderFactory.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/DefaultRtpPayloadReaderFactory.java index 888939b7e89..f24793ff5d1 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/DefaultRtpPayloadReaderFactory.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/DefaultRtpPayloadReaderFactory.java @@ -36,6 +36,9 @@ public RtpPayloadReader createPayloadReader(RtpPayloadFormat payloadFormat) { return new RtpAc3Reader(payloadFormat); case MimeTypes.AUDIO_AAC: return new RtpAacReader(payloadFormat); + case MimeTypes.AUDIO_AMR_NB: + case MimeTypes.AUDIO_AMR_WB: + return new RtpAmrReader(payloadFormat); case MimeTypes.VIDEO_H264: return new RtpH264Reader(payloadFormat); case MimeTypes.VIDEO_H265: diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpAmrReader.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpAmrReader.java new file mode 100644 index 00000000000..8c462e79523 --- /dev/null +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpAmrReader.java @@ -0,0 +1,193 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.exoplayer.rtsp.reader; + +import static androidx.media3.common.util.Assertions.checkArgument; +import static androidx.media3.common.util.Assertions.checkNotNull; + +import androidx.media3.common.C; +import androidx.media3.common.MimeTypes; +import androidx.media3.common.util.Log; +import androidx.media3.common.util.ParsableByteArray; +import androidx.media3.common.util.Util; +import androidx.media3.exoplayer.rtsp.RtpPacket; +import androidx.media3.exoplayer.rtsp.RtpPayloadFormat; +import androidx.media3.extractor.ExtractorOutput; +import androidx.media3.extractor.TrackOutput; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +/** + * Parses an AMR byte stream carried on RTP packets and extracts individual samples. Interleaving + * mode is not supported. + */ +/* package */ final class RtpAmrReader implements RtpPayloadReader { + private static final String TAG = "RtpAmrReader"; + /** + * The frame size in bytes, including header (1 byte), for each of the 16 frame types for AMR + * narrow band. + */ + private static final int[] frameSizeBytesByTypeNb = { + 13, + 14, + 16, + 18, + 20, + 21, + 27, + 32, + 6, // AMR SID + 7, // GSM-EFR SID + 6, // TDMA-EFR SID + 6, // PDC-EFR SID + 1, // Future use + 1, // Future use + 1, // Future use + 1 // No data + }; + + /** + * The frame size in bytes, including header (1 byte), for each of the 16 frame types for AMR wide + * band. + */ + private static final int[] frameSizeBytesByTypeWb = { + 18, + 24, + 33, + 37, + 41, + 47, + 51, + 59, + 61, + 6, // AMR-WB SID + 1, // Future use + 1, // Future use + 1, // Future use + 1, // Future use + 1, // speech lost + 1 // No data + }; + + private final RtpPayloadFormat payloadFormat; + private @MonotonicNonNull TrackOutput trackOutput; + private long firstReceivedTimestamp; + private int previousSequenceNumber; + private long startTimeOffsetUs; + private final int sampleRate; + private boolean isWideBand; + + public RtpAmrReader(RtpPayloadFormat payloadFormat) { + this.payloadFormat = payloadFormat; + firstReceivedTimestamp = C.TIME_UNSET; + previousSequenceNumber = C.INDEX_UNSET; + this.sampleRate = this.payloadFormat.clockRate; + this.isWideBand = (payloadFormat.format.sampleMimeType == MimeTypes.AUDIO_AMR_WB); + } + + // RtpPayloadReader implementation. + @Override + public void createTracks(ExtractorOutput extractorOutput, int trackId) { + trackOutput = extractorOutput.track(trackId, C.TRACK_TYPE_AUDIO); + trackOutput.format(payloadFormat.format); + } + + @Override + public void onReceivingFirstPacket(long timestamp, int sequenceNumber) { + this.firstReceivedTimestamp = timestamp; + } + + @Override + public void consume( + ParsableByteArray data, long timestamp, int sequenceNumber, boolean rtpMarker) { + // Check that this packet is in the sequence of the previous packet. + if (previousSequenceNumber != C.INDEX_UNSET) { + int expectedSequenceNumber = RtpPacket.getNextSequenceNumber(previousSequenceNumber); + if (sequenceNumber != expectedSequenceNumber) { + Log.w( + TAG, + Util.formatInvariant( + "Received RTP packet with unexpected sequence number. Expected: %d; received: %d.", + expectedSequenceNumber, sequenceNumber)); + } + } + checkNotNull(trackOutput); + /** + * AMR as RTP payload RFC4867 Section-4.2 + * +----------------+-------------------+---------------- + * | payload header | table of contents | speech data ... + * +----------------+-------------------+---------------- + * + * Payload header RFC4867 Section-4.4.1 + * As interleaving is not supported currently, our header won't contain ILL and ILP + * +-+-+-+-+-+-+-+ + * | CMR |R|R|R|R| + * +-+-+-+-+-+-+-+ + */ + // skip CMR and reserved bits + data.skipBytes(1); + // Loop over sampleSize to send multiple frames along with appropriate timestamp when compound + // payload support is added + int frameType = (data.peekUnsignedByte() >> 3) & 0x0f; + int frameSize = getFrameSize(frameType, isWideBand); + int sampleSize = data.bytesLeft(); + checkArgument(sampleSize == frameSize, "compound payload not supported currently"); + trackOutput.sampleData(data, sampleSize); + long sampleTimeUs = + toSampleTimeUs(startTimeOffsetUs, timestamp, firstReceivedTimestamp, sampleRate); + trackOutput.sampleMetadata( + sampleTimeUs, + C.BUFFER_FLAG_KEY_FRAME, + sampleSize, + /* offset= */ 0, + /* encryptionData= */ null); + previousSequenceNumber = sequenceNumber; + } + + @Override + public void seek(long nextRtpTimestamp, long timeUs) { + firstReceivedTimestamp = nextRtpTimestamp; + startTimeOffsetUs = timeUs; + } + + // Internal methods. + + private boolean isValidFrameType(int frameType) { + if (frameType < 0 || frameType > 15) { + return false; + } + // For wide band, type 10-13 are for future use. + // For narrow band, type 12-14 are for future use. + return isWideBand ? (frameType < 10 || frameType > 13) : (frameType < 12 || frameType > 14); + } + + public int getFrameSize(int frameType, boolean isWideBand) { + checkArgument( + isValidFrameType(frameType), + "Illegal AMR " + (isWideBand ? "WB" : "NB") + " frame type " + frameType); + + return isWideBand ? frameSizeBytesByTypeWb[frameType] : frameSizeBytesByTypeNb[frameType]; + } + + /** Returns the correct sample time from RTP timestamp, accounting for the AMR sampling rate. */ + private static long toSampleTimeUs( + long startTimeOffsetUs, long rtpTimestamp, long firstReceivedRtpTimestamp, int sampleRate) { + return startTimeOffsetUs + + Util.scaleLargeTimestamp( + rtpTimestamp - firstReceivedRtpTimestamp, + /* multiplier= */ C.MICROS_PER_SECOND, + /* divisor= */ sampleRate); + } +} From 1761b423caaafd133fed72d3669badd6e0fd9ea6 Mon Sep 17 00:00:00 2001 From: Manisha Jajoo Date: Fri, 4 Mar 2022 19:25:09 +0530 Subject: [PATCH 2/2] Fix review comments in RtpAmrReader --- .../media3/exoplayer/rtsp/RtspMediaTrack.java | 5 +- .../exoplayer/rtsp/reader/RtpAmrReader.java | 85 ++++++++++--------- 2 files changed, 50 insertions(+), 40 deletions(-) diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java index 236c921fca8..6d789603bb6 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java @@ -128,10 +128,11 @@ public int hashCode() { checkArgument(channelCount == 1, "multi channel is not supported currently"); checkArgument(!fmtpParameters.isEmpty()); checkArgument( - fmtpParameters.containsKey(PARAMETER_AMR_OCTET_ALIGN), "mode not supported currently"); + fmtpParameters.containsKey(PARAMETER_AMR_OCTET_ALIGN), + "modes other than octet align is not supported currently"); checkArgument( !fmtpParameters.containsKey(PARAMETER_AMR_INTERLEAVING), - "mode not supported currently"); + "interleaving mode is not supported currently"); break; case MimeTypes.VIDEO_H264: checkArgument(!fmtpParameters.isEmpty()); diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpAmrReader.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpAmrReader.java index 8c462e79523..b0cad4e1c78 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpAmrReader.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpAmrReader.java @@ -31,23 +31,26 @@ /** * Parses an AMR byte stream carried on RTP packets and extracts individual samples. Interleaving - * mode is not supported. + * mode is not supported. Refer to RFC4867 for more details. */ /* package */ final class RtpAmrReader implements RtpPayloadReader { private static final String TAG = "RtpAmrReader"; /** * The frame size in bytes, including header (1 byte), for each of the 16 frame types for AMR * narrow band. + * AMR-NB is a multi-mode codec that supports eight narrow band speech encoding modes + * with bit rates between 4.75 and 12.2 kbps RFC4867 Section 3.1 + * Refer to table 1a in 3GPP TS 26.101 */ - private static final int[] frameSizeBytesByTypeNb = { - 13, - 14, - 16, - 18, - 20, - 21, - 27, - 32, + private static final int[] amrNbFrameTypeIndexToFrameSize = { + 13, // 4.75kbps + 14, // 5.15kbps + 16, // 5.90kbps + 18, // 6.70kbps PDC-EFR + 20, // 7.40kbps TDMA-EFR + 21, // 7.95kbps + 27, // 10.2kbps + 32, // 12.2kbps GSM-EFR 6, // AMR SID 7, // GSM-EFR SID 6, // TDMA-EFR SID @@ -61,17 +64,20 @@ /** * The frame size in bytes, including header (1 byte), for each of the 16 frame types for AMR wide * band. + * AMR-WB is a multi-mode codec that supports nine wide band speech encoding modes + * with bit rates between 6.6 to 23.85 kbps RFC4867 Section 3.2 + * Refer to table 1a in 3GPP TS 26.201 */ - private static final int[] frameSizeBytesByTypeWb = { - 18, - 24, - 33, - 37, - 41, - 47, - 51, - 59, - 61, + private static final int[] amrWbFrameTypeIndexToFrameSize = { + 18, // 6.60kbps + 24, // 8.85kbps + 33, // 12.65kbps + 37, // 14.25kbps + 41, // 15.85kbps + 47, // 18.25kbps + 51, // 19.85kbps + 59, // 23.05kbps + 61, // 23.85kbps 6, // AMR-WB SID 1, // Future use 1, // Future use @@ -82,19 +88,22 @@ }; private final RtpPayloadFormat payloadFormat; + private final int sampleRate; + private @MonotonicNonNull TrackOutput trackOutput; private long firstReceivedTimestamp; - private int previousSequenceNumber; private long startTimeOffsetUs; - private final int sampleRate; + private int previousSequenceNumber; private boolean isWideBand; public RtpAmrReader(RtpPayloadFormat payloadFormat) { this.payloadFormat = payloadFormat; - firstReceivedTimestamp = C.TIME_UNSET; - previousSequenceNumber = C.INDEX_UNSET; + this.firstReceivedTimestamp = C.TIME_UNSET; + this.previousSequenceNumber = C.INDEX_UNSET; this.sampleRate = this.payloadFormat.clockRate; - this.isWideBand = (payloadFormat.format.sampleMimeType == MimeTypes.AUDIO_AMR_WB); + + checkNotNull(this.payloadFormat.format.sampleMimeType); + this.isWideBand = this.payloadFormat.format.sampleMimeType == MimeTypes.AUDIO_AMR_WB; } // RtpPayloadReader implementation. @@ -125,12 +134,12 @@ public void consume( } checkNotNull(trackOutput); /** - * AMR as RTP payload RFC4867 Section-4.2 + * AMR as RTP payload RFC4867 Section 4.2 * +----------------+-------------------+---------------- * | payload header | table of contents | speech data ... * +----------------+-------------------+---------------- * - * Payload header RFC4867 Section-4.4.1 + * Payload header RFC4867 Section 4.4.1 * As interleaving is not supported currently, our header won't contain ILL and ILP * +-+-+-+-+-+-+-+ * | CMR |R|R|R|R| @@ -164,21 +173,21 @@ public void seek(long nextRtpTimestamp, long timeUs) { // Internal methods. - private boolean isValidFrameType(int frameType) { - if (frameType < 0 || frameType > 15) { - return false; - } - // For wide band, type 10-13 are for future use. - // For narrow band, type 12-14 are for future use. - return isWideBand ? (frameType < 10 || frameType > 13) : (frameType < 12 || frameType > 14); - } - - public int getFrameSize(int frameType, boolean isWideBand) { + public static int getFrameSize(int frameType, boolean isWideBand) { checkArgument( isValidFrameType(frameType), "Illegal AMR " + (isWideBand ? "WB" : "NB") + " frame type " + frameType); - return isWideBand ? frameSizeBytesByTypeWb[frameType] : frameSizeBytesByTypeNb[frameType]; + return isWideBand + ? amrWbFrameTypeIndexToFrameSize[frameType] + : amrNbFrameTypeIndexToFrameSize[frameType]; + } + + private static boolean isValidFrameType(int frameType) { + if (frameType < 0 || frameType > 15) { + return false; + } + return (frameType < 9 || frameType > 14); } /** Returns the correct sample time from RTP timestamp, accounting for the AMR sampling rate. */