Add support for WAV/PCM output

This is primarily for troubleshooting. With this output format, the entire MediaCodec/MediaMuxer pipeline is bypassed and the raw PCM samples are directly written to the output file. Signed-off-by: Andrew Gunnerson <[email protected]>
chenxiaolong · May 29, 2022 · 00277fd · 00277fd
1 parent 6580265
commit 00277fd
Show file tree

Hide file tree

Showing 11 changed files with 341 additions and 53 deletions.
diff --git a/app/src/main/java/com/chiller3/bcr/AudioFormatExtension.kt b/app/src/main/java/com/chiller3/bcr/AudioFormatExtension.kt
@@ -0,0 +1,13 @@
+package com.chiller3.bcr
+
+import android.media.AudioFormat
+import android.os.Build
+
+val AudioFormat.frameSizeInBytesCompat: Int
+    get() = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
+        frameSizeInBytes
+    } else{
+        // Hardcoded for Android 9 compatibility only
+        assert(encoding == AudioFormat.ENCODING_PCM_16BIT)
+        2 * channelCount
+    }
diff --git a/app/src/main/java/com/chiller3/bcr/FormatBottomSheetFragment.kt b/app/src/main/java/com/chiller3/bcr/FormatBottomSheetFragment.kt
@@ -4,8 +4,10 @@ import android.os.Bundle
 import android.view.LayoutInflater
 import android.view.View
 import android.view.ViewGroup
+import android.widget.LinearLayout
 import android.widget.TextView
 import androidx.core.view.ViewCompat
+import androidx.core.view.isVisible
 import com.chiller3.bcr.format.Format
 import com.chiller3.bcr.format.FormatParamType
 import com.chiller3.bcr.format.Formats
@@ -18,8 +20,9 @@ import com.google.android.material.slider.Slider
 class FormatBottomSheetFragment : BottomSheetDialogFragment(),
     MaterialButtonToggleGroup.OnButtonCheckedListener, LabelFormatter, Slider.OnChangeListener,
     View.OnClickListener {
+    private lateinit var formatParamGroup: LinearLayout
     private lateinit var formatParamTitle: TextView
-    private lateinit var formatParam: Slider
+    private lateinit var formatParamSlider: Slider
     private lateinit var formatReset: MaterialButton
     private lateinit var formatNameGroup: MaterialButtonToggleGroup
     private val buttonIdToFormat = HashMap<Int, Format>()
@@ -33,11 +36,13 @@ class FormatBottomSheetFragment : BottomSheetDialogFragment(),
     ): View? {
         val bottomSheet = inflater.inflate(R.layout.format_bottom_sheet, container, false)
 
+        formatParamGroup = bottomSheet.findViewById(R.id.format_param_group)
+
         formatParamTitle = bottomSheet.findViewById(R.id.format_param_title)
 
-        formatParam = bottomSheet.findViewById(R.id.format_param)
-        formatParam.setLabelFormatter(this)
-        formatParam.addOnChangeListener(this)
+        formatParamSlider = bottomSheet.findViewById(R.id.format_param_slider)
+        formatParamSlider.setLabelFormatter(this)
+        formatParamSlider.addOnChangeListener(this)
 
         formatReset = bottomSheet.findViewById(R.id.format_reset)
         formatReset.setOnClickListener(this)
@@ -88,13 +93,24 @@ class FormatBottomSheetFragment : BottomSheetDialogFragment(),
             FormatParamType.Bitrate -> R.string.bottom_sheet_bitrate
         }
 
-        formatParamTitle.setText(titleResId)
+        formatParamGroup.isVisible = format.paramRange.first != format.paramRange.last
 
-        formatParam.valueFrom = format.paramRange.first.toFloat()
-        formatParam.valueTo = format.paramRange.last.toFloat()
-        formatParam.stepSize = format.paramStepSize.toFloat()
+        formatParamTitle.setText(titleResId)
 
-        formatParam.value = (param ?: format.paramDefault).toFloat()
+        formatParamSlider.valueFrom = format.paramRange.first.toFloat()
+        formatParamSlider.valueTo = format.paramRange.last.toFloat()
+        formatParamSlider.stepSize = format.paramStepSize.toFloat()
+        formatParamSlider.value = (param ?: format.paramDefault).toFloat()
+
+        // Needed due to a bug in the material3 library where the slider label does not disappear
+        // when the slider visibility is set to View.GONE
+        // https://github.com/material-components/material-components-android/issues/2726
+        if (format.paramRange.first == format.paramRange.last) {
+            val ensureLabelsRemoved = formatParamSlider.javaClass.superclass
+                .getDeclaredMethod("ensureLabelsRemoved")
+            ensureLabelsRemoved.isAccessible = true
+            ensureLabelsRemoved.invoke(formatParamSlider)
+        }
     }
 
     override fun onButtonChecked(
@@ -113,7 +129,7 @@ class FormatBottomSheetFragment : BottomSheetDialogFragment(),
 
     override fun onValueChange(slider: Slider, value: Float, fromUser: Boolean) {
         when (slider) {
-            formatParam -> {
+            formatParamSlider -> {
                 val format = buttonIdToFormat[formatNameGroup.checkedButtonId]!!
                 Preferences.setFormatParam(requireContext(), format.name, value.toUInt())
             }

diff --git a/app/src/main/java/com/chiller3/bcr/RecorderThread.kt b/app/src/main/java/com/chiller3/bcr/RecorderThread.kt
@@ -2,13 +2,11 @@ package com.chiller3.bcr
 
 import android.annotation.SuppressLint
 import android.content.Context
-import android.media.AudioFormat
-import android.media.AudioRecord
-import android.media.MediaCodec
-import android.media.MediaRecorder
+import android.media.*
 import android.net.Uri
 import android.os.Build
 import android.os.ParcelFileDescriptor
+import android.system.Os
 import android.telecom.Call
 import android.telecom.PhoneAccount
 import android.util.Log
@@ -18,6 +16,7 @@ import com.chiller3.bcr.format.Formats
 import com.chiller3.bcr.format.Container
 import java.io.IOException
 import java.lang.Integer.min
+import java.nio.ByteBuffer
 import java.time.Instant
 import java.time.ZoneId
 import java.time.ZonedDateTime
@@ -279,25 +278,34 @@ class RecorderThread(
             try {
                 // audioRecord.format has the detected native sample rate
                 val mediaFormat = format.getMediaFormat(audioRecord.format, formatParam)
-                val mediaCodec = format.getMediaCodec(mediaFormat)
+                val mediaCodec = if (!format.passthrough) {
+                    format.getMediaCodec(mediaFormat)
+                } else {
+                    null
+                }
 
                 try {
-                    mediaCodec.start()
+                    mediaCodec?.start()
 
                     try {
                         val container = format.getContainer(pfd.fileDescriptor)
 
                         try {
-                            encodeLoop(audioRecord, mediaCodec, container)
+                            if (mediaCodec != null) {
+                                encodeLoop(audioRecord, mediaCodec, container)
+                            } else {
+                                passthroughLoop(audioRecord, mediaFormat, container)
+                            }
+
                             container.stop()
                         } finally {
                             container.release()
                         }
                     } finally {
-                        mediaCodec.stop()
+                        mediaCodec?.stop()
                     }
                 } finally {
-                    mediaCodec.release()
+                    mediaCodec?.release()
                 }
             } finally {
                 audioRecord.stop()
@@ -307,6 +315,72 @@ class RecorderThread(
         }
     }
 
+    /**
+     * Main loop for capturing raw audio into a PCM-based output file.
+     *
+     * The loop runs forever until [cancel] is called. At that point, no further data will be read
+     * from [audioRecord]. If [audioRecord] fails to capture data, the loop will behave as if
+     * [cancel] was called (ie. abort, but ensuring that the output file is valid).
+     *
+     * The approximate amount of time to cancel reading from the audio source is 100ms. This does
+     * not include the time required to write out the remaining encoded data to the output file.
+     *
+     * @param audioRecord [AudioRecord.startRecording] must have been called
+     * @param container [Container.start] must *not* have been called. It will be left in a started
+     * state after this method returns.
+     */
+    private fun passthroughLoop(
+        audioRecord: AudioRecord,
+        mediaFormat: MediaFormat,
+        container: Container,
+    ) {
+        var inputTimestamp = 0L
+        val bufferInfo = MediaCodec.BufferInfo()
+        val frameSize = audioRecord.format.frameSizeInBytesCompat
+
+        // This is the most we ever read from audioRecord, even if the codec input buffer is
+        // larger. This is purely for fast'ish cancellation and not for latency.
+        val maxSamplesInBytes = audioRecord.sampleRate / 10 * frameSize
+
+        val buffer = ByteBuffer.allocateDirect(maxSamplesInBytes)
+        val trackIndex = container.addTrack(mediaFormat)
+        container.start()
+
+        while (true) {
+            val maxRead = min(maxSamplesInBytes, buffer.remaining())
+            val n = audioRecord.read(buffer, maxRead)
+            if (n < 0) {
+                logE("Error when reading samples from ${audioRecord}: $n")
+                isCancelled = true
+                captureFailed = true
+            } else if (n == 0) {
+                logE( "Unexpected EOF from AudioRecord")
+                isCancelled = true
+            } else {
+                val frames = n / frameSize
+                inputTimestamp += frames * 1_000_000L / audioRecord.sampleRate
+
+                bufferInfo.offset = 0
+                bufferInfo.size = buffer.limit()
+                bufferInfo.presentationTimeUs = 0
+                bufferInfo.flags = if (isCancelled) {
+                    MediaCodec.BUFFER_FLAG_END_OF_STREAM
+                } else {
+                    0
+                }
+
+                container.writeSamples(trackIndex, buffer, bufferInfo)
+                buffer.clear()
+            }
+
+            if (isCancelled) {
+                val duration = "%.1f".format(inputTimestamp / 1_000_000.0)
+                logD("Input complete after ${duration}s")
+                break
+            }
+        }
+    }
+
     /**
      * Main loop for encoding captured raw audio into an output file.
      *
@@ -326,16 +400,16 @@ class RecorderThread(
      * @throws MediaCodec.CodecException if the codec encounters an error
      */
     private fun encodeLoop(audioRecord: AudioRecord, mediaCodec: MediaCodec, container: Container) {
-        // This is the most we ever read from audioRecord, even if the codec input buffer is
-        // larger. This is purely for fast'ish cancellation and not for latency.
-        val maxSamplesInBytes = audioRecord.sampleRate / 10 * getFrameSize(audioRecord.format)
-
         var inputTimestamp = 0L
         var inputComplete = false
         val bufferInfo = MediaCodec.BufferInfo()
-        val frameSize = getFrameSize(audioRecord.format)
+        val frameSize = audioRecord.format.frameSizeInBytesCompat
         var trackIndex = -1
 
+        // This is the most we ever read from audioRecord, even if the codec input buffer is
+        // larger. This is purely for fast'ish cancellation and not for latency.
+        val maxSamplesInBytes = audioRecord.sampleRate / 10 * frameSize
+
         while (true) {
             if (!inputComplete) {
                 val inputBufferId = mediaCodec.dequeueInputBuffer(10000)
@@ -419,16 +493,6 @@ class RecorderThread(
             .appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true)
             .appendOffset("+HHMMss", "+0000")
             .toFormatter()
-
-        private fun getFrameSize(audioFormat: AudioFormat): Int {
-            return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
-                audioFormat.frameSizeInBytes
-            } else{
-                // Hardcoded for Android 9 compatibility only
-                assert(ENCODING == AudioFormat.ENCODING_PCM_16BIT)
-                2 * audioFormat.channelCount
-            }
-        }
     }
 
     interface OnRecordingCompletedListener {

diff --git a/app/src/main/java/com/chiller3/bcr/format/AacFormat.kt b/app/src/main/java/com/chiller3/bcr/format/AacFormat.kt
@@ -1,5 +1,6 @@
 package com.chiller3.bcr.format
 
+import android.media.AudioFormat
 import android.media.MediaCodecInfo
 import android.media.MediaFormat
 import android.media.MediaMuxer
@@ -18,9 +19,14 @@ object AacFormat : Format() {
     // https://datatracker.ietf.org/doc/html/rfc6381#section-3.1
     override val mimeTypeContainer: String = "audio/mp4"
     override val mimeTypeAudio: String = MediaFormat.MIMETYPE_AUDIO_AAC
+    override val passthrough: Boolean = false
     override val supported: Boolean = true
 
-    override fun updateMediaFormat(mediaFormat: MediaFormat, param: UInt) {
+    override fun updateMediaFormat(
+        mediaFormat: MediaFormat,
+        audioFormat: AudioFormat,
+        param: UInt,
+    ) {
         mediaFormat.apply {
             val profile = if (param >= 32_000u) {
                 MediaCodecInfo.CodecProfileLevel.AACObjectLC

diff --git a/app/src/main/java/com/chiller3/bcr/format/FlacFormat.kt b/app/src/main/java/com/chiller3/bcr/format/FlacFormat.kt
@@ -1,9 +1,10 @@
 package com.chiller3.bcr.format
 
+import android.media.AudioFormat
 import android.media.MediaFormat
 import java.io.FileDescriptor
 
-object FlacFormat: Format() {
+object FlacFormat : Format() {
     override val name: String = "FLAC"
     override val paramType: FormatParamType = FormatParamType.CompressionLevel
     override val paramRange: UIntRange = 0u..8u
@@ -12,9 +13,14 @@ object FlacFormat: Format() {
     override val paramDefault: UInt = 8u
     override val mimeTypeContainer: String = MediaFormat.MIMETYPE_AUDIO_FLAC
     override val mimeTypeAudio: String = MediaFormat.MIMETYPE_AUDIO_FLAC
+    override val passthrough: Boolean = false
     override val supported: Boolean = true
 
-    override fun updateMediaFormat(mediaFormat: MediaFormat, param: UInt) {
+    override fun updateMediaFormat(
+        mediaFormat: MediaFormat,
+        audioFormat: AudioFormat,
+        param: UInt,
+    ) {
         mediaFormat.apply {
             // Not relevant for lossless formats
             setInteger(MediaFormat.KEY_BIT_RATE, 0)

diff --git a/app/src/main/java/com/chiller3/bcr/format/Format.kt b/app/src/main/java/com/chiller3/bcr/format/Format.kt
@@ -6,6 +6,7 @@ import android.media.MediaCodecList
 import android.media.MediaFormat
 import android.util.Log
 import java.io.FileDescriptor
+import java.lang.IllegalStateException
 
 sealed class Format {
     /** User-facing name of the format. */
@@ -33,6 +34,9 @@ sealed class Format {
      */
     abstract val mimeTypeAudio: String
 
+    /** Whether the format takes the PCM samples as is without encoding. */
+    abstract val passthrough: Boolean
+
     /** Whether the format is supported on the current device. */
     abstract val supported: Boolean
 
@@ -58,7 +62,7 @@ sealed class Format {
             setInteger(MediaFormat.KEY_SAMPLE_RATE, audioFormat.sampleRate)
         }
 
-        updateMediaFormat(format, param ?: paramDefault)
+        updateMediaFormat(format, audioFormat, param ?: paramDefault)
 
         return format
     }
@@ -68,7 +72,11 @@ sealed class Format {
      *
      * @param param Guaranteed to be within [paramRange]
      */
-    protected abstract fun updateMediaFormat(mediaFormat: MediaFormat, param: UInt)
+    protected abstract fun updateMediaFormat(
+        mediaFormat: MediaFormat,
+        audioFormat: AudioFormat,
+        param: UInt,
+    )
 
     /**
      * Create a [MediaCodec] encoder that produces [mediaFormat] output.
@@ -77,8 +85,13 @@ sealed class Format {
      *
      * @throws Exception if the device does not support encoding with the parameters set in
      * [mediaFormat] or if configuring the [MediaCodec] fails.
+     * @throws IllegalStateException if [passthrough] is true
      */
     fun getMediaCodec(mediaFormat: MediaFormat): MediaCodec {
+        if (passthrough) {
+            throw IllegalStateException("Tried to create MediaCodec for passthrough format")
+        }
+
         val encoder = MediaCodecList(MediaCodecList.REGULAR_CODECS).findEncoderForFormat(mediaFormat)
             ?: throw Exception("No suitable encoder found for $mediaFormat")
         Log.d(TAG, "Audio encoder: $encoder")
@@ -98,7 +111,7 @@ sealed class Format {
     /**
      * Create a container muxer that takes encoded input and writes the muxed output to [fd].
      *
-     * @param fd The container does not take ownership of the file descriptor
+     * @param fd The container does not take ownership of the file descriptor.
      */
     abstract fun getContainer(fd: FileDescriptor): Container
 

diff --git a/app/src/main/java/com/chiller3/bcr/format/Formats.kt b/app/src/main/java/com/chiller3/bcr/format/Formats.kt
@@ -4,7 +4,7 @@ import android.content.Context
 import com.chiller3.bcr.Preferences
 
 object Formats {
-    val all: Array<Format> = arrayOf(OpusFormat, AacFormat, FlacFormat)
+    val all: Array<Format> = arrayOf(OpusFormat, AacFormat, FlacFormat, WaveFormat)
     private val default: Format = all.first { it.supported }
 
     /** Find output format by name. */