Refactor the JNI interface to make it more modular and maintainable (#…

…802)

.github/workflows/apk-asr.yaml

@@ -0,0 +1,174 @@
+name: apk-asr
+
+on:
+  push:
+    tags:
+      - '*'
+
+  workflow_dispatch:
+
+concurrency:
+  group: apk-asr-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: write
+
+jobs:
+  apk_asr:
+    if: github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa'
+    runs-on: ${{ matrix.os }}
+    name: apk for asr ${{ matrix.index }}/${{ matrix.total }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        total: ["1"]
+        index: ["0"]
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      # https://github.com/actions/setup-java
+      - uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin' # See 'Supported distributions' for available options
+          java-version: '21'
+
+      - name: ccache
+        uses: hendrikmuhs/[email protected]
+        with:
+          key: ${{ matrix.os }}-android
+
+      - name: Display NDK HOME
+        shell: bash
+        run: |
+          echo "ANDROID_NDK_LATEST_HOME: ${ANDROID_NDK_LATEST_HOME}"
+          ls -lh ${ANDROID_NDK_LATEST_HOME}
+
+      - name: Install Python dependencies
+        shell: bash
+        run: |
+          python3 -m pip install --upgrade pip jinja2
+
+      - name: Setup build tool version variable
+        shell: bash
+        run: |
+          echo "---"
+          ls -lh /usr/local/lib/android/
+          echo "---"
+
+          ls -lh /usr/local/lib/android/sdk
+          echo "---"
+
+          ls -lh /usr/local/lib/android/sdk/build-tools
+          echo "---"
+
+          BUILD_TOOL_VERSION=$(ls /usr/local/lib/android/sdk/build-tools/ | tail -n 1)
+          echo "BUILD_TOOL_VERSION=$BUILD_TOOL_VERSION" >> $GITHUB_ENV
+          echo "Last build tool version is: $BUILD_TOOL_VERSION"
+
+      - name: Generate build script
+        shell: bash
+        run: |
+          cd scripts/apk
+
+          total=${{ matrix.total }}
+          index=${{ matrix.index }}
+
+          ./generate-asr-apk-script.py --total $total --index $index
+
+          chmod +x build-apk-asr.sh
+          mv -v ./build-apk-asr.sh ../..
+
+      - name: build APK
+        shell: bash
+        run: |
+          export CMAKE_CXX_COMPILER_LAUNCHER=ccache
+          export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
+          cmake --version
+
+          export ANDROID_NDK=$ANDROID_NDK_LATEST_HOME
+          ./build-apk-asr.sh
+
+      - name: Display APK
+        shell: bash
+        run: |
+          ls -lh ./apks/
+          du -h -d1 .
+
+      # https://github.com/marketplace/actions/sign-android-release
+      - uses: r0adkll/sign-android-release@v1
+        name: Sign app APK
+        with:
+          releaseDirectory: ./apks
+          signingKeyBase64: ${{ secrets.ANDROID_SIGNING_KEY }}
+          alias: ${{ secrets.ANDROID_SIGNING_KEY_ALIAS }}
+          keyStorePassword: ${{ secrets.ANDROID_SIGNING_KEY_STORE_PASSWORD }}
+        env:
+          BUILD_TOOLS_VERSION: ${{ env.BUILD_TOOL_VERSION }}
+
+      - name: Display APK after signing
+        shell: bash
+        run: |
+          ls -lh ./apks/
+          du -h -d1 .
+
+      - name: Rename APK after signing
+        shell: bash
+        run: |
+          cd apks
+          rm -fv signingKey.jks
+          rm -fv *.apk.idsig
+          rm -fv *-aligned.apk
+
+          all_apks=$(ls -1 *-signed.apk)
+          echo "----"
+          echo $all_apks
+          echo "----"
+          for apk in ${all_apks[@]}; do
+            n=$(echo $apk | sed -e s/-signed//)
+            mv -v $apk $n
+          done
+
+          cd ..
+
+          ls -lh ./apks/
+          du -h -d1 .
+
+      - name: Display APK after rename
+        shell: bash
+        run: |
+          ls -lh ./apks/
+          du -h -d1 .
+
+      - name: Publish to huggingface
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        uses: nick-fields/retry@v3
+        with:
+          max_attempts: 20
+          timeout_seconds: 200
+          shell: bash
+          command: |
+            git config --global user.email "[email protected]"
+            git config --global user.name "Fangjun Kuang"
+
+            rm -rf huggingface
+            export GIT_LFS_SKIP_SMUDGE=1
+
+            git clone https://huggingface.co/csukuangfj/sherpa-onnx-apk huggingface
+            cd huggingface
+            git fetch
+            git pull
+            git merge -m "merge remote" --ff origin main
+
+            mkdir -p asr
+            cp -v ../apks/*.apk ./asr/
+            git status
+            git lfs track "*.apk"
+            git add .
+            git commit -m "add more apks"
+            git push https://csukuangfj:[email protected]/csukuangfj/sherpa-onnx-apk main

.gitignore

@@ -95,3 +95,4 @@ sherpa-onnx-punct-ct-transformer-zh-en-vocab272727-2024-04-12
 spoken-language-identification-test-wavs
 my-release-key*
 vits-zh-hf-fanchen-C
+sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01

android/SherpaOnnx/app/src/main/AndroidManifest.xml

@@ -16,6 +16,7 @@
         tools:targetApi="31">
         <activity
             android:name=".MainActivity"
+            android:label="ASR: Next-gen Kaldi"
             android:exported="true">
             <intent-filter>
                 <action android:name="android.intent.action.MAIN" />

android/SherpaOnnx/app/src/main/java/com/k2fsa/sherpa/onnx/FeatureConfig.kt

@@ -0,0 +1 @@
+../../../../../../../../../../sherpa-onnx/kotlin-api/FeatureConfig.kt

android/SherpaOnnx/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt

@@ -12,16 +12,19 @@ import android.widget.Button
 import android.widget.TextView
 import androidx.appcompat.app.AppCompatActivity
 import androidx.core.app.ActivityCompat
-import com.k2fsa.sherpa.onnx.*
 import kotlin.concurrent.thread
 
 private const val TAG = "sherpa-onnx"
 private const val REQUEST_RECORD_AUDIO_PERMISSION = 200
 
+// To enable microphone in android emulator, use
+//
+// adb emu avd hostmicon
+
 class MainActivity : AppCompatActivity() {
     private val permissions: Array<String> = arrayOf(Manifest.permission.RECORD_AUDIO)
 
-    private lateinit var model: SherpaOnnx
+    private lateinit var recognizer: OnlineRecognizer
     private var audioRecord: AudioRecord? = null
     private lateinit var recordButton: Button
     private lateinit var textView: TextView
@@ -87,7 +90,6 @@ class MainActivity : AppCompatActivity() {
             audioRecord!!.startRecording()
             recordButton.setText(R.string.stop)
             isRecording = true
-            model.reset(true)
             textView.text = ""
             lastText = ""
             idx = 0
@@ -108,6 +110,7 @@ class MainActivity : AppCompatActivity() {
 
     private fun processSamples() {
         Log.i(TAG, "processing samples")
+        val stream = recognizer.createStream()
 
         val interval = 0.1 // i.e., 100 ms
         val bufferSize = (interval * sampleRateInHz).toInt() // in samples
@@ -117,29 +120,41 @@ class MainActivity : AppCompatActivity() {
             val ret = audioRecord?.read(buffer, 0, buffer.size)
             if (ret != null && ret > 0) {
                 val samples = FloatArray(ret) { buffer[it] / 32768.0f }
-                model.acceptWaveform(samples, sampleRate=sampleRateInHz)
-                while (model.isReady()) {
-                    model.decode()
+                stream.acceptWaveform(samples, sampleRate = sampleRateInHz)
+                while (recognizer.isReady(stream)) {
+                    recognizer.decode(stream)
                 }
 
-                val isEndpoint = model.isEndpoint()
-                val text = model.text
+                val isEndpoint = recognizer.isEndpoint(stream)
+                var text = recognizer.getResult(stream).text
+
+                // For streaming parformer, we need to manually add some
+                // paddings so that it has enough right context to
+                // recognize the last word of this segment
+                if (isEndpoint && recognizer.config.modelConfig.paraformer.encoder.isNotBlank()) {
+                    val tailPaddings = FloatArray((0.8 * sampleRateInHz).toInt())
+                    stream.acceptWaveform(tailPaddings, sampleRate = sampleRateInHz)
+                    while (recognizer.isReady(stream)) {
+                        recognizer.decode(stream)
+                    }
+                    text = recognizer.getResult(stream).text
+                }
 
-                var textToDisplay = lastText;
+                var textToDisplay = lastText
 
-                if(text.isNotBlank()) {
-                    if (lastText.isBlank()) {
-                        textToDisplay = "${idx}: ${text}"
+                if (text.isNotBlank()) {
+                    textToDisplay = if (lastText.isBlank()) {
+                        "${idx}: $text"
                     } else {
-                        textToDisplay = "${lastText}\n${idx}: ${text}"
+                        "${lastText}\n${idx}: $text"
                     }
                 }
 
                 if (isEndpoint) {
-                    model.reset()
+                    recognizer.reset(stream)
                     if (text.isNotBlank()) {
-                        lastText = "${lastText}\n${idx}: ${text}"
-                        textToDisplay = lastText;
+                        lastText = "${lastText}\n${idx}: $text"
+                        textToDisplay = lastText
                         idx += 1
                     }
                 }
@@ -149,6 +164,7 @@ class MainActivity : AppCompatActivity() {
                 }
             }
         }
+        stream.release()
     }
 
     private fun initMicrophone(): Boolean {
@@ -180,7 +196,7 @@ class MainActivity : AppCompatActivity() {
         // See https://k2-fsa.github.io/sherpa/onnx/pretrained_models/index.html
         // for a list of available models
         val type = 0
-        println("Select model type ${type}")
+        Log.i(TAG, "Select model type $type")
         val config = OnlineRecognizerConfig(
             featConfig = getFeatureConfig(sampleRate = sampleRateInHz, featureDim = 80),
             modelConfig = getModelConfig(type = type)!!,
@@ -189,7 +205,7 @@ class MainActivity : AppCompatActivity() {
             enableEndpoint = true,
         )
 
-        model = SherpaOnnx(
+        recognizer = OnlineRecognizer(
             assetManager = application.assets,
             config = config,
         )

android/SherpaOnnx/app/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizer.kt

@@ -0,0 +1 @@
+../../../../../../../../../../sherpa-onnx/kotlin-api/OnlineRecognizer.kt

android/SherpaOnnx/app/src/main/java/com/k2fsa/sherpa/onnx/OnlineStream.kt

@@ -0,0 +1 @@
+../../../../../../../../../../sherpa-onnx/kotlin-api/OnlineStream.kt

android/SherpaOnnx/app/src/main/java/com/k2fsa/sherpa/onnx/WaveReader.kt

@@ -0,0 +1 @@
+../../../../../../../../../../sherpa-onnx/kotlin-api/WaveReader.kt

android/SherpaOnnx2Pass/app/src/main/AndroidManifest.xml

@@ -16,6 +16,7 @@
         tools:targetApi="31">
         <activity
             android:name=".MainActivity"
+            android:label="2pass ASR: Next-gen Kaldi"
             android:exported="true">
             <intent-filter>
                 <action android:name="android.intent.action.MAIN" />
@@ -29,4 +30,4 @@
         </activity>
     </application>
 
-</manifest>
+</manifest>

android/SherpaOnnx2Pass/app/src/main/java/com/k2fsa/sherpa/onnx/FeatureConfig.kt

@@ -0,0 +1 @@
+../../../../../../../../../../sherpa-onnx/kotlin-api/FeatureConfig.kt