From 560652053377bc91b9281b6cd540c42833a7723e Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Wed, 24 Apr 2024 20:48:48 +0800 Subject: [PATCH] Add Java API for non-streaming ASR --- .github/workflows/run-java-test.yaml | 19 ++- java-api-examples/.gitignore | 1 + .../NonStreamingDecodeFileParaformer.java | 49 ++++++++ .../NonStreamingDecodeFileTransducer.java | 57 +++++++++ .../NonStreamingDecodeFileWhisper.java | 50 ++++++++ java-api-examples/README.md | 10 ++ java-api-examples/StreamingDecodeFileCtc.java | 3 - .../StreamingDecodeFileParaformer.java | 3 - .../StreamingDecodeFileTransducer.java | 2 - ...un-non-streaming-decode-file-paraformer.sh | 52 ++++++++ ...un-non-streaming-decode-file-transducer.sh | 52 ++++++++ .../run-non-streaming-decode-file-whisper.sh | 52 ++++++++ sherpa-onnx/java-api/Makefile | 9 ++ .../k2fsa/sherpa/onnx/OfflineModelConfig.java | 117 ++++++++++++++++++ .../onnx/OfflineParaformerModelConfig.java | 32 +++++ .../k2fsa/sherpa/onnx/OfflineRecognizer.java | 55 ++++++++ .../sherpa/onnx/OfflineRecognizerConfig.java | 71 +++++++++++ .../sherpa/onnx/OfflineRecognizerResult.java | 26 ++++ .../com/k2fsa/sherpa/onnx/OfflineStream.java | 49 ++++++++ .../onnx/OfflineTransducerModelConfig.java | 55 ++++++++ .../onnx/OfflineWhisperModelConfig.java | 80 ++++++++++++ .../k2fsa/sherpa/onnx/OnlineModelConfig.java | 1 + .../k2fsa/sherpa/onnx/OnlineRecognizer.java | 15 +-- .../sherpa/onnx/OnlineRecognizerConfig.java | 1 + 24 files changed, 834 insertions(+), 27 deletions(-) create mode 100644 java-api-examples/NonStreamingDecodeFileParaformer.java create mode 100644 java-api-examples/NonStreamingDecodeFileTransducer.java create mode 100644 java-api-examples/NonStreamingDecodeFileWhisper.java create mode 100755 java-api-examples/run-non-streaming-decode-file-paraformer.sh create mode 100755 java-api-examples/run-non-streaming-decode-file-transducer.sh create mode 100755 java-api-examples/run-non-streaming-decode-file-whisper.sh create mode 100644 sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineModelConfig.java create mode 100644 sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineParaformerModelConfig.java create mode 100644 sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineRecognizer.java create mode 100644 sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineRecognizerConfig.java create mode 100644 sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineRecognizerResult.java create mode 100644 sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineStream.java create mode 100644 sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineTransducerModelConfig.java create mode 100644 sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineWhisperModelConfig.java diff --git a/.github/workflows/run-java-test.yaml b/.github/workflows/run-java-test.yaml index 69d0c0c4e..533d868c6 100644 --- a/.github/workflows/run-java-test.yaml +++ b/.github/workflows/run-java-test.yaml @@ -92,13 +92,9 @@ jobs: make -j4 ls -lh lib - - name: Run java test + - name: Run java test (Streaming ASR) shell: bash run: | - export CMAKE_CXX_COMPILER_LAUNCHER=ccache - export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" - cmake --version - cd ./java-api-examples ./run-streaming-decode-file-ctc.sh # Delete model files to save space @@ -109,3 +105,16 @@ jobs: ./run-streaming-decode-file-transducer.sh rm -rf sherpa-onnx-streaming-* + + - name: Run java test (Non-Streaming ASR) + shell: bash + run: | + cd ./java-api-examples + ./run-non-streaming-decode-file-paraformer.sh + rm -rf sherpa-onnx-paraformer-zh-* + + ./run-non-streaming-decode-file-transducer.sh + rm -rf sherpa-onnx-zipformer-* + + ./run-non-streaming-decode-file-whisper.sh + rm -rf sherpa-onnx-whisper-* diff --git a/java-api-examples/.gitignore b/java-api-examples/.gitignore index c9db6d42e..6091eb469 100644 --- a/java-api-examples/.gitignore +++ b/java-api-examples/.gitignore @@ -1,3 +1,4 @@ lib hs_err* !run-streaming*.sh +!run-non-streaming*.sh diff --git a/java-api-examples/NonStreamingDecodeFileParaformer.java b/java-api-examples/NonStreamingDecodeFileParaformer.java new file mode 100644 index 000000000..4a49c8d66 --- /dev/null +++ b/java-api-examples/NonStreamingDecodeFileParaformer.java @@ -0,0 +1,49 @@ +// Copyright 2024 Xiaomi Corporation + +// This file shows how to use an offline paraformer, i.e., non-streaming paraformer, +// to decode files. +import com.k2fsa.sherpa.onnx.*; + +public class NonStreamingDecodeFileTransducer { + public static void main(String[] args) { + // please refer to + // https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-paraformer/paraformer-models.html#csukuangfj-sherpa-onnx-paraformer-zh-2023-03-28-chinese-english + // to download model files + String model = "./sherpa-onnx-paraformer-zh-2023-03-28/model.int8.onnx"; + String tokens = "./sherpa-onnx-paraformer-zh-2023-03-28/tokens.txt"; + + String waveFilename = "./sherpa-onnx-paraformer-zh-2023-03-28/test_wavs/3-sichuan.wav"; + + WaveReader reader = new WaveReader(waveFilename); + + OfflineParaformerModelConfig paraformer = + OfflineParaformerModelConfig.builder().setModel(model).build(); + + OfflineModelConfig modelConfig = + OfflineModelConfig.builder() + .setParaformer(paraformer) + .setTokens(tokens) + .setNumThreads(1) + .setDebug(true) + .build(); + + OfflineRecognizerConfig config = + OfflineRecognizerConfig.builder() + .setOfflineModelConfig(modelConfig) + .setDecodingMethod("greedy_search") + .build(); + + OfflineRecognizer recognizer = new OfflineRecognizer(config); + OfflineStream stream = recognizer.createStream(); + stream.acceptWaveform(reader.getSamples(), reader.getSampleRate()); + + recognizer.decode(stream); + + String text = recognizer.getResult(stream).getText(); + + System.out.printf("filename:%s\nresult:%s\n", waveFilename, text); + + stream.release(); + recognizer.release(); + } +} diff --git a/java-api-examples/NonStreamingDecodeFileTransducer.java b/java-api-examples/NonStreamingDecodeFileTransducer.java new file mode 100644 index 000000000..90bdc69e8 --- /dev/null +++ b/java-api-examples/NonStreamingDecodeFileTransducer.java @@ -0,0 +1,57 @@ +// Copyright 2024 Xiaomi Corporation + +// This file shows how to use an offline transducer, i.e., non-streaming transducer, +// to decode files. +import com.k2fsa.sherpa.onnx.*; + +public class NonStreamingDecodeFileTransducer { + public static void main(String[] args) { + // please refer to + // https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-transducer/zipformer-transducer-models.html#sherpa-onnx-zipformer-gigaspeech-2023-12-12-english + // to download model files + String encoder = + "./sherpa-onnx-zipformer-gigaspeech-2023-12-12/encoder-epoch-30-avg-1.int8.onnx"; + String decoder = "./sherpa-onnx-zipformer-gigaspeech-2023-12-12/decoder-epoch-30-avg-1.onnx"; + String joiner = "./sherpa-onnx-zipformer-gigaspeech-2023-12-12/joiner-epoch-30-avg-1.onnx"; + String tokens = "./sherpa-onnx-zipformer-gigaspeech-2023-12-12/tokens.txt"; + + String waveFilename = + "./sherpa-onnx-zipformer-gigaspeech-2023-12-12/test_wavs/1089-134686-0001.wav"; + + WaveReader reader = new WaveReader(waveFilename); + + OfflineTransducerModelConfig transducer = + OfflineTransducerModelConfig.builder() + .setEncoder(encoder) + .setDecoder(decoder) + .setJoiner(joiner) + .build(); + + OfflineModelConfig modelConfig = + OfflineModelConfig.builder() + .setTransducer(transducer) + .setTokens(tokens) + .setNumThreads(1) + .setDebug(true) + .build(); + + OfflineRecognizerConfig config = + OfflineRecognizerConfig.builder() + .setOfflineModelConfig(modelConfig) + .setDecodingMethod("greedy_search") + .build(); + + OfflineRecognizer recognizer = new OfflineRecognizer(config); + OfflineStream stream = recognizer.createStream(); + stream.acceptWaveform(reader.getSamples(), reader.getSampleRate()); + + recognizer.decode(stream); + + String text = recognizer.getResult(stream).getText(); + + System.out.printf("filename:%s\nresult:%s\n", waveFilename, text); + + stream.release(); + recognizer.release(); + } +} diff --git a/java-api-examples/NonStreamingDecodeFileWhisper.java b/java-api-examples/NonStreamingDecodeFileWhisper.java new file mode 100644 index 000000000..931bb9982 --- /dev/null +++ b/java-api-examples/NonStreamingDecodeFileWhisper.java @@ -0,0 +1,50 @@ +// Copyright 2024 Xiaomi Corporation + +// This file shows how to use an offline whisper, i.e., non-streaming whisper, +// to decode files. +import com.k2fsa.sherpa.onnx.*; + +public class NonStreamingDecodeFileWhisper { + public static void main(String[] args) { + // please refer to + // https://k2-fsa.github.io/sherpa/onnx/pretrained_models/whisper/tiny.en.html + // to download model files + String encoder = "./sherpa-onnx-whisper-tiny.en/tiny.en-encoder.int8.onnx"; + String decoder = "./sherpa-onnx-whisper-tiny.en/tiny.en-decoder.int8.onnx"; + String tokens = "./sherpa-onnx-whisper-tiny.en/tiny.en-tokens.txt"; + + String waveFilename = "./sherpa-onnx-whisper-tiny.en/test_wavs/1.wav"; + + WaveReader reader = new WaveReader(waveFilename); + + OfflineWhisperModelConfig whisper = + OfflineWhisperModelConfig.builder().setEncoder(encoder).setDecoder(decoder).build(); + + OfflineModelConfig modelConfig = + OfflineModelConfig.builder() + .setWhisper(whisper) + .setTokens(tokens) + .setNumThreads(1) + .setDebug(true) + .build(); + + OfflineRecognizerConfig config = + OfflineRecognizerConfig.builder() + .setOfflineModelConfig(modelConfig) + .setDecodingMethod("greedy_search") + .build(); + + OfflineRecognizer recognizer = new OfflineRecognizer(config); + OfflineStream stream = recognizer.createStream(); + stream.acceptWaveform(reader.getSamples(), reader.getSampleRate()); + + recognizer.decode(stream); + + String text = recognizer.getResult(stream).getText(); + + System.out.printf("filename:%s\nresult:%s\n", waveFilename, text); + + stream.release(); + recognizer.release(); + } +} diff --git a/java-api-examples/README.md b/java-api-examples/README.md index 4427b03ec..ae346843a 100755 --- a/java-api-examples/README.md +++ b/java-api-examples/README.md @@ -4,8 +4,18 @@ This directory contains examples for the JAVA API of sherpa-onnx. # Usage +## Streaming Speech recognition + ``` ./run-streaming-decode-file-ctc.sh ./run-streaming-decode-file-paraformer.sh ./run-streaming-decode-file-transducer.sh ``` + +## Non-Streaming Speech recognition + +```bash +./run-non-streaming-decode-file-paraformer.sh +./run-non-streaming-decode-file-transducer.sh +./run-non-streaming-decode-file-whisper.sh +``` diff --git a/java-api-examples/StreamingDecodeFileCtc.java b/java-api-examples/StreamingDecodeFileCtc.java index bb9f9121b..49565c61f 100644 --- a/java-api-examples/StreamingDecodeFileCtc.java +++ b/java-api-examples/StreamingDecodeFileCtc.java @@ -1,4 +1,3 @@ -// Copyright 2022-2023 by zhaoming // Copyright 2024 Xiaomi Corporation // This file shows how to use an online CTC model, i.e., streaming CTC model, @@ -16,8 +15,6 @@ public static void main(String[] args) { String waveFilename = "./sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18/test_wavs/8k.wav"; WaveReader reader = new WaveReader(waveFilename); - System.out.println(reader.getSampleRate()); - System.out.println(reader.getSamples().length); OnlineZipformer2CtcModelConfig ctc = OnlineZipformer2CtcModelConfig.builder().setModel(model).build(); diff --git a/java-api-examples/StreamingDecodeFileParaformer.java b/java-api-examples/StreamingDecodeFileParaformer.java index af44910d8..a9c1ce43b 100644 --- a/java-api-examples/StreamingDecodeFileParaformer.java +++ b/java-api-examples/StreamingDecodeFileParaformer.java @@ -1,4 +1,3 @@ -// Copyright 2022-2023 by zhaoming // Copyright 2024 Xiaomi Corporation // This file shows how to use an online paraformer, i.e., streaming paraformer, @@ -16,8 +15,6 @@ public static void main(String[] args) { String waveFilename = "./sherpa-onnx-streaming-paraformer-bilingual-zh-en/test_wavs/2.wav"; WaveReader reader = new WaveReader(waveFilename); - System.out.println(reader.getSampleRate()); - System.out.println(reader.getSamples().length); OnlineParaformerModelConfig paraformer = OnlineParaformerModelConfig.builder().setEncoder(encoder).setDecoder(decoder).build(); diff --git a/java-api-examples/StreamingDecodeFileTransducer.java b/java-api-examples/StreamingDecodeFileTransducer.java index b1c0288ba..75365728f 100644 --- a/java-api-examples/StreamingDecodeFileTransducer.java +++ b/java-api-examples/StreamingDecodeFileTransducer.java @@ -22,8 +22,6 @@ public static void main(String[] args) { "./sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/test_wavs/0.wav"; WaveReader reader = new WaveReader(waveFilename); - System.out.println(reader.getSampleRate()); - System.out.println(reader.getSamples().length); OnlineTransducerModelConfig transducer = OnlineTransducerModelConfig.builder() diff --git a/java-api-examples/run-non-streaming-decode-file-paraformer.sh b/java-api-examples/run-non-streaming-decode-file-paraformer.sh new file mode 100755 index 000000000..04adbbb58 --- /dev/null +++ b/java-api-examples/run-non-streaming-decode-file-paraformer.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +set -ex + +if [[ ! -f ../build/lib/libsherpa-onnx-jni.dylib && ! -f ../build/lib/libsherpa-onnx-jni.so ]]; then + mkdir -p ../build + pushd ../build + cmake \ + -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ + -DSHERPA_ONNX_ENABLE_TESTS=OFF \ + -DSHERPA_ONNX_ENABLE_CHECK=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ + -DSHERPA_ONNX_ENABLE_JNI=ON \ + .. + + make -j4 + ls -lh lib + popd +fi + +if [ ! -f ../sherpa-onnx/java-api/build/sherpa-onnx.jar ]; then + pushd ../sherpa-onnx/java-api + make + popd +fi + +if [[ ! -f ../build/lib/libsherpa-onnx-jni.dylib && ! -f ../build/lib/libsherpa-onnx-jni.so ]]; then + cmake \ + -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ + -DSHERPA_ONNX_ENABLE_TESTS=OFF \ + -DSHERPA_ONNX_ENABLE_CHECK=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ + -DSHERPA_ONNX_ENABLE_JNI=ON \ + .. + + make -j4 + ls -lh lib +fi + +if [ ! -f ./sherpa-onnx-paraformer-zh-2023-03-28/tokens.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-paraformer-zh-2023-03-28.tar.bz2 + + tar xvf sherpa-onnx-paraformer-zh-2023-03-28.tar.bz2 + rm sherpa-onnx-paraformer-zh-2023-03-28.tar.bz2 +fi + +java \ + -Djava.library.path=$PWD/../build/lib \ + -cp ../sherpa-onnx/java-api/build/sherpa-onnx.jar \ + NonStreamingDecodeFileParaformer.java diff --git a/java-api-examples/run-non-streaming-decode-file-transducer.sh b/java-api-examples/run-non-streaming-decode-file-transducer.sh new file mode 100755 index 000000000..99354e5a5 --- /dev/null +++ b/java-api-examples/run-non-streaming-decode-file-transducer.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +set -ex + +if [[ ! -f ../build/lib/libsherpa-onnx-jni.dylib && ! -f ../build/lib/libsherpa-onnx-jni.so ]]; then + mkdir -p ../build + pushd ../build + cmake \ + -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ + -DSHERPA_ONNX_ENABLE_TESTS=OFF \ + -DSHERPA_ONNX_ENABLE_CHECK=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ + -DSHERPA_ONNX_ENABLE_JNI=ON \ + .. + + make -j4 + ls -lh lib + popd +fi + +if [ ! -f ../sherpa-onnx/java-api/build/sherpa-onnx.jar ]; then + pushd ../sherpa-onnx/java-api + make + popd +fi + +if [[ ! -f ../build/lib/libsherpa-onnx-jni.dylib && ! -f ../build/lib/libsherpa-onnx-jni.so ]]; then + cmake \ + -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ + -DSHERPA_ONNX_ENABLE_TESTS=OFF \ + -DSHERPA_ONNX_ENABLE_CHECK=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ + -DSHERPA_ONNX_ENABLE_JNI=ON \ + .. + + make -j4 + ls -lh lib +fi + +if [ ! -f ./sherpa-onnx-zipformer-gigaspeech-2023-12-12/tokens.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-gigaspeech-2023-12-12.tar.bz2 + + tar xvf sherpa-onnx-zipformer-gigaspeech-2023-12-12.tar.bz2 + rm sherpa-onnx-zipformer-gigaspeech-2023-12-12.tar.bz2 +fi + +java \ + -Djava.library.path=$PWD/../build/lib \ + -cp ../sherpa-onnx/java-api/build/sherpa-onnx.jar \ + NonStreamingDecodeFileTransducer.java diff --git a/java-api-examples/run-non-streaming-decode-file-whisper.sh b/java-api-examples/run-non-streaming-decode-file-whisper.sh new file mode 100755 index 000000000..c60f8b622 --- /dev/null +++ b/java-api-examples/run-non-streaming-decode-file-whisper.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +set -ex + +if [[ ! -f ../build/lib/libsherpa-onnx-jni.dylib && ! -f ../build/lib/libsherpa-onnx-jni.so ]]; then + mkdir -p ../build + pushd ../build + cmake \ + -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ + -DSHERPA_ONNX_ENABLE_TESTS=OFF \ + -DSHERPA_ONNX_ENABLE_CHECK=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ + -DSHERPA_ONNX_ENABLE_JNI=ON \ + .. + + make -j4 + ls -lh lib + popd +fi + +if [ ! -f ../sherpa-onnx/java-api/build/sherpa-onnx.jar ]; then + pushd ../sherpa-onnx/java-api + make + popd +fi + +if [[ ! -f ../build/lib/libsherpa-onnx-jni.dylib && ! -f ../build/lib/libsherpa-onnx-jni.so ]]; then + cmake \ + -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ + -DSHERPA_ONNX_ENABLE_TESTS=OFF \ + -DSHERPA_ONNX_ENABLE_CHECK=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ + -DSHERPA_ONNX_ENABLE_JNI=ON \ + .. + + make -j4 + ls -lh lib +fi + +if [ ! -f ./sherpa-onnx-whisper-tiny.en/tiny.en-tokens.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-whisper-tiny.en.tar.bz2 + + tar xvf sherpa-onnx-whisper-tiny.en.tar.bz2 + rm sherpa-onnx-whisper-tiny.en.tar.bz2 +fi + +java \ + -Djava.library.path=$PWD/../build/lib \ + -cp ../sherpa-onnx/java-api/build/sherpa-onnx.jar \ + NonStreamingDecodeFileWhisper.java diff --git a/sherpa-onnx/java-api/Makefile b/sherpa-onnx/java-api/Makefile index 8b4bf3377..7c28a278d 100644 --- a/sherpa-onnx/java-api/Makefile +++ b/sherpa-onnx/java-api/Makefile @@ -19,6 +19,15 @@ java_files += OnlineRecognizerConfig.java java_files += OnlineRecognizerResult.java java_files += OnlineRecognizer.java +java_files += OfflineTransducerModelConfig.java +java_files += OfflineParaformerModelConfig.java +java_files += OfflineWhisperModelConfig.java +java_files += OfflineModelConfig.java +java_files += OfflineRecognizerConfig.java +java_files += OfflineRecognizerResult.java +java_files += OfflineStream.java +java_files += OfflineRecognizer.java + class_files := $(java_files:%.java=%.class) java_files := $(addprefix src/$(package_dir)/,$(java_files)) diff --git a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineModelConfig.java b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineModelConfig.java new file mode 100644 index 000000000..9de59f7a0 --- /dev/null +++ b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineModelConfig.java @@ -0,0 +1,117 @@ +// Copyright 2024 Xiaomi Corporation +package com.k2fsa.sherpa.onnx; + +public class OfflineModelConfig { + private final OfflineTransducerModelConfig transducer; + private final OfflineParaformerModelConfig paraformer; + private final OfflineWhisperModelConfig whisper; + private final String tokens; + private final int numThreads; + private final boolean debug; + private final String provider; + + private final String modelType; + + private OfflineModelConfig(Builder builder) { + this.transducer = builder.transducer; + this.paraformer = builder.paraformer; + this.whisper = builder.whisper; + this.tokens = builder.tokens; + this.numThreads = builder.numThreads; + this.debug = builder.debug; + this.provider = builder.provider; + this.modelType = builder.modelType; + } + + public static Builder builder() { + return new Builder(); + } + + public OfflineParaformerModelConfig getParaformer() { + return paraformer; + } + + public OfflineTransducerModelConfig getTransducer() { + return transducer; + } + + public OfflineWhisperModelConfig getZipformer2Ctc() { + return whisper; + } + + public String getTokens() { + return tokens; + } + + public int getNumThreads() { + return numThreads; + } + + public boolean getDebug() { + return debug; + } + + public String getProvider() { + return provider; + } + + public String getModelType() { + return modelType; + } + + + public static class Builder { + private OfflineParaformerModelConfig paraformer = OfflineParaformerModelConfig.builder().build(); + private OfflineTransducerModelConfig transducer = OfflineTransducerModelConfig.builder().build(); + private OfflineWhisperModelConfig whisper = OfflineWhisperModelConfig.builder().build(); + private String tokens = ""; + private int numThreads = 1; + private boolean debug = true; + private String provider = "cpu"; + private String modelType = ""; + + public OfflineModelConfig build() { + return new OfflineModelConfig(this); + } + + public Builder setTransducer(OfflineTransducerModelConfig transducer) { + this.transducer = transducer; + return this; + } + + public Builder setParaformer(OfflineParaformerModelConfig paraformer) { + this.paraformer = paraformer; + return this; + } + + public Builder setWhisper(OfflineWhisperModelConfig whisper) { + this.whisper = whisper; + return this; + } + + public Builder setTokens(String tokens) { + this.tokens = tokens; + return this; + } + + public Builder setNumThreads(int numThreads) { + this.numThreads = numThreads; + return this; + } + + public Builder setDebug(boolean debug) { + this.debug = debug; + return this; + } + + public Builder setProvider(String provider) { + this.provider = provider; + return this; + } + + public Builder setModelType(String modelType) { + this.modelType = modelType; + return this; + } + } +} \ No newline at end of file diff --git a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineParaformerModelConfig.java b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineParaformerModelConfig.java new file mode 100644 index 000000000..41f2fd978 --- /dev/null +++ b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineParaformerModelConfig.java @@ -0,0 +1,32 @@ +// Copyright 2024 Xiaomi Corporation +package com.k2fsa.sherpa.onnx; + +public class OfflineParaformerModelConfig { + private final String model; + + private OfflineParaformerModelConfig(Builder builder) { + this.model = builder.model; + } + + public static Builder builder() { + return new Builder(); + } + + public String getModel() { + return model; + } + + + public static class Builder { + private String model = ""; + + public OfflineParaformerModelConfig build() { + return new OfflineParaformerModelConfig(this); + } + + public Builder setModel(String model) { + this.model = model; + return this; + } + } +} diff --git a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineRecognizer.java b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineRecognizer.java new file mode 100644 index 000000000..002a6428d --- /dev/null +++ b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineRecognizer.java @@ -0,0 +1,55 @@ +// Copyright 2024 Xiaomi Corporation +package com.k2fsa.sherpa.onnx; + +public class OfflineRecognizer { + static { + System.loadLibrary("sherpa-onnx-jni"); + } + + private long ptr = 0; // this is the asr engine ptrss + + public OfflineRecognizer(OfflineRecognizerConfig config) { + ptr = newFromFile(config); + } + + public void decode(OfflineStream s) { + decode(ptr, s.getPtr()); + } + + public OfflineStream createStream() { + long p = createStream(ptr); + return new OfflineStream(p); + } + + @Override + protected void finalize() throws Throwable { + release(); + } + + // You'd better call it manually if it is not used anymore + public void release() { + if (this.ptr == 0) { + return; + } + delete(this.ptr); + this.ptr = 0; + } + + public OfflineRecognizerResult getResult(OfflineStream s) { + Object[] arr = getResult(s.getPtr()); + String text = (String) arr[0]; + String[] tokens = (String[]) arr[1]; + float[] timestamps = (float[]) arr[2]; + return new OfflineRecognizerResult(text, tokens, timestamps); + } + + private native void delete(long ptr); + + private native long newFromFile(OfflineRecognizerConfig config); + + private native long createStream(long ptr); + + private native void decode(long ptr, long streamPtr); + + private native Object[] getResult(long streamPtr); +} diff --git a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineRecognizerConfig.java b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineRecognizerConfig.java new file mode 100644 index 000000000..42c9f07af --- /dev/null +++ b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineRecognizerConfig.java @@ -0,0 +1,71 @@ +// Copyright 2024 Xiaomi Corporation +package com.k2fsa.sherpa.onnx; + +public class OfflineRecognizerConfig { + private final FeatureConfig featConfig; + private final OfflineModelConfig modelConfig; + private final String decodingMethod; + private final int maxActivePaths; + private final String hotwordsFile; + private final float hotwordsScore; + + private OfflineRecognizerConfig(Builder builder) { + this.featConfig = builder.featConfig; + this.modelConfig = builder.modelConfig; + this.decodingMethod = builder.decodingMethod; + this.maxActivePaths = builder.maxActivePaths; + this.hotwordsFile = builder.hotwordsFile; + this.hotwordsScore = builder.hotwordsScore; + } + + public static Builder builder() { + return new Builder(); + } + + public OfflineModelConfig getModelConfig() { + return modelConfig; + } + + public static class Builder { + private FeatureConfig featConfig = FeatureConfig.builder().build(); + private OfflineModelConfig modelConfig = OfflineModelConfig.builder().build(); + private String decodingMethod = "greedy_search"; + private int maxActivePaths = 4; + private String hotwordsFile = ""; + private float hotwordsScore = 1.5f; + + public OfflineRecognizerConfig build() { + return new OfflineRecognizerConfig(this); + } + + public Builder setFeatureConfig(FeatureConfig featConfig) { + this.featConfig = featConfig; + return this; + } + + public Builder setOfflineModelConfig(OfflineModelConfig modelConfig) { + this.modelConfig = modelConfig; + return this; + } + + public Builder setDecodingMethod(String decodingMethod) { + this.decodingMethod = decodingMethod; + return this; + } + + public Builder setMaxActivePaths(int maxActivePaths) { + this.maxActivePaths = maxActivePaths; + return this; + } + + public Builder setHotwordsFile(String hotwordsFile) { + this.hotwordsFile = hotwordsFile; + return this; + } + + public Builder setHotwordsScore(float hotwordsScore) { + this.hotwordsScore = hotwordsScore; + return this; + } + } +} diff --git a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineRecognizerResult.java b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineRecognizerResult.java new file mode 100644 index 000000000..fd14b8319 --- /dev/null +++ b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineRecognizerResult.java @@ -0,0 +1,26 @@ +// Copyright 2024 Xiaomi Corporation +package com.k2fsa.sherpa.onnx; + +public class OfflineRecognizerResult { + private final String text; + private final String[] tokens; + private final float[] timestamps; + + public OfflineRecognizerResult(String text, String[] tokens, float[] timestamps) { + this.text = text; + this.tokens = tokens; + this.timestamps = timestamps; + } + + public String getText() { + return text; + } + + public String[] getTokens() { + return tokens; + } + + public float[] getTimestamps() { + return timestamps; + } +} diff --git a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineStream.java b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineStream.java new file mode 100644 index 000000000..6159c8373 --- /dev/null +++ b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineStream.java @@ -0,0 +1,49 @@ +// Copyright 2024 Xiaomi Corporation +package com.k2fsa.sherpa.onnx; + +public class OfflineStream { + static { + System.loadLibrary("sherpa-onnx-jni"); + } + + private long ptr = 0; + + public OfflineStream() { + this.ptr = 0; + } + + public OfflineStream(long ptr) { + this.ptr = ptr; + } + + public long getPtr() { + return ptr; + } + + public void setPtr(long ptr) { + this.ptr = ptr; + } + + public void acceptWaveform(float[] samples, int sampleRate) { + acceptWaveform(this.ptr, samples, sampleRate); + } + + public void release() { + // stream object must be release after used + if (this.ptr == 0) { + return; + } + delete(this.ptr); + this.ptr = 0; + } + + @Override + protected void finalize() throws Throwable { + release(); + super.finalize(); + } + + private native void acceptWaveform(long ptr, float[] samples, int sampleRate); + + private native void delete(long ptr); +} diff --git a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineTransducerModelConfig.java b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineTransducerModelConfig.java new file mode 100644 index 000000000..efa32890b --- /dev/null +++ b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineTransducerModelConfig.java @@ -0,0 +1,55 @@ +// Copyright 2024 Xiaomi Corporation +package com.k2fsa.sherpa.onnx; + +public class OfflineTransducerModelConfig { + private final String encoder; + private final String decoder; + private final String joiner; + + private OfflineTransducerModelConfig(Builder builder) { + this.encoder = builder.encoder; + this.decoder = builder.decoder; + this.joiner = builder.joiner; + } + + public static Builder builder() { + return new Builder(); + } + + public String getEncoder() { + return encoder; + } + + public String getDecoder() { + return decoder; + } + + public String getJoiner() { + return joiner; + } + + public static class Builder { + private String encoder = ""; + private String decoder = ""; + private String joiner = ""; + + public OfflineTransducerModelConfig build() { + return new OfflineTransducerModelConfig(this); + } + + public Builder setEncoder(String encoder) { + this.encoder = encoder; + return this; + } + + public Builder setDecoder(String decoder) { + this.decoder = decoder; + return this; + } + + public Builder setJoiner(String joiner) { + this.joiner = joiner; + return this; + } + } +} diff --git a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineWhisperModelConfig.java b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineWhisperModelConfig.java new file mode 100644 index 000000000..e5ae7649f --- /dev/null +++ b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineWhisperModelConfig.java @@ -0,0 +1,80 @@ +// Copyright 2024 Xiaomi Corporation +package com.k2fsa.sherpa.onnx; + +public class OfflineWhisperModelConfig { + private final String encoder; + private final String decoder; + private final String language; + private final String task; + private final int tailPaddings; + + private OfflineWhisperModelConfig(Builder builder) { + this.encoder = builder.encoder; + this.decoder = builder.decoder; + this.language = builder.language; + this.task = builder.task; + this.tailPaddings = builder.tailPaddings; + } + + public static Builder builder() { + return new Builder(); + } + + public String getEncoder() { + return encoder; + } + + public String getDecoder() { + return decoder; + } + + public String getLanguage() { + return language; + } + + public String getTask() { + return task; + } + + public int getTailPaddings() { + return tailPaddings; + } + + public static class Builder { + private String encoder = ""; + private String decoder = ""; + private String language = "en"; // used only with multilingual models + private String task = "transcribe"; // used only with multilingual models + + private int tailPaddings = 1000; // number of frames to pad + + public OfflineWhisperModelConfig build() { + return new OfflineWhisperModelConfig(this); + } + + public Builder setEncoder(String encoder) { + this.encoder = encoder; + return this; + } + + public Builder setDecoder(String decoder) { + this.decoder = decoder; + return this; + } + + public Builder setLanguage(String language) { + this.language = language; + return this; + } + + public Builder setTask(String task) { + this.task = task; + return this; + } + + public Builder setTailPaddings(int tailPaddings) { + this.tailPaddings = tailPaddings; + return this; + } + } +} diff --git a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OnlineModelConfig.java b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OnlineModelConfig.java index bf2e73dc3..60505576c 100644 --- a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OnlineModelConfig.java +++ b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OnlineModelConfig.java @@ -12,6 +12,7 @@ public class OnlineModelConfig { private final boolean debug; private final String provider; private final String modelType; + private OnlineModelConfig(Builder builder) { this.transducer = builder.transducer; this.paraformer = builder.paraformer; diff --git a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OnlineRecognizer.java b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OnlineRecognizer.java index 87076a830..1271afce3 100644 --- a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OnlineRecognizer.java +++ b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OnlineRecognizer.java @@ -15,19 +15,6 @@ public OnlineRecognizer(OnlineRecognizerConfig config) { ptr = newFromFile(config); } - /* - public static float[] readWavFile(String fileName) { - // read data from the filename - Object[] wavdata = readWave(fileName); - Object data = wavdata[0]; // data[0] is float data, data[1] sample rate - - float[] floatData = (float[]) data; - - return floatData; - } - */ - - public void decode(OnlineStream s) { decode(ptr, s.getPtr()); } @@ -55,7 +42,7 @@ protected void finalize() throws Throwable { release(); } - // recognizer release, you'd better call it manually if not use anymore + // You'd better call it manually if it is not used anymore public void release() { if (this.ptr == 0) { return; diff --git a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OnlineRecognizerConfig.java b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OnlineRecognizerConfig.java index 8a6eeadd5..92bfaa054 100644 --- a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OnlineRecognizerConfig.java +++ b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OnlineRecognizerConfig.java @@ -12,6 +12,7 @@ public class OnlineRecognizerConfig { private final int maxActivePaths; private final String hotwordsFile; private final float hotwordsScore; + private OnlineRecognizerConfig(Builder builder) { this.featConfig = builder.featConfig; this.modelConfig = builder.modelConfig;