WIP: Adding DeepSpeech w/ TFLite

mozilla · Mar 18, 2019 · 77665e3 · 77665e3
1 parent ca9be29
commit 77665e3
Show file tree

Hide file tree

Showing 7 changed files with 539 additions and 16 deletions.
diff --git a/app/build.gradle b/app/build.gradle
@@ -9,6 +9,9 @@ android {
         versionCode 1
         versionName "1.0"
         testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner"
+        ndk {
+            abiFilters 'armeabi-v7a', 'arm64-v8a', 'x86_64'
+        }
     }
     buildTypes {
         release {
@@ -20,6 +23,10 @@ android {
         sourceCompatibility JavaVersion.VERSION_1_8
         targetCompatibility JavaVersion.VERSION_1_8
     }
+
+    lintOptions {
+        abortOnError false
+    }
 }
 
 dependencies {
@@ -28,6 +35,7 @@ dependencies {
     implementation 'com.android.support:appcompat-v7:27.1.1'
     implementation 'com.android.support.constraint:constraint-layout:1.1.2'
     implementation 'com.jjoe64:graphview:4.2.2'
+    implementation 'net.lingala.zip4j:zip4j:1.3.2'
     testImplementation 'junit:junit:4.12'
     androidTestImplementation 'com.android.support.test:runner:1.0.2'
     androidTestImplementation 'com.android.support.test.espresso:espresso-core:3.0.2'

diff --git a/app/src/main/java/com/mozilla/speechapp/MainActivity.java b/app/src/main/java/com/mozilla/speechapp/MainActivity.java
@@ -1,32 +1,58 @@
 package com.mozilla.speechapp;
 
 import android.Manifest;
+
 import android.app.Activity;
+import android.app.DownloadManager;
+
 import android.content.pm.PackageManager;
+import android.content.BroadcastReceiver;
+import android.content.Context;
+import android.content.Intent;
+import android.content.IntentFilter;
+
+import android.database.Cursor;
+
 import android.support.annotation.NonNull;
 import android.support.v4.app.ActivityCompat;
 import android.support.v7.app.AppCompatActivity;
+
+import android.net.Uri;
+
 import android.os.Bundle;
+import android.os.AsyncTask;
+
 import android.util.Log;
+
 import android.view.View;
 import android.view.WindowManager;
+
 import android.widget.Button;
 import android.widget.CompoundButton;
 import android.widget.EditText;
 import android.widget.Switch;
+import android.widget.Toast;
 
 import com.jjoe64.graphview.GraphView;
 import com.jjoe64.graphview.series.DataPoint;
 import com.jjoe64.graphview.series.LineGraphSeries;
+
 import com.mozilla.speechlibrary.ISpeechRecognitionListener;
 import com.mozilla.speechlibrary.MozillaSpeechService;
 import com.mozilla.speechlibrary.STTResult;
 import com.mozilla.speechmodule.R;
 
+import java.io.File;
+
+import net.lingala.zip4j.core.ZipFile;
+
 import static android.support.constraint.Constraints.TAG;
 
 public class MainActivity extends AppCompatActivity implements ISpeechRecognitionListener, CompoundButton.OnCheckedChangeListener {
 
+    private static long sDownloadId;
+    private static DownloadManager sDownloadManager;
+
     private MozillaSpeechService mMozillaSpeechService;
     private GraphView mGraph;
     private long mDtstart;
@@ -47,6 +73,7 @@ private void initialize() {
         EditText txtProdutTag, txtLanguage;
         Switch switchTranscriptions = findViewById(R.id.switchTranscriptions);
         Switch switchSamples = findViewById(R.id.switchSamples);
+        Switch useDeepSpeech = findViewById(R.id.useDeepSpeech);
 
         if (ActivityCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO)
                 != PackageManager.PERMISSION_GRANTED) {
@@ -75,7 +102,12 @@ private void initialize() {
                 mSeries1.resetData(new DataPoint[0]);
                 mMozillaSpeechService.setLanguage(txtLanguage.getText().toString());
                 mMozillaSpeechService.setProductTag(txtProdutTag.getText().toString());
-                mMozillaSpeechService.start(getApplicationContext());
+                mMozillaSpeechService.setModelPath(getExternalFilesDir("models").getAbsolutePath());
+                if (mMozillaSpeechService.ensureModelInstalled()) {
+                    mMozillaSpeechService.start(getApplicationContext());
+                } else {
+                    maybeDownloadOrExtractModel(getExternalFilesDir("models").getAbsolutePath(), mMozillaSpeechService.getLanguageDir());
+                }
             } catch (Exception e) {
                 Log.d(TAG, e.getLocalizedMessage());
                 e.printStackTrace();
@@ -93,8 +125,10 @@ private void initialize() {
 
         switchTranscriptions.setOnCheckedChangeListener(this);
         switchSamples.setOnCheckedChangeListener(this);
+        useDeepSpeech.setOnCheckedChangeListener(this);
         switchTranscriptions.toggle();
         switchSamples.toggle();
+        useDeepSpeech.toggle();
 
         mGraph = findViewById(R.id.graph);
         mSeries1 = new LineGraphSeries<>(new DataPoint[0]);
@@ -154,8 +188,89 @@ public void removeListener() {
     public void onCheckedChanged(CompoundButton buttonView, boolean isChecked) {
         if (buttonView.equals(findViewById(R.id.switchTranscriptions))) {
             mMozillaSpeechService.storeTranscriptions(isChecked);
-        } else {
+        } else if (buttonView.equals(findViewById(R.id.switchSamples))) {
             mMozillaSpeechService.storeSamples(isChecked);
+        } else if (buttonView.equals(findViewById(R.id.useDeepSpeech))) {
+            mMozillaSpeechService.useDeepSpeech(isChecked);
         }
     }
+
+    private class AsyncUnzip extends AsyncTask<String, Void, Boolean> {
+
+        @Override
+        protected void onPreExecute() {
+            Toast noModel = Toast.makeText(getApplicationContext(), "Extracting downloaded model", Toast.LENGTH_LONG);
+            mPlain_text_input.append("Extracting downloaded model\n");
+            noModel.show();
+        }
+
+        @Override
+        protected Boolean doInBackground(String...params) {
+            String aZipFile = params[0], aRootModelsPath = params[1];
+            try {
+                ZipFile zf = new ZipFile(aZipFile);
+                zf.extractAll(aRootModelsPath);
+            } catch (Exception e) {
+                Log.d(TAG, e.getLocalizedMessage());
+                e.printStackTrace();
+            }
+
+            return (new File(aZipFile)).delete();
+        }
+
+        @Override
+        protected void onPostExecute(Boolean result) {
+            Button buttonStart = findViewById(R.id.button_start), buttonCancel = findViewById(R.id.button_cancel);
+            mMozillaSpeechService.start(getApplicationContext());
+            buttonStart.setEnabled(true);
+            buttonCancel.setEnabled(true);
+        }
+
+    }
+
+    public void maybeDownloadOrExtractModel(String aModelsPath, String aLang) {
+        String zipFile   = aModelsPath + "/" + aLang + ".zip";
+        Uri modelZipURL  = Uri.parse(mMozillaSpeechService.getModelDownloadURL());
+        Uri modelZipFile = Uri.parse("file://" + zipFile);
+
+        Button buttonStart = findViewById(R.id.button_start), buttonCancel = findViewById(R.id.button_cancel);
+        buttonStart.setEnabled(false);
+        buttonCancel.setEnabled(false);
+
+        BroadcastReceiver receiver = new BroadcastReceiver() {
+            @Override
+            public void onReceive(Context context, Intent intent) {
+                String action = intent.getAction();
+                if (DownloadManager.ACTION_DOWNLOAD_COMPLETE.equals(action)) {
+                    long downloadId = intent.getLongExtra(DownloadManager.EXTRA_DOWNLOAD_ID, 0);
+                    DownloadManager.Query query = new DownloadManager.Query();
+                    query.setFilterById(downloadId);
+                    Cursor c = sDownloadManager.query(query);
+                    if (c.moveToFirst()) {
+                        int columnIndex = c.getColumnIndex(DownloadManager.COLUMN_STATUS);
+                        if (DownloadManager.STATUS_SUCCESSFUL == c.getInt(columnIndex)) {
+                            Log.d(TAG, "Download successfull");
+
+                            new AsyncUnzip().execute(zipFile, aModelsPath);
+                        }
+                    }
+                }
+            }
+        };
+
+        Toast noModel = Toast.makeText(getApplicationContext(), "No model has been found for language '" + aLang + "'. Triggering download ...", Toast.LENGTH_LONG);
+        mPlain_text_input.append("No model has been found for language '" + aLang + "'. Triggering download ...\n");
+        noModel.show();
+
+        sDownloadManager = (DownloadManager) getSystemService(Context.DOWNLOAD_SERVICE);
+        DownloadManager.Request request = new DownloadManager.Request(modelZipURL);
+        request.setTitle("DeepSpeech " + aLang);
+        request.setDescription("DeepSpeech Model");
+        request.setNotificationVisibility(DownloadManager.Request.VISIBILITY_VISIBLE_NOTIFY_COMPLETED);
+        request.setVisibleInDownloadsUi(false);
+        request.setDestinationUri(modelZipFile);
+        sDownloadId = sDownloadManager.enqueue(request);
+
+        getApplicationContext().registerReceiver(receiver, new IntentFilter(DownloadManager.ACTION_DOWNLOAD_COMPLETE));
+    }
 }
diff --git a/app/src/main/res/layout/activity_main.xml b/app/src/main/res/layout/activity_main.xml
@@ -49,12 +49,21 @@
             android:inputType="textMultiLine"
             android:singleLine="false" />
 
+        <Switch
+            android:id="@+id/useDeepSpeech"
+            android:layout_width="wrap_content"
+            android:layout_height="wrap_content"
+            android:layout_alignParentEnd="true"
+            android:layout_below="@+id/switchTranscriptions"
+            android:text="Use DeepSpeech" />
+
         <Switch
             android:id="@+id/switchTranscriptions"
             android:layout_width="wrap_content"
             android:layout_height="wrap_content"
             android:layout_alignParentEnd="true"
             android:layout_centerVertical="true"
+            android:layout_below="@+id/switchSamples"
             android:text="Store Transcriptions" />
 
         <Switch
@@ -71,7 +80,7 @@
             android:layout_height="wrap_content"
             android:layout_above="@+id/plain_text_input"
             android:layout_alignStart="@+id/graph"
-            android:layout_marginBottom="-71dp"
+            android:layout_marginBottom="-50dp"
             android:ems="10"
             android:inputType="textPersonName"
             android:text="ProductTag" />
@@ -82,9 +91,10 @@
             android:layout_height="wrap_content"
             android:layout_above="@+id/plain_text_input"
             android:layout_alignStart="@+id/graph"
-            android:layout_marginBottom="-120dp"
+            android:layout_marginBottom="-80dp"
             android:ems="10"
             android:inputType="textPersonName"
-            android:text="Language" />
+            android:text="eng" />
+
     </RelativeLayout>
-</android.support.constraint.ConstraintLayout>
+</android.support.constraint.ConstraintLayout>
diff --git a/mozillaspeechlibrary/build.gradle b/mozillaspeechlibrary/build.gradle
@@ -1,9 +1,9 @@
 apply plugin: 'com.android.library'
 apply from: 'maven-push.gradle'
 
-def versionMajor = 1
+def versionMajor = 2
 def versionMinor = 0
-def versionPatch = 4
+def versionPatch = 0
 
 android {
     compileSdkVersion 25
@@ -13,6 +13,10 @@ android {
         versionCode versionMajor * 10000 + versionMinor * 100 + versionPatch
         versionName "${versionMajor}.${versionMinor}.${versionPatch}"
         testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner"
+
+        ndk {
+            abiFilters 'armeabi-v7a', 'arm64-v8a', 'x86_64'
+        }
     }
 
     buildTypes {
@@ -27,15 +31,21 @@ android {
             path 'src/main/cpp/Android.mk'
         }
     }
+
     compileOptions {
         sourceCompatibility = 1.7
         targetCompatibility = 1.7
     }
+
+    lintOptions {
+        abortOnError false
+    }
 }
 
 dependencies {
     implementation fileTree(include: ['*.jar'], dir: 'libs')
     implementation 'com.loopj.android:android-async-http:1.4.9'
+    implementation 'org.mozilla.deepspeech:libdeepspeech:0.5.0-alpha.1@aar'
     testImplementation 'junit:junit:4.12'
     androidTestImplementation 'com.android.support.test:runner:1.0.2'
     androidTestImplementation 'com.android.support.test.espresso:espresso-core:3.0.2'