From c7dd36c830b28a22ad6cd6f6a664fcd9213d5324 Mon Sep 17 00:00:00 2001 From: Manuel Martin Date: Wed, 10 Jun 2020 16:00:17 +0200 Subject: [PATCH] Up update Mozilla Speech v2 (#3340) --- app/build.gradle | 3 +- .../vrbrowser/VRBrowserApplication.java | 9 + .../vrbrowser/downloads/DownloadJob.java | 13 ++ .../vrbrowser/downloads/DownloadsManager.java | 27 ++- .../ui/widgets/AppServicesProvider.java | 4 +- .../vrbrowser/ui/widgets/KeyboardWidget.java | 7 +- .../ui/widgets/NavigationBarWidget.java | 2 +- .../ui/widgets/dialogs/VoiceSearchWidget.java | 185 +++++++++--------- .../main/res/layout/voice_search_dialog.xml | 4 +- versions.gradle | 7 +- 10 files changed, 144 insertions(+), 117 deletions(-) diff --git a/app/build.gradle b/app/build.gradle index af125b09f..28a7b5b47 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -422,7 +422,8 @@ dependencies { annotationProcessor deps.room.compiler // Android Components - implementation deps.mozilla_speech + implementation deps.mozilla_speech.library + implementation deps.mozilla_speech.utils implementation deps.android_components.telemetry implementation deps.android_components.browser_errorpages implementation deps.android_components.browser_search diff --git a/app/src/common/shared/org/mozilla/vrbrowser/VRBrowserApplication.java b/app/src/common/shared/org/mozilla/vrbrowser/VRBrowserApplication.java index c6bc6f089..e354f6afd 100644 --- a/app/src/common/shared/org/mozilla/vrbrowser/VRBrowserApplication.java +++ b/app/src/common/shared/org/mozilla/vrbrowser/VRBrowserApplication.java @@ -9,6 +9,8 @@ import android.content.Context; import android.content.res.Configuration; +import com.mozilla.speechlibrary.SpeechService; + import org.mozilla.vrbrowser.browser.Accounts; import org.mozilla.vrbrowser.browser.Places; import org.mozilla.vrbrowser.browser.Services; @@ -30,6 +32,7 @@ public class VRBrowserApplication extends Application implements AppServicesProv private Places mPlaces; private Accounts mAccounts; private DownloadsManager mDownloadsManager; + private SpeechService mSpeechService; @Override public void onCreate() { @@ -46,6 +49,7 @@ protected void onActivityCreate() { mServices = new Services(this, mPlaces); mAccounts = new Accounts(this); mDownloadsManager = new DownloadsManager(this); + mSpeechService = new SpeechService(this); } @Override @@ -94,4 +98,9 @@ public Accounts getAccounts() { public DownloadsManager getDownloadsManager() { return mDownloadsManager; } + + @Override + public SpeechService getSpeechService() { + return mSpeechService; + } } diff --git a/app/src/common/shared/org/mozilla/vrbrowser/downloads/DownloadJob.java b/app/src/common/shared/org/mozilla/vrbrowser/downloads/DownloadJob.java index fb8e107ff..b01040c90 100644 --- a/app/src/common/shared/org/mozilla/vrbrowser/downloads/DownloadJob.java +++ b/app/src/common/shared/org/mozilla/vrbrowser/downloads/DownloadJob.java @@ -17,9 +17,16 @@ public class DownloadJob { private String mFilename; private String mTitle; private String mDescription; + private String mOutputPath; public static DownloadJob create(@NonNull String uri, @Nullable String contentType, long contentLength, @Nullable String filename) { + return create(uri, contentType, contentLength, filename, null); + } + + public static DownloadJob create(@NonNull String uri, @Nullable String contentType, + long contentLength, @Nullable String filename, + @Nullable String outputPath) { DownloadJob job = new DownloadJob(); job.mUri = uri; job.mContentType = contentType; @@ -37,6 +44,7 @@ public static DownloadJob create(@NonNull String uri, @Nullable String contentTy } job.mTitle = filename; job.mDescription = filename; + job.mOutputPath = outputPath; return job; } @@ -156,4 +164,9 @@ public String getDescription() { public void setDescription(String mDescription) { this.mDescription = mDescription; } + + @Nullable + public String getOutputPath() { + return mOutputPath; + } } diff --git a/app/src/common/shared/org/mozilla/vrbrowser/downloads/DownloadsManager.java b/app/src/common/shared/org/mozilla/vrbrowser/downloads/DownloadsManager.java index b31374858..1cc0e1eb8 100644 --- a/app/src/common/shared/org/mozilla/vrbrowser/downloads/DownloadsManager.java +++ b/app/src/common/shared/org/mozilla/vrbrowser/downloads/DownloadsManager.java @@ -16,6 +16,8 @@ import androidx.annotation.NonNull; import androidx.annotation.Nullable; +import com.mozilla.speechlibrary.utils.StorageUtils; + import org.mozilla.vrbrowser.browser.SettingsStore; import org.mozilla.vrbrowser.utils.UrlUtils; @@ -98,6 +100,10 @@ private void stopUpdates() { } public void startDownload(@NonNull DownloadJob job) { + startDownload(job, SettingsStore.getInstance(mContext).getDownloadsStorage()); + } + + public void startDownload(@NonNull DownloadJob job, @SettingsStore.Storage int storageType) { if (!URLUtil.isHttpUrl(job.getUri()) && !URLUtil.isHttpsUrl(job.getUri())) { notifyDownloadError("Cannot download non http/https files", job.getFilename()); return; @@ -110,16 +116,21 @@ public void startDownload(@NonNull DownloadJob job) { request.setMimeType(job.getContentType()); request.setNotificationVisibility(DownloadManager.Request.VISIBILITY_VISIBLE_NOTIFY_COMPLETED); request.setVisibleInDownloadsUi(false); - if (SettingsStore.getInstance(mContext).getDownloadsStorage() == SettingsStore.EXTERNAL) { - request.setDestinationInExternalPublicDir(Environment.DIRECTORY_DOWNLOADS, job.getFilename()); + if (job.getOutputPath() == null) { + if (storageType == SettingsStore.EXTERNAL) { + request.setDestinationInExternalPublicDir(Environment.DIRECTORY_DOWNLOADS, job.getFilename()); - } else { - String outputPath = getOutputPathForJob(job); - if (outputPath == null) { - notifyDownloadError("Cannot create output file", job.getFilename()); - return; + } else { + String outputPath = getOutputPathForJob(job); + if (outputPath == null) { + notifyDownloadError("Cannot create output file", job.getFilename()); + return; + } + request.setDestinationUri(Uri.parse(outputPath)); } - request.setDestinationUri(Uri.parse(outputPath)); + + } else { + request.setDestinationUri(Uri.parse("file://" + job.getOutputPath())); } if (mDownloadManager != null) { diff --git a/app/src/common/shared/org/mozilla/vrbrowser/ui/widgets/AppServicesProvider.java b/app/src/common/shared/org/mozilla/vrbrowser/ui/widgets/AppServicesProvider.java index 2449592e9..9ee56f057 100644 --- a/app/src/common/shared/org/mozilla/vrbrowser/ui/widgets/AppServicesProvider.java +++ b/app/src/common/shared/org/mozilla/vrbrowser/ui/widgets/AppServicesProvider.java @@ -1,5 +1,7 @@ package org.mozilla.vrbrowser.ui.widgets; +import com.mozilla.speechlibrary.SpeechService; + import org.mozilla.vrbrowser.AppExecutors; import org.mozilla.vrbrowser.browser.Accounts; import org.mozilla.vrbrowser.browser.Places; @@ -19,5 +21,5 @@ public interface AppServicesProvider { BitmapCache getBitmapCache(); Accounts getAccounts(); DownloadsManager getDownloadsManager(); - + SpeechService getSpeechService(); } diff --git a/app/src/common/shared/org/mozilla/vrbrowser/ui/widgets/KeyboardWidget.java b/app/src/common/shared/org/mozilla/vrbrowser/ui/widgets/KeyboardWidget.java index ce0c606cf..8ad3b8566 100644 --- a/app/src/common/shared/org/mozilla/vrbrowser/ui/widgets/KeyboardWidget.java +++ b/app/src/common/shared/org/mozilla/vrbrowser/ui/widgets/KeyboardWidget.java @@ -1257,18 +1257,13 @@ public void onGlobalFocusChanged(View oldFocus, View newFocus) { // VoiceSearch Delegate @Override - public void OnVoiceSearchResult(String aTranscription, float confidance) { + public void OnVoiceSearchResult(String aTranscription, float confidence) { if (aTranscription != null && !aTranscription.isEmpty()) { handleText(aTranscription); } exitVoiceInputMode(); } - @Override - public void OnVoiceSearchCanceled() { - exitVoiceInputMode(); - } - @Override public void OnVoiceSearchError() { exitVoiceInputMode(); diff --git a/app/src/common/shared/org/mozilla/vrbrowser/ui/widgets/NavigationBarWidget.java b/app/src/common/shared/org/mozilla/vrbrowser/ui/widgets/NavigationBarWidget.java index 272f2aa2b..13ea49ecd 100644 --- a/app/src/common/shared/org/mozilla/vrbrowser/ui/widgets/NavigationBarWidget.java +++ b/app/src/common/shared/org/mozilla/vrbrowser/ui/widgets/NavigationBarWidget.java @@ -1030,7 +1030,7 @@ public void onDrmButtonClicked() { // VoiceSearch Delegate @Override - public void OnVoiceSearchResult(String transcription, float confidance) { + public void OnVoiceSearchResult(String transcription, float confidence) { mBinding.navigationBarNavigation.urlBar.handleURLEdit(transcription); } diff --git a/app/src/common/shared/org/mozilla/vrbrowser/ui/widgets/dialogs/VoiceSearchWidget.java b/app/src/common/shared/org/mozilla/vrbrowser/ui/widgets/dialogs/VoiceSearchWidget.java index f11f8515e..50fdf8fe3 100644 --- a/app/src/common/shared/org/mozilla/vrbrowser/ui/widgets/dialogs/VoiceSearchWidget.java +++ b/app/src/common/shared/org/mozilla/vrbrowser/ui/widgets/dialogs/VoiceSearchWidget.java @@ -18,15 +18,18 @@ import android.view.Gravity; import android.view.LayoutInflater; +import androidx.annotation.Nullable; import androidx.core.app.ActivityCompat; import androidx.databinding.DataBindingUtil; -import com.mozilla.speechlibrary.ISpeechRecognitionListener; -import com.mozilla.speechlibrary.MozillaSpeechService; -import com.mozilla.speechlibrary.STTResult; +import com.mozilla.speechlibrary.SpeechResultCallback; +import com.mozilla.speechlibrary.SpeechService; +import com.mozilla.speechlibrary.SpeechServiceSettings; +import com.mozilla.speechlibrary.stt.STTResult; import org.mozilla.vrbrowser.R; import org.mozilla.vrbrowser.VRBrowserActivity; +import org.mozilla.vrbrowser.VRBrowserApplication; import org.mozilla.vrbrowser.browser.SettingsStore; import org.mozilla.vrbrowser.browser.engine.EngineProvider; import org.mozilla.vrbrowser.browser.engine.SessionStore; @@ -43,30 +46,28 @@ public class VoiceSearchWidget extends UIDialog implements WidgetManagerDelegate public enum State { LISTENING, SEARCHING, - ERROR, + SPEECH_ERROR, + MODEL_NOT_FOUND, PERMISSIONS } private static final int VOICE_SEARCH_AUDIO_REQUEST_CODE = 7455; - private static final int ANIMATION_DURATION = 1000; private static int MAX_CLIPPING = 10000; private static int MAX_DB = 130; private static int MIN_DB = 50; public interface VoiceSearchDelegate { - default void OnVoiceSearchResult(String transcription, float confidance) {}; - default void OnVoiceSearchCanceled() {}; + default void OnVoiceSearchResult(String transcription, float confidence) {}; default void OnVoiceSearchError() {}; } private VoiceSearchDialogBinding mBinding; - private MozillaSpeechService mMozillaSpeechService; + private SpeechService mMozillaSpeechService; private VoiceSearchDelegate mDelegate; private ClipDrawable mVoiceInputClipDrawable; private AnimatedVectorDrawable mSearchingAnimation; - private boolean mIsSpeechRecognitionRunning = false; - private boolean mWasSpeechRecognitionRunning = false; + private VRBrowserApplication mApplication; public VoiceSearchWidget(Context aContext) { super(aContext); @@ -87,21 +88,20 @@ private void initialize(Context aContext) { // AnimatedVectorDrawable doesn't work with a Hardware Accelerated canvas, we disable it for this view. setIsHardwareAccelerationEnabled(false); + mApplication = (VRBrowserApplication)aContext.getApplicationContext(); + updateUI(); mWidgetManager.addPermissionListener(this); - mMozillaSpeechService = MozillaSpeechService.getInstance(); - mMozillaSpeechService.setGeckoWebExecutor(EngineProvider.INSTANCE.createGeckoWebExecutor(getContext())); - mMozillaSpeechService.setProductTag(getContext().getString(R.string.voice_app_id)); - mSearchingAnimation = (AnimatedVectorDrawable) mBinding.voiceSearchAnimationSearching.getDrawable(); if (DeviceType.isPicoVR()) { ViewUtils.forceAnimationOnUI(mSearchingAnimation); } - mMozillaSpeechService.addListener(mVoiceSearchListener); - ((Application)aContext.getApplicationContext()).registerActivityLifecycleCallbacks(this); + mMozillaSpeechService = mApplication.getSpeechService(); + + mApplication.registerActivityLifecycleCallbacks(this); } public void updateUI() { @@ -136,8 +136,9 @@ public void setDelegate(VoiceSearchDelegate delegate) { @Override public void releaseWidget() { mWidgetManager.removePermissionListener(this); - mMozillaSpeechService.removeListener(mVoiceSearchListener); - ((Application)getContext().getApplicationContext()).unregisterActivityLifecycleCallbacks(this); + mApplication.unregisterActivityLifecycleCallbacks(this); + + mMozillaSpeechService.stop(); super.releaseWidget(); } @@ -163,68 +164,59 @@ public void setPlacementForKeyboard(int aHandle) { mWidgetPlacement.translationZ = 0; } - private ISpeechRecognitionListener mVoiceSearchListener = new ISpeechRecognitionListener() { + SpeechResultCallback mResultCallback = new SpeechResultCallback() { + @Override + public void onStartListen() { + // Handle when the api successfully opened the microphone and started listening + Log.d(LOGTAG, "===> START_LISTEN"); + } + + @Override + public void onMicActivity(double fftsum) { + // Captures the activity from the microphone + Log.d(LOGTAG, "===> MIC_ACTIVITY"); + double db = (double)fftsum * -1; // the higher the value, quieter the user/environment is + db = db == Double.POSITIVE_INFINITY ? MAX_DB : db; + int level = (int)(MAX_CLIPPING - (((db - MIN_DB) / (MAX_DB - MIN_DB)) * MAX_CLIPPING)); + Log.d(LOGTAG, "===> db: " + db); + Log.d(LOGTAG, "===> level " + level); + mVoiceInputClipDrawable.setLevel(level); + } + + @Override + public void onDecoding() { + // Handle when the speech object changes to decoding state + Log.d(LOGTAG, "===> DECODING"); + setDecodingState(); + } - public void onSpeechStatusChanged(final MozillaSpeechService.SpeechState aState, final Object aPayload){ - if (!mIsSpeechRecognitionRunning) { - return; + @Override + public void onSTTResult(@Nullable STTResult result) { + // When the api finished processing and returned a hypothesis + Log.d(LOGTAG, "===> STT_RESULT"); + String transcription = result.mTranscription; + float confidence = result.mConfidence; + if (mDelegate != null) { + mDelegate.OnVoiceSearchResult(transcription, confidence); + } + hide(KEEP_WIDGET); + } + + @Override + public void onNoVoice() { + // Handle when the api didn't detect any voice + Log.d(LOGTAG, "===> NO_VOICE"); + } + + @Override + public void onError(@ErrorType int errorType, @Nullable String error) { + Log.d(LOGTAG, "===> ERROR: " + error); + setResultState(errorType); + if (mDelegate != null) { + mDelegate.OnVoiceSearchError(); } - ((Activity)getContext()).runOnUiThread(() -> { - switch (aState) { - case DECODING: - // Handle when the speech object changes to decoding state - Log.d(LOGTAG, "===> DECODING"); - setDecodingState(); - break; - case MIC_ACTIVITY: - // Captures the activity from the microphone - Log.d(LOGTAG, "===> MIC_ACTIVITY"); - double db = (double)aPayload * -1; // the higher the value, quieter the user/environment is - db = db == Double.POSITIVE_INFINITY ? MAX_DB : db; - int level = (int)(MAX_CLIPPING - (((db - MIN_DB) / (MAX_DB - MIN_DB)) * MAX_CLIPPING)); - Log.d(LOGTAG, "===> db: " + db); - Log.d(LOGTAG, "===> level " + level); - mVoiceInputClipDrawable.setLevel(level); - break; - case STT_RESULT: - // When the api finished processing and returned a hypothesis - Log.d(LOGTAG, "===> STT_RESULT"); - String transcription = ((STTResult)aPayload).mTranscription; - float confidence = ((STTResult)aPayload).mConfidence; - if (mDelegate != null) { - mDelegate.OnVoiceSearchResult(transcription, confidence); - } - hide(KEEP_WIDGET); - break; - case START_LISTEN: - // Handle when the api successfully opened the microphone and started listening - Log.d(LOGTAG, "===> START_LISTEN"); - break; - case NO_VOICE: - // Handle when the api didn't detect any voice - Log.d(LOGTAG, "===> NO_VOICE"); - setResultState(); - break; - case CANCELED: - // Handle when a cancellation was fully executed - Log.d(LOGTAG, "===> CANCELED"); - if (mDelegate != null) { - mDelegate.OnVoiceSearchCanceled(); - } - break; - case ERROR: - Log.d(LOGTAG, "===> ERROR: " + aPayload.toString()); - setResultState(); - // Handle when any error occurred - if (mDelegate != null) { - mDelegate.OnVoiceSearchError(); - } - break; - default: - break; - } - }); } + }; public void startVoiceSearch() { @@ -234,25 +226,28 @@ public void startVoiceSearch() { VOICE_SEARCH_AUDIO_REQUEST_CODE); } else { String locale = LocaleUtils.getVoiceSearchLanguageTag(getContext()); - mMozillaSpeechService.setLanguage(LocaleUtils.mapToMozillaSpeechLocales(locale)); boolean storeData = SettingsStore.getInstance(getContext()).isSpeechDataCollectionEnabled(); if (SessionStore.get().getActiveSession().isPrivateMode()) { storeData = false; } - mMozillaSpeechService.storeSamples(storeData); - mMozillaSpeechService.storeTranscriptions(storeData); - mMozillaSpeechService.start(getContext().getApplicationContext()); - mIsSpeechRecognitionRunning = true; + + SpeechServiceSettings.Builder builder = new SpeechServiceSettings.Builder() + .withLanguage(locale) + .withStoreSamples(storeData) + .withStoreTranscriptions(storeData) + .withProductTag(getContext().getString(R.string.voice_app_id)); + mMozillaSpeechService.start(builder.build(), + EngineProvider.INSTANCE.getDefaultGeckoWebExecutor(getContext()), + mResultCallback); } } public void stopVoiceSearch() { try { - mMozillaSpeechService.cancel(); - mIsSpeechRecognitionRunning = false; + mMozillaSpeechService.stop(); } catch (Exception e) { - Log.d(LOGTAG, e.getLocalizedMessage()); + Log.d(LOGTAG, e.getLocalizedMessage() != null ? e.getLocalizedMessage() : "Unknown voice error"); e.printStackTrace(); } } @@ -327,15 +322,16 @@ private void setDecodingState() { mBinding.executePendingBindings(); } - private void setResultState() { + private void setResultState(@SpeechResultCallback.ErrorType int errorType) { stopVoiceSearch(); postDelayed(() -> { - mBinding.setState(State.ERROR); + if (errorType == SpeechResultCallback.SPEECH_ERROR) { + mBinding.setState(State.SPEECH_ERROR); + startVoiceSearch(); + } mSearchingAnimation.stop(); mBinding.executePendingBindings(); - - startVoiceSearch(); }, 100); } @@ -345,6 +341,8 @@ private void setPermissionNotGranted() { mBinding.executePendingBindings(); } + // ActivityLifeCycle + @Override public void onActivityCreated(Activity activity, Bundle bundle) { @@ -357,17 +355,12 @@ public void onActivityStarted(Activity activity) { @Override public void onActivityResumed(Activity activity) { - if (mWasSpeechRecognitionRunning) { - startVoiceSearch(); - } + startVoiceSearch(); } @Override public void onActivityPaused(Activity activity) { - mWasSpeechRecognitionRunning = mIsSpeechRecognitionRunning; - if (mIsSpeechRecognitionRunning) { - stopVoiceSearch(); - } + stopVoiceSearch(); } @Override diff --git a/app/src/main/res/layout/voice_search_dialog.xml b/app/src/main/res/layout/voice_search_dialog.xml index a6891f294..959317c42 100644 --- a/app/src/main/res/layout/voice_search_dialog.xml +++ b/app/src/main/res/layout/voice_search_dialog.xml @@ -85,7 +85,7 @@ android:textColor="@color/white" android:textSize="24sp" android:typeface="sans" - visibleInvisible="@{state == State.ERROR}" + visibleInvisible="@{state == State.SPEECH_ERROR}" android:layout_gravity="center_horizontal" android:gravity="center_horizontal" tools:text='@{String.format("%s %s", @string/voice_search_error, @string/voice_search_try_again)}' /> @@ -110,7 +110,7 @@ android:layout_height="wrap_content" android:layout_above="@id/buttonsContainer" android:layout_marginBottom="15dp" - visibleGone="@{state == State.LISTENING || state == State.ERROR}"> + visibleGone="@{state == State.LISTENING || state == State.SPEECH_ERROR}">