-
Notifications
You must be signed in to change notification settings - Fork 2.8k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
[DO_NOT_MERGE] Microphone streaming with a 1 minute duration. #1185
Changes from 2 commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -19,6 +19,9 @@ | |
import com.google.api.gax.longrunning.OperationFuture; | ||
import com.google.api.gax.rpc.ApiStreamObserver; | ||
import com.google.api.gax.rpc.BidiStreamingCallable; | ||
import com.google.api.gax.rpc.ClientStream; | ||
import com.google.api.gax.rpc.ResponseObserver; | ||
import com.google.api.gax.rpc.StreamController; | ||
import com.google.cloud.speech.v1p1beta1.LongRunningRecognizeMetadata; | ||
import com.google.cloud.speech.v1p1beta1.LongRunningRecognizeResponse; | ||
import com.google.cloud.speech.v1p1beta1.RecognitionAudio; | ||
|
@@ -47,6 +50,13 @@ | |
import java.util.ArrayList; | ||
import java.util.List; | ||
|
||
import javax.sound.sampled.AudioFormat; | ||
import javax.sound.sampled.AudioInputStream; | ||
import javax.sound.sampled.AudioSystem; | ||
import javax.sound.sampled.DataLine; | ||
import javax.sound.sampled.DataLine.Info; | ||
import javax.sound.sampled.TargetDataLine; | ||
|
||
public class Recognize { | ||
|
||
/** Run speech recognition tasks. */ | ||
|
@@ -56,9 +66,10 @@ public static void main(String... args) throws Exception { | |
System.out.printf( | ||
"\tjava %s \"<command>\" \"<path-to-image>\"\n" | ||
+ "Commands:\n" | ||
+ "\tsyncrecognize | asyncrecognize | streamrecognize | wordoffsets\n" | ||
+ "\t| model-selection | auto-punctuation | stream-punctuation | enhanced-model\n" | ||
+ "\t| metadata | diarization | multi-channel | multi-language | word-level-conf" | ||
+ "\tsyncrecognize | asyncrecognize | streamrecognize | micstreamrecognize \n" | ||
+ "\t| wordoffsets | model-selection | auto-punctuation | stream-punctuation \n" | ||
+ "\t| enhanced-model| metadata | diarization | multi-channel | multi-language \n" | ||
+ "\t | word-level-conf" | ||
+ "Path:\n\tA file path (ex: ./resources/audio.raw) or a URI " | ||
+ "for a Cloud Storage resource (gs://...)\n", | ||
Recognize.class.getCanonicalName()); | ||
|
@@ -88,6 +99,8 @@ public static void main(String... args) throws Exception { | |
} | ||
} else if (command.equals("streamrecognize")) { | ||
streamingRecognizeFile(path); | ||
} else if (command.equals("micstreamrecognize")) { | ||
streamingMicRecognize(); | ||
} else if (command.equals("model-selection")) { | ||
if (path.startsWith("gs://")) { | ||
transcribeModelSelectionGcs(path); | ||
|
@@ -704,6 +717,99 @@ public SettableFuture<List<T>> future() { | |
} | ||
// [END speech_stream_recognize_punctuation] | ||
|
||
// [START speech_streaming_mic_recognize] | ||
|
||
/** | ||
* Performs microphone streaming speech recognition with a duration of 1 minute. | ||
* | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Delete empty line. |
||
*/ | ||
public static void streamingMicRecognize() throws Exception { | ||
AudioFormat audioFormat = new AudioFormat(16000, 16, 1, true, false); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Can you add a comment for what the numbers mean? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Let's add a comment about what these values mean: I think they're like |
||
DataLine.Info targetInfo = new Info(TargetDataLine.class, audioFormat); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Add comment along the lines of: // Get system information about microphone ? |
||
TargetDataLine targetDataLine; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Is it possible to move the declaration of this variable to when it is instantiated? on line 782. Not sure if there is something with how mic streaming is set up. |
||
int durationMillSec = 60 * 1000; // 60 seconds | ||
if (!AudioSystem.isLineSupported(targetInfo)) { | ||
System.out.println("Microphone not supported"); | ||
System.exit(0); | ||
} | ||
|
||
ResponseObserver<StreamingRecognizeResponse> responseObserver = null; | ||
try (SpeechClient client = SpeechClient.create()) { | ||
|
||
responseObserver = | ||
new ResponseObserver<StreamingRecognizeResponse>() { | ||
ArrayList<StreamingRecognizeResponse> responses = new ArrayList<>(); | ||
|
||
public void onStart(StreamController controller) {} | ||
|
||
public void onResponse(StreamingRecognizeResponse response) { | ||
responses.add(response); | ||
} | ||
|
||
public void onComplete() { | ||
for (StreamingRecognizeResponse response : responses) { | ||
StreamingRecognitionResult result = response.getResultsList().get(0); | ||
SpeechRecognitionAlternative alternative = result.getAlternativesList().get(0); | ||
System.out.printf("Transcript : %s\n", alternative.getTranscript()); | ||
} | ||
} | ||
|
||
public void onError(Throwable t) { | ||
System.out.println(t); | ||
} | ||
}; | ||
|
||
ClientStream<StreamingRecognizeRequest> clientStream = | ||
client.streamingRecognizeCallable().splitCall(responseObserver); | ||
|
||
RecognitionConfig recConfig = | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Let's use the full names, since there are several config things being passed around to add clarity: |
||
RecognitionConfig.newBuilder() | ||
.setEncoding(RecognitionConfig.AudioEncoding.LINEAR16) | ||
.setLanguageCode("en-US") | ||
.setSampleRateHertz(16000) | ||
.build(); | ||
StreamingRecognitionConfig config = | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Same thing for this variable: |
||
StreamingRecognitionConfig.newBuilder().setConfig(recConfig).build(); | ||
|
||
StreamingRecognizeRequest request = | ||
StreamingRecognizeRequest.newBuilder() | ||
.setStreamingConfig(config) | ||
.build(); // The first request in a streaming call has to be a config | ||
|
||
clientStream.send(request); | ||
|
||
// Get the target data line | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Short description of what a Target Data Line is. Is is the data from the microphone? |
||
targetDataLine = (TargetDataLine) AudioSystem.getLine(targetInfo); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. TargetDataLine targetDataLine = ... |
||
targetDataLine.open(audioFormat); | ||
targetDataLine.start(); | ||
System.out.println("Start speaking"); | ||
long startTime = System.currentTimeMillis(); | ||
// Audio Input Stream | ||
AudioInputStream audio = new AudioInputStream(targetDataLine); | ||
while (true) { | ||
long estimatedTime = System.currentTimeMillis() - startTime; | ||
byte[] data = new byte[6400]; | ||
audio.read(data); | ||
if (estimatedTime > durationMillSec) { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. if (estimatedTime > 60000) { // 60 seconds |
||
System.out.println("Stop speaking."); | ||
targetDataLine.stop(); | ||
targetDataLine.close(); | ||
break; | ||
} | ||
request = | ||
StreamingRecognizeRequest.newBuilder() | ||
.setAudioContent(ByteString.copyFrom(data)) | ||
.build(); | ||
clientStream.send(request); | ||
} | ||
} catch (Exception e) { | ||
System.out.println(e); | ||
} | ||
responseObserver.onComplete(); | ||
} | ||
|
||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Remove empty line |
||
// [END speech_streaming_mic_recognize] | ||
|
||
// [START speech_transcribe_file_with_enhanced_model] | ||
/** | ||
* Transcribe the given audio file using an enhanced model. | ||
|
@@ -833,8 +939,9 @@ public static void transcribeDiarization(String fileName) throws Exception { | |
SpeechRecognitionAlternative alternative = result.getAlternatives(0); | ||
System.out.format("Transcript : %s\n", alternative.getTranscript()); | ||
// The words array contains the entire transcript up until that point. | ||
//Referencing the last spoken word to get the associated Speaker tag | ||
System.out.format("Speaker Tag %s: %s\n", | ||
// Referencing the last spoken word to get the associated Speaker tag | ||
System.out.format( | ||
"Speaker Tag %s: %s\n", | ||
alternative.getWords((alternative.getWordsCount() - 1)).getSpeakerTag(), | ||
alternative.getTranscript()); | ||
} | ||
|
@@ -877,8 +984,9 @@ public static void transcribeDiarizationGcs(String gcsUri) throws Exception { | |
// use the first (most likely) one here. | ||
SpeechRecognitionAlternative alternative = result.getAlternatives(0); | ||
// The words array contains the entire transcript up until that point. | ||
//Referencing the last spoken word to get the associated Speaker tag | ||
System.out.format("Speaker Tag %s:%s\n", | ||
// Referencing the last spoken word to get the associated Speaker tag | ||
System.out.format( | ||
"Speaker Tag %s:%s\n", | ||
alternative.getWords((alternative.getWordsCount() - 1)).getSpeakerTag(), | ||
alternative.getTranscript()); | ||
} | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Let's remove the space, so that when importing to the docs, there isn't a blank line at the top. :)