From c65a0da5bad145814f85eb9c2c1cfc4da7c68ce3 Mon Sep 17 00:00:00 2001 From: Joey Wang Date: Sat, 29 Jul 2023 08:43:29 -0700 Subject: [PATCH] ios: Support multi-language (#220) --- .../rac/rac/Interactive/InteractiveView.swift | 1 + .../Interactive/Voice/SpeechRecognizer.swift | 4 +- .../Interactive/Voice/VoiceMessageView.swift | 3 +- .../ios/rac/rac/Network/WebSocketClient.swift | 13 +++-- .../Welcome/Settings/PreferenceSettings.swift | 8 +++ .../rac/Welcome/Settings/SettingsView.swift | 55 +++++++++++++++++++ .../ios/rac/rac/Welcome/WelcomeView.swift | 11 +++- 7 files changed, 86 insertions(+), 9 deletions(-) diff --git a/client/mobile/ios/rac/rac/Interactive/InteractiveView.swift b/client/mobile/ios/rac/rac/Interactive/InteractiveView.swift index e7977be5c..834e2a482 100644 --- a/client/mobile/ios/rac/rac/Interactive/InteractiveView.swift +++ b/client/mobile/ios/rac/rac/Interactive/InteractiveView.swift @@ -55,6 +55,7 @@ struct InteractiveView: View { VoiceMessageView(openMic: openMic, messages: $messages, state: $voiceState, + speechRecognizer: SpeechRecognizer(locale: preferenceSettings.languageOption.locale), onUpdateUserMessage: { message in if messages.last?.role == .user { messages[messages.count - 1].content = message diff --git a/client/mobile/ios/rac/rac/Interactive/Voice/SpeechRecognizer.swift b/client/mobile/ios/rac/rac/Interactive/Voice/SpeechRecognizer.swift index f4b0bcc3a..067448b22 100644 --- a/client/mobile/ios/rac/rac/Interactive/Voice/SpeechRecognizer.swift +++ b/client/mobile/ios/rac/rac/Interactive/Voice/SpeechRecognizer.swift @@ -38,8 +38,8 @@ actor SpeechRecognizer: ObservableObject { Initializes a new speech recognizer. If this is the first time you've used the class, it requests access to the speech recognizer and the microphone. */ - init() { - recognizer = SFSpeechRecognizer() + init(locale: Locale = .current) { + recognizer = SFSpeechRecognizer(locale: locale) guard recognizer != nil else { transcribe(RecognizerError.nilRecognizer) return diff --git a/client/mobile/ios/rac/rac/Interactive/Voice/VoiceMessageView.swift b/client/mobile/ios/rac/rac/Interactive/Voice/VoiceMessageView.swift index b72e58e20..de884e48d 100644 --- a/client/mobile/ios/rac/rac/Interactive/Voice/VoiceMessageView.swift +++ b/client/mobile/ios/rac/rac/Interactive/Voice/VoiceMessageView.swift @@ -82,7 +82,7 @@ struct VoiceMessageView: View { let openMic: Bool @Binding var messages: [ChatMessage] @Binding var state: VoiceState - @StateObject var speechRecognizer = SpeechRecognizer() + @StateObject var speechRecognizer: SpeechRecognizer @State private var isInputUpdated: Bool = true @State private var timer: Timer? = nil @@ -325,6 +325,7 @@ struct VoiceMessageView_Previews: PreviewProvider { ChatMessage(id: UUID(), role: .assistant, content: "Well thank you, Karina! I like your nam too. Now tell me, where do you live?") ]), state: .constant(.idle(streamingEnded: true)), + speechRecognizer: SpeechRecognizer(), onUpdateUserMessage: { _ in }, onSendUserMessage: { _ in }, onTapVoiceButton: { }) diff --git a/client/mobile/ios/rac/rac/Network/WebSocketClient.swift b/client/mobile/ios/rac/rac/Network/WebSocketClient.swift index 8a4c3acf1..4ee6735e9 100644 --- a/client/mobile/ios/rac/rac/Network/WebSocketClient.swift +++ b/client/mobile/ios/rac/rac/Network/WebSocketClient.swift @@ -24,7 +24,7 @@ protocol WebSocket: NSObject, ObservableObject { var onStringReceived: ((String) -> Void)? { get set } var onDataReceived: ((Data) -> Void)? { get set } var onErrorReceived: ((Error) -> Void)? { get set } - func connectSession(llmOption: LlmOption, characterId: String, userId: String?, token: String?) + func connectSession(languageOption: LanguageOption, llmOption: LlmOption, characterId: String, userId: String?, token: String?) func reconnectSession() func closeSession() func send(message: String) @@ -47,6 +47,7 @@ class WebSocketClient: NSObject, WebSocket, URLSessionWebSocketDelegate { private lazy var session = URLSession(configuration: .default, delegate: self, delegateQueue: OperationQueue()) + private var lastUsedLanguageOption: LanguageOption = .english private var lastUsedLlmOption: LlmOption = .gpt35 private var lastUsedCharacterId: String = "" private var lastUsedUserId: String? = nil @@ -86,7 +87,8 @@ class WebSocketClient: NSObject, WebSocket, URLSessionWebSocketDelegate { super.init() } - func connectSession(llmOption: LlmOption, characterId: String, userId: String?, token: String?) { + func connectSession(languageOption: LanguageOption, llmOption: LlmOption, characterId: String, userId: String?, token: String?) { + lastUsedLanguageOption = languageOption lastUsedLlmOption = llmOption lastUsedCharacterId = characterId // TODO: Use userId once it's ready @@ -96,6 +98,7 @@ class WebSocketClient: NSObject, WebSocket, URLSessionWebSocketDelegate { connectWebSocket(session: session, serverUrl: serverUrl, characterId: characterId, + languageOption: languageOption, llmOption: llmOption, clientId: clientId, token: token) @@ -105,6 +108,7 @@ class WebSocketClient: NSObject, WebSocket, URLSessionWebSocketDelegate { connectWebSocket(session: session, serverUrl: serverUrl, characterId: lastUsedCharacterId, + languageOption: lastUsedLanguageOption, llmOption: lastUsedLlmOption, clientId: lastUsedUserId ?? String(Int.random(in: 0...1010000000)), token: lastUsedToken) @@ -113,6 +117,7 @@ class WebSocketClient: NSObject, WebSocket, URLSessionWebSocketDelegate { private func connectWebSocket(session: URLSession, serverUrl: URL, characterId: String, + languageOption: LanguageOption, llmOption: LlmOption, clientId: String, token: String?) { @@ -122,7 +127,7 @@ class WebSocketClient: NSObject, WebSocket, URLSessionWebSocketDelegate { lastConnectingDate = Date() let wsScheme = serverUrl.scheme == "https" ? "wss" : "ws" - let wsPath = "\(wsScheme)://\(serverUrl.host ?? "")\(serverUrl.port.flatMap { ":\($0)" } ?? "")/ws/\(clientId)?platform=mobile&character_id=\(characterId)&llm_model=\(llmOption.rawValue)&token=\(token ?? "")" + let wsPath = "\(wsScheme)://\(serverUrl.host ?? "")\(serverUrl.port.flatMap { ":\($0)" } ?? "")/ws/\(clientId)?platform=mobile&language=\(languageOption.rawValue)&character_id=\(characterId)&llm_model=\(llmOption.rawValue)&token=\(token ?? "")" print("Connecting websocket: \(wsPath)") webSocket = session.webSocketTask(with: URL(string: wsPath)!) webSocket.resume() @@ -296,7 +301,7 @@ class MockWebSocket: NSObject, WebSocket { var onErrorReceived: ((Error) -> Void)? - func connectSession(llmOption: LlmOption, characterId: String, userId: String?, token: String?) { + func connectSession(languageOption: LanguageOption, llmOption: LlmOption, characterId: String, userId: String?, token: String?) { } func reconnectSession() { diff --git a/client/mobile/ios/rac/rac/Welcome/Settings/PreferenceSettings.swift b/client/mobile/ios/rac/rac/Welcome/Settings/PreferenceSettings.swift index 507104878..40725fec9 100644 --- a/client/mobile/ios/rac/rac/Welcome/Settings/PreferenceSettings.swift +++ b/client/mobile/ios/rac/rac/Welcome/Settings/PreferenceSettings.swift @@ -14,6 +14,7 @@ class PreferenceSettings: ObservableObject { static let hapticFeedbackKey = "enableHapticFeedback" static let useSearch = "useSearch" static let llmOptionKey = "largeLanguageModel" + static let languageOption = "languageOption" static let includedCommunityCharacterIds = "includedCommunityCharacterIds" } @@ -38,6 +39,13 @@ class PreferenceSettings: ObservableObject { print("Saved large language model preference: \(llmOption.displayName)") } } + @Published var languageOption: LanguageOption = LanguageOption(rawValue: UserDefaults.standard.string(forKey: Constants.languageOption) ?? "en-US") ?? .english { + didSet { + guard languageOption != oldValue else { return } + UserDefaults.standard.set(languageOption.rawValue, forKey: Constants.languageOption) + print("Saved language preference: \(languageOption.displayName)") + } + } @Published var includedCommunityCharacterIds: [String] = (UserDefaults.standard.string(forKey: Constants.includedCommunityCharacterIds) ?? "").split(separator: ",").map { String($0) } { didSet { guard includedCommunityCharacterIds != oldValue else { return } diff --git a/client/mobile/ios/rac/rac/Welcome/Settings/SettingsView.swift b/client/mobile/ios/rac/rac/Welcome/Settings/SettingsView.swift index 936f53ad9..b3f3594c3 100644 --- a/client/mobile/ios/rac/rac/Welcome/Settings/SettingsView.swift +++ b/client/mobile/ios/rac/rac/Welcome/Settings/SettingsView.swift @@ -47,6 +47,44 @@ enum LlmOption: RawRepresentable, Hashable, CaseIterable, Identifiable, Codable } } +enum LanguageOption: RawRepresentable, Hashable, CaseIterable, Identifiable, Codable { + + case english, spanish + + init?(rawValue: String) { + for option in LanguageOption.allCases { + if rawValue == option.rawValue { + self = option + return + } + } + return nil + } + + var id: String { rawValue } + var rawValue: String { + switch self { + case .english: + return "en-US" + case .spanish: + return "es-ES" + } + } + + var displayName: String { + switch self { + case .english: + return "English" + case .spanish: + return "Spanish" + } + } + + var locale: Locale { + Locale(identifier: rawValue) + } +} + struct SettingsView: View { @EnvironmentObject private var userSettings: UserSettings @EnvironmentObject private var preferenceSettings: PreferenceSettings @@ -98,6 +136,23 @@ struct SettingsView: View { Font.custom("Prompt", size: 18).weight(.medium) ) + Text("Conversation Language?") + .font( + Font.custom("Prompt", size: 16) + ) + + Picker("Conversation Language", selection: $preferenceSettings.languageOption) { + ForEach(LanguageOption.allCases) { languageOption in + Text(languageOption.displayName) + .font( + Font.custom("Prompt", size: 16) + ) + .tag(languageOption) + } + } + .padding(.bottom, 2) + .pickerStyle(.segmented) + Text("LLM Model?") .font( Font.custom("Prompt", size: 16) diff --git a/client/mobile/ios/rac/rac/Welcome/WelcomeView.swift b/client/mobile/ios/rac/rac/Welcome/WelcomeView.swift index 077542033..976820887 100644 --- a/client/mobile/ios/rac/rac/Welcome/WelcomeView.swift +++ b/client/mobile/ios/rac/rac/Welcome/WelcomeView.swift @@ -80,7 +80,8 @@ struct WelcomeView: View { VStack { Button { if webSocketConnectionStatusObserver.status == .disconnected, let characterId = character?.id { - webSocket.connectSession(llmOption: preferenceSettings.llmOption, + webSocket.connectSession(languageOption: preferenceSettings.languageOption, + llmOption: preferenceSettings.llmOption, characterId: characterId, userId: userSettings.userId, token: userSettings.userToken) @@ -122,6 +123,11 @@ struct WelcomeView: View { reconnectWebSocket(characterId: characterId) } } + .onChange(of: preferenceSettings.languageOption) { newValue in + if let characterId = character?.id { + reconnectWebSocket(characterId: characterId) + } + } .onChange(of: userSettings.isLoggedIn) { newValue in if let characterId = character?.id { reconnectWebSocket(characterId: characterId) @@ -138,7 +144,8 @@ struct WelcomeView: View { webSocket.onConnectionChanged = { status in self.webSocketConnectionStatusObserver.update(status: webSocket.status) } - webSocket.connectSession(llmOption: preferenceSettings.llmOption, + webSocket.connectSession(languageOption: preferenceSettings.languageOption, + llmOption: preferenceSettings.llmOption, characterId: characterId, userId: userSettings.userId, token: userSettings.userToken)