diff --git a/maestro-ai/src/main/java/maestro/ai/Prediction.kt b/maestro-ai/src/main/java/maestro/ai/Prediction.kt index c9cca754c3..332c6bb79d 100644 --- a/maestro-ai/src/main/java/maestro/ai/Prediction.kt +++ b/maestro-ai/src/main/java/maestro/ai/Prediction.kt @@ -12,22 +12,36 @@ data class Defect( ) @Serializable -private data class ModelResponse( +private data class AskForDefectsResponse( val defects: List, ) +@Serializable +private data class ExtractTextResponse( + val text: String? +) + object Prediction { + private val askForDefectsSchema by lazy { + readSchema("askForDefects") + } + + private val extractTextSchema by lazy { + readSchema("extractText") + } + /** * We use JSON mode/Structured Outputs to define the schema of the response we expect from the LLM. * - OpenAI: https://platform.openai.com/docs/guides/structured-outputs * - Gemini: https://ai.google.dev/gemini-api/docs/json-mode */ - private val askForDefectsSchema: String = run { - val resourceStream = this::class.java.getResourceAsStream("/askForDefects_schema.json") - ?: throw IllegalStateException("Could not find askForDefects_schema.json in resources") + private fun readSchema(name: String): String { + val fileName = "/${name}_schema.json" + val resourceStream = this::class.java.getResourceAsStream(fileName) + ?: throw IllegalStateException("Could not find $fileName in resources") - resourceStream.bufferedReader().use { it.readText() } + return resourceStream.bufferedReader().use { it.readText() } } private val json = Json { ignoreUnknownKeys = true } @@ -126,7 +140,7 @@ object Prediction { println("--- RAW RESPONSE END ---") } - val defects = json.decodeFromString(aiResponse.response) + val defects = json.decodeFromString(aiResponse.response) return defects.defects } @@ -208,7 +222,52 @@ object Prediction { println("--- RAW RESPONSE END ---") } - val response = json.decodeFromString(aiResponse.response) + val response = json.decodeFromString(aiResponse.response) return response.defects.firstOrNull() } + + suspend fun extractText( + aiClient: AI, + screen: ByteArray, + query: String, + ): String { + val prompt = buildString { + append("What text on the screen matches the following query: $query") + + append( + """ + | + |RULES: + |* Provide response as a valid JSON, with structure described below. + """.trimMargin("|") + ) + + append( + """ + | + |* You must provide result as a valid JSON object, matching this structure: + | + | { + | "text": + | } + | + |DO NOT output any other information in the JSON object. + """.trimMargin("|") + ) + } + + val aiResponse = aiClient.chatCompletion( + prompt, + model = aiClient.defaultModel, + maxTokens = 4096, + identifier = "perform-assertion", + imageDetail = "high", + images = listOf(screen), + jsonSchema = if (aiClient is OpenAI) json.parseToJsonElement(extractTextSchema).jsonObject else null, + ) + + val response = json.decodeFromString(aiResponse.response) + return response.text ?: "" + } + } diff --git a/maestro-ai/src/main/resources/extractText_schema.json b/maestro-ai/src/main/resources/extractText_schema.json new file mode 100644 index 0000000000..7fc613d322 --- /dev/null +++ b/maestro-ai/src/main/resources/extractText_schema.json @@ -0,0 +1,17 @@ +{ + "name": "extractText", + "description": "Extracts text from an image based on a given query", + "strict": true, + "schema": { + "type": "object", + "required": [ + "text" + ], + "additionalProperties": false, + "properties": { + "text": { + "type": "string" + } + } + } +} \ No newline at end of file diff --git a/maestro-orchestra-models/src/main/java/maestro/orchestra/Commands.kt b/maestro-orchestra-models/src/main/java/maestro/orchestra/Commands.kt index 2a8c1b7643..5d9ca617aa 100644 --- a/maestro-orchestra-models/src/main/java/maestro/orchestra/Commands.kt +++ b/maestro-orchestra-models/src/main/java/maestro/orchestra/Commands.kt @@ -37,7 +37,7 @@ sealed interface Command { fun visible(): Boolean = true val label: String? - + val optional: Boolean } @@ -65,18 +65,23 @@ data class SwipeCommand( label != null -> { label } + elementSelector != null && direction != null -> { "Swiping in $direction direction on ${elementSelector.description()}" } + direction != null -> { "Swiping in $direction direction in $duration ms" } + startPoint != null && endPoint != null -> { "Swipe from (${startPoint.x},${startPoint.y}) to (${endPoint.x},${endPoint.y}) in $duration ms" } + startRelative != null && endRelative != null -> { "Swipe from ($startRelative) to ($endRelative) in $duration ms" } + else -> "Invalid input to swipe command" } } @@ -113,11 +118,15 @@ data class ScrollUntilVisibleCommand( private fun String.speedToDuration(): String { val duration = ((1000 * (100 - this.toLong()).toDouble() / 100).toLong() + 1) - return if (duration < 0) { DEFAULT_SCROLL_DURATION } else duration.toString() + return if (duration < 0) { + DEFAULT_SCROLL_DURATION + } else duration.toString() } private fun String.timeoutToMillis(): String { - return if (this.toLong() < 0) { DEFAULT_TIMEOUT_IN_MILLIS } else this + return if (this.toLong() < 0) { + DEFAULT_TIMEOUT_IN_MILLIS + } else this } override fun description(): String { @@ -336,7 +345,7 @@ data class AssertCommand( ) : Command { override fun description(): String { - if (label != null){ + if (label != null) { return label } val timeoutStr = timeout?.let { " within $timeout ms" } ?: "" @@ -382,7 +391,8 @@ data class AssertConditionCommand( } override fun description(): String { - val optional = if (optional || condition.visible?.optional == true || condition.notVisible?.optional == true ) "(Optional) " else "" + val optional = + if (optional || condition.visible?.optional == true || condition.notVisible?.optional == true) "(Optional) " else "" return label ?: "Assert that $optional${condition.description()}" } @@ -425,6 +435,25 @@ data class AssertWithAICommand( } } +data class ExtractTextWithAICommand( + val query: String, + val outputVariable: String, + override val optional: Boolean = true, + override val label: String? = null +) : Command { + override fun description(): String { + if (label != null) return label + + return "Extract text with AI: $query" + } + + override fun evaluateScripts(jsEngine: JsEngine): Command { + return copy( + query = query.evaluateScripts(jsEngine), + ) + } +} + data class InputTextCommand( val text: String, override val label: String? = null, @@ -454,7 +483,7 @@ data class LaunchAppCommand( ) : Command { override fun description(): String { - if (label != null){ + if (label != null) { return label } @@ -782,12 +811,15 @@ data class RepeatCommand( label != null -> { label } + condition != null && timesInt > 1 -> { "Repeat while ${condition.description()} (up to $timesInt times)" } + condition != null -> { "Repeat while ${condition.description()}" } + timesInt > 1 -> "Repeat $timesInt times" else -> "Repeat indefinitely" } @@ -824,6 +856,7 @@ data class RetryCommand( label != null -> { label } + else -> "Retry $maxAttempts times" } } @@ -943,8 +976,8 @@ data class TravelCommand( val dLon = Math.toRadians(aLon - oLon) val a = Math.sin(dLat / 2) * Math.sin(dLat / 2) + - Math.cos(Math.toRadians(oLat)) * Math.cos(Math.toRadians(aLat)) * - Math.sin(dLon / 2) * Math.sin(dLon / 2) + Math.cos(Math.toRadians(oLat)) * Math.cos(Math.toRadians(aLat)) * + Math.sin(dLon / 2) * Math.sin(dLon / 2) val c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a)) val distance = earthRadius * c * 1000 // convert to meters @@ -960,7 +993,12 @@ data class TravelCommand( override fun evaluateScripts(jsEngine: JsEngine): Command { return copy( - points = points.map { it.copy(latitude = it.latitude.evaluateScripts(jsEngine), longitude = it.longitude.evaluateScripts(jsEngine)) } + points = points.map { + it.copy( + latitude = it.latitude.evaluateScripts(jsEngine), + longitude = it.longitude.evaluateScripts(jsEngine) + ) + } ) } @@ -987,7 +1025,7 @@ data class AddMediaCommand( val mediaPaths: List, override val label: String? = null, override val optional: Boolean = false, -): Command { +) : Command { override fun description(): String { return label ?: "Adding media files(${mediaPaths.size}) to the device" diff --git a/maestro-orchestra-models/src/main/java/maestro/orchestra/MaestroCommand.kt b/maestro-orchestra-models/src/main/java/maestro/orchestra/MaestroCommand.kt index 6a20ded467..7f2cc86ff3 100644 --- a/maestro-orchestra-models/src/main/java/maestro/orchestra/MaestroCommand.kt +++ b/maestro-orchestra-models/src/main/java/maestro/orchestra/MaestroCommand.kt @@ -39,6 +39,7 @@ data class MaestroCommand( val assertConditionCommand: AssertConditionCommand? = null, val assertNoDefectsWithAICommand: AssertNoDefectsWithAICommand? = null, val assertWithAICommand: AssertWithAICommand? = null, + val extractTextWithAICommand: ExtractTextWithAICommand? = null, val inputTextCommand: InputTextCommand? = null, val inputRandomTextCommand: InputRandomCommand? = null, val launchAppCommand: LaunchAppCommand? = null, @@ -82,6 +83,7 @@ data class MaestroCommand( assertConditionCommand = command as? AssertConditionCommand, assertNoDefectsWithAICommand = command as? AssertNoDefectsWithAICommand, assertWithAICommand = command as? AssertWithAICommand, + extractTextWithAICommand = command as? ExtractTextWithAICommand, inputTextCommand = command as? InputTextCommand, inputRandomTextCommand = command as? InputRandomCommand, launchAppCommand = command as? LaunchAppCommand, @@ -125,6 +127,7 @@ data class MaestroCommand( assertConditionCommand != null -> assertConditionCommand assertNoDefectsWithAICommand != null -> assertNoDefectsWithAICommand assertWithAICommand != null -> assertWithAICommand + extractTextWithAICommand != null -> extractTextWithAICommand inputTextCommand != null -> inputTextCommand inputRandomTextCommand != null -> inputRandomTextCommand launchAppCommand != null -> launchAppCommand diff --git a/maestro-orchestra/src/main/java/maestro/orchestra/Orchestra.kt b/maestro-orchestra/src/main/java/maestro/orchestra/Orchestra.kt index 53c2735637..25dfe7afb9 100644 --- a/maestro-orchestra/src/main/java/maestro/orchestra/Orchestra.kt +++ b/maestro-orchestra/src/main/java/maestro/orchestra/Orchestra.kt @@ -84,7 +84,7 @@ class Orchestra( private val onCommandStart: (Int, MaestroCommand) -> Unit = { _, _ -> }, private val onCommandComplete: (Int, MaestroCommand) -> Unit = { _, _ -> }, private val onCommandFailed: (Int, MaestroCommand, Throwable) -> ErrorResolution = { _, _, e -> throw e }, - private val onCommandWarned: (Int, MaestroCommand) -> Unit = { _, _ -> }, + private val onCommandWarned: (Int, MaestroCommand) -> Unit = { _, _ -> }, private val onCommandSkipped: (Int, MaestroCommand) -> Unit = { _, _ -> }, private val onCommandReset: (MaestroCommand) -> Unit = {}, private val onCommandMetadataUpdate: (MaestroCommand, CommandMetadata) -> Unit = { _, _ -> }, @@ -197,7 +197,8 @@ class Orchestra( executeCommand(evaluatedCommand, config) onCommandComplete(index, command) } catch (e: MaestroException) { - val isOptional = command.asCommand()?.optional == true || command.elementSelector()?.optional == true + val isOptional = + command.asCommand()?.optional == true || command.elementSelector()?.optional == true if (isOptional) throw CommandWarned(e.message) else throw e } @@ -255,11 +256,11 @@ class Orchestra( return when (command) { is TapOnElementCommand -> { tapOnElement( - command = command, - maestroCommand = maestroCommand, - retryIfNoChange = command.retryIfNoChange ?: true, - waitUntilVisible = command.waitUntilVisible ?: false, - config = config, + command = command, + maestroCommand = maestroCommand, + retryIfNoChange = command.retryIfNoChange ?: true, + waitUntilVisible = command.waitUntilVisible ?: false, + config = config, ) } @@ -276,6 +277,7 @@ class Orchestra( is AssertConditionCommand -> assertConditionCommand(command) is AssertNoDefectsWithAICommand -> assertNoDefectsWithAICommand(command) is AssertWithAICommand -> assertWithAICommand(command) + is ExtractTextWithAICommand -> extractTextWithAICommand(command) is InputTextCommand -> inputTextCommand(command) is InputRandomCommand -> inputTextRandomCommand(command) is LaunchAppCommand -> launchAppCommand(command) @@ -413,6 +415,26 @@ class Orchestra( false } + private fun extractTextWithAICommand(command: ExtractTextWithAICommand): Boolean = runBlocking { + // Extract text from the screen using AI + if (ai == null) { + throw MaestroException.AINotAvailable("AI client is not available. Did you export $AI_KEY_ENV_VAR?") + } + + val imageData = Buffer() + maestro.takeScreenshot(imageData, compressed = false) + + val text = Prediction.extractText( + aiClient = ai, + screen = imageData.copy().readByteArray(), + query = command.query, + ) + + jsEngine.putEnv(command.outputVariable, text) + + false + } + private fun evalScriptCommand(command: EvalScriptCommand): Boolean { command.scriptString.evaluateScripts(jsEngine) @@ -508,15 +530,19 @@ class Orchestra( } val oldMetadata = getMetadata(maestroCommand); val metadata = oldMetadata.copy( - action = Action.MultipleSwipePoint( - direction = direction, - points = (oldMetadata.action as? Action.MultipleSwipePoint)?.points?.toMutableList()?.apply { - add(Point(deviceInfo.widthGrid / 2, deviceInfo.heightGrid / 2)) - } ?: listOf(Point(deviceInfo.widthGrid / 2, deviceInfo.heightGrid / 2)) - ), + action = Action.MultipleSwipePoint( + direction = direction, + points = (oldMetadata.action as? Action.MultipleSwipePoint)?.points?.toMutableList()?.apply { + add(Point(deviceInfo.widthGrid / 2, deviceInfo.heightGrid / 2)) + } ?: listOf(Point(deviceInfo.widthGrid / 2, deviceInfo.heightGrid / 2)) + ), ) updateMetadata(maestroCommand, metadata); - maestro.swipeFromCenter(direction, durationMs = command.scrollDuration.toLong(), waitToSettleTimeoutMs = command.waitToSettleTimeoutMs) + maestro.swipeFromCenter( + direction, + durationMs = command.scrollDuration.toLong(), + waitToSettleTimeoutMs = command.waitToSettleTimeoutMs + ) } while (System.currentTimeMillis() < endTime) throw MaestroException.ElementNotFound( @@ -578,7 +604,7 @@ class Orchestra( val maxRetries = (command.maxRetries?.toIntOrNull() ?: 1).coerceAtMost(MAX_RETRIES_ALLOWED) var attempt = 0 - while(attempt <= maxRetries) { + while (attempt <= maxRetries) { try { return runSubFlow(command.commands, config, command.config) } catch (exception: Throwable) { @@ -587,8 +613,9 @@ class Orchestra( break } - val message = "Retrying the commands due to an error: ${exception.message} while execution (Attempt ${attempt + 1})" - logger.error("Attempt ${attempt +1} failed for retry command", exception) + val message = + "Retrying the commands due to an error: ${exception.message} while execution (Attempt ${attempt + 1})" + logger.error("Attempt ${attempt + 1} failed for retry command", exception) insights.report(Insight(message = message, Insight.Level.WARNING)) } attempt++ @@ -724,7 +751,8 @@ class Orchestra( onCommandComplete(index, command) } } catch (exception: MaestroException) { - val isOptional = command.asCommand()?.optional == true || command.elementSelector()?.optional == true + val isOptional = + command.asCommand()?.optional == true || command.elementSelector()?.optional == true if (isOptional) throw CommandWarned(exception.message) else throw exception } @@ -898,7 +926,7 @@ class Orchestra( ): Boolean { val result = findElement(command.selector, optional = command.optional) val metadata = getMetadata(maestroCommand).copy( - action = Action.TapPoint(Point(x = result.element.bounds.center().x, y = result.element.bounds.center().y)), + action = Action.TapPoint(Point(x = result.element.bounds.center().x, y = result.element.bounds.center().y)), ) updateMetadata(maestroCommand, metadata); @@ -948,7 +976,12 @@ class Orchestra( } val metadata = getMetadata(maestroCommand).copy( - action = Action.TapPoint(Point(deviceInfo.widthGrid * percentX / 100, deviceInfo.heightGrid * percentY / 100)), + action = Action.TapPoint( + Point( + deviceInfo.widthGrid * percentX / 100, + deviceInfo.heightGrid * percentY / 100 + ) + ), ) updateMetadata(maestroCommand, metadata); maestro.tapOnRelative( @@ -966,7 +999,7 @@ class Orchestra( } val metadata = getMetadata(maestroCommand).copy( - action = Action.TapPoint(Point(x, y)), + action = Action.TapPoint(Point(x, y)), ) updateMetadata(maestroCommand, metadata); maestro.tap( @@ -1190,45 +1223,66 @@ class Orchestra( val metadata = getMetadata(maestroCommand).copy( action = Action.SwipePoint.WithDirection( direction = direction, - startPoint = Point(uiElement.element.bounds.center().x, uiElement.element.bounds.center().y), + startPoint = Point(uiElement.element.bounds.center().x, uiElement.element.bounds.center().y), ), ) updateMetadata(maestroCommand, metadata); - maestro.swipe(direction, uiElement.element, command.duration, waitToSettleTimeoutMs = command.waitToSettleTimeoutMs) + maestro.swipe( + direction, + uiElement.element, + command.duration, + waitToSettleTimeoutMs = command.waitToSettleTimeoutMs + ) } startRelative != null && endRelative != null -> { val startPoints = startRelative.replace("%", "") - .split(",").map { it.trim().toInt() } + .split(",").map { it.trim().toInt() } val endPoint = endRelative.replace("%", "") - .split(",").map { it.trim().toInt() } + .split(",").map { it.trim().toInt() } val metadata = getMetadata(maestroCommand).copy( - action = Action.SwipePoint.WithEndPoint( - startPoint = Point(startPoints[0] * deviceInfo.widthGrid / 100, startPoints[1] * deviceInfo.widthGrid / 100), - endPoint = Point(endPoint[0] * deviceInfo.widthGrid / 100, endPoint[1] * deviceInfo.widthGrid / 100), - ), + action = Action.SwipePoint.WithEndPoint( + startPoint = Point( + startPoints[0] * deviceInfo.widthGrid / 100, + startPoints[1] * deviceInfo.widthGrid / 100 + ), + endPoint = Point( + endPoint[0] * deviceInfo.widthGrid / 100, + endPoint[1] * deviceInfo.widthGrid / 100 + ), + ), ) updateMetadata(maestroCommand, metadata); - maestro.swipe(startRelative = startRelative, endRelative = endRelative, duration = command.duration, waitToSettleTimeoutMs = command.waitToSettleTimeoutMs) + maestro.swipe( + startRelative = startRelative, + endRelative = endRelative, + duration = command.duration, + waitToSettleTimeoutMs = command.waitToSettleTimeoutMs + ) } direction != null -> { - val metadata = getMetadata(maestroCommand).copy( - action = Action.SwipePoint.WithDirection( - direction = direction, - startPoint = Point(deviceInfo.widthGrid / 2, deviceInfo.heightGrid / 2), - ), - ) - updateMetadata(maestroCommand, metadata); - maestro.swipe(swipeDirection = direction, duration = command.duration, waitToSettleTimeoutMs = command.waitToSettleTimeoutMs) + val metadata = getMetadata(maestroCommand).copy( + action = Action.SwipePoint.WithDirection( + direction = direction, + startPoint = Point(deviceInfo.widthGrid / 2, deviceInfo.heightGrid / 2), + ), + ) + updateMetadata(maestroCommand, metadata); + maestro.swipe( + swipeDirection = direction, + duration = command.duration, + waitToSettleTimeoutMs = command.waitToSettleTimeoutMs + ) } + start != null && end != null -> { - maestro.swipe( - startPoint = start, - endPoint = end, - duration = command.duration, - waitToSettleTimeoutMs = command.waitToSettleTimeoutMs - ) + maestro.swipe( + startPoint = start, + endPoint = end, + duration = command.duration, + waitToSettleTimeoutMs = command.waitToSettleTimeoutMs + ) } else -> error("Illegal arguments for swiping") @@ -1244,7 +1298,7 @@ class Orchestra( private fun copyTextFromCommand(command: CopyTextFromCommand, maestroCommand: MaestroCommand): Boolean { val result = findElement(command.selector, optional = command.optional) val metadata = getMetadata(maestroCommand).copy( - action = Action.TapPoint(Point(result.element.bounds.center().x, result.element.bounds.center().y)), + action = Action.TapPoint(Point(result.element.bounds.center().x, result.element.bounds.center().y)), ) updateMetadata(maestroCommand, metadata); copiedText = resolveText(result.element.treeNode.attributes) diff --git a/maestro-orchestra/src/main/java/maestro/orchestra/yaml/YamlExtractTextWithAI.kt b/maestro-orchestra/src/main/java/maestro/orchestra/yaml/YamlExtractTextWithAI.kt new file mode 100644 index 0000000000..27f57083fb --- /dev/null +++ b/maestro-orchestra/src/main/java/maestro/orchestra/yaml/YamlExtractTextWithAI.kt @@ -0,0 +1,23 @@ +package maestro.orchestra.yaml + +import com.fasterxml.jackson.annotation.JsonCreator + +data class YamlExtractTextWithAI( + val query: String, + val outputVariable: String = "aiOutput", + val optional: Boolean = true, + val label: String? = null, +) { + + companion object { + + @JvmStatic + @JsonCreator(mode = JsonCreator.Mode.DELEGATING) + fun parse(query: String): YamlExtractTextWithAI { + return YamlExtractTextWithAI( + query = query, + optional = true, + ) + } + } +} \ No newline at end of file diff --git a/maestro-orchestra/src/main/java/maestro/orchestra/yaml/YamlFluentCommand.kt b/maestro-orchestra/src/main/java/maestro/orchestra/yaml/YamlFluentCommand.kt index 514221b672..237138120e 100644 --- a/maestro-orchestra/src/main/java/maestro/orchestra/yaml/YamlFluentCommand.kt +++ b/maestro-orchestra/src/main/java/maestro/orchestra/yaml/YamlFluentCommand.kt @@ -23,7 +23,49 @@ import com.fasterxml.jackson.annotation.JsonCreator import maestro.KeyCode import maestro.Point import maestro.TapRepeat -import maestro.orchestra.* +import maestro.orchestra.AddMediaCommand +import maestro.orchestra.AssertConditionCommand +import maestro.orchestra.AssertNoDefectsWithAICommand +import maestro.orchestra.AssertWithAICommand +import maestro.orchestra.BackPressCommand +import maestro.orchestra.ClearKeychainCommand +import maestro.orchestra.ClearStateCommand +import maestro.orchestra.Condition +import maestro.orchestra.CopyTextFromCommand +import maestro.orchestra.ElementSelector +import maestro.orchestra.ElementTrait +import maestro.orchestra.EraseTextCommand +import maestro.orchestra.EvalScriptCommand +import maestro.orchestra.ExtractTextWithAICommand +import maestro.orchestra.HideKeyboardCommand +import maestro.orchestra.InputRandomCommand +import maestro.orchestra.InputRandomType +import maestro.orchestra.InputTextCommand +import maestro.orchestra.KillAppCommand +import maestro.orchestra.LaunchAppCommand +import maestro.orchestra.MaestroCommand +import maestro.orchestra.MaestroConfig +import maestro.orchestra.OpenLinkCommand +import maestro.orchestra.PasteTextCommand +import maestro.orchestra.PressKeyCommand +import maestro.orchestra.RepeatCommand +import maestro.orchestra.RetryCommand +import maestro.orchestra.RunFlowCommand +import maestro.orchestra.RunScriptCommand +import maestro.orchestra.ScrollCommand +import maestro.orchestra.ScrollUntilVisibleCommand +import maestro.orchestra.SetAirplaneModeCommand +import maestro.orchestra.SetLocationCommand +import maestro.orchestra.StartRecordingCommand +import maestro.orchestra.StopAppCommand +import maestro.orchestra.StopRecordingCommand +import maestro.orchestra.SwipeCommand +import maestro.orchestra.TakeScreenshotCommand +import maestro.orchestra.TapOnElementCommand +import maestro.orchestra.TapOnPointV2Command +import maestro.orchestra.ToggleAirplaneModeCommand +import maestro.orchestra.TravelCommand +import maestro.orchestra.WaitForAnimationToEndCommand import maestro.orchestra.error.InvalidFlowFile import maestro.orchestra.error.MediaFileNotFound import maestro.orchestra.error.SyntaxError @@ -43,6 +85,7 @@ data class YamlFluentCommand( val assertTrue: YamlAssertTrue? = null, val assertNoDefectsWithAI: YamlAssertNoDefectsWithAI? = null, val assertWithAI: YamlAssertWithAI? = null, + val extractTextWithAI: YamlExtractTextWithAI? = null, val back: YamlActionBack? = null, val clearKeychain: YamlActionClearKeychain? = null, val hideKeyboard: YamlActionHideKeyboard? = null, @@ -99,6 +142,7 @@ data class YamlFluentCommand( ) ) ) + assertNotVisible != null -> listOf( MaestroCommand( AssertConditionCommand( @@ -110,6 +154,7 @@ data class YamlFluentCommand( ) ) ) + assertTrue != null -> listOf( MaestroCommand( AssertConditionCommand( @@ -121,6 +166,7 @@ data class YamlFluentCommand( ) ) ) + assertNoDefectsWithAI != null -> listOf( MaestroCommand( AssertNoDefectsWithAICommand( @@ -129,6 +175,7 @@ data class YamlFluentCommand( ) ) ) + assertWithAI != null -> listOf( MaestroCommand( AssertWithAICommand( @@ -138,19 +185,99 @@ data class YamlFluentCommand( ) ) ) + + extractTextWithAI != null -> listOf( + MaestroCommand( + ExtractTextWithAICommand( + query = extractTextWithAI.query, + outputVariable = extractTextWithAI.outputVariable, + optional = extractTextWithAI.optional, + label = extractTextWithAI.label, + ) + ) + ) + addMedia != null -> listOf( MaestroCommand( addMediaCommand = addMediaCommand(addMedia, flowPath) ) ) - inputText != null -> listOf(MaestroCommand(InputTextCommand(text = inputText.text, label = inputText.label, optional = inputText.optional))) - inputRandomText != null -> listOf(MaestroCommand(InputRandomCommand(inputType = InputRandomType.TEXT, length = inputRandomText.length, label = inputRandomText.label, optional = inputRandomText.optional))) - inputRandomNumber != null -> listOf(MaestroCommand(InputRandomCommand(inputType = InputRandomType.NUMBER, length = inputRandomNumber.length, label = inputRandomNumber.label, optional = inputRandomNumber.optional))) - inputRandomEmail != null -> listOf(MaestroCommand(InputRandomCommand(inputType = InputRandomType.TEXT_EMAIL_ADDRESS, label = inputRandomEmail.label, optional = inputRandomEmail.optional))) - inputRandomPersonName != null -> listOf(MaestroCommand(InputRandomCommand(inputType = InputRandomType.TEXT_PERSON_NAME, label = inputRandomPersonName.label, optional = inputRandomPersonName.optional))) + + inputText != null -> listOf( + MaestroCommand( + InputTextCommand( + text = inputText.text, + label = inputText.label, + optional = inputText.optional + ) + ) + ) + + inputRandomText != null -> listOf( + MaestroCommand( + InputRandomCommand( + inputType = InputRandomType.TEXT, + length = inputRandomText.length, + label = inputRandomText.label, + optional = inputRandomText.optional + ) + ) + ) + + inputRandomNumber != null -> listOf( + MaestroCommand( + InputRandomCommand( + inputType = InputRandomType.NUMBER, + length = inputRandomNumber.length, + label = inputRandomNumber.label, + optional = inputRandomNumber.optional + ) + ) + ) + + inputRandomEmail != null -> listOf( + MaestroCommand( + InputRandomCommand( + inputType = InputRandomType.TEXT_EMAIL_ADDRESS, + label = inputRandomEmail.label, + optional = inputRandomEmail.optional + ) + ) + ) + + inputRandomPersonName != null -> listOf( + MaestroCommand( + InputRandomCommand( + inputType = InputRandomType.TEXT_PERSON_NAME, + label = inputRandomPersonName.label, + optional = inputRandomPersonName.optional + ) + ) + ) + swipe != null -> listOf(swipeCommand(swipe)) - openLink != null -> listOf(MaestroCommand(OpenLinkCommand(link = openLink.link, autoVerify = openLink.autoVerify, browser = openLink.browser, label = openLink.label, optional = openLink.optional))) - pressKey != null -> listOf(MaestroCommand(PressKeyCommand(code = KeyCode.getByName(pressKey.key) ?: throw SyntaxError("Unknown key name: $pressKey"), label = pressKey.label, optional = pressKey.optional))) + openLink != null -> listOf( + MaestroCommand( + OpenLinkCommand( + link = openLink.link, + autoVerify = openLink.autoVerify, + browser = openLink.browser, + label = openLink.label, + optional = openLink.optional + ) + ) + ) + + pressKey != null -> listOf( + MaestroCommand( + PressKeyCommand( + code = KeyCode.getByName(pressKey.key) ?: throw SyntaxError("Unknown key name: $pressKey"), + label = pressKey.label, + optional = pressKey.optional + ) + ) + ) + eraseText != null -> listOf(eraseCommand(eraseText)) action != null -> listOf( when (action) { @@ -162,12 +289,46 @@ data class YamlFluentCommand( else -> error("Unknown navigation target: $action") } ) + back != null -> listOf(MaestroCommand(BackPressCommand(label = back.label, optional = back.optional))) - clearKeychain != null -> listOf(MaestroCommand(ClearKeychainCommand(label = clearKeychain.label, optional = clearKeychain.optional))) - hideKeyboard != null -> listOf(MaestroCommand(HideKeyboardCommand(label = hideKeyboard.label, optional = hideKeyboard.optional))) - pasteText != null -> listOf(MaestroCommand(PasteTextCommand(label = pasteText.label, optional = pasteText.optional))) + clearKeychain != null -> listOf( + MaestroCommand( + ClearKeychainCommand( + label = clearKeychain.label, + optional = clearKeychain.optional + ) + ) + ) + + hideKeyboard != null -> listOf( + MaestroCommand( + HideKeyboardCommand( + label = hideKeyboard.label, + optional = hideKeyboard.optional + ) + ) + ) + + pasteText != null -> listOf( + MaestroCommand( + PasteTextCommand( + label = pasteText.label, + optional = pasteText.optional + ) + ) + ) + scroll != null -> listOf(MaestroCommand(ScrollCommand(label = scroll.label, optional = scroll.optional))) - takeScreenshot != null -> listOf(MaestroCommand(TakeScreenshotCommand(path = takeScreenshot.path, label = takeScreenshot.label, optional = takeScreenshot.optional))) + takeScreenshot != null -> listOf( + MaestroCommand( + TakeScreenshotCommand( + path = takeScreenshot.path, + label = takeScreenshot.label, + optional = takeScreenshot.optional + ) + ) + ) + extendedWaitUntil != null -> listOf(extendedWait(extendedWaitUntil)) stopApp != null -> listOf( MaestroCommand( @@ -178,6 +339,7 @@ data class YamlFluentCommand( ) ) ) + killApp != null -> listOf( MaestroCommand( KillAppCommand( @@ -187,6 +349,7 @@ data class YamlFluentCommand( ) ) ) + clearState != null -> listOf( MaestroCommand( ClearStateCommand( @@ -196,6 +359,7 @@ data class YamlFluentCommand( ) ) ) + runFlow != null -> listOf(runFlowCommand(appId, flowPath, runFlow)) setLocation != null -> listOf( MaestroCommand( @@ -207,12 +371,15 @@ data class YamlFluentCommand( ) ) ) + repeat != null -> listOf( repeatCommand(repeat, flowPath, appId) ) + retry != null -> listOf( retryCommand(retry, flowPath, appId) ) + copyTextFrom != null -> listOf(copyTextFromCommand(copyTextFrom)) runScript != null -> listOf( MaestroCommand( @@ -227,6 +394,7 @@ data class YamlFluentCommand( ) ) ) + waitForAnimationToEnd != null -> listOf( MaestroCommand( WaitForAnimationToEndCommand( @@ -236,6 +404,7 @@ data class YamlFluentCommand( ) ) ) + evalScript != null -> listOf( MaestroCommand( EvalScriptCommand( @@ -245,18 +414,55 @@ data class YamlFluentCommand( ) ) ) + scrollUntilVisible != null -> listOf(scrollUntilVisibleCommand(scrollUntilVisible)) travel != null -> listOf(travelCommand(travel)) - startRecording != null -> listOf(MaestroCommand(StartRecordingCommand(startRecording.path, startRecording.label, startRecording.optional))) - stopRecording != null -> listOf(MaestroCommand(StopRecordingCommand(stopRecording.label, stopRecording.optional))) + startRecording != null -> listOf( + MaestroCommand( + StartRecordingCommand( + startRecording.path, + startRecording.label, + startRecording.optional + ) + ) + ) + + stopRecording != null -> listOf( + MaestroCommand( + StopRecordingCommand( + stopRecording.label, + stopRecording.optional + ) + ) + ) + doubleTapOn != null -> { val yamlDelay = (doubleTapOn as? YamlElementSelector)?.delay?.toLong() - val delay = if (yamlDelay != null && yamlDelay >= 0) yamlDelay else TapOnElementCommand.DEFAULT_REPEAT_DELAY + val delay = + if (yamlDelay != null && yamlDelay >= 0) yamlDelay else TapOnElementCommand.DEFAULT_REPEAT_DELAY val tapRepeat = TapRepeat(2, delay) listOf(tapCommand(doubleTapOn, tapRepeat = tapRepeat)) } - setAirplaneMode != null -> listOf(MaestroCommand(SetAirplaneModeCommand(setAirplaneMode.value, setAirplaneMode.label, setAirplaneMode.optional))) - toggleAirplaneMode != null -> listOf(MaestroCommand(ToggleAirplaneModeCommand(toggleAirplaneMode.label, toggleAirplaneMode.optional))) + + setAirplaneMode != null -> listOf( + MaestroCommand( + SetAirplaneModeCommand( + setAirplaneMode.value, + setAirplaneMode.label, + setAirplaneMode.optional + ) + ) + ) + + toggleAirplaneMode != null -> listOf( + MaestroCommand( + ToggleAirplaneModeCommand( + toggleAirplaneMode.label, + toggleAirplaneMode.optional + ) + ) + ) + else -> throw SyntaxError("Invalid command: No mapping provided for $this") } } @@ -364,8 +570,10 @@ data class YamlFluentCommand( throw SyntaxError("Invalid travel point: $point") } - val latitude = spitPoint[0].toDoubleOrNull() ?: throw SyntaxError("Invalid travel point latitude: $point") - val longitude = spitPoint[1].toDoubleOrNull() ?: throw SyntaxError("Invalid travel point longitude: $point") + val latitude = + spitPoint[0].toDoubleOrNull() ?: throw SyntaxError("Invalid travel point latitude: $point") + val longitude = + spitPoint[1].toDoubleOrNull() ?: throw SyntaxError("Invalid travel point longitude: $point") TravelCommand.GeoPoint( latitude = latitude.toString(), @@ -392,9 +600,21 @@ data class YamlFluentCommand( private fun eraseCommand(eraseText: YamlEraseText): MaestroCommand { return if (eraseText.charactersToErase != null) { - MaestroCommand(EraseTextCommand(charactersToErase = eraseText.charactersToErase, label = eraseText.label, optional = eraseText.optional)) + MaestroCommand( + EraseTextCommand( + charactersToErase = eraseText.charactersToErase, + label = eraseText.label, + optional = eraseText.optional + ) + ) } else { - MaestroCommand(EraseTextCommand(charactersToErase = null, label = eraseText.label, optional = eraseText.optional)) + MaestroCommand( + EraseTextCommand( + charactersToErase = null, + label = eraseText.label, + optional = eraseText.optional + ) + ) } } @@ -448,7 +668,10 @@ data class YamlFluentCommand( flowPath.resolveSibling(path).toAbsolutePath() } if (resolvedPath.equals(flowPath.toAbsolutePath())) { - throw InvalidFlowFile("Referenced Flow file can't be the same as the main Flow file: ${resolvedPath.toUri()}", resolvedPath) + throw InvalidFlowFile( + "Referenced Flow file can't be the same as the main Flow file: ${resolvedPath.toUri()}", + resolvedPath + ) } if (!resolvedPath.exists()) { throw InvalidFlowFile("Flow file does not exist: ${resolvedPath.toUri()}", resolvedPath) @@ -547,7 +770,16 @@ data class YamlFluentCommand( private fun swipeCommand(swipe: YamlSwipe): MaestroCommand { when (swipe) { - is YamlSwipeDirection -> return MaestroCommand(SwipeCommand(direction = swipe.direction, duration = swipe.duration, label = swipe.label, optional = swipe.optional, waitToSettleTimeoutMs = swipe.waitToSettleTimeoutMs)) + is YamlSwipeDirection -> return MaestroCommand( + SwipeCommand( + direction = swipe.direction, + duration = swipe.duration, + label = swipe.label, + optional = swipe.optional, + waitToSettleTimeoutMs = swipe.waitToSettleTimeoutMs + ) + ) + is YamlCoordinateSwipe -> { val start = swipe.start val end = swipe.end @@ -566,18 +798,36 @@ data class YamlFluentCommand( } endPoint = Point(endPoints[0], endPoints[1]) - return MaestroCommand(SwipeCommand(startPoint = startPoint, endPoint = endPoint, duration = swipe.duration, label = swipe.label, optional = swipe.optional, waitToSettleTimeoutMs = swipe.waitToSettleTimeoutMs)) + return MaestroCommand( + SwipeCommand( + startPoint = startPoint, + endPoint = endPoint, + duration = swipe.duration, + label = swipe.label, + optional = swipe.optional, + waitToSettleTimeoutMs = swipe.waitToSettleTimeoutMs + ) + ) } + is YamlRelativeCoordinateSwipe -> { return MaestroCommand( - SwipeCommand(startRelative = swipe.start, endRelative = swipe.end, duration = swipe.duration, label = swipe.label, optional = swipe.optional, waitToSettleTimeoutMs = swipe.waitToSettleTimeoutMs) + SwipeCommand( + startRelative = swipe.start, + endRelative = swipe.end, + duration = swipe.duration, + label = swipe.label, + optional = swipe.optional, + waitToSettleTimeoutMs = swipe.waitToSettleTimeoutMs + ) ) } + is YamlSwipeElement -> return swipeElementCommand(swipe) else -> { throw IllegalStateException( "Provide swipe direction UP, DOWN, RIGHT OR LEFT or by giving explicit " + - "start and end coordinates." + "start and end coordinates." ) } } @@ -663,7 +913,8 @@ data class YamlFluentCommand( } private fun scrollUntilVisibleCommand(yaml: YamlScrollUntilVisible): MaestroCommand { - val visibility = if (yaml.visibilityPercentage < 0) 0 else if (yaml.visibilityPercentage > 100) 100 else yaml.visibilityPercentage + val visibility = + if (yaml.visibilityPercentage < 0) 0 else if (yaml.visibilityPercentage > 100) 100 else yaml.visibilityPercentage return MaestroCommand( ScrollUntilVisibleCommand( selector = toElementSelector(yaml.element), diff --git a/recipes/web/xmas.yaml b/recipes/web/xmas.yaml index 451e760bb5..ffb63ebeb1 100644 --- a/recipes/web/xmas.yaml +++ b/recipes/web/xmas.yaml @@ -1,18 +1,26 @@ url: https://amazon.com --- - launchApp +- extractTextWithAI: CAPTCHA value +- tapOn: Type characters +- inputText: ${aiOutput} +- tapOn: Continue shopping - tapOn: .*Dismiss.* - tapOn: "Search Amazon" - inputText: "Ugly Christmas Sweater With Darth Vader" - pressKey: "Enter" - assertWithAI: - assertion: All results are Star Wars themed + assertion: All sweaters have Darth Vader's mask on them - assertWithAI: - assertion: At least one result is Star Wars themed -- tapOn: 39 + assertion: At least one result is Star Wars themed +- extractTextWithAI: Dollar price without cents and currency of the first item +- tapOn: ${aiOutput} - assertWithAI: - assertion: User is shown a product detail page that fits in the screen + assertion: User is shown a product detail page that fits in the screen +- swipe: + start: 50%,50% + end: 20%,50% - tapOn: "Add to Cart" - tapOn: "Proceed to checkout" - assertWithAI: - assertion: User is asked to sign in \ No newline at end of file + assertion: User is asked to sign in