diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index b91c7d1299..5952122c04 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -42,7 +42,8 @@ micrometerCore = "1.13.4" mockk = "1.12.0" mozillaRhino = "1.7.14" picocli = "4.6.3" -selenium = "4.13.0" +selenium = "4.26.0" +selenium-devtools = "4.26.0" skiko = "0.8.18" slf4j = "1.7.36" squareOkhttp = "4.12.0" @@ -114,6 +115,7 @@ mozilla-rhino = { module = "org.mozilla:rhino", version.ref = "mozillaRhino" } picocli = { module = "info.picocli:picocli", version.ref = "picocli" } picocli-codegen = { module = "info.picocli:picocli-codegen", version.ref = "picocli" } selenium = { module = "org.seleniumhq.selenium:selenium-java", version.ref = "selenium" } +selenium-devtools = { module = "org.seleniumhq.selenium:selenium-devtools-v130", version.ref = "selenium-devtools" } skiko-macos-arm64 = { module = "org.jetbrains.skiko:skiko-awt-runtime-macos-arm64", version.ref = "skiko" } skiko-macos-x64 = { module = "org.jetbrains.skiko:skiko-awt-runtime-macos-x64", version.ref = "skiko" } skiko-linux-arm64 = { module = "org.jetbrains.skiko:skiko-awt-runtime-linux-arm64", version.ref = "skiko" } diff --git a/installLocally.sh b/installLocally.sh new file mode 100755 index 0000000000..03311d85bc --- /dev/null +++ b/installLocally.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +./gradlew :maestro-cli:installDist + +rm -rf ~/.maestro/bin +rm -rf ~/.maestro/lib + +cp -r ./maestro-cli/build/install/maestro/bin ~/.maestro/bin +cp -r ./maestro-cli/build/install/maestro/lib ~/.maestro/lib \ No newline at end of file diff --git a/maestro-cli/src/main/java/maestro/cli/App.kt b/maestro-cli/src/main/java/maestro/cli/App.kt index c50a238f9a..915eaf57ac 100644 --- a/maestro-cli/src/main/java/maestro/cli/App.kt +++ b/maestro-cli/src/main/java/maestro/cli/App.kt @@ -19,6 +19,7 @@ package maestro.cli +import maestro.MaestroException import maestro.cli.analytics.Analytics import maestro.cli.command.BugReportCommand import maestro.cli.command.CloudCommand @@ -33,6 +34,7 @@ import maestro.cli.command.StudioCommand import maestro.cli.command.TestCommand import maestro.cli.command.UploadCommand import maestro.cli.update.Updates +import maestro.cli.util.ChangeLogUtils import maestro.cli.util.ErrorReporter import maestro.cli.view.box import maestro.debuglog.DebugLogStore @@ -40,9 +42,8 @@ import picocli.AutoComplete.GenerateCompletion import picocli.CommandLine import picocli.CommandLine.Command import picocli.CommandLine.Option -import java.util.Properties +import java.util.* import kotlin.system.exitProcess -import maestro.cli.util.ChangeLogUtils @Command( name = "maestro", @@ -127,8 +128,7 @@ fun main(args: Array) { cmd.colorScheme.errorText(ex.message) ) - // Print stack trace - if (ex !is CliError) { + if (ex !is CliError && ex !is MaestroException.UnsupportedJavaVersion) { cmd.err.println("\nThe stack trace was:") cmd.err.println(ex.stackTraceToString()) } @@ -150,14 +150,16 @@ fun main(args: Array) { System.err.println() val changelog = Updates.getChangelog() val anchor = newVersion.toString().replace(".", "") - System.err.println(listOf( - "A new version of the Maestro CLI is available ($newVersion).\n", - "See what's new:", - "https://github.com/mobile-dev-inc/maestro/blob/main/CHANGELOG.md#$anchor", - ChangeLogUtils.print(changelog), - "Upgrade command:", - "curl -Ls \"https://get.maestro.mobile.dev\" | bash", - ).joinToString("\n").box()) + System.err.println( + listOf( + "A new version of the Maestro CLI is available ($newVersion).\n", + "See what's new:", + "https://github.com/mobile-dev-inc/maestro/blob/main/CHANGELOG.md#$anchor", + ChangeLogUtils.print(changelog), + "Upgrade command:", + "curl -Ls \"https://get.maestro.mobile.dev\" | bash", + ).joinToString("\n").box() + ) } if (commandLine.isVersionHelpRequested) { diff --git a/maestro-cli/src/main/java/maestro/cli/command/RecordCommand.kt b/maestro-cli/src/main/java/maestro/cli/command/RecordCommand.kt index 2e15e985b1..2e66592240 100644 --- a/maestro-cli/src/main/java/maestro/cli/command/RecordCommand.kt +++ b/maestro-cli/src/main/java/maestro/cli/command/RecordCommand.kt @@ -30,6 +30,7 @@ import maestro.cli.report.TestDebugReporter import maestro.cli.runner.TestRunner import maestro.cli.runner.resultview.AnsiResultView import maestro.cli.session.MaestroSessionManager +import maestro.cli.util.FileUtils.isWebFlow import okio.sink import picocli.CommandLine import picocli.CommandLine.Option @@ -98,11 +99,17 @@ class RecordCommand : Callable { TestDebugReporter.install(debugOutputPathAsString = debugOutput, printToConsole = parent?.verbose == true) val path = TestDebugReporter.getDebugOutputPath() + val deviceId = if (flowFile.isWebFlow()) { + throw CliError("'record' command does not support web flows yet.") + } else { + parent?.deviceId + } + return MaestroSessionManager.newSession( host = parent?.host, port = parent?.port, driverHostPort = parent?.port, - deviceId = parent?.deviceId, + deviceId = deviceId, platform = parent?.platform, ) { session -> val maestro = session.maestro diff --git a/maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt b/maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt index 2c29571330..c068743268 100644 --- a/maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt +++ b/maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt @@ -40,6 +40,7 @@ import maestro.cli.runner.TestSuiteInteractor import maestro.cli.runner.resultview.AnsiResultView import maestro.cli.runner.resultview.PlainTextResultView import maestro.cli.session.MaestroSessionManager +import maestro.cli.util.FileUtils.isWebFlow import maestro.cli.util.PrintUtils import maestro.cli.view.box import maestro.orchestra.error.ValidationError @@ -47,7 +48,6 @@ import maestro.orchestra.util.Env.withDefaultEnvVars import maestro.orchestra.util.Env.withInjectedShellEnvVars import maestro.orchestra.workspace.WorkspaceExecutionPlanner import maestro.orchestra.workspace.WorkspaceExecutionPlanner.ExecutionPlan -import maestro.orchestra.yaml.YamlCommandReader import maestro.utils.isSingleFile import okio.sink import org.slf4j.LoggerFactory @@ -158,8 +158,7 @@ class TestCommand : Callable { private fun isWebFlow(): Boolean { if (flowFiles.isSingleFile) { - val config = YamlCommandReader.readConfig(flowFiles.first().toPath()) - return Regex("http(s?)://").containsMatchIn(config.appId) + return flowFiles.first().isWebFlow() } return false @@ -210,7 +209,8 @@ class TestCommand : Callable { val onlySequenceFlows = plan.sequence.flows.isNotEmpty() && plan.flowsToRun.isEmpty() // An edge case - val availableDevices = DeviceService.listConnectedDevices().map { it.instanceId }.toSet() + val availableDevices = + DeviceService.listConnectedDevices(includeWeb = isWebFlow()).map { it.instanceId }.toSet() val deviceIds = getPassedOptionsDeviceIds() .filter { device -> if (device !in availableDevices) { @@ -413,7 +413,7 @@ class TestCommand : Callable { private fun getPassedOptionsDeviceIds(): List { val arguments = if (isWebFlow()) { - PrintUtils.warn("Web support is an experimental feature and may be removed in future versions.\n") + PrintUtils.warn("Web support is in Beta. We would appreciate your feedback!\n") "chromium" } else parent?.deviceId val deviceIds = arguments diff --git a/maestro-cli/src/main/java/maestro/cli/device/DeviceService.kt b/maestro-cli/src/main/java/maestro/cli/device/DeviceService.kt index a62f36c23f..91deb1e925 100644 --- a/maestro-cli/src/main/java/maestro/cli/device/DeviceService.kt +++ b/maestro-cli/src/main/java/maestro/cli/device/DeviceService.kt @@ -16,13 +16,17 @@ import util.LocalSimulatorUtils import util.LocalSimulatorUtils.SimctlError import util.SimctlList import java.io.File -import java.util.UUID +import java.util.* import java.util.concurrent.TimeUnit import java.util.concurrent.TimeoutException object DeviceService { private val logger = LoggerFactory.getLogger(DeviceService::class.java) - fun startDevice(device: Device.AvailableForLaunch, driverHostPort: Int?, connectedDevices: Set = setOf()): Device.Connected { + fun startDevice( + device: Device.AvailableForLaunch, + driverHostPort: Int?, + connectedDevices: Set = setOf() + ): Device.Connected { when (device.platform) { Platform.IOS -> { try { @@ -113,31 +117,35 @@ object DeviceService { } } - fun listConnectedDevices(): List { - return listDevices() + fun listConnectedDevices(includeWeb: Boolean = false): List { + return listDevices(includeWeb = includeWeb) .filterIsInstance() } fun List.withPlatform(platform: Platform?) = filter { platform == null || it.platform == platform } - fun listAvailableForLaunchDevices(): List { - return listDevices() + fun listAvailableForLaunchDevices(includeWeb: Boolean = false): List { + return listDevices(includeWeb = includeWeb) .filterIsInstance() } - private fun listDevices(): List { - return listAndroidDevices() + listIOSDevices() + listWebDevices() + private fun listDevices(includeWeb: Boolean): List { + return listAndroidDevices() + + listIOSDevices() + + if (includeWeb) { + listWebDevices() + } else { + listOf() + } } private fun listWebDevices(): List { return listOf( - Device.AvailableForLaunch( + Device.Connected( platform = Platform.WEB, description = "Chromium Desktop Browser (Experimental)", - modelId = "chromium", - language = null, - country = null, + instanceId = "chromium" ) ) } @@ -234,10 +242,17 @@ object DeviceService { Platform.IOS -> listIOSDevices() .filterIsInstance() .find { it.description.contains(deviceName, ignoreCase = true) } + else -> runCatching { (Dadb.list() + AdbServer.listDadbs(adbServerPort = 5038)) .mapNotNull { dadb -> runCatching { dadb.shell("getprop ro.kernel.qemu.avd_name").output }.getOrNull() } - .map { output -> Device.Connected(instanceId = output, description = output, platform = Platform.ANDROID) } + .map { output -> + Device.Connected( + instanceId = output, + description = output, + platform = Platform.ANDROID + ) + } .find { connectedDevice -> connectedDevice.description.contains(deviceName, ignoreCase = true) } }.getOrNull() } @@ -316,7 +331,7 @@ object DeviceService { shardIndex: Int? = null, ): String { val avd = requireAvdManagerBinary() - val name = "${deviceName}${"_${(shardIndex ?: 0)+1}"}" + val name = "${deviceName}${"_${(shardIndex ?: 0) + 1}"}" val command = mutableListOf( avd.absolutePath, "create", "avd", diff --git a/maestro-cli/src/main/java/maestro/cli/util/FileUtils.kt b/maestro-cli/src/main/java/maestro/cli/util/FileUtils.kt index 37760ab6f6..9cbd7ca10a 100644 --- a/maestro-cli/src/main/java/maestro/cli/util/FileUtils.kt +++ b/maestro-cli/src/main/java/maestro/cli/util/FileUtils.kt @@ -1,5 +1,6 @@ package maestro.cli.util +import maestro.orchestra.yaml.YamlCommandReader import java.io.File import java.util.zip.ZipInputStream @@ -14,4 +15,9 @@ object FileUtils { } } + fun File.isWebFlow(): Boolean { + val config = YamlCommandReader.readConfig(toPath()) + return Regex("http(s?)://").containsMatchIn(config.appId) + } + } \ No newline at end of file diff --git a/maestro-client/build.gradle.kts b/maestro-client/build.gradle.kts index fd765d8eb2..582a99a085 100644 --- a/maestro-client/build.gradle.kts +++ b/maestro-client/build.gradle.kts @@ -80,9 +80,12 @@ dependencies { api(libs.apk.parser) implementation(project(":maestro-ios")) + implementation(project(":maestro-web")) implementation(libs.google.findbugs) implementation(libs.axml) implementation(libs.selenium) + implementation(libs.selenium.devtools) + implementation(libs.jcodec) api(libs.slf4j) api(libs.logback) { exclude(group = "org.slf4j", module = "slf4j-api") diff --git a/maestro-client/src/main/java/maestro/Errors.kt b/maestro-client/src/main/java/maestro/Errors.kt index e39a9969c0..5a9b766979 100644 --- a/maestro-client/src/main/java/maestro/Errors.kt +++ b/maestro-client/src/main/java/maestro/Errors.kt @@ -56,6 +56,8 @@ sealed class MaestroException(override val message: String) : RuntimeException(m class DeprecatedCommand(message: String) : MaestroException(message) class NoRootAccess(message: String) : MaestroException(message) + + class UnsupportedJavaVersion(message: String) : MaestroException(message) } sealed class MaestroDriverStartupException(override val message: String): RuntimeException() { diff --git a/maestro-client/src/main/java/maestro/KeyCode.kt b/maestro-client/src/main/java/maestro/KeyCode.kt index eac7b51412..9873e6ebf8 100644 --- a/maestro-client/src/main/java/maestro/KeyCode.kt +++ b/maestro-client/src/main/java/maestro/KeyCode.kt @@ -1,7 +1,5 @@ package maestro -import org.openqa.selenium.Keys - enum class KeyCode( val description: String, ) { @@ -42,14 +40,6 @@ enum class KeyCode( val lowercaseName = name.lowercase() return values().find { it.description.lowercase() == lowercaseName } } - - fun mapToSeleniumKey(code: KeyCode): Keys? { - return when (code) { - ENTER -> Keys.ENTER - BACKSPACE -> Keys.BACK_SPACE - else -> null - } - } } } diff --git a/maestro-client/src/main/java/maestro/Maestro.kt b/maestro-client/src/main/java/maestro/Maestro.kt index 4f7fafe455..1d20db7f07 100644 --- a/maestro-client/src/main/java/maestro/Maestro.kt +++ b/maestro-client/src/main/java/maestro/Maestro.kt @@ -635,6 +635,17 @@ class Maestro( } fun web(isStudio: Boolean): Maestro { + // Check that JRE is at least 11 + val version = System.getProperty("java.version") + if (version.startsWith("1.")) { + val majorVersion = version.substring(2, 3).toInt() + if (majorVersion < 11) { + throw MaestroException.UnsupportedJavaVersion( + "Maestro Web requires Java 11 or later. Current version: $version" + ) + } + } + val driver = WebDriver(isStudio) driver.open() return Maestro(driver) diff --git a/maestro-client/src/main/java/maestro/drivers/WebDriver.kt b/maestro-client/src/main/java/maestro/drivers/WebDriver.kt index e0b02dde7d..9be0251b25 100644 --- a/maestro-client/src/main/java/maestro/drivers/WebDriver.kt +++ b/maestro-client/src/main/java/maestro/drivers/WebDriver.kt @@ -12,8 +12,9 @@ import maestro.SwipeDirection import maestro.TreeNode import maestro.ViewHierarchy import maestro.utils.ScreenshotUtils +import maestro.web.record.JcodecVideoEncoder +import maestro.web.record.WebScreenRecorder import okio.Sink -import maestro.NamedSource import okio.buffer import org.openqa.selenium.By import org.openqa.selenium.JavascriptExecutor @@ -24,21 +25,27 @@ import org.openqa.selenium.chrome.ChromeDriver import org.openqa.selenium.chrome.ChromeDriverService import org.openqa.selenium.chrome.ChromeOptions import org.openqa.selenium.chromium.ChromiumDriverLogLevel +import org.openqa.selenium.devtools.HasDevTools +import org.openqa.selenium.devtools.v130.emulation.Emulation import org.openqa.selenium.interactions.Actions import org.openqa.selenium.interactions.PointerInput import org.openqa.selenium.remote.RemoteWebDriver import org.openqa.selenium.support.ui.WebDriverWait +import org.slf4j.LoggerFactory import java.io.File import java.time.Duration -import java.util.Random -import java.util.UUID +import java.util.* import java.util.logging.Level import java.util.logging.Logger + class WebDriver(val isStudio: Boolean) : Driver { private var seleniumDriver: org.openqa.selenium.WebDriver? = null private var maestroWebScript: String? = null + private var lastSeenWindowHandles = setOf() + + private var webScreenRecorder: WebScreenRecorder? = null init { Maestro::class.java.getResourceAsStream("/maestro-web.js")?.let { @@ -67,13 +74,20 @@ class WebDriver(val isStudio: Boolean) : Driver { ChromeOptions().apply { addArguments("--remote-allow-origins=*") addArguments("--disable-search-engine-choice-screen") + addArguments("--lang=en") if (isStudio) { addArguments("--headless=new") addArguments("--window-size=1024,768") + setExperimentalOption("detach", true) } } ) + seleniumDriver + ?.let { it as? HasDevTools } + ?.devTools + ?.createSessionIfThereIsNotOne() + if (isStudio) { seleniumDriver?.get("https://maestro.mobile.dev") } @@ -123,21 +137,30 @@ class WebDriver(val isStudio: Boolean) : Driver { } override fun close() { - seleniumDriver?.quit() + try { + seleniumDriver?.quit() + webScreenRecorder?.close() + } catch (e: Exception) { + // Swallow the exception to avoid crashing the whole process + } + seleniumDriver = null + lastSeenWindowHandles = setOf() + webScreenRecorder = null } override fun deviceInfo(): DeviceInfo { - val driver = ensureOpen() + val driver = ensureOpen() as JavascriptExecutor - val windowSize = driver.manage().window().size + val width = driver.executeScript("return window.innerWidth;") as Long + val height = driver.executeScript("return window.innerHeight;") as Long return DeviceInfo( platform = Platform.WEB, - widthPixels = windowSize.width, - heightPixels = windowSize.height, - widthGrid = windowSize.width, - heightGrid = windowSize.height, + widthPixels = width.toInt(), + heightPixels = height.toInt(), + widthGrid = width.toInt(), + heightGrid = height.toInt(), ) } @@ -170,6 +193,8 @@ class WebDriver(val isStudio: Boolean) : Driver { override fun contentDescriptor(excludeKeyboardElements: Boolean): TreeNode { ensureOpen() + detectWindowChange() + // retrieve view hierarchy from DOM // There are edge cases where executeJS returns null, and we cannot get the hierarchy. In this situation // we retry multiple times until throwing an error eventually. (See issue #1936) @@ -205,6 +230,25 @@ class WebDriver(val isStudio: Boolean) : Driver { return parse(contentDesc as Map) } + private fun detectWindowChange() { + // Checks whether there are any new window handles available and, if so, switches Selenium driver focus to it + val driver = ensureOpen() + + if (lastSeenWindowHandles != driver.windowHandles) { + val newHandles = driver.windowHandles - lastSeenWindowHandles + lastSeenWindowHandles = driver.windowHandles + + if (newHandles.isNotEmpty()) { + val newHandle = newHandles.first(); + LOGGER.info("Detected a window change, switching to new window handle $newHandle") + + driver.switchTo().window(newHandle) + + webScreenRecorder?.onWindowChange() + } + } + } + override fun clearAppState(appId: String) { // Do nothing } @@ -219,7 +263,14 @@ class WebDriver(val isStudio: Boolean) : Driver { val mouse = PointerInput(PointerInput.Kind.MOUSE, "default mouse") val actions = org.openqa.selenium.interactions.Sequence(mouse, 1) - .addAction(mouse.createPointerMove(Duration.ofMillis(400), PointerInput.Origin.viewport(), point.x, point.y - pixelsScrolled.toInt())) + .addAction( + mouse.createPointerMove( + Duration.ofMillis(400), + PointerInput.Origin.viewport(), + point.x, + point.y - pixelsScrolled.toInt() + ) + ) (driver as RemoteWebDriver).perform(listOf(actions)) @@ -242,21 +293,27 @@ class WebDriver(val isStudio: Boolean) : Driver { val xPath = executeJS("return window.maestro.createXPathFromElement(document.activeElement)") as String val element = driver.findElement(By.ByXPath(xPath)) - val key = KeyCode.mapToSeleniumKey(code) - ?: throw IllegalArgumentException("Keycode $code is not supported on web") + val key = mapToSeleniumKey(code) element.sendKeys(key) } + private fun mapToSeleniumKey(code: KeyCode): Keys { + return when (code) { + KeyCode.ENTER -> Keys.ENTER + KeyCode.BACKSPACE -> Keys.BACK_SPACE + else -> error("Keycode $code is not supported on web") + } + } + override fun scrollVertical() { scroll("window.scrollY + Math.round(window.innerHeight / 2)", "window.scrollX") } override fun isKeyboardVisible(): Boolean { - TODO("Not yet implemented") + return false } override fun swipe(start: Point, end: Point, durationMs: Long) { - // TODO validate implementation and ensure it works properly val driver = ensureOpen() val finger = PointerInput(PointerInput.Kind.TOUCH, "finger") @@ -283,7 +340,6 @@ class WebDriver(val isStudio: Boolean) : Driver { } override fun swipe(swipeDirection: SwipeDirection, durationMs: Long) { - // TODO validate implementation and ensure it works properly when (swipeDirection) { SwipeDirection.UP -> scroll("window.scrollY + Math.round(window.innerHeight / 2)", "window.scrollX") SwipeDirection.DOWN -> scroll("window.scrollY - Math.round(window.innerHeight / 2)", "window.scrollX") @@ -297,10 +353,6 @@ class WebDriver(val isStudio: Boolean) : Driver { swipe(direction, durationMs) } - fun swipe(start: Point, end: Point) { - TODO("Not yet implemented") - } - override fun backPress() { val driver = ensureOpen() driver.navigate().back() @@ -336,11 +388,32 @@ class WebDriver(val isStudio: Boolean) : Driver { } override fun startScreenRecording(out: Sink): ScreenRecording { - TODO("Not yet implemented") + val driver = ensureOpen() + webScreenRecorder = WebScreenRecorder( + JcodecVideoEncoder(), + driver + ) + webScreenRecorder?.startScreenRecording(out) + + return object : ScreenRecording { + override fun close() { + webScreenRecorder?.close() + } + } } override fun setLocation(latitude: Double, longitude: Double) { - TODO("Not yet implemented") + val driver = ensureOpen() as HasDevTools + + driver.devTools.createSessionIfThereIsNotOne() + + driver.devTools.send( + Emulation.setGeolocationOverride( + Optional.of(latitude), + Optional.of(longitude), + Optional.of(0.0) + ) + ) } override fun eraseText(charactersToErase: Int) { @@ -357,11 +430,11 @@ class WebDriver(val isStudio: Boolean) : Driver { } override fun setProxy(host: String, port: Int) { - TODO("Not yet implemented") + // Do nothing } override fun resetProxy() { - TODO("Not yet implemented") + // Do nothing } override fun isShutdown(): Boolean { @@ -396,15 +469,17 @@ class WebDriver(val isStudio: Boolean) : Driver { } override fun isAirplaneModeEnabled(): Boolean { - TODO("Not yet implemented") + return false; } override fun setAirplaneMode(enabled: Boolean) { - TODO("Not yet implemented") + // Do nothing } companion object { private const val SCREENSHOT_DIFF_THRESHOLD = 0.005 private const val RETRY_FETCHING_CONTENT_DESCRIPTION = 10 + + private val LOGGER = LoggerFactory.getLogger(maestro.drivers.WebDriver::class.java) } } diff --git a/maestro-orchestra/src/main/java/maestro/orchestra/yaml/YamlConfig.kt b/maestro-orchestra/src/main/java/maestro/orchestra/yaml/YamlConfig.kt index 985fab6956..d3a6121852 100644 --- a/maestro-orchestra/src/main/java/maestro/orchestra/yaml/YamlConfig.kt +++ b/maestro-orchestra/src/main/java/maestro/orchestra/yaml/YamlConfig.kt @@ -1,5 +1,6 @@ package maestro.orchestra.yaml +import com.fasterxml.jackson.annotation.JsonAlias import com.fasterxml.jackson.annotation.JsonAnySetter import maestro.orchestra.ApplyConfigurationCommand import maestro.orchestra.MaestroCommand @@ -10,6 +11,7 @@ import java.nio.file.Path data class YamlConfig( val name: String?, + @JsonAlias("url") val appId: String, val tags: List? = emptyList(), val env: Map = emptyMap(), diff --git a/maestro-orchestra/src/main/java/maestro/orchestra/yaml/YamlLaunchApp.kt b/maestro-orchestra/src/main/java/maestro/orchestra/yaml/YamlLaunchApp.kt index 5f9a3ec837..f6523c07eb 100644 --- a/maestro-orchestra/src/main/java/maestro/orchestra/yaml/YamlLaunchApp.kt +++ b/maestro-orchestra/src/main/java/maestro/orchestra/yaml/YamlLaunchApp.kt @@ -19,9 +19,11 @@ package maestro.orchestra.yaml +import com.fasterxml.jackson.annotation.JsonAlias import com.fasterxml.jackson.annotation.JsonCreator data class YamlLaunchApp( + @JsonAlias("url") val appId: String?, val clearState: Boolean?, val clearKeychain: Boolean?, diff --git a/maestro-studio/web/src/components/interact/InteractPageLayout.tsx b/maestro-studio/web/src/components/interact/InteractPageLayout.tsx index 9f8d97826a..76f5d37601 100644 --- a/maestro-studio/web/src/components/interact/InteractPageLayout.tsx +++ b/maestro-studio/web/src/components/interact/InteractPageLayout.tsx @@ -11,6 +11,7 @@ import DeviceWrapperAspectRatio from "../device-and-device-elements/DeviceWrappe import { useDeviceContext } from "../../context/DeviceContext"; import { Spinner } from "../design-system/spinner"; import { useRepl } from '../../context/ReplContext'; +import { DeviceScreen } from "../../helpers/models"; const InteractPageLayout = () => { const { @@ -52,6 +53,8 @@ const InteractPageLayout = () => { if (!deviceScreen) return null; + var widthClass = computeWidthClass(deviceScreen, showElementsPanel); + return (
{showElementsPanel && ( @@ -59,10 +62,8 @@ const InteractPageLayout = () => { )}
{!showElementsPanel && ( @@ -95,4 +96,26 @@ const InteractPageLayout = () => { ); }; +function computeWidthClass(deviceScreen: DeviceScreen, showElementsPanel: boolean) { + const wideDevice = deviceScreen.width > deviceScreen.height; + + var widthModifier = "basis-1/2"; + if (showElementsPanel) { + widthModifier += " max-w-[33.333333%]"; + + if (wideDevice) { + widthModifier += " lg:basis-5/12"; + } + } else { + if (wideDevice) { + widthModifier += " max-w-[80%]"; + } else { + widthModifier += " lg:basis-4/12 max-w-[41.666667%]"; + } + } + + return widthModifier; +} + export default InteractPageLayout; + diff --git a/maestro-web/build.gradle.kts b/maestro-web/build.gradle.kts new file mode 100644 index 0000000000..f1dd9145c8 --- /dev/null +++ b/maestro-web/build.gradle.kts @@ -0,0 +1,38 @@ +import com.vanniktech.maven.publish.SonatypeHost +import org.jetbrains.kotlin.gradle.tasks.KotlinCompilationTask + +plugins { + id("maven-publish") + alias(libs.plugins.kotlin.jvm) + alias(libs.plugins.kotlin.serialization) + alias(libs.plugins.mavenPublish) +} + +dependencies { + implementation(libs.square.okio) + + implementation(libs.selenium) + implementation(libs.selenium.devtools) + implementation(libs.jcodec) + implementation(libs.jcodec.awt) +} + +java { + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 +} + +tasks.named("compileKotlin", KotlinCompilationTask::class.java) { + compilerOptions { + freeCompilerArgs.addAll("-Xjdk-release=1.8") + } +} + +mavenPublishing { + publishToMavenCentral(SonatypeHost.S01) +} + +tasks.named("test") { + useJUnitPlatform() + environment.put("PROJECT_DIR", projectDir.absolutePath) +} diff --git a/maestro-web/gradle.properties b/maestro-web/gradle.properties new file mode 100644 index 0000000000..1f4cfaaa9e --- /dev/null +++ b/maestro-web/gradle.properties @@ -0,0 +1,3 @@ +POM_NAME=Maestro Web +POM_ARTIFACT_ID=maestro-web +POM_PACKAGING=jar \ No newline at end of file diff --git a/maestro-web/src/main/kotlin/maestro/web/record/JcodecVideoEncoder.kt b/maestro-web/src/main/kotlin/maestro/web/record/JcodecVideoEncoder.kt new file mode 100644 index 0000000000..6c88bc5994 --- /dev/null +++ b/maestro-web/src/main/kotlin/maestro/web/record/JcodecVideoEncoder.kt @@ -0,0 +1,46 @@ +package maestro.web.record + +import okio.Sink +import okio.buffer +import okio.source +import org.jcodec.api.SequenceEncoder +import org.jcodec.scale.AWTUtil +import java.io.ByteArrayInputStream +import java.io.File +import javax.imageio.ImageIO + +class JcodecVideoEncoder : VideoEncoder { + + private lateinit var sequenceEncoder: SequenceEncoder + private lateinit var tempFile: File + private lateinit var out: Sink + + override fun start(out: Sink) { + tempFile = File.createTempFile("maestro_jcodec", ".mp4") + + sequenceEncoder = SequenceEncoder.create2997Fps(tempFile) + + this.out = out + } + + override fun encodeFrame(frame: ByteArray) { + val image = ByteArrayInputStream(frame).use { ImageIO.read(it) } + + val picture = AWTUtil.fromBufferedImageRGB(image) + + sequenceEncoder.encodeNativeFrame(picture) + } + + override fun close() { + sequenceEncoder.finish() + + try { + out.buffer().use { + it.writeAll(tempFile.source().buffer()) + } + } finally { + tempFile.delete() + } + } + +} \ No newline at end of file diff --git a/maestro-web/src/main/kotlin/maestro/web/record/VideoEncoder.kt b/maestro-web/src/main/kotlin/maestro/web/record/VideoEncoder.kt new file mode 100644 index 0000000000..9828c5c4c3 --- /dev/null +++ b/maestro-web/src/main/kotlin/maestro/web/record/VideoEncoder.kt @@ -0,0 +1,11 @@ +package maestro.web.record + +import okio.Sink + +interface VideoEncoder : AutoCloseable { + + fun start(out: Sink) + + fun encodeFrame(frame: ByteArray) + +} \ No newline at end of file diff --git a/maestro-web/src/main/kotlin/maestro/web/record/WebScreenRecorder.kt b/maestro-web/src/main/kotlin/maestro/web/record/WebScreenRecorder.kt new file mode 100644 index 0000000000..81308226da --- /dev/null +++ b/maestro-web/src/main/kotlin/maestro/web/record/WebScreenRecorder.kt @@ -0,0 +1,102 @@ +package maestro.web.record + +import okio.Sink +import org.openqa.selenium.WebDriver +import org.openqa.selenium.devtools.HasDevTools +import org.openqa.selenium.devtools.v130.page.Page +import java.util.* +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit + +class WebScreenRecorder( + private val videoEncoder: VideoEncoder, + private val seleniumDriver: WebDriver +) : AutoCloseable { + + private val screenRecordingSessions = mutableListOf() + private lateinit var recordingExecutor: ExecutorService + + private var closed = false + + fun startScreenRecording(out: Sink) { + ensureNotClosed() + + recordingExecutor = Executors.newSingleThreadExecutor() + videoEncoder.start(out) + + startScreenRecordingForCurrentWindow() + } + + fun onWindowChange() { + if (closed) { + return + } + + startScreenRecordingForCurrentWindow() + } + + override fun close() { + if (closed) { + return + } + closed = true + + closeScreenRecordingSessions() + + recordingExecutor.shutdown() + recordingExecutor.awaitTermination(2, TimeUnit.MINUTES) + + videoEncoder.close() + } + + private fun startScreenRecordingForCurrentWindow() { + closeScreenRecordingSessions() + + val driver = seleniumDriver as HasDevTools + + val seleniumDevTools = driver.devTools + + seleniumDevTools.createSessionIfThereIsNotOne() + + seleniumDevTools.send(Page.enable()) + + seleniumDevTools.send( + Page.startScreencast( + Optional.of(Page.StartScreencastFormat.JPEG), + Optional.of(80), + Optional.of(1280), + Optional.of(1280), + Optional.of(1) + ) + ) + + seleniumDevTools.addListener(Page.screencastFrame()) { frame -> + recordingExecutor.submit { + val imageData = frame.data + val imageBytes = Base64.getDecoder().decode(imageData) + + videoEncoder.encodeFrame(imageBytes) + + seleniumDevTools.send(Page.screencastFrameAck(frame.sessionId)) + } + } + + val session = AutoCloseable { seleniumDevTools.send(Page.stopScreencast()) } + screenRecordingSessions.add(session) + } + + private fun closeScreenRecordingSessions() { + screenRecordingSessions.forEach { + it.close() + } + screenRecordingSessions.clear() + } + + private fun ensureNotClosed() { + if (closed) { + error("Screen recorder is already closed") + } + } + +} \ No newline at end of file diff --git a/recipes/web/xmas.yaml b/recipes/web/xmas.yaml new file mode 100644 index 0000000000..451e760bb5 --- /dev/null +++ b/recipes/web/xmas.yaml @@ -0,0 +1,18 @@ +url: https://amazon.com +--- +- launchApp +- tapOn: .*Dismiss.* +- tapOn: "Search Amazon" +- inputText: "Ugly Christmas Sweater With Darth Vader" +- pressKey: "Enter" +- assertWithAI: + assertion: All results are Star Wars themed +- assertWithAI: + assertion: At least one result is Star Wars themed +- tapOn: 39 +- assertWithAI: + assertion: User is shown a product detail page that fits in the screen +- tapOn: "Add to Cart" +- tapOn: "Proceed to checkout" +- assertWithAI: + assertion: User is asked to sign in \ No newline at end of file diff --git a/settings.gradle.kts b/settings.gradle.kts index 8d1a50ae39..cd54f1510f 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -31,3 +31,4 @@ include("maestro-studio:server") include("maestro-studio:web") include("maestro-test") include("maestro-ai") +include("maestro-web")