diff --git a/.cirrus.yml b/.cirrus.yml index a20a9223e1..f26f927fbd 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -33,7 +33,7 @@ env: DEBIAN_NAME: "debian-13" # Image identifiers - IMAGE_SUFFIX: "c20230928t004553z-f39f38d13" + IMAGE_SUFFIX: "c20231004t194547z-f39f38d13" # EC2 images FEDORA_AMI: "fedora-aws-${IMAGE_SUFFIX}" @@ -52,7 +52,7 @@ env: #### N/B: Required ALL of these are set for every single task. #### TEST_FLAVOR: # int, sys, ext_svc, validate, automation, etc. - TEST_ENVIRON: host # 'host', or 'container' + TEST_ENVIRON: host # 'host', or 'container'. PODBIN_NAME: podman # 'podman' or 'remote' PRIV_NAME: root # 'root' or 'rootless' DISTRO_NV: # any {PRIOR_,}{FEDORA,DEBIAN}_NAME value @@ -367,6 +367,8 @@ alt_build_task: - env: ALT_NAME: 'Build Each Commit' - env: + # TODO: Replace with task using `winmake` to build + # binary and archive installation zip file. ALT_NAME: 'Windows Cross' - env: ALT_NAME: 'Alt Arch. x86 Cross' @@ -397,19 +399,50 @@ win_installer_task: $CIRRUS_BASE_BRANCH !=~ 'v[0-9\.]+-rhel' depends_on: - alt_build - ec2_instance: + ec2_instance: &windows image: "${WINDOWS_AMI}" type: m5.large region: us-east-1 platform: windows - env: - PATH: "${PATH};C:\\ProgramData\\chocolatey\\bin" + env: &winenv + CIRRUS_WORKING_DIR: &wincwd "${LOCALAPPDATA}\\cirrus-ci-build" CIRRUS_SHELL: powershell + PATH: "${PATH};C:\\ProgramData\\chocolatey\\bin" + CI_DESIRED_DATABASE: "" + DISTRO_NV: "windows" + PRIV_NAME: "rootless" # Fake version, we are only testing the installer functions, so version doesn't matter WIN_INST_VER: 9.9.9 - CIRRUS_WORKING_DIR: "${LOCALAPPDATA}\\cirrus-ci-build" - install_script: '.\contrib\cirrus\win-installer-install.ps1' - main_script: '.\contrib\cirrus\win-installer-main.ps1' + # It's HIGHLY desireable to use the same binary throughout CI. Otherwise, if + # there's a toolchain or build-environment specific problem, it can be incredibly + # difficult (and non-obvious) to debug. + clone_script: &winclone | + $ErrorActionPreference = 'Stop' + $ProgressPreference = 'SilentlyContinue' + New-Item -ItemType Directory -Force -Path "$ENV:CIRRUS_WORKING_DIR" + Set-Location "$ENV:CIRRUS_WORKING_DIR" + $uri = "${ENV:ART_URL}/Windows Cross/repo/repo.tbz" + Write-Host "Downloading $uri" + For($i = 0;;) { + Try { + Invoke-WebRequest -UseBasicParsing -ErrorAction Stop -OutFile "repo.tbz2" ` + -Uri "$uri" + Break + } Catch { + if (++$i -gt 6) { + throw $_.Exception + } + Write-Host "Download failed - retrying:" $_.Exception.Response.StatusCode + Start-Sleep -Seconds 10 + } + } + arc unarchive repo.tbz2 .\ + if ($LASTEXITCODE -ne 0) { + throw "Unarchive repo.tbz2 failed" + Exit 1 + } + Get-ChildItem -Path .\repo + main_script: ".\\repo\\contrib\\cirrus\\win-installer-main.ps1" # Confirm building the remote client, natively on a Mac OS-X VM. @@ -585,32 +618,6 @@ compose_test_task: always: *logs_artifacts -# Execute the podman integration tests on all primary platforms and release -windows_smoke_test_task: - name: "Windows Smoke Test" - alias: windows_smoke_test - # Only run for non-docs/copr PRs and non-release branch builds - # and never for tags. Docs: ./contrib/cirrus/CIModes.md - only_if: >- - $CIRRUS_TAG == '' && - $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' && - $CIRRUS_BRANCH !=~ 'v[0-9\.]+-rhel' && - $CIRRUS_BASE_BRANCH !=~ 'v[0-9\.]+-rhel' - depends_on: - - alt_build - ec2_instance: - image: "${WINDOWS_AMI}" - type: m5zn.metal - region: us-east-1 - platform: windows - env: - PATH: "${PATH};C:\\ProgramData\\chocolatey\\bin" - CIRRUS_SHELL: powershell - # Fake version, we are only testing the installer functions, so version doesn't matter - CIRRUS_WORKING_DIR: "${LOCALAPPDATA}\\Temp\\cirrus-ci-build" - main_script: 'contrib/cirrus/win-podman-machine-main.ps1' - - # versions, as root, without involving the podman-remote client. local_integration_test_task: &local_integration_test_task # Integration-test task name convention: @@ -723,7 +730,7 @@ podman_machine_task: region: us-east-1 env: EC2_INST_TYPE: "m5zn.metal" # Bare-metal instance is required - TEST_FLAVOR: "machine" + TEST_FLAVOR: "machine-linux" PRIV_NAME: "rootless" # intended use-case DISTRO_NV: "${FEDORA_NAME}" VM_IMAGE_NAME: "${FEDORA_AMI}" @@ -748,7 +755,7 @@ podman_machine_aarch64_task: ec2_instance: <<: *standard_build_ec2_aarch64 env: - TEST_FLAVOR: "machine" + TEST_FLAVOR: "machine-linux" EC2_INST_TYPE: c6g.metal PRIV_NAME: "rootless" # intended use-case DISTRO_NV: "${FEDORA_AARCH64_NAME}" @@ -760,6 +767,41 @@ podman_machine_aarch64_task: always: *int_logs_artifacts +podman_machine_windows_task: + name: *std_name_fmt + alias: podman_machine_windows + # TODO: These tests are new and mostly fail. Disable all failures impacting overall CI status + # until the tests, scripts, and environment stabalize. + allow_failures: $CI == $CI + # Only run for non-docs/copr PRs and non-release branch builds + # and never for tags. Docs: ./contrib/cirrus/CIModes.md + only_if: >- + $CIRRUS_TAG == '' && + $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' && + $CIRRUS_BRANCH !=~ 'v[0-9\.]+-rhel' && + $CIRRUS_BASE_BRANCH !=~ 'v[0-9\.]+-rhel' + depends_on: + - alt_build + - build + - win_installer + - local_integration_test + - remote_integration_test + - container_integration_test + - rootless_integration_test + ec2_instance: + <<: *windows + type: m5zn.metal + platform: windows + env: *winenv + matrix: + - env: + TEST_FLAVOR: "machine-wsl" + - env: + TEST_FLAVOR: "machine-hyperv" + clone_script: *winclone + main_script: ".\\repo\\contrib\\cirrus\\win-podman-machine-main.ps1" + + # Always run subsequent to integration tests. While parallelism is lost # with runtime, debugging system-test failures can be more challenging # for some golang developers. Otherwise the following tasks run across @@ -994,7 +1036,6 @@ success_task: - osx_alt_build - freebsd_alt_build - win_installer - - windows_smoke_test - docker-py_test - unit_test - apiv2_test @@ -1005,6 +1046,7 @@ success_task: - rootless_integration_test - podman_machine - podman_machine_aarch64 + - podman_machine_windows - local_system_test - local_system_test_aarch64 - remote_system_test diff --git a/contrib/cirrus/runner.sh b/contrib/cirrus/runner.sh index 6ee803fe1d..798bac6bde 100755 --- a/contrib/cirrus/runner.sh +++ b/contrib/cirrus/runner.sh @@ -418,7 +418,7 @@ dotest() { |& logformatter } -_run_machine() { +_run_machine-linux() { # N/B: Can't use _bail_if_test_can_be_skipped here b/c content isn't under test/ showrun make localmachine |& logformatter } diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh index d416299b4c..0f4929f161 100755 --- a/contrib/cirrus/setup_environment.sh +++ b/contrib/cirrus/setup_environment.sh @@ -413,7 +413,7 @@ case "$TEST_FLAVOR" in showrun minikube config set driver podman install_test_configs ;; - machine) + machine-linux) showrun dnf install -y podman-gvproxy* remove_packaged_podman_files showrun make install PREFIX=/usr ETCDIR=/etc diff --git a/contrib/cirrus/win-installer-install.ps1 b/contrib/cirrus/win-installer-install.ps1 deleted file mode 100644 index 6e20b8f994..0000000000 --- a/contrib/cirrus/win-installer-install.ps1 +++ /dev/null @@ -1,27 +0,0 @@ -# Update service is required for dotnet 3.5 (dep of wix) -Set-Service -Name wuauserv -StartupType "Manual" -function retryInstall { - param([Parameter(ValueFromRemainingArguments)] [string[]] $pkgs) - - foreach ($pkg in $pkgs) { - for ($retries = 0; ; $retries++) { - if ($retries -gt 5) { - throw "Could not install package $pkg" - } - - if ($pkg -match '(.[^\@]+)@(.+)') { - $pkg = @("--version", $Matches.2, $Matches.1) - } - - choco install -y --allow-downgrade $pkg - if ($LASTEXITCODE -eq 0) { - break - } - Write-Output "Error installing, waiting before retry..." - Start-Sleep -Seconds 6 - } - } -} -# Force mingw version 11.2 since later versions are incompatible -# with CGO on some versions of golang -retryInstall wixtoolset mingw@11.2 golang archiver diff --git a/contrib/cirrus/win-installer-main.ps1 b/contrib/cirrus/win-installer-main.ps1 index ed5ed67d5a..8bf6764ac3 100644 --- a/contrib/cirrus/win-installer-main.ps1 +++ b/contrib/cirrus/win-installer-main.ps1 @@ -1,52 +1,12 @@ - # Powershell doesn't exit after - function CheckExit { - if ($LASTEXITCODE -ne 0) { - throw "Exit code = $LASTEXITCODE" - } -} -function DownloadFile { - param( - [Parameter(Mandatory)] - [string]$url, - [Parameter(Mandatory)] - [string]$file, - [Int]$retries=5, - [Int]$delay=8 - ) - $ProgressPreference = 'SilentlyContinue'; - Write-Host "Downloading $url to $file" - For($i = 0;;) { - Try { - Invoke-WebRequest -UseBasicParsing -ErrorAction Stop -Uri $url -OutFile $file - Break - } Catch { - if (++$i -gt $retries) { - throw $_.Exception - } - Write-Host "Download failed - retrying:" $_.Exception.Response.StatusCode - Start-Sleep -Seconds $delay - } - } -} -# Drop global envs which have unix paths, defaults are fine -Remove-Item Env:\GOPATH -ErrorAction:Ignore -Remove-Item Env:\GOSRC -ErrorAction:Ignore -Remove-Item Env:\GOCACHE -ErrorAction:Ignore - -# Drop large known env variables (an env > 32k will break MSI/ICE validation) -Remove-Item Env:\CIRRUS_COMMIT_MESSAGE -ErrorAction:Ignore -Remove-Item Env:\CIRRUS_PR_BODY -ErrorAction:Ignore +#!/usr/bin/env powershell -Set-Location contrib\win-installer +. $PSScriptRoot\win-lib.ps1 -# Download and extract alt_build win release zip -$url = "${ENV:ART_URL}/Windows Cross/repo/repo.tbz" -# Arc requires extension to be "tbz2" -DownloadFile "$url" "repo.tbz2" -arc unarchive repo.tbz2 .; CheckExit +Set-Location "$ENV:CIRRUS_WORKING_DIR\repo\contrib\win-installer" # Build Installer -.\build.ps1 $Env:WIN_INST_VER dev repo; CheckExit +# Note: consumes podman-remote-release-windows_amd64.zip from repo.tbz2 +Run-Command ".\build.ps1 $Env:WIN_INST_VER dev `"$ENV:CIRRUS_WORKING_DIR\repo`"" # Run the installer silently and WSL install option disabled (prevent reboots, wsl requirements) # We need AllowOldWin=1 for server 2019 (cirrus image), can be dropped after server 2022 diff --git a/contrib/cirrus/win-lib.ps1 b/contrib/cirrus/win-lib.ps1 new file mode 100644 index 0000000000..4d87ba23df --- /dev/null +++ b/contrib/cirrus/win-lib.ps1 @@ -0,0 +1,72 @@ +#!/usr/bin/env powershell + +# This powershell script is intended to be "dot-sourced" by other scripts. +# It's purpose is identical to that of the `lib.sh` script for Linux environments. + +# Behave similar to `set -e` in bash, but ONLY for powershell commandlets! +# For all legacy, program, and script calls use Run-Command() or Check-Exit() +$ErrorActionPreference = 'Stop' + +# Any golang compilation needs to know what it's building for. +$Env:GOOS = "windows" +$Env:GOARCH = "amd64" + +# Unnecessary and intrusive. They claim parameter/variable +# values aren't collected, but there could be a bug leading +# to a concern over leaking of some sensitive-value. Stop this. +$Env:POWERSHELL_TELEMETRY_OPTOUT = "true" + +# Unnecessary and potentially disruptive. Powershell will +# never ever be updated during automation execution. Stop this. +$Env:POWERSHELL_UPDATECHECK = "off" + +# Color in output may confuse tooling and makes logs hard to read. +# TODO: There are probably other places where color needs to be disabled +# in a slightly different way :( +$Env:NO_COLOR = "true" + +# Items only relevant in a CI environment. +if ($Env:CI -eq "true") { + # Defined by .cirrus.yml for use by all the linux tasks. + # Drop all global envs which have unix paths, defaults are fine. + Remove-Item Env:\GOPATH -ErrorAction:Ignore + Remove-Item Env:\GOSRC -ErrorAction:Ignore + Remove-Item Env:\GOCACHE -ErrorAction:Ignore + + # Defined by Cirrus-CI + # Drop large known env variables (an env > 32k will break MSI/ICE validation) + Remove-Item Env:\CIRRUS_COMMIT_MESSAGE -ErrorAction:Ignore + Remove-Item Env:\CIRRUS_CHANGE_MESSAGE -ErrorAction:Ignore + Remove-Item Env:\CIRRUS_PR_BODY -ErrorAction:Ignore +} + +# Non-powershell commands do not halt execution on error! This helper +# should be called after every critical operation to check and halt on a +# non-zero exit code. Be careful not to use this for powershell commandlets +# (builtins)! They set '$?' to "True" (failed) or "False" success so calling +# this would mask failures. Rely on $ErrorActionPreference = 'Stop' instead. +function Check-Exit { + $result = $LASTEXITCODE # WARNING: might not be a number! + if ( ($result -ne $null) -and ($result -ne 0) ) { + # https://learn.microsoft.com/en-us/dotnet/api/system.management.automation.callstackframe + $caller = (Get-PSCallStack)[1] + Write-Host "Exit code = '$result' from $($caller.ScriptName):$($caller.ScriptLineNumber)" + Exit $result + } +} + +# Small helper to avoid needing to write 'Check-Exit' after every +# non-powershell instruction. It simply prints then executes the _QUOTED_ +# argument followed by Check-Exit. +# N/B: Escape any nested quotes with back-tick ("`") characters. +# WARNING: DO NOT use this with powershell builtins! It will not do what you expect! +function Run-Command { + param ( + [string] $command + ) + + Write-Host $command + + Invoke-Expression $command + Check-Exit +} diff --git a/contrib/cirrus/win-podman-machine-main.ps1 b/contrib/cirrus/win-podman-machine-main.ps1 index fb7690297b..bee9250c7d 100644 --- a/contrib/cirrus/win-podman-machine-main.ps1 +++ b/contrib/cirrus/win-podman-machine-main.ps1 @@ -1,39 +1,20 @@ -$ErrorActionPreference = 'Stop' +#!/usr/bin/env powershell -# Powershell doesn't exit after command failures -# Note, due to a bug in cirrus that does not correctly evaluate exit -# code, error conditions should always be thrown -function CheckExit { - if ($LASTEXITCODE -ne 0) { - throw "Exit code failure = $LASTEXITCODE" - } -} +. $PSScriptRoot\win-lib.ps1 -# Drop global envs which have unix paths, defaults are fine -Remove-Item Env:\GOPATH -ErrorAction:Ignore -Remove-Item Env:\GOSRC -ErrorAction:Ignore -Remove-Item Env:\GOCACHE -ErrorAction:Ignore +Set-Location "$ENV:CIRRUS_WORKING_DIR\repo" -mkdir tmp -Set-Location tmp - -# Download and extract alt_build win release zip -$url = "${ENV:ART_URL}/Windows%20Cross/repo/repo.tbz" -Write-Output "URL: $url" -# Arc requires extension to be "tbz2" -curl.exe -L -o repo.tbz2 "$url"; CheckExit -arc unarchive repo.tbz2 .; CheckExit -Set-Location repo -Expand-Archive -Path "podman-remote-release-windows_amd64.zip" ` - -DestinationPath extracted -Set-Location extracted -$x = Get-ChildItem -Path bin -Recurse -Set-Location $x +Write-Host "Saving selection of CI env. vars." +# Env. vars will not pass through win-sess-launch.ps1 +Get-ChildItem -Path "Env:\*" -include @("PATH", "Chocolatey*", "CIRRUS*", "TEST_*", "CI_*") ` + | Export-CLIXML "$ENV:TEMP\envars.xml" # Recent versions of WSL are packaged as a Windows store app running in # an appX container, which is incompatible with non-interactive # session 0 execution (where the cirrus agent runs). # Run verification under an interactive session instead. -powershell.exe -File "$PSScriptRoot\wsl-env-launch.ps1" ` - "$PSScriptRoot\win-podman-machine-verify.ps1" -CheckExit +Write-Host "Spawning new session to execute $PSScriptRoot\win-podman-machine-test.ps1" +# Can't use Run-Command(), would need overly-complex nested quoting +powershell.exe -File "$PSScriptRoot\win-sess-launch.ps1" ` + "$PSScriptRoot\win-podman-machine-test.ps1" +Check-Exit diff --git a/contrib/cirrus/win-podman-machine-test.ps1 b/contrib/cirrus/win-podman-machine-test.ps1 new file mode 100644 index 0000000000..5833d5bcc6 --- /dev/null +++ b/contrib/cirrus/win-podman-machine-test.ps1 @@ -0,0 +1,33 @@ +#!/usr/bin/env powershell + +. $PSScriptRoot\win-lib.ps1 + +Write-Host "Recovering env. vars." +Import-CLIXML "$ENV:TEMP\envars.xml" | % { + Write-Host " $($_.Name) = $($_.Value)" + Set-Item "Env:$($_.Name)" "$($_.Value)" +} + +if ($Env:TEST_FLAVOR -eq "machine-wsl") { + # FIXME: Test-modes should be definitively set and positively asserted. + # Otherwise if the var. goes out-of-scope, defaults change, or definition + # fails: Suddenly assumed behavior != actual behaviorr, esp. if/when only + # quickly glancing at a green status check-mark. + $Env:CONTAINERS_MACHINE_PROVIDER = "" # IMPLIES WSL +} elseif ($Env:TEST_FLAVOR -eq "machine-hyperv") { + $Env:CONTAINERS_MACHINE_PROVIDER = "hyperv" +} else { + Write-Host "Unsupported value for `$TEST_FLAVOR '$Env:TEST_FLAVOR'" + Exit 1 +} +# Make sure an observer knows the value of this critical variable (consumed by tests). +Write-Host " CONTAINERS_MACHINE_PROVIDER = $Env:CONTAINERS_MACHINE_PROVIDER" +Write-Host "`n" + +# The repo.tbz artifact was extracted here +Set-Location "$ENV:CIRRUS_WORKING_DIR\repo" +# Tests hard-code this location for podman-remote binary, make sure it actually runs. +Run-Command ".\bin\windows\podman.exe --version" + +Write-Host "`nRunning podman-machine e2e tests" +Run-Command ".\winmake localmachine" diff --git a/contrib/cirrus/win-podman-machine-verify.ps1 b/contrib/cirrus/win-podman-machine-verify.ps1 deleted file mode 100644 index 80587b99a3..0000000000 --- a/contrib/cirrus/win-podman-machine-verify.ps1 +++ /dev/null @@ -1,27 +0,0 @@ -$ErrorActionPreference = 'Stop' -function CheckExit { - if ($LASTEXITCODE -ne 0) { - throw "Exit code failure = $LASTEXITCODE" - } -} - -# Verify extracted podman binary -Write-Output `n"Starting init...`n" -.\podman machine init; CheckExit -Write-Output "`nStarting podman machine...`n" -.\podman machine start; CheckExit -Write-Output "`nDumping info...`n" -for ($i =0; $i -lt 60; $i++) { - .\podman info - if ($LASTEXITCODE -eq 0) { - break - } - Start-Sleep -Seconds 2 -} -Write-Output "`nRunning container...`n" -.\podman run ubi8-micro sh -c "exit 123" -if ($LASTEXITCODE -ne 123) { - throw "Expected 123, got $LASTEXITCODE" -} -Write-Host "`nMachine verification is successful!`n" -Exit 0 diff --git a/contrib/cirrus/wsl-env-launch.ps1 b/contrib/cirrus/win-sess-launch.ps1 similarity index 100% rename from contrib/cirrus/wsl-env-launch.ps1 rename to contrib/cirrus/win-sess-launch.ps1 diff --git a/winmake.ps1 b/winmake.ps1 index 228e573ef6..e3e12b7ff1 100644 --- a/winmake.ps1 +++ b/winmake.ps1 @@ -1,4 +1,5 @@ -$ErrorActionPreference = 'Stop' + +. ./contrib/cirrus/win-lib.ps1 # Targets function Podman-Remote{ @@ -68,26 +69,10 @@ function Git-Commit{ return $commit } -function Run-Command { - param ( - [string] $command - ) - - Write-Host $command - - Invoke-Expression $command - $result = $LASTEXITCODE - if ($result -ne 0) { - Write-Host "Command failed (exit: $result)" - Exit $result - } -} - # Init script $target = $args[0] $remotetags = "remote exclude_graphdriver_btrfs btrfs_noversion exclude_graphdriver_devicemapper containers_image_openpgp" -$Env:GOOS = "windows"; $Env:GOARCH = "amd64" switch ($target) { {$_ -in '', 'podman-remote', 'podman'} {