feat: sentry-cli integration test scripts (#54)

.github/workflows/updater-scripts-tests.yml → .github/workflows/script-tests.yml

@@ -1,12 +1,12 @@
-# This isn't a reusable workflow but an actual CI action for this repo itself - to test scripts.
-name: Updater Script Tests
+# This isn't a reusable workflow but a CI action for this repo itself - testing the contained workflows & scripts.
+name: Script Tests
 
 on:
   push:
 
 jobs:
-  test:
-    name: ${{ matrix.host }}
+  updater:
+    name: Updater @ ${{ matrix.host }}
     runs-on: ${{ matrix.host }}-latest
     strategy:
       fail-fast: false

.github/workflows/workflow-tests.yml

@@ -0,0 +1,62 @@
+# This isn't a reusable workflow but an actual CI action for this repo itself - to test the workflows.
+name: Workflow Tests
+
+on:
+  push:
+
+jobs:
+  updater-create-pr:
+    uses: ./.github/workflows/updater.yml
+    with:
+      path: updater/tests/sentry-cli.properties
+      name: CLI
+      pattern: '^2\.0\.'
+      pr-strategy: update
+      _workflow_version: ${{ github.sha }}
+    secrets:
+      api-token: ${{ github.token }}
+
+  updater-test-args:
+    uses: ./.github/workflows/updater.yml
+    with:
+      path: updater/tests/workflow-args.sh
+      name: Workflow args test script
+      runs-on: macos-latest
+      pattern: '.*'
+      _workflow_version: ${{ github.sha }}
+    secrets:
+      api-token: ${{ github.token }}
+
+  updater-test-outputs:
+    runs-on: ubuntu-latest
+    needs:
+      - updater-create-pr
+      - updater-test-args
+    steps:
+      - run: "[[ '${{ needs.updater-create-pr.outputs.baseBranch }}' == 'main' ]]"
+      - run: "[[ '${{ needs.updater-create-pr.outputs.originalTag }}' == '2.0.0' ]]"
+      - run: "[[ '${{ needs.updater-create-pr.outputs.latestTag }}' =~ ^[0-9.]+$ ]]"
+      - run: "[[ '${{ needs.updater-create-pr.outputs.prUrl }}' =~ ^https://github.com/getsentry/github-workflows/pull/[0-9]+$ ]]"
+      - run: "[[ '${{ needs.updater-create-pr.outputs.prBranch }}' == 'deps/updater/tests/sentry-cli.properties' ]]"
+
+      - run: "[[ '${{ needs.updater-test-args.outputs.baseBranch }}' == '' ]]"
+      - run: "[[ '${{ needs.updater-test-args.outputs.originalTag }}' == 'latest' ]]"
+      - run: "[[ '${{ needs.updater-test-args.outputs.latestTag }}' == 'latest' ]]"
+      - run: "[[ '${{ needs.updater-test-args.outputs.prUrl }}' == '' ]]"
+      - run: "[[ '${{ needs.updater-test-args.outputs.prBranch }}' == '' ]]"
+
+  cli-integration:
+    runs-on: ${{ matrix.host }}-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        host:
+          - ubuntu
+          - macos
+          - windows
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: ./sentry-cli/integration-test/
+        with:
+          path: sentry-cli/integration-test/tests/

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## Unreleased
+
+### Features
+
+- Sentry-CLI integration test action ([#54](https://github.com/getsentry/github-workflows/pull/54))
+
 ## 2.6.0
 
 ### Features

sentry-cli/integration-test/.gitignore

@@ -0,0 +1 @@
+*-output.txt

sentry-cli/integration-test/action.psm1

@@ -0,0 +1,167 @@
+# Executes the given block starting a dummy Sentry server that collects and logs requests.
+# The block is given the server URL as a first argument.
+# Returns the dummy server logs.
+
+$ServerUri = "http://127.0.0.1:8000"
+
+class InvokeSentryResult
+{
+    [string[]]$ServerStdOut
+    [string[]]$ServerStdErr
+    [string[]]$ScriptOutput
+
+    # It is common to test debug files uploaded to the server so this function gives you a list.
+    [string[]]UploadedDebugFiles()
+    {
+        $prefix = "upload-dif:"
+        return @($this.ServerStdOut | Where-Object { $_.StartsWith($prefix) } | ForEach-Object { $_.Substring($prefix.Length).Trim() })
+    }
+
+    [bool]HasErrors()
+    {
+        return $this.ServerStdErr.Length -gt 0
+    }
+}
+
+function IsNullOrEmpty([string] $value)
+{
+    "$value".Trim().Length -eq 0
+}
+
+function OutputToArray($output, [string] $uri = $null)
+{
+    if ($output -isnot [system.array])
+    {
+        $output = ("$output".Trim() -replace "`r`n", "`n") -split "`n"
+    }
+
+    if (!(IsNullOrEmpty $uri))
+    {
+        $output = $output -replace $uri, "<ServerUri>"
+    }
+    $output | ForEach-Object { "$_".Trim() }
+}
+
+function RunApiServer([string] $ServerScript, [string] $Uri = $ServerUri)
+{
+    $result = "" | Select-Object -Property process, outFile, errFile, stop, output, dispose
+    Write-Host "Starting the $ServerScript on $Uri" -ForegroundColor DarkYellow
+    $stopwatch = [system.diagnostics.stopwatch]::StartNew()
+
+    $result.outFile = New-TemporaryFile
+    $result.errFile = New-TemporaryFile
+
+    $result.process = Start-Process "python3" -ArgumentList @("$PSScriptRoot/$ServerScript.py", $Uri) `
+        -NoNewWindow -PassThru -RedirectStandardOutput $result.outFile -RedirectStandardError $result.errFile
+
+    $out = New-Object InvokeSentryResult
+    $out.ServerStdOut = @()
+    $out.ServerStdErr = @()
+
+    # We must reassign functions as variables to make them available in a block scope together with GetNewClosure().
+    $OutputToArray = { OutputToArray $args[0] $args[1] }
+    $IsNullOrEmpty = { IsNullOrEmpty $args[0] }
+
+    $result.dispose = {
+        $result.stop.Invoke()
+
+        $stdout = Get-Content $result.outFile -Raw
+        Write-Host "Server stdout:" -ForegroundColor Yellow
+        Write-Host $stdout
+
+        $out.ServerStdOut += & $OutputToArray $stdout $Uri
+
+        $stderr = Get-Content $result.errFile -Raw
+        if (!(& $IsNullOrEmpty $stderr))
+        {
+            Write-Host "Server stderr:" -ForegroundColor Yellow
+            Write-Host $stderr
+            $out.ServerStdErr += & $OutputToArray $stderr $Uri
+        }
+
+        Remove-Item $result.outFile -ErrorAction Continue
+        Remove-Item $result.errFile -ErrorAction Continue
+        return $out
+    }.GetNewClosure()
+
+    $result.stop = {
+        # Stop the HTTP server
+        Write-Host "Stopping the $ServerScript ... " -NoNewline
+        try
+        {
+            Write-Host (Invoke-WebRequest -Uri "$Uri/STOP").StatusDescription
+        }
+        catch
+        {
+            Write-Host "/STOP request failed: $_ - killing the server process instead"
+            $result.process | Stop-Process -Force -ErrorAction SilentlyContinue
+        }
+        $result.process | Wait-Process -Timeout 10 -ErrorAction Continue
+        $result.stop = {}
+    }.GetNewClosure()
+
+    $startupFailed = $false
+    while ($true)
+    {
+        Start-Sleep -Milliseconds 100
+        try
+        {
+            if ((Invoke-WebRequest -Uri "$Uri/_check" -SkipHttpErrorCheck -Method Head).StatusCode -eq 999)
+            {
+                $msg = "Server started successfully in $($stopwatch.ElapsedMilliseconds) ms."
+                Write-Host $additionalOutput -ForegroundColor Green
+                $out.ServerStdOut += $msg
+                break;
+            }
+        }
+        catch
+        {}
+        if ($stopwatch.ElapsedMilliseconds -gt 60000)
+        {
+            $msg = "Server startup timed out."
+            Write-Warning $msg
+            $out.ServerStdErr += $msg
+            $startupFailed = $true;
+            break;
+        }
+        else
+        {
+            Write-Host "Waiting for server to become available..."
+        }
+    }
+
+    if ($result.process.HasExited -or $startupFailed)
+    {
+        $result.stop.Invoke()
+        $result.dispose.Invoke()
+        throw Write-Host "Couldn't start the $ServerScript"
+    }
+
+    return $result
+}
+
+function Invoke-SentryServer([ScriptBlock] $Callback)
+{
+    # start the server
+    $httpServer = RunApiServer "sentry-server"
+
+    $result = $null
+    $output = $null
+    try
+    {
+        # run the test
+        $output = & $Callback $ServerUri
+    }
+    finally
+    {
+        $result = $httpServer.dispose.Invoke()[0]
+    }
+
+    if ($null -ne $result)
+    {
+        $result.ScriptOutput = OutputToArray $output
+    }
+    return $result
+}
+
+Export-ModuleMember -Function Invoke-SentryServer

sentry-cli/integration-test/action.yml

@@ -0,0 +1,21 @@
+name: Sentry CLI integration test
+
+description: |
+  Action to test Sentry CLI integration & symbol upload. This action simply runs all the https://github.com/pester/Pester
+  tests in the given directory. The tests can make use of a dummy Sentry server that collects uploaded symbols.
+  This server is made available as a PowerShell module to your tests.
+
+inputs:
+  path:
+    description: The directory containing all the tests.
+    required: true
+
+runs:
+  using: composite
+
+  steps:
+    - name: Run tests
+      shell: pwsh
+      run: |
+        Import-Module -Name ${{ github.action_path }}/action.psm1 -Force
+        Invoke-Pester -Output Detailed '${{ inputs.path }}'

sentry-cli/integration-test/assets/artifact.json

@@ -0,0 +1,10 @@
+{
+    "id": "fixture-id",
+    "sha1": "fixture-sha1",
+    "name": "fixture-name",
+    "size": 1,
+    "dist": null,
+    "headers": {
+        "fixture-header-key": "fixture-header-value"
+    }
+}

sentry-cli/integration-test/assets/artifacts.json

@@ -0,0 +1,22 @@
+[
+    {
+        "id": "6796495645",
+        "name": "~/dist/bundle.min.js",
+        "dist": "foo",
+        "headers": {
+            "Sourcemap": "dist/bundle.min.js.map"
+        },
+        "size": 497,
+        "sha1": "2fb719956748ab7ec5ae9bcb47606733f5589b72",
+        "dateCreated": "2022-05-12T11:08:01.520199Z"
+    },
+    {
+        "id": "6796495646",
+        "name": "~/dist/bundle.min.js.map",
+        "dist": "foo",
+        "headers": {},
+        "size": 1522,
+        "sha1": "f818059cbf617a8fae9b4e46d08f6c0246bb1624",
+        "dateCreated": "2022-05-12T11:08:01.496220Z"
+    }
+]

sentry-cli/integration-test/assets/associate-dsyms.json

@@ -0,0 +1,17 @@
+{
+    "associatedDsymFiles": [
+        {
+            "uuid": null,
+            "debugId": null,
+            "objectName": "fixture-objectName",
+            "cpuName": "fixture-cpuName",
+            "sha1": "fixture-sha1",
+            "data": {
+                "type": null,
+                "features": [
+                    "fixture-feature"
+                ]
+            }
+        }
+    ]
+}