Skip to content

Commit

Permalink
Add rekor test harness to presubmit tests (sigstore#921)
Browse files Browse the repository at this point in the history
* Add rekor test harness to presubmit tests

This will test critical user journeys against the last three versions of rekor released.

Signed-off-by: Priya Wadhwa <[email protected]>

* Set up matrix for testing

Signed-off-by: Priya Wadhwa <[email protected]>

* Code review comments

Signed-off-by: Priya Wadhwa <[email protected]>
  • Loading branch information
priyawadhwa authored and bobcallaway committed Aug 3, 2022
1 parent d3fa7da commit b531591
Show file tree
Hide file tree
Showing 3 changed files with 123 additions and 1 deletion.
19 changes: 19 additions & 0 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -132,3 +132,22 @@ jobs:
with:
name: Docker Compose logs
path: /tmp/*docker-compose.log

harness:
runs-on: ubuntu-20.04
needs: build
steps:
- uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v3.0.2
- name: Extract version of Go to use
run: echo "GOVERSION=$(cat Dockerfile|grep golang | awk ' { print $2 } ' | cut -d '@' -f 1 | cut -d ':' -f 2 | uniq)" >> $GITHUB_ENV
- uses: actions/setup-go@b22fbbc2921299758641fab08929b4ac52b32923 # v3.1.0
with:
go-version: ${{ env.GOVERSION }}
- name: Run test harness
run: ./tests/rekor-harness.sh
- name: Upload logs if they exist
uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8 # v3
if: failure()
with:
name: E2E Docker Compose logs
path: /tmp/docker-compose.log
2 changes: 1 addition & 1 deletion tests/e2e_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -203,7 +203,7 @@ type getOut struct {
IntegratedTime int64
}

func TestGet(t *testing.T) {
func TestGetCLI(t *testing.T) {
// Create something and add it to the log
artifactPath := filepath.Join(t.TempDir(), "artifact")
sigPath := filepath.Join(t.TempDir(), "signature.asc")
Expand Down
103 changes: 103 additions & 0 deletions tests/rekor-harness.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
#!/bin/bash
#
# Copyright 2022 The Sigstore Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -e

function start_server () {
server_version=$1
current_branch=$(git rev-parse --abbrev-ref HEAD)
git checkout $server_version
if [ $(docker-compose ps | grep -c "(healthy)") == 0 ]; then
echo "starting services with version $server_version"
docker-compose up -d --build
else
echo "turning down rekor and restarting at version $server_version"
docker stop $(docker ps --filter name=rekor-server --format {{.ID}})
docker-compose up -d --build rekor-server
fi
git checkout $current_branch

count=0
echo -n "waiting up to 60 sec for system to start"
until [ $(docker-compose ps | grep -c "(healthy)") == 3 ];
do
if [ $count -eq 6 ]; then
echo "! timeout reached"
exit 1
else
echo -n "."
sleep 10
let 'count+=1'
fi
done
echo
}

function build_cli () {
echo "Building CLI at version $cli_version"
cli_version=$1
current_branch=$(git rev-parse --abbrev-ref HEAD)
git checkout $cli_version
make rekor-cli
git checkout $current_branch
}

function run_tests () {
REKORTMPDIR="$(mktemp -d -t rekor_test.XXXXXX)"
touch $REKORTMPDIR.rekor.yaml
trap "rm -rf $REKORTMPDIR" EXIT


go clean -testcache
for test in $HARNESS_TESTS
do
if ! REKORTMPDIR=$REKORTMPDIR go test -run $test -v -tags=e2e ./tests/ > $REKORTMPDIR/logs ; then
cat $REKORTMPDIR/logs
docker-compose logs --no-color > /tmp/docker-compose.log
exit 1
fi
if docker-compose logs --no-color | grep -q "panic: runtime error:" ; then
# if we're here, we found a panic
echo "Failing due to panics detected in logs"
docker-compose logs --no-color > /tmp/docker-compose.log
exit 1
fi
done
}

# Get last 3 server versions
git fetch origin
VERSIONS=$(git tag --sort=-version:refname | head -n 3 | tac)
echo $VERSIONS

HARNESS_TESTS="TestUploadVerify TestLogInfo TestGetCLI TestSSH TestJAR TestAPK TestIntoto TestX509 TestEntryUpload"

for server_version in $VERSIONS
do
start_server $server_version
for cli_version in $VERSIONS
do
echo "======================================================="
echo "Running tests with server version $server_version and CLI version $cli_version"

build_cli $cli_version
run_tests

echo "Tests passed successfully."
echo "======================================================="
done
done

echo "Harness testing successful :)"

0 comments on commit b531591

Please sign in to comment.