From 2964ff0ff6c1018748d7bd909c344bdf18c31652 Mon Sep 17 00:00:00 2001 From: Michele Sciabarra Date: Thu, 8 Dec 2022 12:39:45 +0000 Subject: [PATCH 01/39] github action initial commit --- .github/workflows/build.yml | 49 +++++++++++++++ tools/github/README.md | 36 +++++++++++ tools/github/box-upload.py | 89 ++++++++++++++++++++++++++++ tools/github/checkAndUploadLogs.sh | 48 +++++++++++++++ tools/github/debugAction.sh | 39 ++++++++++++ tools/github/distDocker.sh | 36 +++++++++++ tools/github/docker.conf | 19 ++++++ tools/github/docker.sh | 56 +++++++++++++++++ tools/github/flake8.sh | 42 +++++++++++++ tools/github/runLeanSystemTests.sh | 36 +++++++++++ tools/github/runMultiRuntimeTests.sh | 36 +++++++++++ tools/github/runSchedulerTests.sh | 35 +++++++++++ tools/github/runStandaloneTests.sh | 64 ++++++++++++++++++++ tools/github/runSystemTests.sh | 36 +++++++++++ tools/github/runTests.sh | 32 ++++++++++ tools/github/runUnitTests.sh | 38 ++++++++++++ tools/github/scan.sh | 52 ++++++++++++++++ tools/github/setup-docker.py | 78 ++++++++++++++++++++++++ tools/github/setup.sh | 53 +++++++++++++++++ tools/github/setupLeanSystem.sh | 33 +++++++++++ tools/github/setupPrereq.sh | 38 ++++++++++++ tools/github/setupSystem.sh | 36 +++++++++++ tools/github/waitIfDebug.sh | 9 +++ 23 files changed, 990 insertions(+) create mode 100644 .github/workflows/build.yml create mode 100644 tools/github/README.md create mode 100755 tools/github/box-upload.py create mode 100755 tools/github/checkAndUploadLogs.sh create mode 100755 tools/github/debugAction.sh create mode 100755 tools/github/distDocker.sh create mode 100644 tools/github/docker.conf create mode 100755 tools/github/docker.sh create mode 100755 tools/github/flake8.sh create mode 100755 tools/github/runLeanSystemTests.sh create mode 100755 tools/github/runMultiRuntimeTests.sh create mode 100755 tools/github/runSchedulerTests.sh create mode 100755 tools/github/runStandaloneTests.sh create mode 100755 tools/github/runSystemTests.sh create mode 100755 tools/github/runTests.sh create mode 100755 tools/github/runUnitTests.sh create mode 100755 tools/github/scan.sh create mode 100755 tools/github/setup-docker.py create mode 100755 tools/github/setup.sh create mode 100755 tools/github/setupLeanSystem.sh create mode 100755 tools/github/setupPrereq.sh create mode 100755 tools/github/setupSystem.sh create mode 100755 tools/github/waitIfDebug.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000000..71d183f62b1 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +name: OpenWhisk + +jobs: + build: + name: Build + runs-on: ubuntu-latest + steps: + - name: Set up environment variables + env: + ANSIBLE_CMD: "ansible-playbook -i environments/local -e docker_image_prefix=testing" + GRADLE_PROJS_SKIP: "" + + - name: Checkout + uses: actions/checkout@v2 + - name: Add java + uses: actions/setup-java@v1 + with: + java-version: 11 + - name: NgrokDebug + run: |- + if test -e .github/debug + then bash tools/github/debugAction.sh + fi + env: + NGROK_TOKEN: ${{ secrets.NGROK_TOKEN }} + NGROK_PASSWORD: ${{ secrets.NGROK_PASSWORD }} + - name: WaitForNgrokDebug + run: |- + if test -e .github/debug + then bash tools/github/waitIfDebug.sh + fi + diff --git a/tools/github/README.md b/tools/github/README.md new file mode 100644 index 00000000000..8d0fbad9580 --- /dev/null +++ b/tools/github/README.md @@ -0,0 +1,36 @@ + + +# Travis Setup + +Travis build is configured to perform build of this repo in multiple parallel jobs as listed below. + +1. Unit Tests - Runs the test which only need database service. +2. System Tests - Runs those tests which need complete OpenWhisk system up and running. +3. Performance test suite - Run basic performance tests with the objective to check if tests are working or not. + +These jobs make use of following scripts + +1. `scan.sh` - Performs various code scan task like python flake scan, scala formatting etc. +2. `setupPrereq.sh` - Performs setup if basis prerequisites like database setup and property file generation. +3. `distDocker.sh` - Builds the various docker containers. +4. `setupSystem.sh` - Runs the various containers which are part of an OpenWhisk setup like Controller, Invoker etc. +5. `runTests.sh` - Runs the tests. It make use of `ORG_GRADLE_PROJECT_testSetName` env setting to determine which test + suite to run. +6. `checkAndUploadLogs.sh` - Collects the logs, checks them and uploads them to https://openwhisk.box.com/v/travis-logs. diff --git a/tools/github/box-upload.py b/tools/github/box-upload.py new file mode 100755 index 00000000000..fad503974f4 --- /dev/null +++ b/tools/github/box-upload.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python +"""Executable Python script for compressing folders to Box. + +Compresses the contents of a folder and upload the result to Box. + + Run this script as: + $ upload-logs.py LOG_DIR DEST_NAME + + e.g.: $ upload-logs.py /tmp/wsklogs logs-5512.tar.gz + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +""" + +from __future__ import print_function + +import os +import subprocess +import sys +import tempfile +import urllib +import humanize +import requests +import hashlib + + +def upload_file(local_file, remote_file): + """Upload file.""" + if remote_file[0] == '/': + remote_file = remote_file[1:] + + url = "http://DamCYhF8.mybluemix.net/upload?%s" % \ + urllib.parse.urlencode({"name": remote_file}) + + r = requests.post(url, + headers={"Content-Type": "application/gzip"}, + data=open(local_file, 'rb')) + + print("Posting result", r) + print(r.text) + + +def tar_gz_dir(dir_path): + """Create TAR (ZIP) of path and its contents.""" + _, dst = tempfile.mkstemp(suffix=".tar.gz") + subprocess.call(["tar", "-cvzf", dst, dir_path]) + return dst + + +def print_tarball_size(tarball): + """Get and print the size of the tarball""" + tarballsize = os.path.getsize(tarball) + print("Size of tarball", tarball, "is", humanize.naturalsize(tarballsize)) + + sha256_hash = hashlib.sha256() + with open(tarball, "rb") as f: + for byte_block in iter(lambda: f.read(4096), b""): + sha256_hash.update(byte_block) + print("SHA256 hash of tarball is", sha256_hash.hexdigest()) + + +if __name__ == "__main__": + dir_path = sys.argv[1] + dst_path = sys.argv[2] + + if not os.path.isdir(dir_path): + print("Directory doesn't exist: %s." % dir_path) + sys.exit(0) + + print("Compressing logs dir...") + tar = tar_gz_dir(dir_path) + print_tarball_size(tar) + + print("Uploading to Box...") + upload_file(tar, dst_path) diff --git a/tools/github/checkAndUploadLogs.sh b/tools/github/checkAndUploadLogs.sh new file mode 100755 index 00000000000..36ce72a8c3e --- /dev/null +++ b/tools/github/checkAndUploadLogs.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Disable abort script at first error as we require the logs to be uploaded +# even if check and log collection fails +# set -e + +SECONDS=0 +SCRIPTDIR=$(cd $(dirname "$0") && pwd) +ROOTDIR="$SCRIPTDIR/../.." + +cd $ROOTDIR + +LOG_NAME=$1 +TAGS=${2-""} +LOG_TAR_NAME="${LOG_NAME}_${TRAVIS_BUILD_ID}-$TRAVIS_BRANCH.tar.gz" + +# Perf logs are typically about 20MB and thus rapidly fill our box account. +# Disable upload to reduce the interval at which we need to manually clean logs from box. +if [ "$LOG_NAME" == "perf" ]; then + echo "Skipping upload of perf logs to conserve space" + exit 0 +fi + +ansible-playbook -i ansible/environments/local ansible/logs.yml + +./tools/build/checkLogs.py logs "$TAGS" + +./tools/travis/box-upload.py "$TRAVIS_BUILD_DIR/logs" "$LOG_TAR_NAME" + +echo "Uploaded Logs with name $LOG_TAR_NAME" +echo "Time taken for ${0##*/} is $SECONDS secs" diff --git a/tools/github/debugAction.sh b/tools/github/debugAction.sh new file mode 100755 index 00000000000..75df4a233e7 --- /dev/null +++ b/tools/github/debugAction.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +if [[ -z "$NGROK_TOKEN" ]]; then + echo "Please set 'NGROK_TOKEN'" + exit 2 +fi + +if [[ -z "$NGROK_PASSWORD" ]]; then + echo "Please set 'NGROK_PASS' for user: $USER" + exit 3 +fi + +echo "### Install ngrok ###" + +wget -q https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-386.zip +unzip ngrok-stable-linux-386.zip +chmod +x ./ngrok + +echo "### Update user: $USER password ###" +echo -e "$NGROK_PASSWORD\n$NGROK_PASSWORD" | sudo passwd "$USER" + +echo "### Start ngrok proxy for 22 port ###" + +rm -f .ngrok.log +./ngrok authtoken "$NGROK_TOKEN" +./ngrok tcp 22 --log ".ngrok.log" & + +sleep 10 +HAS_ERRORS=$(grep "command failed" < .ngrok.log) + +if [[ -z "$HAS_ERRORS" ]]; then + echo "" + echo "==========================================" + echo "To connect: $(grep -o -E "tcp://(.+)" < .ngrok.log | sed "s/tcp:\/\//ssh $USER@/" | sed "s/:/ -p /")" + echo "==========================================" +else + echo "$HAS_ERRORS" + exit 4 +fi diff --git a/tools/github/distDocker.sh b/tools/github/distDocker.sh new file mode 100755 index 00000000000..13c8450d98b --- /dev/null +++ b/tools/github/distDocker.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e + +# Build script for Travis-CI. + +SECONDS=0 +SCRIPTDIR=$(cd $(dirname "$0") && pwd) +ROOTDIR="$SCRIPTDIR/../.." + +cd $ROOTDIR +TERM=dumb ./gradlew clean # Run a clean step before build +TERM=dumb ./gradlew distDocker -PdockerImagePrefix=testing $GRADLE_PROJS_SKIP + +TERM=dumb ./gradlew :core:controller:distDockerCoverage -PdockerImagePrefix=testing +TERM=dumb ./gradlew :core:scheduler:distDockerCoverage -PdockerImagePrefix=testing +TERM=dumb ./gradlew :core:invoker:distDockerCoverage -PdockerImagePrefix=testing +TERM=dumb ./gradlew :core:standalone:build + +echo "Time taken for ${0##*/} is $SECONDS secs" diff --git a/tools/github/docker.conf b/tools/github/docker.conf new file mode 100644 index 00000000000..4b9d8df63ae --- /dev/null +++ b/tools/github/docker.conf @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +[Service] +ExecStart= +ExecStart=/usr/bin/dockerd diff --git a/tools/github/docker.sh b/tools/github/docker.sh new file mode 100755 index 00000000000..21eed7c2b16 --- /dev/null +++ b/tools/github/docker.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +BASEDIR=$(dirname "$0") +echo "$BASEDIR" + +sudo gpasswd -a travis docker +sudo usermod -aG docker travis +#sudo -E bash -c 'echo '\''DOCKER_OPTS="-H tcp://0.0.0.0:4243 -H unix:///var/run/docker.sock --storage-driver=overlay --userns-remap=default"'\'' > /etc/default/docker' + +# Docker +sudo apt-get clean +sudo apt-get update + +# Need to update dpkg due to known issue: https://bugs.launchpad.net/ubuntu/+source/dpkg/+bug/1730627 +sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common dpkg +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - +sudo apt-key fingerprint 0EBFCD88 + +# This is required because libseccomp2 (>= 2.3.0) is not provided in trusty by default +sudo add-apt-repository -y ppa:ubuntu-sdk-team/ppa + +sudo add-apt-repository \ + "deb [arch=$(uname -m | sed -e 's/x86_64/amd64/g')] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + +sudo apt-get update +sudo apt-get -o Dpkg::Options::="--force-confold" --force-yes -y install docker-ce=18.06.3~ce~3-0~ubuntu containerd.io +# daemon.json and flags does not work together. Overwritting the docker.service file +# to remove the host flags. - https://docs.docker.com/config/daemon/#troubleshoot-conflicts-between-the-daemonjson-and-startup-scripts +sudo mkdir -p /etc/systemd/system/docker.service.d +sudo cp $BASEDIR/docker.conf /etc/systemd/system/docker.service.d/docker.conf +# setup-docker will add configs to /etc/docker/daemon.json +sudo python $BASEDIR/setup-docker.py +sudo cat /etc/docker/daemon.json +sudo systemctl daemon-reload +sudo systemctl restart docker +sudo systemctl status docker.service +echo "Docker Version:" +docker version +echo "Docker Info:" +docker info diff --git a/tools/github/flake8.sh b/tools/github/flake8.sh new file mode 100755 index 00000000000..98b7c0d9c0c --- /dev/null +++ b/tools/github/flake8.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +pip3 install --user --upgrade flake8 + +# These files do not have a .py extension so flake8 will not scan them +declare -a PYTHON_FILES=("." + "./tools/admin/wskadmin" + "./tools/build/citool" + "./tools/build/redo") + +echo 'Flake8: first round (fast fail) stops the build if there are any Python 3 syntax errors...' +for i in "${PYTHON_FILES[@]}" +do + flake8 "$i" --select=E999,F821 --statistics + RETURN_CODE=$? + if [ $RETURN_CODE != 0 ]; then + echo 'Flake8 found Python 3 syntax errors above. See: https://docs.python.org/3/howto/pyporting.html' + exit $RETURN_CODE + fi +done + +echo 'Flake8: second round to find any other stylistic issues...' +for i in "${PYTHON_FILES[@]}" +do + flake8 "$i" --ignore=E,W503,W504,W605 --max-line-length=127 --statistics +done diff --git a/tools/github/runLeanSystemTests.sh b/tools/github/runLeanSystemTests.sh new file mode 100755 index 00000000000..372fbc1e409 --- /dev/null +++ b/tools/github/runLeanSystemTests.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e + +SCRIPTDIR=$(cd $(dirname "$0") && pwd) +ROOTDIR="$SCRIPTDIR/../.." + +cd $ROOTDIR/tools/travis + +export ORG_GRADLE_PROJECT_testSetName="REQUIRE_LEAN_SYSTEM" +export GRADLE_COVERAGE=true + +./setupPrereq.sh /ansible/files/runtimes-nodeonly.json + +./distDocker-lean.sh + +./setupLeanSystem.sh /ansible/files/runtimes-nodeonly.json + +./runTests.sh diff --git a/tools/github/runMultiRuntimeTests.sh b/tools/github/runMultiRuntimeTests.sh new file mode 100755 index 00000000000..e2de0c59bd4 --- /dev/null +++ b/tools/github/runMultiRuntimeTests.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e + +SCRIPTDIR=$(cd $(dirname "$0") && pwd) +ROOTDIR="$SCRIPTDIR/../.." + +cd $ROOTDIR/tools/travis + +export ORG_GRADLE_PROJECT_testSetName="REQUIRE_MULTI_RUNTIME" +export GRADLE_COVERAGE=true + +./setupPrereq.sh + +./distDocker.sh + +./setupSystem.sh + +./runTests.sh diff --git a/tools/github/runSchedulerTests.sh b/tools/github/runSchedulerTests.sh new file mode 100755 index 00000000000..000de8080e5 --- /dev/null +++ b/tools/github/runSchedulerTests.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e + +SCRIPTDIR=$(cd $(dirname "$0") && pwd) +ROOTDIR="$SCRIPTDIR/../.." + +cd $ROOTDIR/tools/travis +export ORG_GRADLE_PROJECT_testSetName="REQUIRE_SCHEDULER" +export GRADLE_COVERAGE=true + +./setupPrereq.sh /ansible/files/runtimes-nodeonly.json + +./distDocker.sh + +./setupSystem.sh /ansible/files/runtimes-nodeonly.json + +./runTests.sh diff --git a/tools/github/runStandaloneTests.sh b/tools/github/runStandaloneTests.sh new file mode 100755 index 00000000000..ba10c5b4715 --- /dev/null +++ b/tools/github/runStandaloneTests.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e + +SCRIPTDIR=$(cd $(dirname "$0") && pwd) +ROOTDIR="$SCRIPTDIR/../.." + +export ORG_GRADLE_PROJECT_testSetName="REQUIRE_STANDALONE" +export GRADLE_COVERAGE=true + +cd $ROOTDIR/ansible +$ANSIBLE_CMD setup.yml +$ANSIBLE_CMD properties.yml -e manifest_file="/ansible/files/runtimes-nodeonly.json" +$ANSIBLE_CMD downloadcli-github.yml + +# Install kubectl +curl -Lo ./kubectl https://storage.googleapis.com/kubernetes-release/release/v1.16.1/bin/linux/amd64/kubectl +chmod +x kubectl +sudo cp kubectl /usr/local/bin/kubectl + +# Install kind +curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/v0.5.1/kind-linux-amd64 +chmod +x kind +sudo cp kind /usr/local/bin/kind + +kind create cluster --wait 5m +export KUBECONFIG="$(kind get kubeconfig-path)" +kubectl config set-context --current --namespace=default + +# This is required because it is timed out to pull the image during the test. +docker pull openwhisk/action-nodejs-v14:nightly +docker pull openwhisk/dockerskeleton:nightly +docker pull openwhisk/example:nightly +docker pull openwhisk/apigateway:0.11.0 + +cd $ROOTDIR +TERM=dumb ./gradlew :core:standalone:build \ + :core:monitoring:user-events:distDocker + +cd $ROOTDIR +TERM=dumb ./gradlew :core:standalone:cleanTest \ + :core:standalone:test \ + :core:monitoring:user-events:reportTestScoverage + +# Run test in end as it publishes the coverage also +cd $ROOTDIR/tools/travis +./runTests.sh diff --git a/tools/github/runSystemTests.sh b/tools/github/runSystemTests.sh new file mode 100755 index 00000000000..8325caddbae --- /dev/null +++ b/tools/github/runSystemTests.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e + +SCRIPTDIR=$(cd $(dirname "$0") && pwd) +ROOTDIR="$SCRIPTDIR/../.." + +cd $ROOTDIR/tools/travis + +export ORG_GRADLE_PROJECT_testSetName="REQUIRE_SYSTEM" +export GRADLE_COVERAGE=true + +./setupPrereq.sh /ansible/files/runtimes-nodeonly.json + +./distDocker.sh + +./setupSystem.sh /ansible/files/runtimes-nodeonly.json + +./runTests.sh diff --git a/tools/github/runTests.sh b/tools/github/runTests.sh new file mode 100755 index 00000000000..e2a7f7a4b32 --- /dev/null +++ b/tools/github/runTests.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e + +# Build script for Travis-CI. + +SECONDS=0 +SCRIPTDIR=$(cd "$(dirname "$0")" && pwd) +ROOTDIR="$SCRIPTDIR/../.." + +cd $ROOTDIR +cat whisk.properties +TERM=dumb ./gradlew :tests:testCoverageLean :tests:reportCoverage :tests:testSwaggerCodegen + +bash <(curl -s https://codecov.io/bash) +echo "Time taken for ${0##*/} is $SECONDS secs" diff --git a/tools/github/runUnitTests.sh b/tools/github/runUnitTests.sh new file mode 100755 index 00000000000..81fc0623b10 --- /dev/null +++ b/tools/github/runUnitTests.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e + +SCRIPTDIR=$(cd $(dirname "$0") && pwd) +ROOTDIR="$SCRIPTDIR/../.." + +cd $ROOTDIR/tools/travis +export TESTCONTAINERS_RYUK_DISABLED="true" +export ORG_GRADLE_PROJECT_testSetName="REQUIRE_ONLY_DB" + +./setupPrereq.sh + +cat "$ROOTDIR/tests/src/test/resources/application.conf" + +./distDocker.sh + +# yet another hack to hit docker rate limits early... +docker pull alpine:3.5 + +./runTests.sh diff --git a/tools/github/scan.sh b/tools/github/scan.sh new file mode 100755 index 00000000000..9a69413d30c --- /dev/null +++ b/tools/github/scan.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e + +# Build script for Travis-CI. +SECONDS=0 +SCRIPTDIR=$(cd $(dirname "$0") && pwd) +ROOTDIR="$SCRIPTDIR/../.." +HOMEDIR="$SCRIPTDIR/../../../" +UTILDIR="$HOMEDIR/openwhisk-utilities/" + +cd $ROOTDIR +./tools/travis/flake8.sh # Check Python files for style and stop the build on syntax errors + +# clone the openwhisk utilities repo. +cd $HOMEDIR +git clone https://github.com/apache/openwhisk-utilities.git + +# run the scancode util. against project source code starting at its root +cd $UTILDIR +scancode/scanCode.py --config scancode/ASF-Release.cfg $ROOTDIR + +# run scalafmt checks +cd $ROOTDIR +TERM=dumb ./gradlew checkScalafmtAll + +# lint tests to all be actually runnable +MISSING_TESTS=$(grep -rL "RunWith" --include="*Tests.scala" tests || true) +if [ -n "$MISSING_TESTS" ] +then + echo "The following tests are missing the 'RunWith' annotation" + echo $MISSING_TESTS + exit 1 +fi + +echo "Time taken for ${0##*/} is $SECONDS secs" diff --git a/tools/github/setup-docker.py b/tools/github/setup-docker.py new file mode 100755 index 00000000000..525e6f44479 --- /dev/null +++ b/tools/github/setup-docker.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python +"""Executable Python script for setting up docker daemon. + +Add docker daemon configuration options in /etc/docker/daemon.json + + Run this script as: + $python setup-docker.py + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +""" + +from __future__ import print_function + +import json +import traceback + +DOCKER_DAEMON_FILE = "/etc/docker/daemon.json" + +# Read the file. + +DOCKER_OPTS = { + "hosts": [ + "tcp://0.0.0.0:4243", + "unix:///var/run/docker.sock" + ], + "storage-driver": "overlay", + "userns-remap": "default" +} + + +def get_daemon_content(): + data = {} + with open(DOCKER_DAEMON_FILE) as json_file: + data = json.load(json_file) + return data + + +def add_content(data): + for config in DOCKER_OPTS.items(): + # config will be a tuple of key, value + # ('hosts', ['tcp://0.0.0.0:4243', 'unix:///var/run/docker.sock']) + key, value = config + data[key] = value + return data + + +def write_to_daemon_conf(data): + try: + with open(DOCKER_DAEMON_FILE, 'w') as fp: + json.dump(data, fp) + except Exception as e: + print("Failed to write to daemon file") + print(e) + traceback.print_exc() + + +if __name__ == "__main__": + current_data = get_daemon_content() + print(current_data) + updated_data = add_content(current_data) + print(updated_data) + write_to_daemon_conf(updated_data) + print("Successfully Configured Docker daemon.json") diff --git a/tools/github/setup.sh b/tools/github/setup.sh new file mode 100755 index 00000000000..6d67941e340 --- /dev/null +++ b/tools/github/setup.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# retries a command for five times and exits with the non-zero exit if even after +# the retries the command did not succeed. +function retry() { + local exitcode=0 + for i in {1..5}; + do + exitcode=0 + "$@" && break || exitcode=$? && echo "$i. attempt failed. Will retry $((5-i)) more times!" && sleep 1; + done + if [ $exitcode -ne 0 ]; then + exit $exitcode + fi +} + +# Python +python -m pip install --user couchdb + +# Ansible +python -m pip install --user ansible==2.8.18 + +# Azure CosmosDB +python -m pip install --user pydocumentdb + +# Support the revises log upload script +python -m pip install --user humanize requests + +# Scan code before compiling the code +./scan.sh + +# Basic check that all code compiles and depdendencies are downloaded correctly. +# Compiling the tests will compile all components as well. +# +# Downloads the gradle wrapper, dependencies and tries to compile the code. +# Retried 5 times in case there are network hiccups. +TERM=dumb retry ./gradlew :tests:compileTestScala diff --git a/tools/github/setupLeanSystem.sh b/tools/github/setupLeanSystem.sh new file mode 100755 index 00000000000..89568b21c0a --- /dev/null +++ b/tools/github/setupLeanSystem.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e + +# Build script for Travis-CI. +SECONDS=0 +SCRIPTDIR=$(cd $(dirname "$0") && pwd) +ROOTDIR="$SCRIPTDIR/../.." +RUNTIMES_MANIFEST=${1:-"/ansible/files/runtimes.json"} + +cd $ROOTDIR/ansible + +$ANSIBLE_CMD openwhisk.yml -e manifest_file="$RUNTIMES_MANIFEST" -e lean=true +$ANSIBLE_CMD apigateway.yml +$ANSIBLE_CMD routemgmt.yml + +echo "Time taken for ${0##*/} is $SECONDS secs" diff --git a/tools/github/setupPrereq.sh b/tools/github/setupPrereq.sh new file mode 100755 index 00000000000..264fc3da047 --- /dev/null +++ b/tools/github/setupPrereq.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e + +# Build script for Travis-CI. +SECONDS=0 +SCRIPTDIR=$(cd $(dirname "$0") && pwd) +ROOTDIR="$SCRIPTDIR/../.." +RUNTIMES_MANIFEST=${1:-"/ansible/files/runtimes.json"} + +cd $ROOTDIR/ansible + +$ANSIBLE_CMD setup.yml -e mode=HA +$ANSIBLE_CMD prereq.yml +$ANSIBLE_CMD couchdb.yml +$ANSIBLE_CMD initdb.yml +$ANSIBLE_CMD wipe.yml +$ANSIBLE_CMD elasticsearch.yml +$ANSIBLE_CMD etcd.yml + +$ANSIBLE_CMD properties.yml -e manifest_file="$RUNTIMES_MANIFEST" +echo "Time taken for ${0##*/} is $SECONDS secs" diff --git a/tools/github/setupSystem.sh b/tools/github/setupSystem.sh new file mode 100755 index 00000000000..387d45c6fa6 --- /dev/null +++ b/tools/github/setupSystem.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e + +# Build script for Travis-CI. +SECONDS=0 +SCRIPTDIR=$(cd $(dirname "$0") && pwd) +ROOTDIR="$SCRIPTDIR/../.." +RUNTIMES_MANIFEST=${1:-"/ansible/files/runtimes.json"} + +# This is required because it is timed out to pull the image during the test. +docker pull openwhisk/example + +cd $ROOTDIR/ansible + +$ANSIBLE_CMD openwhisk.yml -e manifest_file="$RUNTIMES_MANIFEST" -e db_activation_backend=ElasticSearch +$ANSIBLE_CMD apigateway.yml +$ANSIBLE_CMD routemgmt.yml + +echo "Time taken for ${0##*/} is $SECONDS secs" diff --git a/tools/github/waitIfDebug.sh b/tools/github/waitIfDebug.sh new file mode 100755 index 00000000000..5c886a59d9e --- /dev/null +++ b/tools/github/waitIfDebug.sh @@ -0,0 +1,9 @@ +#!/bin/bash +echo "You have an hour to debug this build." +echo "remove .github/debug to stop the wait now." +for i in $(seq 1 60) +do if test -e .github/debug + sleep 60 + echo "ngrok active since $i minutes" + fi +done From e2177d3f50c1ae6c2375990f0e4c2aeb79a754e9 Mon Sep 17 00:00:00 2001 From: Michele Sciabarra Date: Thu, 8 Dec 2022 12:46:09 +0000 Subject: [PATCH 02/39] fix blank --- .github/workflows/build.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 71d183f62b1..a20b6b6b6f8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -26,7 +26,6 @@ jobs: env: ANSIBLE_CMD: "ansible-playbook -i environments/local -e docker_image_prefix=testing" GRADLE_PROJS_SKIP: "" - - name: Checkout uses: actions/checkout@v2 - name: Add java @@ -34,7 +33,7 @@ jobs: with: java-version: 11 - name: NgrokDebug - run: |- + run: |- if test -e .github/debug then bash tools/github/debugAction.sh fi From 5aa489ae7f59776828624a2b1ef2e70153212ce2 Mon Sep 17 00:00:00 2001 From: Michele Sciabarra Date: Thu, 8 Dec 2022 12:55:57 +0000 Subject: [PATCH 03/39] debug --- .github/debug | 0 .github/workflows/build.yml | 13 +++++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 .github/debug diff --git a/.github/debug b/.github/debug new file mode 100644 index 00000000000..e69de29bb2d diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a20b6b6b6f8..c6611ecb0ea 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,21 +17,24 @@ # name: OpenWhisk +env: + ANSIBLE_CMD: "ansible-playbook -i environments/local -e docker_image_prefix=testing" + GRADLE_PROJS_SKIP: "" + jobs: build: name: Build runs-on: ubuntu-latest steps: - - name: Set up environment variables - env: - ANSIBLE_CMD: "ansible-playbook -i environments/local -e docker_image_prefix=testing" - GRADLE_PROJS_SKIP: "" - name: Checkout uses: actions/checkout@v2 - name: Add java uses: actions/setup-java@v1 with: java-version: 11 + # this section is to debug the action with ngrok + # you need to add the relevant secrets + # and enable it with touch .github/debug and push it - name: NgrokDebug run: |- if test -e .github/debug @@ -40,6 +43,8 @@ jobs: env: NGROK_TOKEN: ${{ secrets.NGROK_TOKEN }} NGROK_PASSWORD: ${{ secrets.NGROK_PASSWORD }} + # if ngrok debuggins is enabled this will wait for an hour to let debug + # you can sto the waiting and resume execution with rm .github/debug - name: WaitForNgrokDebug run: |- if test -e .github/debug From 57f76fd9f08a927a48535e8f9872aa02de131d1f Mon Sep 17 00:00:00 2001 From: Michele Sciabarra Date: Thu, 8 Dec 2022 13:18:50 +0000 Subject: [PATCH 04/39] fix push --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c6611ecb0ea..52f1a13a563 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,6 +16,8 @@ # under the License. # name: OpenWhisk +on: + push: env: ANSIBLE_CMD: "ansible-playbook -i environments/local -e docker_image_prefix=testing" From 3bdc6cdef55fac7a565a58617eadf6e3afc677d8 Mon Sep 17 00:00:00 2001 From: Michele Sciabarra Date: Thu, 8 Dec 2022 13:37:01 +0000 Subject: [PATCH 05/39] fix java --- .github/workflows/build.yml | 4 +++- tools/github/waitIfDebug.sh | 5 ++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 52f1a13a563..2cfa47c6464 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,9 +31,11 @@ jobs: - name: Checkout uses: actions/checkout@v2 - name: Add java - uses: actions/setup-java@v1 + uses: actions/setup-java@v2 with: + distribution: adopt-openj9 java-version: 11 + cache: 'gradle' # this section is to debug the action with ngrok # you need to add the relevant secrets # and enable it with touch .github/debug and push it diff --git a/tools/github/waitIfDebug.sh b/tools/github/waitIfDebug.sh index 5c886a59d9e..b1b113e59a5 100755 --- a/tools/github/waitIfDebug.sh +++ b/tools/github/waitIfDebug.sh @@ -1,9 +1,8 @@ #!/bin/bash echo "You have an hour to debug this build." -echo "remove .github/debug to stop the wait now." +echo "Remove .github/debug to stop the wait now." for i in $(seq 1 60) do if test -e .github/debug - sleep 60 - echo "ngrok active since $i minutes" + then sleep 60 ; echo "$i minutes elapsed" fi done From f400f1e9f9ab04aed40c3ac0fda8dbe7fa8e5716 Mon Sep 17 00:00:00 2001 From: Michele Sciabarra Date: Thu, 8 Dec 2022 14:22:32 +0000 Subject: [PATCH 06/39] add setup --- .github/workflows/build.yml | 6 ++++++ tools/github/setup.sh | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2cfa47c6464..6a0893bb692 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -36,6 +36,10 @@ jobs: distribution: adopt-openj9 java-version: 11 cache: 'gradle' + - name: "Setup" + run: ./tools/github/setup.sh + - name: "Scala 2.13 compilation" + run: TERM=dumb OW_SCALA_VERSION=2.13 ./gradlew :tests:compileTestScala # this section is to debug the action with ngrok # you need to add the relevant secrets # and enable it with touch .github/debug and push it @@ -54,4 +58,6 @@ jobs: if test -e .github/debug then bash tools/github/waitIfDebug.sh fi + + diff --git a/tools/github/setup.sh b/tools/github/setup.sh index 6d67941e340..cc3ce9a05a9 100755 --- a/tools/github/setup.sh +++ b/tools/github/setup.sh @@ -43,7 +43,7 @@ python -m pip install --user pydocumentdb python -m pip install --user humanize requests # Scan code before compiling the code -./scan.sh +tools/github/scan.sh # Basic check that all code compiles and depdendencies are downloaded correctly. # Compiling the tests will compile all components as well. From 44d4d2418d5662d8af48411d379d651006cb376c Mon Sep 17 00:00:00 2001 From: Michele Sciabarra Date: Thu, 8 Dec 2022 15:58:43 +0000 Subject: [PATCH 07/39] triggering build --- .github/debug | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/debug b/.github/debug index e69de29bb2d..d00491fd7e5 100644 --- a/.github/debug +++ b/.github/debug @@ -0,0 +1 @@ +1 From 0c8993a5d3166cb248cb9965156c3b347467f87e Mon Sep 17 00:00:00 2001 From: Michele Sciabarra Date: Thu, 8 Dec 2022 17:09:54 +0000 Subject: [PATCH 08/39] run unit test --- .github/workflows/build.yml | 3 +++ ansible/group_vars/all | 2 -- tools/github/runUnitTests.sh | 2 +- tools/travis/setup.sh | 1 + 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6a0893bb692..a505845b554 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -40,6 +40,9 @@ jobs: run: ./tools/github/setup.sh - name: "Scala 2.13 compilation" run: TERM=dumb OW_SCALA_VERSION=2.13 ./gradlew :tests:compileTestScala + + - name: echo "localhost" >ansible/environments/local/hosts + # this section is to debug the action with ngrok # you need to add the relevant secrets # and enable it with touch .github/debug and push it diff --git a/ansible/group_vars/all b/ansible/group_vars/all index 072e4aebb15..bf61a4dc7bb 100644 --- a/ansible/group_vars/all +++ b/ansible/group_vars/all @@ -551,5 +551,3 @@ scheduler: throttlingFraction: "{{ scheduler_queue_throttlingFraction | default(0.9) }}" durationBufferSize: "{{ scheduler_queue_durationBufferSize | default(10) }}" deployment_ignore_error: "{{ scheduler_deployment_ignore_error | default('False') }}" - dataManagementService: - retryInterval: "{{ scheduler_dataManagementService_retryInterval | default('1 second') }}" diff --git a/tools/github/runUnitTests.sh b/tools/github/runUnitTests.sh index 81fc0623b10..8ca5b31bcf5 100755 --- a/tools/github/runUnitTests.sh +++ b/tools/github/runUnitTests.sh @@ -22,7 +22,7 @@ set -e SCRIPTDIR=$(cd $(dirname "$0") && pwd) ROOTDIR="$SCRIPTDIR/../.." -cd $ROOTDIR/tools/travis +cd $ROOTDIR/tools/github export TESTCONTAINERS_RYUK_DISABLED="true" export ORG_GRADLE_PROJECT_testSetName="REQUIRE_ONLY_DB" diff --git a/tools/travis/setup.sh b/tools/travis/setup.sh index 6d67941e340..f6a3abf361b 100755 --- a/tools/travis/setup.sh +++ b/tools/travis/setup.sh @@ -51,3 +51,4 @@ python -m pip install --user humanize requests # Downloads the gradle wrapper, dependencies and tries to compile the code. # Retried 5 times in case there are network hiccups. TERM=dumb retry ./gradlew :tests:compileTestScala + From 906b1a092082698d88baf1f5bcc3ede94cc2933e Mon Sep 17 00:00:00 2001 From: Michele Sciabarra Date: Thu, 8 Dec 2022 17:17:52 +0000 Subject: [PATCH 09/39] ooops --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a505845b554..f959ae4927f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -40,8 +40,8 @@ jobs: run: ./tools/github/setup.sh - name: "Scala 2.13 compilation" run: TERM=dumb OW_SCALA_VERSION=2.13 ./gradlew :tests:compileTestScala - - - name: echo "localhost" >ansible/environments/local/hosts + - name: Run Unit Tests + run: ./tools/github/runUnitTests.sh # this section is to debug the action with ngrok # you need to add the relevant secrets From 5c421d164a0f69ac0041da895414635427f06504 Mon Sep 17 00:00:00 2001 From: Michele Sciabarra Date: Thu, 8 Dec 2022 19:01:51 +0000 Subject: [PATCH 10/39] updating debugAction --- .github/debug | 1 - .github/workflows/build.yml | 66 ++++++++++++++++++++++++------------- tools/github/debugAction.sh | 27 ++++++++++++--- 3 files changed, 65 insertions(+), 29 deletions(-) delete mode 100644 .github/debug diff --git a/.github/debug b/.github/debug deleted file mode 100644 index d00491fd7e5..00000000000 --- a/.github/debug +++ /dev/null @@ -1 +0,0 @@ -1 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f959ae4927f..6e0ef3adc0a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,9 +20,16 @@ on: push: env: + # uncomment this to enable an ngrok access to the vm on test failure + # enabling this you will get an ssh command to access the vm, active for one hour + NGROK_DEBUG: yes + # you need to add as secrets an ngrok token and a password to access the terminal + NGROK_TOKEN: ${{ secrets.NGROK_TOKEN }} + NGROK_PASSWORD: ${{ secrets.NGROK_PASSWORD }} + # openwhisk env ANSIBLE_CMD: "ansible-playbook -i environments/local -e docker_image_prefix=testing" GRADLE_PROJS_SKIP: "" - + jobs: build: name: Build @@ -36,31 +43,44 @@ jobs: distribution: adopt-openj9 java-version: 11 cache: 'gradle' + - run: true || ./tools/github/debugAction.sh + - run: false || ./tools/github/debugAction.sh + - run: false || ./tools/github/debugAction.sh - name: "Setup" run: ./tools/github/setup.sh - name: "Scala 2.13 compilation" run: TERM=dumb OW_SCALA_VERSION=2.13 ./gradlew :tests:compileTestScala - name: Run Unit Tests - run: ./tools/github/runUnitTests.sh + run: ./tools/github/runUnitTests.sh || ./tools/github/debugAction.sh + #- name: upload logs + # run: ./tools/github/checkAndUploadLogs.sh unit db + - name: Run System Tests + run: ./tools/travis/runSystemTests.sh || ./tools/github/debugAction.sh + #- name: upload logs + # run: ./tools/github/checkAndUploadLogs.sh system + - name: Run MultiSystem Tests + run: ./tools/travis/runMultiRuntimeTests.sh || ./tools/github/debugAction.sh + #- name: upload logs + # run: ./tools/github/checkAndUploadLogs.sh multi-runtime + - name: Run Standalone + run: ./tools/travis/runStandaloneTests.sh || ./tools/github/debugAction.sh + #- name: upload logs + # run: ./tools/github/checkAndUploadLogs.sh standalone + - name: Performance tests + run: ./tests/performance/preparation/deploy.sh || ./tools/github/debugAction.sh + - run: TERM=dumb ./tests/performance/wrk_tests/latency.sh "https://172.17.0.1:10001" "$(cat ansible/files/auth.guest)" ./tests/performance/preparation/actions/noop.js 2m + - run: TERM=dumb ./tests/performance/wrk_tests/latency.sh "https://172.17.0.1:10001" "$(cat ansible/files/auth.guest)" ./tests/performance/preparation/actions/async.js 2m + - run: TERM=dumb ./tests/performance/wrk_tests/throughput.sh "https://172.17.0.1:10001" "$(cat ansible/files/auth.guest)" ./tests/performance/preparation/actions/noop.js 4 1 2 2m + - run: TERM=dumb ./tests/performance/wrk_tests/throughput.sh "https://172.17.0.1:10001" "$(cat ansible/files/auth.guest)" ./tests/performance/preparation/actions/async.js 4 1 2 2m + - run: TERM=dumb ./tests/performance/wrk_tests/throughput.sh "https://172.17.0.1:10001" "$(cat ansible/files/auth.guest)" ./tests/performance/preparation/actions/noop.js 100 110 2 2m + - run: TERM=dumb ./tests/performance/wrk_tests/throughput.sh "https://172.17.0.1:10001" "$(cat ansible/files/auth.guest)" ./tests/performance/preparation/actions/async.js 100 110 2 2m + - run: OPENWHISK_HOST="172.17.0.1" CONNECTIONS="100" REQUESTS_PER_SEC="1" ./gradlew gatlingRun-org.apache.openwhisk.ApiV1Simulation + - run: OPENWHISK_HOST="172.17.0.1" MEAN_RESPONSE_TIME="1000" API_KEY="$(cat ansible/files/auth.guest)" EXCLUDED_KINDS="python:default,java:default,swift:default" PAUSE_BETWEEN_INVOKES="100" ./gradlew gatlingRun-org.apache.openwhisk.LatencySimulation + - run: OPENWHISK_HOST="172.17.0.1" API_KEY="$(cat ansible/files/auth.guest)" CONNECTIONS="100" REQUESTS_PER_SEC="1" ./gradlew gatlingRun-org.apache.openwhisk.BlockingInvokeOneActionSimulation + - run: OPENWHISK_HOST="172.17.0.1" API_KEY="$(cat ansible/files/auth.guest)" CONNECTIONS="100" REQUESTS_PER_SEC="1" ASYNC="true" ./gradlew gatlingRun-org.apache.openwhisk.BlockingInvokeOneActionSimulation + # The following configuration does not make much sense. But we do not have enough users. But it's good to verify, that the test is still working. + - run: OPENWHISK_HOST="172.17.0.1" USERS="1" REQUESTS_PER_SEC="1" ./gradlew gatlingRun-org.apache.openwhisk.ColdBlockingInvokeSimulation + #- name: upload logs + # run: ./tools/github/checkAndUploadLogs.sh perf - # this section is to debug the action with ngrok - # you need to add the relevant secrets - # and enable it with touch .github/debug and push it - - name: NgrokDebug - run: |- - if test -e .github/debug - then bash tools/github/debugAction.sh - fi - env: - NGROK_TOKEN: ${{ secrets.NGROK_TOKEN }} - NGROK_PASSWORD: ${{ secrets.NGROK_PASSWORD }} - # if ngrok debuggins is enabled this will wait for an hour to let debug - # you can sto the waiting and resume execution with rm .github/debug - - name: WaitForNgrokDebug - run: |- - if test -e .github/debug - then bash tools/github/waitIfDebug.sh - fi - - - + \ No newline at end of file diff --git a/tools/github/debugAction.sh b/tools/github/debugAction.sh index 75df4a233e7..b065d1c9ff8 100755 --- a/tools/github/debugAction.sh +++ b/tools/github/debugAction.sh @@ -1,20 +1,26 @@ #!/bin/bash +if [[ -z "$NGROK_DEBUG" ]]; then + exit 1 +fi + if [[ -z "$NGROK_TOKEN" ]]; then echo "Please set 'NGROK_TOKEN'" exit 2 fi if [[ -z "$NGROK_PASSWORD" ]]; then - echo "Please set 'NGROK_PASS' for user: $USER" + echo "Please set 'NGROK_PASSWORD'" exit 3 fi echo "### Install ngrok ###" - -wget -q https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-386.zip -unzip ngrok-stable-linux-386.zip -chmod +x ./ngrok +if ! test -e ./ngrok +then + wget -q https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-386.zip + unzip ngrok-stable-linux-386.zip + chmod +x ./ngrok +fi echo "### Update user: $USER password ###" echo -e "$NGROK_PASSWORD\n$NGROK_PASSWORD" | sudo passwd "$USER" @@ -37,3 +43,14 @@ else echo "$HAS_ERRORS" exit 4 fi + +#!/bin/bash +echo "You have an hour to debug this build." +echo "Do touch /tmp/continue to continue." +echo "Do touch /tmp/abort to abort." +for i in $(seq 1 12) +do + if test -e /tmp/continue ; then exit 0 ; fi + if test -e /tmp/abort ; then exit 1 ; fi + sleep 300; echo "$i/12 still waiting..." +done From 1408f8133bcf4cd3f552ff6d50d2f72e439bb5f0 Mon Sep 17 00:00:00 2001 From: Michele Sciabarra Date: Thu, 8 Dec 2022 19:27:35 +0000 Subject: [PATCH 11/39] waitifdebug --- .github/workflows/build.yml | 8 ++++++++ tools/github/debugAction.sh | 11 ----------- tools/github/waitIfDebug.sh | 17 +++++++++++++---- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6e0ef3adc0a..7f6660f030d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -44,30 +44,38 @@ jobs: java-version: 11 cache: 'gradle' - run: true || ./tools/github/debugAction.sh + - run: ./tools/github/waitIfDebug.sh - run: false || ./tools/github/debugAction.sh + - run: ./tools/github/waitIfDebug.sh - run: false || ./tools/github/debugAction.sh + - run: ./tools/github/waitIfDebug.sh - name: "Setup" run: ./tools/github/setup.sh - name: "Scala 2.13 compilation" run: TERM=dumb OW_SCALA_VERSION=2.13 ./gradlew :tests:compileTestScala - name: Run Unit Tests run: ./tools/github/runUnitTests.sh || ./tools/github/debugAction.sh + - run: ./tools/github/waitIfDebug.sh #- name: upload logs # run: ./tools/github/checkAndUploadLogs.sh unit db - name: Run System Tests run: ./tools/travis/runSystemTests.sh || ./tools/github/debugAction.sh + - run: ./tools/github/waitIfDebug.sh #- name: upload logs # run: ./tools/github/checkAndUploadLogs.sh system - name: Run MultiSystem Tests run: ./tools/travis/runMultiRuntimeTests.sh || ./tools/github/debugAction.sh + - run: ./tools/github/waitIfDebug.sh #- name: upload logs # run: ./tools/github/checkAndUploadLogs.sh multi-runtime - name: Run Standalone run: ./tools/travis/runStandaloneTests.sh || ./tools/github/debugAction.sh + - run: ./tools/github/waitIfDebug.sh #- name: upload logs # run: ./tools/github/checkAndUploadLogs.sh standalone - name: Performance tests run: ./tests/performance/preparation/deploy.sh || ./tools/github/debugAction.sh + - run: ./tools/github/waitIfDebug.sh - run: TERM=dumb ./tests/performance/wrk_tests/latency.sh "https://172.17.0.1:10001" "$(cat ansible/files/auth.guest)" ./tests/performance/preparation/actions/noop.js 2m - run: TERM=dumb ./tests/performance/wrk_tests/latency.sh "https://172.17.0.1:10001" "$(cat ansible/files/auth.guest)" ./tests/performance/preparation/actions/async.js 2m - run: TERM=dumb ./tests/performance/wrk_tests/throughput.sh "https://172.17.0.1:10001" "$(cat ansible/files/auth.guest)" ./tests/performance/preparation/actions/noop.js 4 1 2 2m diff --git a/tools/github/debugAction.sh b/tools/github/debugAction.sh index b065d1c9ff8..29a466730af 100755 --- a/tools/github/debugAction.sh +++ b/tools/github/debugAction.sh @@ -43,14 +43,3 @@ else echo "$HAS_ERRORS" exit 4 fi - -#!/bin/bash -echo "You have an hour to debug this build." -echo "Do touch /tmp/continue to continue." -echo "Do touch /tmp/abort to abort." -for i in $(seq 1 12) -do - if test -e /tmp/continue ; then exit 0 ; fi - if test -e /tmp/abort ; then exit 1 ; fi - sleep 300; echo "$i/12 still waiting..." -done diff --git a/tools/github/waitIfDebug.sh b/tools/github/waitIfDebug.sh index b1b113e59a5..2a14be2cf44 100755 --- a/tools/github/waitIfDebug.sh +++ b/tools/github/waitIfDebug.sh @@ -1,8 +1,17 @@ #!/bin/bash +if ! test -e .ngrok.log +then exit 0 +fi echo "You have an hour to debug this build." -echo "Remove .github/debug to stop the wait now." +echo "Do touch /tmp/continue to continue." +echo "Do touch /tmp/abort to abort." +EXIT=0 for i in $(seq 1 60) -do if test -e .github/debug - then sleep 60 ; echo "$i minutes elapsed" - fi +do + if test -e /tmp/continue ; then EXIT=0 ; break ; fi + if test -e /tmp/abort ; then EXIT=1 ; break ; fi + sleep 60; echo "$i/60 still waiting..." done +killall ngrok +rm .ngrok.log +exit $EXIT From fc4212d3d55755456f4419d1d4cc586b8ef1270f Mon Sep 17 00:00:00 2001 From: Michele Sciabarra Date: Thu, 8 Dec 2022 19:42:33 +0000 Subject: [PATCH 12/39] fix cleanup --- tools/github/waitIfDebug.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/github/waitIfDebug.sh b/tools/github/waitIfDebug.sh index 2a14be2cf44..c4742f7d38b 100755 --- a/tools/github/waitIfDebug.sh +++ b/tools/github/waitIfDebug.sh @@ -10,8 +10,9 @@ for i in $(seq 1 60) do if test -e /tmp/continue ; then EXIT=0 ; break ; fi if test -e /tmp/abort ; then EXIT=1 ; break ; fi - sleep 60; echo "$i/60 still waiting..." + echo "$i/60 still waiting..." + sleep 60; done killall ngrok -rm .ngrok.log +rm -f .ngrok.log /tmp/continue /tmp/abort exit $EXIT From 880ab52a9e209f2f3fe75e40e45677e37e598cdc Mon Sep 17 00:00:00 2001 From: Michele Sciabarra Date: Thu, 8 Dec 2022 19:47:43 +0000 Subject: [PATCH 13/39] final test --- .github/workflows/build.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7f6660f030d..4a019f7cf81 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -43,12 +43,6 @@ jobs: distribution: adopt-openj9 java-version: 11 cache: 'gradle' - - run: true || ./tools/github/debugAction.sh - - run: ./tools/github/waitIfDebug.sh - - run: false || ./tools/github/debugAction.sh - - run: ./tools/github/waitIfDebug.sh - - run: false || ./tools/github/debugAction.sh - - run: ./tools/github/waitIfDebug.sh - name: "Setup" run: ./tools/github/setup.sh - name: "Scala 2.13 compilation" @@ -90,5 +84,3 @@ jobs: - run: OPENWHISK_HOST="172.17.0.1" USERS="1" REQUESTS_PER_SEC="1" ./gradlew gatlingRun-org.apache.openwhisk.ColdBlockingInvokeSimulation #- name: upload logs # run: ./tools/github/checkAndUploadLogs.sh perf - - \ No newline at end of file From 3150456bc85904ce4ab4b9aee16866ec74c38457 Mon Sep 17 00:00:00 2001 From: Michele Sciabarra Date: Thu, 8 Dec 2022 20:06:42 +0000 Subject: [PATCH 14/39] disable ngrok --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4a019f7cf81..6b840270939 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,7 +22,7 @@ on: env: # uncomment this to enable an ngrok access to the vm on test failure # enabling this you will get an ssh command to access the vm, active for one hour - NGROK_DEBUG: yes + #NGROK_DEBUG: yes # you need to add as secrets an ngrok token and a password to access the terminal NGROK_TOKEN: ${{ secrets.NGROK_TOKEN }} NGROK_PASSWORD: ${{ secrets.NGROK_PASSWORD }} From b1534d0ecc2bdacac6d395478632de971749fb28 Mon Sep 17 00:00:00 2001 From: Michele Sciabarra Date: Fri, 9 Dec 2022 06:07:47 +0000 Subject: [PATCH 15/39] reworked the PR - multiple jobs and slack --- .github/workflows/build.yml | 95 ++++++++++++++++++++++++++-- tools/github/README.md | 36 ----------- tools/github/box-upload.py | 89 -------------------------- tools/github/checkAndUploadLogs.sh | 48 -------------- tools/github/debugAction.sh | 16 +++++ tools/github/distDocker.sh | 36 ----------- tools/github/docker.conf | 19 ------ tools/github/docker.sh | 56 ---------------- tools/github/flake8.sh | 42 ------------ tools/github/runLeanSystemTests.sh | 36 ----------- tools/github/runMultiRuntimeTests.sh | 36 ----------- tools/github/runSchedulerTests.sh | 35 ---------- tools/github/runStandaloneTests.sh | 64 ------------------- tools/github/runSystemTests.sh | 36 ----------- tools/github/runTests.sh | 32 ---------- tools/github/runUnitTests.sh | 38 ----------- tools/github/scan.sh | 52 --------------- tools/github/setup-docker.py | 78 ----------------------- tools/github/setup.sh | 53 ---------------- tools/github/setupLeanSystem.sh | 33 ---------- tools/github/setupPrereq.sh | 38 ----------- tools/github/setupSystem.sh | 36 ----------- tools/github/waitIfDebug.sh | 21 +++++- 23 files changed, 123 insertions(+), 902 deletions(-) delete mode 100644 tools/github/README.md delete mode 100755 tools/github/box-upload.py delete mode 100755 tools/github/checkAndUploadLogs.sh delete mode 100755 tools/github/distDocker.sh delete mode 100644 tools/github/docker.conf delete mode 100755 tools/github/docker.sh delete mode 100755 tools/github/flake8.sh delete mode 100755 tools/github/runLeanSystemTests.sh delete mode 100755 tools/github/runMultiRuntimeTests.sh delete mode 100755 tools/github/runSchedulerTests.sh delete mode 100755 tools/github/runStandaloneTests.sh delete mode 100755 tools/github/runSystemTests.sh delete mode 100755 tools/github/runTests.sh delete mode 100755 tools/github/runUnitTests.sh delete mode 100755 tools/github/scan.sh delete mode 100755 tools/github/setup-docker.py delete mode 100755 tools/github/setup.sh delete mode 100755 tools/github/setupLeanSystem.sh delete mode 100755 tools/github/setupPrereq.sh delete mode 100755 tools/github/setupSystem.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6b840270939..9124c9fd3a1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,11 +29,14 @@ env: # openwhisk env ANSIBLE_CMD: "ansible-playbook -i environments/local -e docker_image_prefix=testing" GRADLE_PROJS_SKIP: "" + # put a slack incoming webook secret to be notified + SLACK_WEBHOOK_URL: ${{secrets.SLACK_WEBOOK}} + SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK jobs: - build: - name: Build - runs-on: ubuntu-latest + + unit: + runs-on: ubuntu-22.04 steps: - name: Checkout uses: actions/checkout@v2 @@ -44,32 +47,107 @@ jobs: java-version: 11 cache: 'gradle' - name: "Setup" - run: ./tools/github/setup.sh + run: ./tools/travis/setup.sh - name: "Scala 2.13 compilation" run: TERM=dumb OW_SCALA_VERSION=2.13 ./gradlew :tests:compileTestScala - name: Run Unit Tests - run: ./tools/github/runUnitTests.sh || ./tools/github/debugAction.sh + run: ./tools/travis/runUnitTests.sh || ./tools/github/debugAction.sh - run: ./tools/github/waitIfDebug.sh #- name: upload logs # run: ./tools/github/checkAndUploadLogs.sh unit db + - uses: slackapi/slack-github-action@v1.23.0 + with: + slack-message: "GitHub Action build result: ${{ job.status }}\n${{ github.event.pull_request.html_url || github.event.head_commit.url }}" + + system: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Add java + uses: actions/setup-java@v2 + with: + distribution: adopt-openj9 + java-version: 11 + cache: 'gradle' + - name: "Setup" + run: ./tools/travis/setup.sh + - name: "Scala 2.13 compilation" + run: TERM=dumb OW_SCALA_VERSION=2.13 ./gradlew :tests:compileTestScala - name: Run System Tests run: ./tools/travis/runSystemTests.sh || ./tools/github/debugAction.sh - run: ./tools/github/waitIfDebug.sh #- name: upload logs # run: ./tools/github/checkAndUploadLogs.sh system + - uses: slackapi/slack-github-action@v1.23.0 + with: + slack-message: "GitHub Action build result: ${{ job.status }}\n${{ github.event.pull_request.html_url || github.event.head_commit.url }}" + + multi-system: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Add java + uses: actions/setup-java@v2 + with: + distribution: adopt-openj9 + java-version: 11 + cache: 'gradle' + - name: "Setup" + run: ./tools/travis/setup.sh + - name: "Scala 2.13 compilation" + run: TERM=dumb OW_SCALA_VERSION=2.13 ./gradlew :tests:compileTestScala - name: Run MultiSystem Tests run: ./tools/travis/runMultiRuntimeTests.sh || ./tools/github/debugAction.sh - run: ./tools/github/waitIfDebug.sh #- name: upload logs # run: ./tools/github/checkAndUploadLogs.sh multi-runtime + - uses: slackapi/slack-github-action@v1.23.0 + with: + slack-message: "GitHub Action build result: ${{ job.status }}\n${{ github.event.pull_request.html_url || github.event.head_commit.url }}" + + standalone: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Add java + uses: actions/setup-java@v2 + with: + distribution: adopt-openj9 + java-version: 11 + cache: 'gradle' + - name: "Setup" + run: ./tools/travis/setup.sh + - name: "Scala 2.13 compilation" + run: TERM=dumb OW_SCALA_VERSION=2.13 ./gradlew :tests:compileTestScala - name: Run Standalone run: ./tools/travis/runStandaloneTests.sh || ./tools/github/debugAction.sh - run: ./tools/github/waitIfDebug.sh #- name: upload logs # run: ./tools/github/checkAndUploadLogs.sh standalone + - uses: slackapi/slack-github-action@v1.23.0 + with: + slack-message: "GitHub Action build result: ${{ job.status }}\n${{ github.event.pull_request.html_url || github.event.head_commit.url }}" + + performance: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Add java + uses: actions/setup-java@v2 + with: + distribution: adopt-openj9 + java-version: 11 + cache: 'gradle' + - name: "Setup" + run: ./tools/travis/setup.sh + - name: "Scala 2.13 compilation" + run: TERM=dumb OW_SCALA_VERSION=2.13 ./gradlew :tests:compileTestScala - name: Performance tests - run: ./tests/performance/preparation/deploy.sh || ./tools/github/debugAction.sh - - run: ./tools/github/waitIfDebug.sh + run: ./tests/performance/preparation/deploy.sh - run: TERM=dumb ./tests/performance/wrk_tests/latency.sh "https://172.17.0.1:10001" "$(cat ansible/files/auth.guest)" ./tests/performance/preparation/actions/noop.js 2m - run: TERM=dumb ./tests/performance/wrk_tests/latency.sh "https://172.17.0.1:10001" "$(cat ansible/files/auth.guest)" ./tests/performance/preparation/actions/async.js 2m - run: TERM=dumb ./tests/performance/wrk_tests/throughput.sh "https://172.17.0.1:10001" "$(cat ansible/files/auth.guest)" ./tests/performance/preparation/actions/noop.js 4 1 2 2m @@ -84,3 +162,6 @@ jobs: - run: OPENWHISK_HOST="172.17.0.1" USERS="1" REQUESTS_PER_SEC="1" ./gradlew gatlingRun-org.apache.openwhisk.ColdBlockingInvokeSimulation #- name: upload logs # run: ./tools/github/checkAndUploadLogs.sh perf + - uses: slackapi/slack-github-action@v1.23.0 + with: + slack-message: "GitHub Action build result: ${{ job.status }}\n${{ github.event.pull_request.html_url || github.event.head_commit.url }}" diff --git a/tools/github/README.md b/tools/github/README.md deleted file mode 100644 index 8d0fbad9580..00000000000 --- a/tools/github/README.md +++ /dev/null @@ -1,36 +0,0 @@ - - -# Travis Setup - -Travis build is configured to perform build of this repo in multiple parallel jobs as listed below. - -1. Unit Tests - Runs the test which only need database service. -2. System Tests - Runs those tests which need complete OpenWhisk system up and running. -3. Performance test suite - Run basic performance tests with the objective to check if tests are working or not. - -These jobs make use of following scripts - -1. `scan.sh` - Performs various code scan task like python flake scan, scala formatting etc. -2. `setupPrereq.sh` - Performs setup if basis prerequisites like database setup and property file generation. -3. `distDocker.sh` - Builds the various docker containers. -4. `setupSystem.sh` - Runs the various containers which are part of an OpenWhisk setup like Controller, Invoker etc. -5. `runTests.sh` - Runs the tests. It make use of `ORG_GRADLE_PROJECT_testSetName` env setting to determine which test - suite to run. -6. `checkAndUploadLogs.sh` - Collects the logs, checks them and uploads them to https://openwhisk.box.com/v/travis-logs. diff --git a/tools/github/box-upload.py b/tools/github/box-upload.py deleted file mode 100755 index fad503974f4..00000000000 --- a/tools/github/box-upload.py +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env python -"""Executable Python script for compressing folders to Box. - -Compresses the contents of a folder and upload the result to Box. - - Run this script as: - $ upload-logs.py LOG_DIR DEST_NAME - - e.g.: $ upload-logs.py /tmp/wsklogs logs-5512.tar.gz - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -""" - -from __future__ import print_function - -import os -import subprocess -import sys -import tempfile -import urllib -import humanize -import requests -import hashlib - - -def upload_file(local_file, remote_file): - """Upload file.""" - if remote_file[0] == '/': - remote_file = remote_file[1:] - - url = "http://DamCYhF8.mybluemix.net/upload?%s" % \ - urllib.parse.urlencode({"name": remote_file}) - - r = requests.post(url, - headers={"Content-Type": "application/gzip"}, - data=open(local_file, 'rb')) - - print("Posting result", r) - print(r.text) - - -def tar_gz_dir(dir_path): - """Create TAR (ZIP) of path and its contents.""" - _, dst = tempfile.mkstemp(suffix=".tar.gz") - subprocess.call(["tar", "-cvzf", dst, dir_path]) - return dst - - -def print_tarball_size(tarball): - """Get and print the size of the tarball""" - tarballsize = os.path.getsize(tarball) - print("Size of tarball", tarball, "is", humanize.naturalsize(tarballsize)) - - sha256_hash = hashlib.sha256() - with open(tarball, "rb") as f: - for byte_block in iter(lambda: f.read(4096), b""): - sha256_hash.update(byte_block) - print("SHA256 hash of tarball is", sha256_hash.hexdigest()) - - -if __name__ == "__main__": - dir_path = sys.argv[1] - dst_path = sys.argv[2] - - if not os.path.isdir(dir_path): - print("Directory doesn't exist: %s." % dir_path) - sys.exit(0) - - print("Compressing logs dir...") - tar = tar_gz_dir(dir_path) - print_tarball_size(tar) - - print("Uploading to Box...") - upload_file(tar, dst_path) diff --git a/tools/github/checkAndUploadLogs.sh b/tools/github/checkAndUploadLogs.sh deleted file mode 100755 index 36ce72a8c3e..00000000000 --- a/tools/github/checkAndUploadLogs.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Disable abort script at first error as we require the logs to be uploaded -# even if check and log collection fails -# set -e - -SECONDS=0 -SCRIPTDIR=$(cd $(dirname "$0") && pwd) -ROOTDIR="$SCRIPTDIR/../.." - -cd $ROOTDIR - -LOG_NAME=$1 -TAGS=${2-""} -LOG_TAR_NAME="${LOG_NAME}_${TRAVIS_BUILD_ID}-$TRAVIS_BRANCH.tar.gz" - -# Perf logs are typically about 20MB and thus rapidly fill our box account. -# Disable upload to reduce the interval at which we need to manually clean logs from box. -if [ "$LOG_NAME" == "perf" ]; then - echo "Skipping upload of perf logs to conserve space" - exit 0 -fi - -ansible-playbook -i ansible/environments/local ansible/logs.yml - -./tools/build/checkLogs.py logs "$TAGS" - -./tools/travis/box-upload.py "$TRAVIS_BUILD_DIR/logs" "$LOG_TAR_NAME" - -echo "Uploaded Logs with name $LOG_TAR_NAME" -echo "Time taken for ${0##*/} is $SECONDS secs" diff --git a/tools/github/debugAction.sh b/tools/github/debugAction.sh index 29a466730af..d0d1876db3b 100755 --- a/tools/github/debugAction.sh +++ b/tools/github/debugAction.sh @@ -1,4 +1,20 @@ #!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# if [[ -z "$NGROK_DEBUG" ]]; then exit 1 diff --git a/tools/github/distDocker.sh b/tools/github/distDocker.sh deleted file mode 100755 index 13c8450d98b..00000000000 --- a/tools/github/distDocker.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -set -e - -# Build script for Travis-CI. - -SECONDS=0 -SCRIPTDIR=$(cd $(dirname "$0") && pwd) -ROOTDIR="$SCRIPTDIR/../.." - -cd $ROOTDIR -TERM=dumb ./gradlew clean # Run a clean step before build -TERM=dumb ./gradlew distDocker -PdockerImagePrefix=testing $GRADLE_PROJS_SKIP - -TERM=dumb ./gradlew :core:controller:distDockerCoverage -PdockerImagePrefix=testing -TERM=dumb ./gradlew :core:scheduler:distDockerCoverage -PdockerImagePrefix=testing -TERM=dumb ./gradlew :core:invoker:distDockerCoverage -PdockerImagePrefix=testing -TERM=dumb ./gradlew :core:standalone:build - -echo "Time taken for ${0##*/} is $SECONDS secs" diff --git a/tools/github/docker.conf b/tools/github/docker.conf deleted file mode 100644 index 4b9d8df63ae..00000000000 --- a/tools/github/docker.conf +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -[Service] -ExecStart= -ExecStart=/usr/bin/dockerd diff --git a/tools/github/docker.sh b/tools/github/docker.sh deleted file mode 100755 index 21eed7c2b16..00000000000 --- a/tools/github/docker.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -BASEDIR=$(dirname "$0") -echo "$BASEDIR" - -sudo gpasswd -a travis docker -sudo usermod -aG docker travis -#sudo -E bash -c 'echo '\''DOCKER_OPTS="-H tcp://0.0.0.0:4243 -H unix:///var/run/docker.sock --storage-driver=overlay --userns-remap=default"'\'' > /etc/default/docker' - -# Docker -sudo apt-get clean -sudo apt-get update - -# Need to update dpkg due to known issue: https://bugs.launchpad.net/ubuntu/+source/dpkg/+bug/1730627 -sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common dpkg -curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - -sudo apt-key fingerprint 0EBFCD88 - -# This is required because libseccomp2 (>= 2.3.0) is not provided in trusty by default -sudo add-apt-repository -y ppa:ubuntu-sdk-team/ppa - -sudo add-apt-repository \ - "deb [arch=$(uname -m | sed -e 's/x86_64/amd64/g')] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - -sudo apt-get update -sudo apt-get -o Dpkg::Options::="--force-confold" --force-yes -y install docker-ce=18.06.3~ce~3-0~ubuntu containerd.io -# daemon.json and flags does not work together. Overwritting the docker.service file -# to remove the host flags. - https://docs.docker.com/config/daemon/#troubleshoot-conflicts-between-the-daemonjson-and-startup-scripts -sudo mkdir -p /etc/systemd/system/docker.service.d -sudo cp $BASEDIR/docker.conf /etc/systemd/system/docker.service.d/docker.conf -# setup-docker will add configs to /etc/docker/daemon.json -sudo python $BASEDIR/setup-docker.py -sudo cat /etc/docker/daemon.json -sudo systemctl daemon-reload -sudo systemctl restart docker -sudo systemctl status docker.service -echo "Docker Version:" -docker version -echo "Docker Info:" -docker info diff --git a/tools/github/flake8.sh b/tools/github/flake8.sh deleted file mode 100755 index 98b7c0d9c0c..00000000000 --- a/tools/github/flake8.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pip3 install --user --upgrade flake8 - -# These files do not have a .py extension so flake8 will not scan them -declare -a PYTHON_FILES=("." - "./tools/admin/wskadmin" - "./tools/build/citool" - "./tools/build/redo") - -echo 'Flake8: first round (fast fail) stops the build if there are any Python 3 syntax errors...' -for i in "${PYTHON_FILES[@]}" -do - flake8 "$i" --select=E999,F821 --statistics - RETURN_CODE=$? - if [ $RETURN_CODE != 0 ]; then - echo 'Flake8 found Python 3 syntax errors above. See: https://docs.python.org/3/howto/pyporting.html' - exit $RETURN_CODE - fi -done - -echo 'Flake8: second round to find any other stylistic issues...' -for i in "${PYTHON_FILES[@]}" -do - flake8 "$i" --ignore=E,W503,W504,W605 --max-line-length=127 --statistics -done diff --git a/tools/github/runLeanSystemTests.sh b/tools/github/runLeanSystemTests.sh deleted file mode 100755 index 372fbc1e409..00000000000 --- a/tools/github/runLeanSystemTests.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -set -e - -SCRIPTDIR=$(cd $(dirname "$0") && pwd) -ROOTDIR="$SCRIPTDIR/../.." - -cd $ROOTDIR/tools/travis - -export ORG_GRADLE_PROJECT_testSetName="REQUIRE_LEAN_SYSTEM" -export GRADLE_COVERAGE=true - -./setupPrereq.sh /ansible/files/runtimes-nodeonly.json - -./distDocker-lean.sh - -./setupLeanSystem.sh /ansible/files/runtimes-nodeonly.json - -./runTests.sh diff --git a/tools/github/runMultiRuntimeTests.sh b/tools/github/runMultiRuntimeTests.sh deleted file mode 100755 index e2de0c59bd4..00000000000 --- a/tools/github/runMultiRuntimeTests.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -set -e - -SCRIPTDIR=$(cd $(dirname "$0") && pwd) -ROOTDIR="$SCRIPTDIR/../.." - -cd $ROOTDIR/tools/travis - -export ORG_GRADLE_PROJECT_testSetName="REQUIRE_MULTI_RUNTIME" -export GRADLE_COVERAGE=true - -./setupPrereq.sh - -./distDocker.sh - -./setupSystem.sh - -./runTests.sh diff --git a/tools/github/runSchedulerTests.sh b/tools/github/runSchedulerTests.sh deleted file mode 100755 index 000de8080e5..00000000000 --- a/tools/github/runSchedulerTests.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -set -e - -SCRIPTDIR=$(cd $(dirname "$0") && pwd) -ROOTDIR="$SCRIPTDIR/../.." - -cd $ROOTDIR/tools/travis -export ORG_GRADLE_PROJECT_testSetName="REQUIRE_SCHEDULER" -export GRADLE_COVERAGE=true - -./setupPrereq.sh /ansible/files/runtimes-nodeonly.json - -./distDocker.sh - -./setupSystem.sh /ansible/files/runtimes-nodeonly.json - -./runTests.sh diff --git a/tools/github/runStandaloneTests.sh b/tools/github/runStandaloneTests.sh deleted file mode 100755 index ba10c5b4715..00000000000 --- a/tools/github/runStandaloneTests.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env bash - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -set -e - -SCRIPTDIR=$(cd $(dirname "$0") && pwd) -ROOTDIR="$SCRIPTDIR/../.." - -export ORG_GRADLE_PROJECT_testSetName="REQUIRE_STANDALONE" -export GRADLE_COVERAGE=true - -cd $ROOTDIR/ansible -$ANSIBLE_CMD setup.yml -$ANSIBLE_CMD properties.yml -e manifest_file="/ansible/files/runtimes-nodeonly.json" -$ANSIBLE_CMD downloadcli-github.yml - -# Install kubectl -curl -Lo ./kubectl https://storage.googleapis.com/kubernetes-release/release/v1.16.1/bin/linux/amd64/kubectl -chmod +x kubectl -sudo cp kubectl /usr/local/bin/kubectl - -# Install kind -curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/v0.5.1/kind-linux-amd64 -chmod +x kind -sudo cp kind /usr/local/bin/kind - -kind create cluster --wait 5m -export KUBECONFIG="$(kind get kubeconfig-path)" -kubectl config set-context --current --namespace=default - -# This is required because it is timed out to pull the image during the test. -docker pull openwhisk/action-nodejs-v14:nightly -docker pull openwhisk/dockerskeleton:nightly -docker pull openwhisk/example:nightly -docker pull openwhisk/apigateway:0.11.0 - -cd $ROOTDIR -TERM=dumb ./gradlew :core:standalone:build \ - :core:monitoring:user-events:distDocker - -cd $ROOTDIR -TERM=dumb ./gradlew :core:standalone:cleanTest \ - :core:standalone:test \ - :core:monitoring:user-events:reportTestScoverage - -# Run test in end as it publishes the coverage also -cd $ROOTDIR/tools/travis -./runTests.sh diff --git a/tools/github/runSystemTests.sh b/tools/github/runSystemTests.sh deleted file mode 100755 index 8325caddbae..00000000000 --- a/tools/github/runSystemTests.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -set -e - -SCRIPTDIR=$(cd $(dirname "$0") && pwd) -ROOTDIR="$SCRIPTDIR/../.." - -cd $ROOTDIR/tools/travis - -export ORG_GRADLE_PROJECT_testSetName="REQUIRE_SYSTEM" -export GRADLE_COVERAGE=true - -./setupPrereq.sh /ansible/files/runtimes-nodeonly.json - -./distDocker.sh - -./setupSystem.sh /ansible/files/runtimes-nodeonly.json - -./runTests.sh diff --git a/tools/github/runTests.sh b/tools/github/runTests.sh deleted file mode 100755 index e2a7f7a4b32..00000000000 --- a/tools/github/runTests.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -set -e - -# Build script for Travis-CI. - -SECONDS=0 -SCRIPTDIR=$(cd "$(dirname "$0")" && pwd) -ROOTDIR="$SCRIPTDIR/../.." - -cd $ROOTDIR -cat whisk.properties -TERM=dumb ./gradlew :tests:testCoverageLean :tests:reportCoverage :tests:testSwaggerCodegen - -bash <(curl -s https://codecov.io/bash) -echo "Time taken for ${0##*/} is $SECONDS secs" diff --git a/tools/github/runUnitTests.sh b/tools/github/runUnitTests.sh deleted file mode 100755 index 8ca5b31bcf5..00000000000 --- a/tools/github/runUnitTests.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env bash - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -set -e - -SCRIPTDIR=$(cd $(dirname "$0") && pwd) -ROOTDIR="$SCRIPTDIR/../.." - -cd $ROOTDIR/tools/github -export TESTCONTAINERS_RYUK_DISABLED="true" -export ORG_GRADLE_PROJECT_testSetName="REQUIRE_ONLY_DB" - -./setupPrereq.sh - -cat "$ROOTDIR/tests/src/test/resources/application.conf" - -./distDocker.sh - -# yet another hack to hit docker rate limits early... -docker pull alpine:3.5 - -./runTests.sh diff --git a/tools/github/scan.sh b/tools/github/scan.sh deleted file mode 100755 index 9a69413d30c..00000000000 --- a/tools/github/scan.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -set -e - -# Build script for Travis-CI. -SECONDS=0 -SCRIPTDIR=$(cd $(dirname "$0") && pwd) -ROOTDIR="$SCRIPTDIR/../.." -HOMEDIR="$SCRIPTDIR/../../../" -UTILDIR="$HOMEDIR/openwhisk-utilities/" - -cd $ROOTDIR -./tools/travis/flake8.sh # Check Python files for style and stop the build on syntax errors - -# clone the openwhisk utilities repo. -cd $HOMEDIR -git clone https://github.com/apache/openwhisk-utilities.git - -# run the scancode util. against project source code starting at its root -cd $UTILDIR -scancode/scanCode.py --config scancode/ASF-Release.cfg $ROOTDIR - -# run scalafmt checks -cd $ROOTDIR -TERM=dumb ./gradlew checkScalafmtAll - -# lint tests to all be actually runnable -MISSING_TESTS=$(grep -rL "RunWith" --include="*Tests.scala" tests || true) -if [ -n "$MISSING_TESTS" ] -then - echo "The following tests are missing the 'RunWith' annotation" - echo $MISSING_TESTS - exit 1 -fi - -echo "Time taken for ${0##*/} is $SECONDS secs" diff --git a/tools/github/setup-docker.py b/tools/github/setup-docker.py deleted file mode 100755 index 525e6f44479..00000000000 --- a/tools/github/setup-docker.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env python -"""Executable Python script for setting up docker daemon. - -Add docker daemon configuration options in /etc/docker/daemon.json - - Run this script as: - $python setup-docker.py - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -""" - -from __future__ import print_function - -import json -import traceback - -DOCKER_DAEMON_FILE = "/etc/docker/daemon.json" - -# Read the file. - -DOCKER_OPTS = { - "hosts": [ - "tcp://0.0.0.0:4243", - "unix:///var/run/docker.sock" - ], - "storage-driver": "overlay", - "userns-remap": "default" -} - - -def get_daemon_content(): - data = {} - with open(DOCKER_DAEMON_FILE) as json_file: - data = json.load(json_file) - return data - - -def add_content(data): - for config in DOCKER_OPTS.items(): - # config will be a tuple of key, value - # ('hosts', ['tcp://0.0.0.0:4243', 'unix:///var/run/docker.sock']) - key, value = config - data[key] = value - return data - - -def write_to_daemon_conf(data): - try: - with open(DOCKER_DAEMON_FILE, 'w') as fp: - json.dump(data, fp) - except Exception as e: - print("Failed to write to daemon file") - print(e) - traceback.print_exc() - - -if __name__ == "__main__": - current_data = get_daemon_content() - print(current_data) - updated_data = add_content(current_data) - print(updated_data) - write_to_daemon_conf(updated_data) - print("Successfully Configured Docker daemon.json") diff --git a/tools/github/setup.sh b/tools/github/setup.sh deleted file mode 100755 index cc3ce9a05a9..00000000000 --- a/tools/github/setup.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# retries a command for five times and exits with the non-zero exit if even after -# the retries the command did not succeed. -function retry() { - local exitcode=0 - for i in {1..5}; - do - exitcode=0 - "$@" && break || exitcode=$? && echo "$i. attempt failed. Will retry $((5-i)) more times!" && sleep 1; - done - if [ $exitcode -ne 0 ]; then - exit $exitcode - fi -} - -# Python -python -m pip install --user couchdb - -# Ansible -python -m pip install --user ansible==2.8.18 - -# Azure CosmosDB -python -m pip install --user pydocumentdb - -# Support the revises log upload script -python -m pip install --user humanize requests - -# Scan code before compiling the code -tools/github/scan.sh - -# Basic check that all code compiles and depdendencies are downloaded correctly. -# Compiling the tests will compile all components as well. -# -# Downloads the gradle wrapper, dependencies and tries to compile the code. -# Retried 5 times in case there are network hiccups. -TERM=dumb retry ./gradlew :tests:compileTestScala diff --git a/tools/github/setupLeanSystem.sh b/tools/github/setupLeanSystem.sh deleted file mode 100755 index 89568b21c0a..00000000000 --- a/tools/github/setupLeanSystem.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -set -e - -# Build script for Travis-CI. -SECONDS=0 -SCRIPTDIR=$(cd $(dirname "$0") && pwd) -ROOTDIR="$SCRIPTDIR/../.." -RUNTIMES_MANIFEST=${1:-"/ansible/files/runtimes.json"} - -cd $ROOTDIR/ansible - -$ANSIBLE_CMD openwhisk.yml -e manifest_file="$RUNTIMES_MANIFEST" -e lean=true -$ANSIBLE_CMD apigateway.yml -$ANSIBLE_CMD routemgmt.yml - -echo "Time taken for ${0##*/} is $SECONDS secs" diff --git a/tools/github/setupPrereq.sh b/tools/github/setupPrereq.sh deleted file mode 100755 index 264fc3da047..00000000000 --- a/tools/github/setupPrereq.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -set -e - -# Build script for Travis-CI. -SECONDS=0 -SCRIPTDIR=$(cd $(dirname "$0") && pwd) -ROOTDIR="$SCRIPTDIR/../.." -RUNTIMES_MANIFEST=${1:-"/ansible/files/runtimes.json"} - -cd $ROOTDIR/ansible - -$ANSIBLE_CMD setup.yml -e mode=HA -$ANSIBLE_CMD prereq.yml -$ANSIBLE_CMD couchdb.yml -$ANSIBLE_CMD initdb.yml -$ANSIBLE_CMD wipe.yml -$ANSIBLE_CMD elasticsearch.yml -$ANSIBLE_CMD etcd.yml - -$ANSIBLE_CMD properties.yml -e manifest_file="$RUNTIMES_MANIFEST" -echo "Time taken for ${0##*/} is $SECONDS secs" diff --git a/tools/github/setupSystem.sh b/tools/github/setupSystem.sh deleted file mode 100755 index 387d45c6fa6..00000000000 --- a/tools/github/setupSystem.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -set -e - -# Build script for Travis-CI. -SECONDS=0 -SCRIPTDIR=$(cd $(dirname "$0") && pwd) -ROOTDIR="$SCRIPTDIR/../.." -RUNTIMES_MANIFEST=${1:-"/ansible/files/runtimes.json"} - -# This is required because it is timed out to pull the image during the test. -docker pull openwhisk/example - -cd $ROOTDIR/ansible - -$ANSIBLE_CMD openwhisk.yml -e manifest_file="$RUNTIMES_MANIFEST" -e db_activation_backend=ElasticSearch -$ANSIBLE_CMD apigateway.yml -$ANSIBLE_CMD routemgmt.yml - -echo "Time taken for ${0##*/} is $SECONDS secs" diff --git a/tools/github/waitIfDebug.sh b/tools/github/waitIfDebug.sh index c4742f7d38b..6c92b42945c 100755 --- a/tools/github/waitIfDebug.sh +++ b/tools/github/waitIfDebug.sh @@ -1,4 +1,21 @@ #!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + if ! test -e .ngrok.log then exit 0 fi @@ -6,12 +23,12 @@ echo "You have an hour to debug this build." echo "Do touch /tmp/continue to continue." echo "Do touch /tmp/abort to abort." EXIT=0 -for i in $(seq 1 60) +for i in $(seq 1 60) do if test -e /tmp/continue ; then EXIT=0 ; break ; fi if test -e /tmp/abort ; then EXIT=1 ; break ; fi echo "$i/60 still waiting..." - sleep 60; + sleep 60 done killall ngrok rm -f .ngrok.log /tmp/continue /tmp/abort From f8cec72bd114f20b0cfe60f942bb408cc0bb6724 Mon Sep 17 00:00:00 2001 From: Michele Sciabarra Date: Fri, 9 Dec 2022 11:44:34 +0000 Subject: [PATCH 16/39] split in multiple build --- .github/workflows/multi-system.yaml | 62 +++++++++++ .../workflows/{build.yml => performance.yaml} | 100 +----------------- .github/workflows/standalone.yaml | 62 +++++++++++ .github/workflows/system.yaml | 62 +++++++++++ .github/workflows/unit.yaml | 62 +++++++++++ 5 files changed, 250 insertions(+), 98 deletions(-) create mode 100644 .github/workflows/multi-system.yaml rename .github/workflows/{build.yml => performance.yaml} (54%) create mode 100644 .github/workflows/standalone.yaml create mode 100644 .github/workflows/system.yaml create mode 100644 .github/workflows/unit.yaml diff --git a/.github/workflows/multi-system.yaml b/.github/workflows/multi-system.yaml new file mode 100644 index 00000000000..56e1fc450d5 --- /dev/null +++ b/.github/workflows/multi-system.yaml @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +name: Multi-System Tests +on: + push: + +env: + # uncomment this to enable an ngrok access to the vm on test failure + # enabling this you will get an ssh command to access the vm, active for one hour + #NGROK_DEBUG: yes + # you need to add as secrets an ngrok token and a password to access the terminal + NGROK_TOKEN: ${{ secrets.NGROK_TOKEN }} + NGROK_PASSWORD: ${{ secrets.NGROK_PASSWORD }} + # openwhisk env + ANSIBLE_CMD: "ansible-playbook -i environments/local -e docker_image_prefix=testing" + GRADLE_PROJS_SKIP: "" + # put a slack incoming webook secret to be notified + SLACK_WEBHOOK_URL: ${{secrets.SLACK_WEHBOOK}} + SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK + +jobs: + + multi-system: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Add java + uses: actions/setup-java@v2 + with: + distribution: adopt-openj9 + java-version: 11 + cache: 'gradle' + - name: "Setup" + run: ./tools/travis/setup.sh + - name: "Scala 2.13 compilation" + run: TERM=dumb OW_SCALA_VERSION=2.13 ./gradlew :tests:compileTestScala + - name: Run MultiSystem Tests + run: ./tools/travis/runMultiRuntimeTests.sh || ./tools/github/debugAction.sh + - run: ./tools/github/waitIfDebug.sh + #- name: upload logs + # run: ./tools/github/checkAndUploadLogs.sh multi-runtime + - uses: slackapi/slack-github-action@v1.23.0 + with: + slack-message: "GitHub Action build result: ${{ job.status }}\n${{ github.event.pull_request.html_url || github.event.head_commit.url }}" + + \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/performance.yaml similarity index 54% rename from .github/workflows/build.yml rename to .github/workflows/performance.yaml index 9124c9fd3a1..d9678268766 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/performance.yaml @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. # -name: OpenWhisk +name: Performance Tests on: push: @@ -30,107 +30,11 @@ env: ANSIBLE_CMD: "ansible-playbook -i environments/local -e docker_image_prefix=testing" GRADLE_PROJS_SKIP: "" # put a slack incoming webook secret to be notified - SLACK_WEBHOOK_URL: ${{secrets.SLACK_WEBOOK}} + SLACK_WEBHOOK_URL: ${{secrets.SLACK_WEBHOOK}} SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK jobs: - unit: - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Add java - uses: actions/setup-java@v2 - with: - distribution: adopt-openj9 - java-version: 11 - cache: 'gradle' - - name: "Setup" - run: ./tools/travis/setup.sh - - name: "Scala 2.13 compilation" - run: TERM=dumb OW_SCALA_VERSION=2.13 ./gradlew :tests:compileTestScala - - name: Run Unit Tests - run: ./tools/travis/runUnitTests.sh || ./tools/github/debugAction.sh - - run: ./tools/github/waitIfDebug.sh - #- name: upload logs - # run: ./tools/github/checkAndUploadLogs.sh unit db - - uses: slackapi/slack-github-action@v1.23.0 - with: - slack-message: "GitHub Action build result: ${{ job.status }}\n${{ github.event.pull_request.html_url || github.event.head_commit.url }}" - - system: - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Add java - uses: actions/setup-java@v2 - with: - distribution: adopt-openj9 - java-version: 11 - cache: 'gradle' - - name: "Setup" - run: ./tools/travis/setup.sh - - name: "Scala 2.13 compilation" - run: TERM=dumb OW_SCALA_VERSION=2.13 ./gradlew :tests:compileTestScala - - name: Run System Tests - run: ./tools/travis/runSystemTests.sh || ./tools/github/debugAction.sh - - run: ./tools/github/waitIfDebug.sh - #- name: upload logs - # run: ./tools/github/checkAndUploadLogs.sh system - - uses: slackapi/slack-github-action@v1.23.0 - with: - slack-message: "GitHub Action build result: ${{ job.status }}\n${{ github.event.pull_request.html_url || github.event.head_commit.url }}" - - multi-system: - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Add java - uses: actions/setup-java@v2 - with: - distribution: adopt-openj9 - java-version: 11 - cache: 'gradle' - - name: "Setup" - run: ./tools/travis/setup.sh - - name: "Scala 2.13 compilation" - run: TERM=dumb OW_SCALA_VERSION=2.13 ./gradlew :tests:compileTestScala - - name: Run MultiSystem Tests - run: ./tools/travis/runMultiRuntimeTests.sh || ./tools/github/debugAction.sh - - run: ./tools/github/waitIfDebug.sh - #- name: upload logs - # run: ./tools/github/checkAndUploadLogs.sh multi-runtime - - uses: slackapi/slack-github-action@v1.23.0 - with: - slack-message: "GitHub Action build result: ${{ job.status }}\n${{ github.event.pull_request.html_url || github.event.head_commit.url }}" - - standalone: - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Add java - uses: actions/setup-java@v2 - with: - distribution: adopt-openj9 - java-version: 11 - cache: 'gradle' - - name: "Setup" - run: ./tools/travis/setup.sh - - name: "Scala 2.13 compilation" - run: TERM=dumb OW_SCALA_VERSION=2.13 ./gradlew :tests:compileTestScala - - name: Run Standalone - run: ./tools/travis/runStandaloneTests.sh || ./tools/github/debugAction.sh - - run: ./tools/github/waitIfDebug.sh - #- name: upload logs - # run: ./tools/github/checkAndUploadLogs.sh standalone - - uses: slackapi/slack-github-action@v1.23.0 - with: - slack-message: "GitHub Action build result: ${{ job.status }}\n${{ github.event.pull_request.html_url || github.event.head_commit.url }}" - performance: runs-on: ubuntu-22.04 steps: diff --git a/.github/workflows/standalone.yaml b/.github/workflows/standalone.yaml new file mode 100644 index 00000000000..116741163c0 --- /dev/null +++ b/.github/workflows/standalone.yaml @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +name: Standalone Tests +on: + push: + +env: + # uncomment this to enable an ngrok access to the vm on test failure + # enabling this you will get an ssh command to access the vm, active for one hour + #NGROK_DEBUG: yes + # you need to add as secrets an ngrok token and a password to access the terminal + NGROK_TOKEN: ${{ secrets.NGROK_TOKEN }} + NGROK_PASSWORD: ${{ secrets.NGROK_PASSWORD }} + # openwhisk env + ANSIBLE_CMD: "ansible-playbook -i environments/local -e docker_image_prefix=testing" + GRADLE_PROJS_SKIP: "" + # put a slack incoming webook secret to be notified + SLACK_WEBHOOK_URL: ${{secrets.SLACK_WEBHOOK}} + SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK + +jobs: + + standalone: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Add java + uses: actions/setup-java@v2 + with: + distribution: adopt-openj9 + java-version: 11 + cache: 'gradle' + - name: "Setup" + run: ./tools/travis/setup.sh + - name: "Scala 2.13 compilation" + run: TERM=dumb OW_SCALA_VERSION=2.13 ./gradlew :tests:compileTestScala + - name: Run Standalone + run: ./tools/travis/runStandaloneTests.sh || ./tools/github/debugAction.sh + - run: ./tools/github/waitIfDebug.sh + #- name: upload logs + # run: ./tools/github/checkAndUploadLogs.sh standalone + - uses: slackapi/slack-github-action@v1.23.0 + with: + slack-message: "GitHub Action build result: ${{ job.status }}\n${{ github.event.pull_request.html_url || github.event.head_commit.url }}" + + \ No newline at end of file diff --git a/.github/workflows/system.yaml b/.github/workflows/system.yaml new file mode 100644 index 00000000000..b3821849b3c --- /dev/null +++ b/.github/workflows/system.yaml @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +name: System Tests +on: + push: + +env: + # uncomment this to enable an ngrok access to the vm on test failure + # enabling this you will get an ssh command to access the vm, active for one hour + #NGROK_DEBUG: yes + # you need to add as secrets an ngrok token and a password to access the terminal + NGROK_TOKEN: ${{ secrets.NGROK_TOKEN }} + NGROK_PASSWORD: ${{ secrets.NGROK_PASSWORD }} + # openwhisk env + ANSIBLE_CMD: "ansible-playbook -i environments/local -e docker_image_prefix=testing" + GRADLE_PROJS_SKIP: "" + # put a slack incoming webook secret to be notified + SLACK_WEBHOOK_URL: ${{secrets.SLACK_WEBHOOK}} + SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK + +jobs: + + system: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Add java + uses: actions/setup-java@v2 + with: + distribution: adopt-openj9 + java-version: 11 + cache: 'gradle' + - name: "Setup" + run: ./tools/travis/setup.sh + - name: "Scala 2.13 compilation" + run: TERM=dumb OW_SCALA_VERSION=2.13 ./gradlew :tests:compileTestScala + - name: Run System Tests + run: ./tools/travis/runSystemTests.sh || ./tools/github/debugAction.sh + - run: ./tools/github/waitIfDebug.sh + #- name: upload logs + # run: ./tools/github/checkAndUploadLogs.sh system + - uses: slackapi/slack-github-action@v1.23.0 + with: + slack-message: "GitHub Action build result: ${{ job.status }}\n${{ github.event.pull_request.html_url || github.event.head_commit.url }}" + + \ No newline at end of file diff --git a/.github/workflows/unit.yaml b/.github/workflows/unit.yaml new file mode 100644 index 00000000000..0f574f8546d --- /dev/null +++ b/.github/workflows/unit.yaml @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +name: Unit Tests +on: + push: + +env: + # uncomment this to enable an ngrok access to the vm on test failure + # enabling this you will get an ssh command to access the vm, active for one hour + #NGROK_DEBUG: yes + # you need to add as secrets an ngrok token and a password to access the terminal + NGROK_TOKEN: ${{ secrets.NGROK_TOKEN }} + NGROK_PASSWORD: ${{ secrets.NGROK_PASSWORD }} + # openwhisk env + ANSIBLE_CMD: "ansible-playbook -i environments/local -e docker_image_prefix=testing" + GRADLE_PROJS_SKIP: "" + # put a slack incoming webook secret to be notified + SLACK_WEBHOOK_URL: ${{secrets.SLACK_WEBHOOK}} + SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK + +jobs: + + unit: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Add java + uses: actions/setup-java@v2 + with: + distribution: adopt-openj9 + java-version: 11 + cache: 'gradle' + - name: "Setup" + run: ./tools/travis/setup.sh + - name: "Scala 2.13 compilation" + run: TERM=dumb OW_SCALA_VERSION=2.13 ./gradlew :tests:compileTestScala + - name: Run Unit Tests + run: ./tools/travis/runUnitTests.sh || ./tools/github/debugAction.sh + - run: ./tools/github/waitIfDebug.sh + #- name: upload logs + # run: ./tools/github/checkAndUploadLogs.sh unit db + - uses: slackapi/slack-github-action@v1.23.0 + with: + slack-message: "GitHub Action build result: ${{ job.status }}\n${{ github.event.pull_request.html_url || github.event.head_commit.url }}" + + From 0da4b9f77c75063332c2a61064a099f64af73151 Mon Sep 17 00:00:00 2001 From: Michele Sciabarra Date: Sun, 11 Dec 2022 18:34:05 +0000 Subject: [PATCH 17/39] github actions updated --- .github/workflows/0-on-demand.yaml | 95 +++++++++++++++++++ .github/workflows/1-unit.yaml | 70 ++++++++++++++ .github/workflows/2-system.yaml | 70 ++++++++++++++ .github/workflows/3-multi-runtime.yaml | 70 ++++++++++++++ .github/workflows/4-standalone.yaml | 70 ++++++++++++++ .github/workflows/5-scheduler.yaml | 70 ++++++++++++++ .../{performance.yaml => 6-performance.yaml} | 74 ++++++++------- .github/workflows/README.md | 56 +++++++++++ .github/workflows/multi-system.yaml | 62 ------------ .github/workflows/standalone.yaml | 62 ------------ .github/workflows/system.yaml | 62 ------------ .github/workflows/unit.yaml | 62 ------------ tools/github/checkAndUploadLogs.sh | 59 ++++++++++++ tools/github/debugAction.sh | 18 ++-- tools/github/flake8.sh | 42 ++++++++ tools/github/runDummyKOTests.sh | 26 +++++ tools/github/runDummyTests.sh | 26 +++++ tools/github/runLeanSystemTests.sh | 36 +++++++ tools/github/runMultiRuntimeTests.sh | 36 +++++++ tools/github/runSchedulerTests.sh | 35 +++++++ tools/github/runStandaloneTests.sh | 64 +++++++++++++ tools/github/runSystemTests.sh | 36 +++++++ tools/github/runUnitTests.sh | 34 +++++++ tools/github/s3-upload.sh | 57 +++++++++++ tools/github/scan.sh | 52 ++++++++++ tools/github/setup.sh | 61 ++++++++++++ tools/github/waitIfDebug.sh | 2 + tools/github/writeOnSlack.sh | 28 ++++++ 28 files changed, 1146 insertions(+), 289 deletions(-) create mode 100644 .github/workflows/0-on-demand.yaml create mode 100644 .github/workflows/1-unit.yaml create mode 100644 .github/workflows/2-system.yaml create mode 100644 .github/workflows/3-multi-runtime.yaml create mode 100644 .github/workflows/4-standalone.yaml create mode 100644 .github/workflows/5-scheduler.yaml rename .github/workflows/{performance.yaml => 6-performance.yaml} (65%) create mode 100644 .github/workflows/README.md delete mode 100644 .github/workflows/multi-system.yaml delete mode 100644 .github/workflows/standalone.yaml delete mode 100644 .github/workflows/system.yaml delete mode 100644 .github/workflows/unit.yaml create mode 100755 tools/github/checkAndUploadLogs.sh create mode 100755 tools/github/flake8.sh create mode 100755 tools/github/runDummyKOTests.sh create mode 100755 tools/github/runDummyTests.sh create mode 100755 tools/github/runLeanSystemTests.sh create mode 100755 tools/github/runMultiRuntimeTests.sh create mode 100755 tools/github/runSchedulerTests.sh create mode 100755 tools/github/runStandaloneTests.sh create mode 100755 tools/github/runSystemTests.sh create mode 100755 tools/github/runUnitTests.sh create mode 100755 tools/github/s3-upload.sh create mode 100755 tools/github/scan.sh create mode 100755 tools/github/setup.sh create mode 100755 tools/github/writeOnSlack.sh diff --git a/.github/workflows/0-on-demand.yaml b/.github/workflows/0-on-demand.yaml new file mode 100644 index 00000000000..c74a0fa4850 --- /dev/null +++ b/.github/workflows/0-on-demand.yaml @@ -0,0 +1,95 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +name: On Demand Tests + +on: + workflow_dispatch: + inputs: + enable_ngrok_debug: + description: "Enable Ngrok Debugging" + required: true + type: boolean + default: false + test_suite: + description: Select Test Suite to run + type: choice + options: + - Unit + - System + - MultiRuntime + - Standalone + - Scheduler + ## TODO + #- Performance + ## Uncomment those to debug the build + - Dummy + - DummyKO + +env: + # openwhisk env + TEST_SUITE: ${{ inputs.test_suite }} + JAVA_DISTRO: adopt-openj9 + ANSIBLE_CMD: "ansible-playbook -i environments/local -e docker_image_prefix=testing" + GRADLE_PROJS_SKIP: "" + + ## secrets + # (optional) slack incoming wehbook for notifications + SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}} + + # (optional) s3 log upload + S3_LOG_BUCKET: ${{ secrets.S3_LOG_BUCKET }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.AWS_REGION }} + + # (optional) you need to add as secrets an ngrok token and a password to debug a build on demand + NGROK_DEBUG: ${{ inputs.enable_ngrok_debug }} + NGROK_TOKEN: ${{ secrets.NGROK_TOKEN }} + NGROK_PASSWORD: ${{ secrets.NGROK_PASSWORD }} + + # github + GH_BUILD: ${{ github.run_id }}-${{ github.event_name }} + GH_BRANCH: ${{ github.head_ref || github.ref_name }} + +jobs: + openwhisk: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: "Setup" + run: ./tools/github/setup.sh + - id: tests + name: Run Tests + run: "./tools/github/run${{ env.TEST_SUITE }}Tests.sh" + continue-on-error: true + - id: logs + name: Upload Logs + run: ./tools/github/checkAndUploadLogs.sh ${{ env.TEST_SUITE }} + - name: Slack Notification + run: > + ./tools/github/writeOnSlack.sh "[$TEST_SUITE]" + $GH_BUILD: ${{ steps.tests.outcome }} + $'\nbranch:' $GH_BRANCH + $'\nlogurl:' ${{ steps.logs.outputs.logurl }} + - name: Debug Action (if requested) + run: ./tools/github/debugAction.sh + - name: Wait for Debug (if requested) + run: ./tools/github/waitIfDebug.sh + - name: Results + run: test "${{ steps.tests.outcome }}" == "success" diff --git a/.github/workflows/1-unit.yaml b/.github/workflows/1-unit.yaml new file mode 100644 index 00000000000..5b5ec9b717e --- /dev/null +++ b/.github/workflows/1-unit.yaml @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +name: Unit Tests + +on: + # build on tagged + push: + tags: '*' + # build on pull requests + pull_request: + +env: + # openwhisk env + TEST_SUITE: Unit + JAVA_DISTRO: adopt-openj9 + ANSIBLE_CMD: "ansible-playbook -i environments/local -e docker_image_prefix=testing" + GRADLE_PROJS_SKIP: "" + + ## secrets + # (optional) slack incoming wehbook for notifications + SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}} + + # (optional) s3 log upload + S3_LOG_BUCKET: ${{ secrets.S3_LOG_BUCKET }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.AWS_REGION }} + + # github + GH_BUILD: ${{ github.run_id }}-${{ github.event_name }} + GH_BRANCH: ${{ github.head_ref || github.ref_name }} + +jobs: + openwhisk: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: "Setup" + run: ./tools/github/setup.sh + - id: tests + name: Run Tests + run: "./tools/github/run${{ env.TEST_SUITE }}Tests.sh" + continue-on-error: true + - id: logs + name: Upload Logs + run: ./tools/github/checkAndUploadLogs.sh ${{ env.TEST_SUITE }} + - name: Slack Notification + run: > + ./tools/github/writeOnSlack.sh "[$TEST_SUITE]" + $GH_BUILD: ${{ steps.tests.outcome }} + $'\nbranch:' $GH_BRANCH + $'\nlogurl:' ${{ steps.logs.outputs.logurl }} + - name: Results + run: test "${{ steps.tests.outcome }}" == "success" diff --git a/.github/workflows/2-system.yaml b/.github/workflows/2-system.yaml new file mode 100644 index 00000000000..ff4490fea17 --- /dev/null +++ b/.github/workflows/2-system.yaml @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +name: System Tests + +on: + # build on tagged + push: + tags: '*' + # build on pull requests + pull_request: + +env: + # openwhisk env + TEST_SUITE: System + JAVA_DISTRO: adopt-openj9 + ANSIBLE_CMD: "ansible-playbook -i environments/local -e docker_image_prefix=testing" + GRADLE_PROJS_SKIP: "" + + ## secrets + # (optional) slack incoming wehbook for notifications + SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}} + + # (optional) s3 log upload + S3_LOG_BUCKET: ${{ secrets.S3_LOG_BUCKET }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.AWS_REGION }} + + # github + GH_BUILD: ${{ github.run_id }}-${{ github.event_name }} + GH_BRANCH: ${{ github.head_ref || github.ref_name }} + +jobs: + openwhisk: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: "Setup" + run: ./tools/github/setup.sh + - id: tests + name: Run Tests + run: "./tools/github/run${{ env.TEST_SUITE }}Tests.sh" + continue-on-error: true + - id: logs + name: Upload Logs + run: ./tools/github/checkAndUploadLogs.sh ${{ env.TEST_SUITE }} + - name: Slack Notification + run: > + ./tools/github/writeOnSlack.sh "[$TEST_SUITE]" + $GH_BUILD: ${{ steps.tests.outcome }} + $'\nbranch:' $GH_BRANCH + $'\nlogurl:' ${{ steps.logs.outputs.logurl }} + - name: Results + run: test "${{ steps.tests.outcome }}" == "success" diff --git a/.github/workflows/3-multi-runtime.yaml b/.github/workflows/3-multi-runtime.yaml new file mode 100644 index 00000000000..81cbbb53042 --- /dev/null +++ b/.github/workflows/3-multi-runtime.yaml @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +name: MultiRuntime Tests + +on: + # build on tagged + push: + tags: '*' + # build on pull requests + pull_request: + +env: + # openwhisk env + TEST_SUITE: MultiRuntime + JAVA_DISTRO: adopt-openj9 + ANSIBLE_CMD: "ansible-playbook -i environments/local -e docker_image_prefix=testing" + GRADLE_PROJS_SKIP: "" + + ## secrets + # (optional) slack incoming wehbook for notifications + SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}} + + # (optional) s3 log upload + S3_LOG_BUCKET: ${{ secrets.S3_LOG_BUCKET }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.AWS_REGION }} + + # github + GH_BUILD: ${{ github.run_id }}-${{ github.event_name }} + GH_BRANCH: ${{ github.head_ref || github.ref_name }} + +jobs: + openwhisk: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: "Setup" + run: ./tools/github/setup.sh + - id: tests + name: Run Tests + run: "./tools/github/run${{ env.TEST_SUITE }}Tests.sh" + continue-on-error: true + - id: logs + name: Upload Logs + run: ./tools/github/checkAndUploadLogs.sh ${{ env.TEST_SUITE }} + - name: Slack Notification + run: > + ./tools/github/writeOnSlack.sh "[$TEST_SUITE]" + $GH_BUILD: ${{ steps.tests.outcome }} + $'\nbranch:' $GH_BRANCH + $'\nlogurl:' ${{ steps.logs.outputs.logurl }} + - name: Results + run: test "${{ steps.tests.outcome }}" == "success" diff --git a/.github/workflows/4-standalone.yaml b/.github/workflows/4-standalone.yaml new file mode 100644 index 00000000000..ada2cd3c9c3 --- /dev/null +++ b/.github/workflows/4-standalone.yaml @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +name: Standalone Tests + +on: + # build on tagged + push: + tags: '*' + # build on pull requests + pull_request: + +env: + # openwhisk env + TEST_SUITE: Standalone + JAVA_DISTRO: adopt-openj9 + ANSIBLE_CMD: "ansible-playbook -i environments/local -e docker_image_prefix=testing" + GRADLE_PROJS_SKIP: "" + + ## secrets + # (optional) slack incoming wehbook for notifications + SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}} + + # (optional) s3 log upload + S3_LOG_BUCKET: ${{ secrets.S3_LOG_BUCKET }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.AWS_REGION }} + + # github + GH_BUILD: ${{ github.run_id }}-${{ github.event_name }} + GH_BRANCH: ${{ github.head_ref || github.ref_name }} + +jobs: + openwhisk: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: "Setup" + run: ./tools/github/setup.sh + - id: tests + name: Run Tests + run: "./tools/github/run${{ env.TEST_SUITE }}Tests.sh" + continue-on-error: true + - id: logs + name: Upload Logs + run: ./tools/github/checkAndUploadLogs.sh ${{ env.TEST_SUITE }} + - name: Slack Notification + run: > + ./tools/github/writeOnSlack.sh "[$TEST_SUITE]" + $GH_BUILD: ${{ steps.tests.outcome }} + $'\nbranch:' $GH_BRANCH + $'\nlogurl:' ${{ steps.logs.outputs.logurl }} + - name: Results + run: test "${{ steps.tests.outcome }}" == "success" diff --git a/.github/workflows/5-scheduler.yaml b/.github/workflows/5-scheduler.yaml new file mode 100644 index 00000000000..d87368e529b --- /dev/null +++ b/.github/workflows/5-scheduler.yaml @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +name: Scheduler Tests + +on: + # build on tagged + push: + tags: '*' + # build on pull requests + pull_request: + +env: + # openwhisk env + TEST_SUITE: Scheduler + JAVA_DISTRO: adopt-openj9 + ANSIBLE_CMD: "ansible-playbook -i environments/local -e docker_image_prefix=testing" + GRADLE_PROJS_SKIP: "" + + ## secrets + # (optional) slack incoming wehbook for notifications + SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}} + + # (optional) s3 log upload + S3_LOG_BUCKET: ${{ secrets.S3_LOG_BUCKET }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.AWS_REGION }} + + # github + GH_BUILD: ${{ github.run_id }}-${{ github.event_name }} + GH_BRANCH: ${{ github.head_ref || github.ref_name }} + +jobs: + openwhisk: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: "Setup" + run: ./tools/github/setup.sh + - id: tests + name: Run Tests + run: "./tools/github/run${{ env.TEST_SUITE }}Tests.sh" + continue-on-error: true + - id: logs + name: Upload Logs + run: ./tools/github/checkAndUploadLogs.sh ${{ env.TEST_SUITE }} + - name: Slack Notification + run: > + ./tools/github/writeOnSlack.sh "[$TEST_SUITE]" + $GH_BUILD: ${{ steps.tests.outcome }} + $'\nbranch:' $GH_BRANCH + $'\nlogurl:' ${{ steps.logs.outputs.logurl }} + - name: Results + run: test "${{ steps.tests.outcome }}" == "success" diff --git a/.github/workflows/performance.yaml b/.github/workflows/6-performance.yaml similarity index 65% rename from .github/workflows/performance.yaml rename to .github/workflows/6-performance.yaml index d9678268766..ae6dd73af38 100644 --- a/.github/workflows/performance.yaml +++ b/.github/workflows/6-performance.yaml @@ -16,56 +16,66 @@ # under the License. # name: Performance Tests + on: + # build on tagged push: + tags: '*' + # build on pull requests + pull_request: env: - # uncomment this to enable an ngrok access to the vm on test failure - # enabling this you will get an ssh command to access the vm, active for one hour - #NGROK_DEBUG: yes - # you need to add as secrets an ngrok token and a password to access the terminal - NGROK_TOKEN: ${{ secrets.NGROK_TOKEN }} - NGROK_PASSWORD: ${{ secrets.NGROK_PASSWORD }} # openwhisk env + TEST_SUITE: Performance + JAVA_DISTRO: adopt-openj9 ANSIBLE_CMD: "ansible-playbook -i environments/local -e docker_image_prefix=testing" GRADLE_PROJS_SKIP: "" - # put a slack incoming webook secret to be notified - SLACK_WEBHOOK_URL: ${{secrets.SLACK_WEBHOOK}} - SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK - + + ## secrets + # (optional) slack incoming wehbook for notifications + SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}} + + # (optional) s3 log upload + S3_LOG_BUCKET: ${{ secrets.S3_LOG_BUCKET }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.AWS_REGION }} + + # github + GH_BUILD: ${{ github.run_id }}-${{ github.event_name }} + GH_BRANCH: ${{ github.head_ref || github.ref_name }} + jobs: - - performance: + openwhisk: runs-on: ubuntu-22.04 - steps: + steps: - name: Checkout - uses: actions/checkout@v2 - - name: Add java - uses: actions/setup-java@v2 - with: - distribution: adopt-openj9 - java-version: 11 - cache: 'gradle' + uses: actions/checkout@v3 - name: "Setup" - run: ./tools/travis/setup.sh - - name: "Scala 2.13 compilation" - run: TERM=dumb OW_SCALA_VERSION=2.13 ./gradlew :tests:compileTestScala - - name: Performance tests - run: ./tests/performance/preparation/deploy.sh + run: ./tools/github/setup.sh + - run: ./tests/performance/preparation/deploy.sh - run: TERM=dumb ./tests/performance/wrk_tests/latency.sh "https://172.17.0.1:10001" "$(cat ansible/files/auth.guest)" ./tests/performance/preparation/actions/noop.js 2m + continue-on-error: true - run: TERM=dumb ./tests/performance/wrk_tests/latency.sh "https://172.17.0.1:10001" "$(cat ansible/files/auth.guest)" ./tests/performance/preparation/actions/async.js 2m + continue-on-error: true - run: TERM=dumb ./tests/performance/wrk_tests/throughput.sh "https://172.17.0.1:10001" "$(cat ansible/files/auth.guest)" ./tests/performance/preparation/actions/noop.js 4 1 2 2m + continue-on-error: true - run: TERM=dumb ./tests/performance/wrk_tests/throughput.sh "https://172.17.0.1:10001" "$(cat ansible/files/auth.guest)" ./tests/performance/preparation/actions/async.js 4 1 2 2m + continue-on-error: true - run: TERM=dumb ./tests/performance/wrk_tests/throughput.sh "https://172.17.0.1:10001" "$(cat ansible/files/auth.guest)" ./tests/performance/preparation/actions/noop.js 100 110 2 2m + continue-on-error: true - run: TERM=dumb ./tests/performance/wrk_tests/throughput.sh "https://172.17.0.1:10001" "$(cat ansible/files/auth.guest)" ./tests/performance/preparation/actions/async.js 100 110 2 2m + continue-on-error: true - run: OPENWHISK_HOST="172.17.0.1" CONNECTIONS="100" REQUESTS_PER_SEC="1" ./gradlew gatlingRun-org.apache.openwhisk.ApiV1Simulation + continue-on-error: true - run: OPENWHISK_HOST="172.17.0.1" MEAN_RESPONSE_TIME="1000" API_KEY="$(cat ansible/files/auth.guest)" EXCLUDED_KINDS="python:default,java:default,swift:default" PAUSE_BETWEEN_INVOKES="100" ./gradlew gatlingRun-org.apache.openwhisk.LatencySimulation + continue-on-error: true - run: OPENWHISK_HOST="172.17.0.1" API_KEY="$(cat ansible/files/auth.guest)" CONNECTIONS="100" REQUESTS_PER_SEC="1" ./gradlew gatlingRun-org.apache.openwhisk.BlockingInvokeOneActionSimulation + continue-on-error: true - run: OPENWHISK_HOST="172.17.0.1" API_KEY="$(cat ansible/files/auth.guest)" CONNECTIONS="100" REQUESTS_PER_SEC="1" ASYNC="true" ./gradlew gatlingRun-org.apache.openwhisk.BlockingInvokeOneActionSimulation - # The following configuration does not make much sense. But we do not have enough users. But it's good to verify, that the test is still working. - - run: OPENWHISK_HOST="172.17.0.1" USERS="1" REQUESTS_PER_SEC="1" ./gradlew gatlingRun-org.apache.openwhisk.ColdBlockingInvokeSimulation - #- name: upload logs - # run: ./tools/github/checkAndUploadLogs.sh perf - - uses: slackapi/slack-github-action@v1.23.0 - with: - slack-message: "GitHub Action build result: ${{ job.status }}\n${{ github.event.pull_request.html_url || github.event.head_commit.url }}" + continue-on-error: true + - name: Slack Notification + run: > + ./tools/github/writeOnSlack.sh "[$TEST_SUITE]" + $'\nbranch:' $GH_BRANCH + $'\nCheck GitHub logs for results' \ No newline at end of file diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 00000000000..c454e35e09e --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,56 @@ +# How to use those workflows + +There are a few [GitHub secrets](https://docs.github.com/en/actions/security-guides/encrypted-secrets) to configure to fully leverage the build. + +You can use and set the followings secrets also in your fork. + +## Ngrok Debugging + +You can debug a GitHub Action build using [NGROK](https://ngrok.com/). + +It is disabled for automated build triggered by push and pull_requests. + +You can trigger a workflow run manually enabling ngrok debugging. + +It will open an ssh connection to the VM and keep it up and running for one hour. +The connection url is showns in the log for debugAction.sh + +You can then connect to the build vm, and debug it. +You need to use a password of your choice to access it. + +You can continue the build with `touch /tmp/continue`. +You can abort the build with `touch /tmp/abort`. + +To enable this option you have to register to Ngrok, using the fee account and get the NGROK Token. + +Then set the following secrets: + +- `NGROK_TOKEN` to the ngrok token. +- `NGROK_PASSWORD` to a password of choice to access the build with the ssh command generated. + +## Log Upload + +The build uploads the logs to an s3 bucket allowing to inspect them with a browser. + +You need to create the bucket with the following commands: + +``` +S3_LOG_BUCKET= +AWS_REGION= +aws s3 mb s3://$S3_LOG_BUCKET --region $AWS_REGION +aws s3 website s3://$S3_LOG_BUCKET/ --index-document index.html +aws s3api put-bucket-acl --acl public-read --bucket $S3_LOG_BUCKET +``` + +To enable upload to the created bucket you need to set the following secrets: + +- `S3_LOG_BUCKET`: name of your bucket in s3 (just the name, without `s3://`); create it before. +- `AWS_ACCESS_KEY_ID`: your aws access key. +- `AWS_SECRET_ACCESS_KEY`: your aws secret key. +- `AWS_REGION`: important: the region where your bucket is. + +## Slack notification + +If you want to get notified of what happens on slack, create an [Incoming Web Hook](https://api.slack.com/messaging/webhooks) and then set the following secret: + +- `SLACK_WEBHOOK`: the incoming webhook url provided by slack. diff --git a/.github/workflows/multi-system.yaml b/.github/workflows/multi-system.yaml deleted file mode 100644 index 56e1fc450d5..00000000000 --- a/.github/workflows/multi-system.yaml +++ /dev/null @@ -1,62 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -name: Multi-System Tests -on: - push: - -env: - # uncomment this to enable an ngrok access to the vm on test failure - # enabling this you will get an ssh command to access the vm, active for one hour - #NGROK_DEBUG: yes - # you need to add as secrets an ngrok token and a password to access the terminal - NGROK_TOKEN: ${{ secrets.NGROK_TOKEN }} - NGROK_PASSWORD: ${{ secrets.NGROK_PASSWORD }} - # openwhisk env - ANSIBLE_CMD: "ansible-playbook -i environments/local -e docker_image_prefix=testing" - GRADLE_PROJS_SKIP: "" - # put a slack incoming webook secret to be notified - SLACK_WEBHOOK_URL: ${{secrets.SLACK_WEHBOOK}} - SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK - -jobs: - - multi-system: - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Add java - uses: actions/setup-java@v2 - with: - distribution: adopt-openj9 - java-version: 11 - cache: 'gradle' - - name: "Setup" - run: ./tools/travis/setup.sh - - name: "Scala 2.13 compilation" - run: TERM=dumb OW_SCALA_VERSION=2.13 ./gradlew :tests:compileTestScala - - name: Run MultiSystem Tests - run: ./tools/travis/runMultiRuntimeTests.sh || ./tools/github/debugAction.sh - - run: ./tools/github/waitIfDebug.sh - #- name: upload logs - # run: ./tools/github/checkAndUploadLogs.sh multi-runtime - - uses: slackapi/slack-github-action@v1.23.0 - with: - slack-message: "GitHub Action build result: ${{ job.status }}\n${{ github.event.pull_request.html_url || github.event.head_commit.url }}" - - \ No newline at end of file diff --git a/.github/workflows/standalone.yaml b/.github/workflows/standalone.yaml deleted file mode 100644 index 116741163c0..00000000000 --- a/.github/workflows/standalone.yaml +++ /dev/null @@ -1,62 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -name: Standalone Tests -on: - push: - -env: - # uncomment this to enable an ngrok access to the vm on test failure - # enabling this you will get an ssh command to access the vm, active for one hour - #NGROK_DEBUG: yes - # you need to add as secrets an ngrok token and a password to access the terminal - NGROK_TOKEN: ${{ secrets.NGROK_TOKEN }} - NGROK_PASSWORD: ${{ secrets.NGROK_PASSWORD }} - # openwhisk env - ANSIBLE_CMD: "ansible-playbook -i environments/local -e docker_image_prefix=testing" - GRADLE_PROJS_SKIP: "" - # put a slack incoming webook secret to be notified - SLACK_WEBHOOK_URL: ${{secrets.SLACK_WEBHOOK}} - SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK - -jobs: - - standalone: - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Add java - uses: actions/setup-java@v2 - with: - distribution: adopt-openj9 - java-version: 11 - cache: 'gradle' - - name: "Setup" - run: ./tools/travis/setup.sh - - name: "Scala 2.13 compilation" - run: TERM=dumb OW_SCALA_VERSION=2.13 ./gradlew :tests:compileTestScala - - name: Run Standalone - run: ./tools/travis/runStandaloneTests.sh || ./tools/github/debugAction.sh - - run: ./tools/github/waitIfDebug.sh - #- name: upload logs - # run: ./tools/github/checkAndUploadLogs.sh standalone - - uses: slackapi/slack-github-action@v1.23.0 - with: - slack-message: "GitHub Action build result: ${{ job.status }}\n${{ github.event.pull_request.html_url || github.event.head_commit.url }}" - - \ No newline at end of file diff --git a/.github/workflows/system.yaml b/.github/workflows/system.yaml deleted file mode 100644 index b3821849b3c..00000000000 --- a/.github/workflows/system.yaml +++ /dev/null @@ -1,62 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -name: System Tests -on: - push: - -env: - # uncomment this to enable an ngrok access to the vm on test failure - # enabling this you will get an ssh command to access the vm, active for one hour - #NGROK_DEBUG: yes - # you need to add as secrets an ngrok token and a password to access the terminal - NGROK_TOKEN: ${{ secrets.NGROK_TOKEN }} - NGROK_PASSWORD: ${{ secrets.NGROK_PASSWORD }} - # openwhisk env - ANSIBLE_CMD: "ansible-playbook -i environments/local -e docker_image_prefix=testing" - GRADLE_PROJS_SKIP: "" - # put a slack incoming webook secret to be notified - SLACK_WEBHOOK_URL: ${{secrets.SLACK_WEBHOOK}} - SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK - -jobs: - - system: - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Add java - uses: actions/setup-java@v2 - with: - distribution: adopt-openj9 - java-version: 11 - cache: 'gradle' - - name: "Setup" - run: ./tools/travis/setup.sh - - name: "Scala 2.13 compilation" - run: TERM=dumb OW_SCALA_VERSION=2.13 ./gradlew :tests:compileTestScala - - name: Run System Tests - run: ./tools/travis/runSystemTests.sh || ./tools/github/debugAction.sh - - run: ./tools/github/waitIfDebug.sh - #- name: upload logs - # run: ./tools/github/checkAndUploadLogs.sh system - - uses: slackapi/slack-github-action@v1.23.0 - with: - slack-message: "GitHub Action build result: ${{ job.status }}\n${{ github.event.pull_request.html_url || github.event.head_commit.url }}" - - \ No newline at end of file diff --git a/.github/workflows/unit.yaml b/.github/workflows/unit.yaml deleted file mode 100644 index 0f574f8546d..00000000000 --- a/.github/workflows/unit.yaml +++ /dev/null @@ -1,62 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -name: Unit Tests -on: - push: - -env: - # uncomment this to enable an ngrok access to the vm on test failure - # enabling this you will get an ssh command to access the vm, active for one hour - #NGROK_DEBUG: yes - # you need to add as secrets an ngrok token and a password to access the terminal - NGROK_TOKEN: ${{ secrets.NGROK_TOKEN }} - NGROK_PASSWORD: ${{ secrets.NGROK_PASSWORD }} - # openwhisk env - ANSIBLE_CMD: "ansible-playbook -i environments/local -e docker_image_prefix=testing" - GRADLE_PROJS_SKIP: "" - # put a slack incoming webook secret to be notified - SLACK_WEBHOOK_URL: ${{secrets.SLACK_WEBHOOK}} - SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK - -jobs: - - unit: - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Add java - uses: actions/setup-java@v2 - with: - distribution: adopt-openj9 - java-version: 11 - cache: 'gradle' - - name: "Setup" - run: ./tools/travis/setup.sh - - name: "Scala 2.13 compilation" - run: TERM=dumb OW_SCALA_VERSION=2.13 ./gradlew :tests:compileTestScala - - name: Run Unit Tests - run: ./tools/travis/runUnitTests.sh || ./tools/github/debugAction.sh - - run: ./tools/github/waitIfDebug.sh - #- name: upload logs - # run: ./tools/github/checkAndUploadLogs.sh unit db - - uses: slackapi/slack-github-action@v1.23.0 - with: - slack-message: "GitHub Action build result: ${{ job.status }}\n${{ github.event.pull_request.html_url || github.event.head_commit.url }}" - - diff --git a/tools/github/checkAndUploadLogs.sh b/tools/github/checkAndUploadLogs.sh new file mode 100755 index 00000000000..6681202dbb5 --- /dev/null +++ b/tools/github/checkAndUploadLogs.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# check variables +for i in S3_LOG_BUCKET AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION +do + if test -z "${!i}" + then echo "Required Environment Variable Missing: $i" ; exit 1 + fi +done + +# Disable abort script at first error as we require the logs to be uploaded +# even if check and log collection fails +# set -e + + +SECONDS=0 +SCRIPTDIR=$(cd $(dirname "$0") && pwd) +ROOTDIR="$SCRIPTDIR/../.." + +cd $ROOTDIR + +LOG_NAME="$1" + +# tags is db only when the test is unit +TAGS="" +[[ "$2" == "Unit" ]] && TAGS="db" + +LOG_DIR="$(date +%Y-%m-%d)/${LOG_NAME}-${GH_BUILD}-${GH_BRANCH}" + +# Perf logs are typically about 20MB and thus rapidly fill our box account. +# Disable upload to reduce the interval at which we need to manually clean logs from box. +if [ "$LOG_NAME" == "Performance" ]; then + echo "Skipping upload of perf logs to conserve space" + exit 0 +fi + +ansible-playbook -i ansible/environments/local ansible/logs.yml + +./tools/build/checkLogs.py logs "$TAGS" + +./tools/github/s3-upload.sh "$PWD/logs" "$LOG_DIR" + +echo "Time taken for ${0##*/} is $SECONDS secs" diff --git a/tools/github/debugAction.sh b/tools/github/debugAction.sh index d0d1876db3b..e1691d13162 100755 --- a/tools/github/debugAction.sh +++ b/tools/github/debugAction.sh @@ -16,18 +16,18 @@ # limitations under the License. # -if [[ -z "$NGROK_DEBUG" ]]; then - exit 1 +if [[ -z "$NGROK_DEBUG" ]] || [[ "$NGROK_DEBUG" == "false" ]] +then exit 0 fi -if [[ -z "$NGROK_TOKEN" ]]; then - echo "Please set 'NGROK_TOKEN'" - exit 2 +if [[ -z "$NGROK_TOKEN" ]] +then echo "Please set 'NGROK_TOKEN'" + exit 1 fi -if [[ -z "$NGROK_PASSWORD" ]]; then - echo "Please set 'NGROK_PASSWORD'" - exit 3 +if [[ -z "$NGROK_PASSWORD" ]] +then echo "Please set 'NGROK_PASSWORD'" + exit 1 fi echo "### Install ngrok ###" @@ -57,5 +57,5 @@ if [[ -z "$HAS_ERRORS" ]]; then echo "==========================================" else echo "$HAS_ERRORS" - exit 4 + exit 1 fi diff --git a/tools/github/flake8.sh b/tools/github/flake8.sh new file mode 100755 index 00000000000..98b7c0d9c0c --- /dev/null +++ b/tools/github/flake8.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +pip3 install --user --upgrade flake8 + +# These files do not have a .py extension so flake8 will not scan them +declare -a PYTHON_FILES=("." + "./tools/admin/wskadmin" + "./tools/build/citool" + "./tools/build/redo") + +echo 'Flake8: first round (fast fail) stops the build if there are any Python 3 syntax errors...' +for i in "${PYTHON_FILES[@]}" +do + flake8 "$i" --select=E999,F821 --statistics + RETURN_CODE=$? + if [ $RETURN_CODE != 0 ]; then + echo 'Flake8 found Python 3 syntax errors above. See: https://docs.python.org/3/howto/pyporting.html' + exit $RETURN_CODE + fi +done + +echo 'Flake8: second round to find any other stylistic issues...' +for i in "${PYTHON_FILES[@]}" +do + flake8 "$i" --ignore=E,W503,W504,W605 --max-line-length=127 --statistics +done diff --git a/tools/github/runDummyKOTests.sh b/tools/github/runDummyKOTests.sh new file mode 100755 index 00000000000..e7d2e7d674d --- /dev/null +++ b/tools/github/runDummyKOTests.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e + +SCRIPTDIR=$(cd $(dirname "$0") && pwd) +ROOTDIR="$SCRIPTDIR/../.." + +mkdir $ROOTDIR/logs +echo "

$(date)

" >$ROOTDIR/logs/index.html +exit 1 \ No newline at end of file diff --git a/tools/github/runDummyTests.sh b/tools/github/runDummyTests.sh new file mode 100755 index 00000000000..1e33153beef --- /dev/null +++ b/tools/github/runDummyTests.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e + +SCRIPTDIR=$(cd $(dirname "$0") && pwd) +ROOTDIR="$SCRIPTDIR/../.." + +mkdir $ROOTDIR/logs +echo "

$(date)

" >$ROOTDIR/logs/now.html +exit 0 \ No newline at end of file diff --git a/tools/github/runLeanSystemTests.sh b/tools/github/runLeanSystemTests.sh new file mode 100755 index 00000000000..372fbc1e409 --- /dev/null +++ b/tools/github/runLeanSystemTests.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e + +SCRIPTDIR=$(cd $(dirname "$0") && pwd) +ROOTDIR="$SCRIPTDIR/../.." + +cd $ROOTDIR/tools/travis + +export ORG_GRADLE_PROJECT_testSetName="REQUIRE_LEAN_SYSTEM" +export GRADLE_COVERAGE=true + +./setupPrereq.sh /ansible/files/runtimes-nodeonly.json + +./distDocker-lean.sh + +./setupLeanSystem.sh /ansible/files/runtimes-nodeonly.json + +./runTests.sh diff --git a/tools/github/runMultiRuntimeTests.sh b/tools/github/runMultiRuntimeTests.sh new file mode 100755 index 00000000000..e2de0c59bd4 --- /dev/null +++ b/tools/github/runMultiRuntimeTests.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e + +SCRIPTDIR=$(cd $(dirname "$0") && pwd) +ROOTDIR="$SCRIPTDIR/../.." + +cd $ROOTDIR/tools/travis + +export ORG_GRADLE_PROJECT_testSetName="REQUIRE_MULTI_RUNTIME" +export GRADLE_COVERAGE=true + +./setupPrereq.sh + +./distDocker.sh + +./setupSystem.sh + +./runTests.sh diff --git a/tools/github/runSchedulerTests.sh b/tools/github/runSchedulerTests.sh new file mode 100755 index 00000000000..000de8080e5 --- /dev/null +++ b/tools/github/runSchedulerTests.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e + +SCRIPTDIR=$(cd $(dirname "$0") && pwd) +ROOTDIR="$SCRIPTDIR/../.." + +cd $ROOTDIR/tools/travis +export ORG_GRADLE_PROJECT_testSetName="REQUIRE_SCHEDULER" +export GRADLE_COVERAGE=true + +./setupPrereq.sh /ansible/files/runtimes-nodeonly.json + +./distDocker.sh + +./setupSystem.sh /ansible/files/runtimes-nodeonly.json + +./runTests.sh diff --git a/tools/github/runStandaloneTests.sh b/tools/github/runStandaloneTests.sh new file mode 100755 index 00000000000..ba10c5b4715 --- /dev/null +++ b/tools/github/runStandaloneTests.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e + +SCRIPTDIR=$(cd $(dirname "$0") && pwd) +ROOTDIR="$SCRIPTDIR/../.." + +export ORG_GRADLE_PROJECT_testSetName="REQUIRE_STANDALONE" +export GRADLE_COVERAGE=true + +cd $ROOTDIR/ansible +$ANSIBLE_CMD setup.yml +$ANSIBLE_CMD properties.yml -e manifest_file="/ansible/files/runtimes-nodeonly.json" +$ANSIBLE_CMD downloadcli-github.yml + +# Install kubectl +curl -Lo ./kubectl https://storage.googleapis.com/kubernetes-release/release/v1.16.1/bin/linux/amd64/kubectl +chmod +x kubectl +sudo cp kubectl /usr/local/bin/kubectl + +# Install kind +curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/v0.5.1/kind-linux-amd64 +chmod +x kind +sudo cp kind /usr/local/bin/kind + +kind create cluster --wait 5m +export KUBECONFIG="$(kind get kubeconfig-path)" +kubectl config set-context --current --namespace=default + +# This is required because it is timed out to pull the image during the test. +docker pull openwhisk/action-nodejs-v14:nightly +docker pull openwhisk/dockerskeleton:nightly +docker pull openwhisk/example:nightly +docker pull openwhisk/apigateway:0.11.0 + +cd $ROOTDIR +TERM=dumb ./gradlew :core:standalone:build \ + :core:monitoring:user-events:distDocker + +cd $ROOTDIR +TERM=dumb ./gradlew :core:standalone:cleanTest \ + :core:standalone:test \ + :core:monitoring:user-events:reportTestScoverage + +# Run test in end as it publishes the coverage also +cd $ROOTDIR/tools/travis +./runTests.sh diff --git a/tools/github/runSystemTests.sh b/tools/github/runSystemTests.sh new file mode 100755 index 00000000000..8325caddbae --- /dev/null +++ b/tools/github/runSystemTests.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e + +SCRIPTDIR=$(cd $(dirname "$0") && pwd) +ROOTDIR="$SCRIPTDIR/../.." + +cd $ROOTDIR/tools/travis + +export ORG_GRADLE_PROJECT_testSetName="REQUIRE_SYSTEM" +export GRADLE_COVERAGE=true + +./setupPrereq.sh /ansible/files/runtimes-nodeonly.json + +./distDocker.sh + +./setupSystem.sh /ansible/files/runtimes-nodeonly.json + +./runTests.sh diff --git a/tools/github/runUnitTests.sh b/tools/github/runUnitTests.sh new file mode 100755 index 00000000000..b10418c9e4d --- /dev/null +++ b/tools/github/runUnitTests.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e + +SCRIPTDIR=$(cd $(dirname "$0") && pwd) +ROOTDIR="$SCRIPTDIR/../.." + +cd $ROOTDIR/tools/travis +export TESTCONTAINERS_RYUK_DISABLED="true" +export ORG_GRADLE_PROJECT_testSetName="REQUIRE_ONLY_DB" + +./setupPrereq.sh + +cat "$ROOTDIR/tests/src/test/resources/application.conf" + +./distDocker.sh + +./runTests.sh diff --git a/tools/github/s3-upload.sh b/tools/github/s3-upload.sh new file mode 100755 index 00000000000..6918f285950 --- /dev/null +++ b/tools/github/s3-upload.sh @@ -0,0 +1,57 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# check variables +for i in S3_LOG_BUCKET AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY +do + if test -z "${!i}" + then echo "Please set $i" ; exit 1 + fi +done + +if [[ -z "$1" ]] || [[ -z "$2" ]] +then echo "usage: " ; exit 1 +fi + +FROM="$1" +TO="$2" + +BROWSER="https://raw.githubusercontent.com/qoomon/aws-s3-bucket-browser/master/index.html" +BUCKET_URL="https://$S3_LOG_BUCKET.s3.$AWS_REGION.amazonaws.com/" + +# install rclone +if ! which rclone +then curl https://rclone.org/install.sh | sudo bash +fi + +RCLONE="rclone --config /dev/null \ + --s3-provider AWS \ + --s3-region $AWS_REGION \ + --s3-acl public-read \ + --s3-access-key-id $AWS_ACCESS_KEY_ID \ + --s3-secret-access-key $AWS_SECRET_ACCESS_KEY" + +curl -s "$BROWSER" |\ + sed -e 's!bucketUrl: undefined!bucketUrl: "'$BUCKET_URL'"!' |\ + $RCLONE rcat ":s3:$S3_LOG_BUCKET/index.html" + +$RCLONE copyto "$FROM" ":s3:$S3_LOG_BUCKET/$TO/" +echo "logurl=${BUCKET_URL}index.html#$TO" >>"${GITHUB_OUTPUT:-/dev/stdout}" + + + diff --git a/tools/github/scan.sh b/tools/github/scan.sh new file mode 100755 index 00000000000..42b3fd57332 --- /dev/null +++ b/tools/github/scan.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e + +# Build script for Travis-CI. +SECONDS=0 +SCRIPTDIR=$(cd $(dirname "$0") && pwd) +ROOTDIR="$SCRIPTDIR/../.." +HOMEDIR="$SCRIPTDIR/../../../" +UTILDIR="$HOMEDIR/openwhisk-utilities/" + +cd $ROOTDIR +./tools/github/flake8.sh # Check Python files for style and stop the build on syntax errors + +# clone the openwhisk utilities repo. +cd $HOMEDIR +git clone https://github.com/apache/openwhisk-utilities.git + +# run the scancode util. against project source code starting at its root +cd $UTILDIR +scancode/scanCode.py --config scancode/ASF-Release.cfg $ROOTDIR + +# run scalafmt checks +cd $ROOTDIR +TERM=dumb ./gradlew checkScalafmtAll + +# lint tests to all be actually runnable +MISSING_TESTS=$(grep -rL "RunWith" --include="*Tests.scala" tests || true) +if [ -n "$MISSING_TESTS" ] +then + echo "The following tests are missing the 'RunWith' annotation" + echo $MISSING_TESTS + exit 1 +fi + +echo "Time taken for ${0##*/} is $SECONDS secs" diff --git a/tools/github/setup.sh b/tools/github/setup.sh new file mode 100755 index 00000000000..59151292126 --- /dev/null +++ b/tools/github/setup.sh @@ -0,0 +1,61 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +if [[ $TEST_SUITE =~ Dummy ]] +then echo skiping setup ; exit 0 +fi + +# retries a command for five times and exits with the non-zero exit if even after +# the retries the command did not succeed. +function retry() { + local exitcode=0 + for i in {1..5}; + do + exitcode=0 + "$@" && break || exitcode=$? && echo "$i. attempt failed. Will retry $((5-i)) more times!" && sleep 1; + done + if [ $exitcode -ne 0 ]; then + exit $exitcode + fi +} + +# Python +python -m pip install --user couchdb + +# Ansible (warning you need jinja < 3.1 with this version) +python -m pip install --user 'jinja2<3.1' ansible==2.8.18 + +# Azure CosmosDB +python -m pip install --user pydocumentdb + +# Support the revises log upload script +python -m pip install --user humanize requests + +# Scan code before compiling the code +tools/github/scan.sh + +# Preload alpine 3.5 to avoid issues with depending images +retry docker pull alpine:3.5 + +# Basic check that all code compiles and depdendencies are downloaded correctly. +# Compiling the tests will compile all components as well. +# +# Downloads the gradle wrapper, dependencies and tries to compile the code. +# Retried 5 times in case there are network hiccups. +TERM=dumb retry ./gradlew :tests:compileTestScala + diff --git a/tools/github/waitIfDebug.sh b/tools/github/waitIfDebug.sh index 6c92b42945c..30f791b2964 100755 --- a/tools/github/waitIfDebug.sh +++ b/tools/github/waitIfDebug.sh @@ -22,6 +22,7 @@ fi echo "You have an hour to debug this build." echo "Do touch /tmp/continue to continue." echo "Do touch /tmp/abort to abort." + EXIT=0 for i in $(seq 1 60) do @@ -30,6 +31,7 @@ do echo "$i/60 still waiting..." sleep 60 done + killall ngrok rm -f .ngrok.log /tmp/continue /tmp/abort exit $EXIT diff --git a/tools/github/writeOnSlack.sh b/tools/github/writeOnSlack.sh new file mode 100755 index 00000000000..da8f2a8f2f3 --- /dev/null +++ b/tools/github/writeOnSlack.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +if test -z "$SLACK_WEBHOOK" +then echo "Please create an incoming webhook for slack and set SLACK_WEBHOOK" + exit 0 +fi + +echo -n '{"text":' >/tmp/msg$$ +echo -n "$@" | jq -Rsa . >>/tmp/msg$$ +echo -n '}' >>/tmp/msg$$ + +curl -X POST -H 'Content-type: application/json' --data "@/tmp/msg$$" "$SLACK_WEBHOOK" From fe8cf67979202a513c6029d2d7cd7c0ad0391ac6 Mon Sep 17 00:00:00 2001 From: Michele Sciabarra Date: Mon, 12 Dec 2022 06:55:49 +0000 Subject: [PATCH 18/39] scancode fixes --- tools/github/checkAndUploadLogs.sh | 2 +- tools/github/runDummyKOTests.sh | 2 +- tools/github/runDummyTests.sh | 2 +- tools/macos/README.md | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/github/checkAndUploadLogs.sh b/tools/github/checkAndUploadLogs.sh index 6681202dbb5..74803b91a25 100755 --- a/tools/github/checkAndUploadLogs.sh +++ b/tools/github/checkAndUploadLogs.sh @@ -19,7 +19,7 @@ # check variables for i in S3_LOG_BUCKET AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION do - if test -z "${!i}" + if test -z "${!i}" then echo "Required Environment Variable Missing: $i" ; exit 1 fi done diff --git a/tools/github/runDummyKOTests.sh b/tools/github/runDummyKOTests.sh index e7d2e7d674d..c30a791b60a 100755 --- a/tools/github/runDummyKOTests.sh +++ b/tools/github/runDummyKOTests.sh @@ -23,4 +23,4 @@ ROOTDIR="$SCRIPTDIR/../.." mkdir $ROOTDIR/logs echo "

$(date)

" >$ROOTDIR/logs/index.html -exit 1 \ No newline at end of file +exit 1 diff --git a/tools/github/runDummyTests.sh b/tools/github/runDummyTests.sh index 1e33153beef..11564531e56 100755 --- a/tools/github/runDummyTests.sh +++ b/tools/github/runDummyTests.sh @@ -23,4 +23,4 @@ ROOTDIR="$SCRIPTDIR/../.." mkdir $ROOTDIR/logs echo "

$(date)

" >$ROOTDIR/logs/now.html -exit 0 \ No newline at end of file +exit 0 diff --git a/tools/macos/README.md b/tools/macos/README.md index 8ff60fb266a..964bb949f73 100644 --- a/tools/macos/README.md +++ b/tools/macos/README.md @@ -75,7 +75,7 @@ Follow instructions in [ansible/README.md](../../ansible/README.md) #### Using brew ```bash -brew install wsk +brew install wsk wsk property set --apihost https://localhost wsk property set --auth `cat ansible/files/auth.guest` ``` @@ -109,9 +109,9 @@ ansible-playbook wipe.yml ansible-playbook properties.yml ``` -To run the unit tests execute the command bellow from the project's root folder: +To run the unit tests execute the command bellow from the project's root folder: ```bash # go back to project's root folder cd ../ -./gradlew -PtestSetName="REQUIRE_ONLY_DB" :tests:testCoverageLean +./gradlew -PtestSetName="REQUIRE_ONLY_DB" :tests:testCoverageLean ``` From 00308f96c65ed2e1aece8718039eb87c58ec4390 Mon Sep 17 00:00:00 2001 From: Michele Sciabarra Date: Mon, 12 Dec 2022 19:48:37 +0000 Subject: [PATCH 19/39] forcing a corresponding version of jdk in ci and docker image and renaming S3_LOG_BUCKET to AWS_BUCKET to fix tests --- .github/workflows/0-on-demand.yaml | 3 +-- .github/workflows/1-unit.yaml | 3 +-- .github/workflows/2-system.yaml | 3 +-- .github/workflows/3-multi-runtime.yaml | 3 +-- .github/workflows/4-standalone.yaml | 3 +-- .github/workflows/5-scheduler.yaml | 3 +-- .github/workflows/6-performance.yaml | 3 +-- .github/workflows/README.md | 10 +++++----- common/scala/Dockerfile | 3 ++- tools/github/checkAndUploadLogs.sh | 2 +- tools/github/debugAction.sh | 10 +++++++++- tools/github/s3-upload.sh | 8 ++++---- tools/github/setup.sh | 8 ++++++++ 13 files changed, 36 insertions(+), 26 deletions(-) diff --git a/.github/workflows/0-on-demand.yaml b/.github/workflows/0-on-demand.yaml index c74a0fa4850..77f6a7d8330 100644 --- a/.github/workflows/0-on-demand.yaml +++ b/.github/workflows/0-on-demand.yaml @@ -43,7 +43,6 @@ on: env: # openwhisk env TEST_SUITE: ${{ inputs.test_suite }} - JAVA_DISTRO: adopt-openj9 ANSIBLE_CMD: "ansible-playbook -i environments/local -e docker_image_prefix=testing" GRADLE_PROJS_SKIP: "" @@ -52,7 +51,7 @@ env: SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}} # (optional) s3 log upload - S3_LOG_BUCKET: ${{ secrets.S3_LOG_BUCKET }} + AWS_BUCKET: ${{ secrets.AWS_BUCKET }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.AWS_REGION }} diff --git a/.github/workflows/1-unit.yaml b/.github/workflows/1-unit.yaml index 5b5ec9b717e..93cab949650 100644 --- a/.github/workflows/1-unit.yaml +++ b/.github/workflows/1-unit.yaml @@ -27,7 +27,6 @@ on: env: # openwhisk env TEST_SUITE: Unit - JAVA_DISTRO: adopt-openj9 ANSIBLE_CMD: "ansible-playbook -i environments/local -e docker_image_prefix=testing" GRADLE_PROJS_SKIP: "" @@ -36,7 +35,7 @@ env: SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}} # (optional) s3 log upload - S3_LOG_BUCKET: ${{ secrets.S3_LOG_BUCKET }} + AWS_BUCKET: ${{ secrets.AWS_BUCKET }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.AWS_REGION }} diff --git a/.github/workflows/2-system.yaml b/.github/workflows/2-system.yaml index ff4490fea17..19ba441d776 100644 --- a/.github/workflows/2-system.yaml +++ b/.github/workflows/2-system.yaml @@ -27,7 +27,6 @@ on: env: # openwhisk env TEST_SUITE: System - JAVA_DISTRO: adopt-openj9 ANSIBLE_CMD: "ansible-playbook -i environments/local -e docker_image_prefix=testing" GRADLE_PROJS_SKIP: "" @@ -36,7 +35,7 @@ env: SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}} # (optional) s3 log upload - S3_LOG_BUCKET: ${{ secrets.S3_LOG_BUCKET }} + AWS_BUCKET: ${{ secrets.AWS_BUCKET }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.AWS_REGION }} diff --git a/.github/workflows/3-multi-runtime.yaml b/.github/workflows/3-multi-runtime.yaml index 81cbbb53042..e81c0156c89 100644 --- a/.github/workflows/3-multi-runtime.yaml +++ b/.github/workflows/3-multi-runtime.yaml @@ -27,7 +27,6 @@ on: env: # openwhisk env TEST_SUITE: MultiRuntime - JAVA_DISTRO: adopt-openj9 ANSIBLE_CMD: "ansible-playbook -i environments/local -e docker_image_prefix=testing" GRADLE_PROJS_SKIP: "" @@ -36,7 +35,7 @@ env: SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}} # (optional) s3 log upload - S3_LOG_BUCKET: ${{ secrets.S3_LOG_BUCKET }} + AWS_BUCKET: ${{ secrets.AWS_BUCKET }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.AWS_REGION }} diff --git a/.github/workflows/4-standalone.yaml b/.github/workflows/4-standalone.yaml index ada2cd3c9c3..e2b9d3c317c 100644 --- a/.github/workflows/4-standalone.yaml +++ b/.github/workflows/4-standalone.yaml @@ -27,7 +27,6 @@ on: env: # openwhisk env TEST_SUITE: Standalone - JAVA_DISTRO: adopt-openj9 ANSIBLE_CMD: "ansible-playbook -i environments/local -e docker_image_prefix=testing" GRADLE_PROJS_SKIP: "" @@ -36,7 +35,7 @@ env: SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}} # (optional) s3 log upload - S3_LOG_BUCKET: ${{ secrets.S3_LOG_BUCKET }} + AWS_BUCKET: ${{ secrets.AWS_BUCKET }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.AWS_REGION }} diff --git a/.github/workflows/5-scheduler.yaml b/.github/workflows/5-scheduler.yaml index d87368e529b..d5b1b1e43f5 100644 --- a/.github/workflows/5-scheduler.yaml +++ b/.github/workflows/5-scheduler.yaml @@ -27,7 +27,6 @@ on: env: # openwhisk env TEST_SUITE: Scheduler - JAVA_DISTRO: adopt-openj9 ANSIBLE_CMD: "ansible-playbook -i environments/local -e docker_image_prefix=testing" GRADLE_PROJS_SKIP: "" @@ -36,7 +35,7 @@ env: SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}} # (optional) s3 log upload - S3_LOG_BUCKET: ${{ secrets.S3_LOG_BUCKET }} + AWS_BUCKET: ${{ secrets.AWS_BUCKET }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.AWS_REGION }} diff --git a/.github/workflows/6-performance.yaml b/.github/workflows/6-performance.yaml index ae6dd73af38..b89ffad566c 100644 --- a/.github/workflows/6-performance.yaml +++ b/.github/workflows/6-performance.yaml @@ -27,7 +27,6 @@ on: env: # openwhisk env TEST_SUITE: Performance - JAVA_DISTRO: adopt-openj9 ANSIBLE_CMD: "ansible-playbook -i environments/local -e docker_image_prefix=testing" GRADLE_PROJS_SKIP: "" @@ -36,7 +35,7 @@ env: SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}} # (optional) s3 log upload - S3_LOG_BUCKET: ${{ secrets.S3_LOG_BUCKET }} + AWS_BUCKET: ${{ secrets.AWS_BUCKET }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.AWS_REGION }} diff --git a/.github/workflows/README.md b/.github/workflows/README.md index c454e35e09e..55473d04e86 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -35,16 +35,16 @@ The build uploads the logs to an s3 bucket allowing to inspect them with a brows You need to create the bucket with the following commands: ``` -S3_LOG_BUCKET= +AWS_BUCKET= AWS_REGION= -aws s3 mb s3://$S3_LOG_BUCKET --region $AWS_REGION -aws s3 website s3://$S3_LOG_BUCKET/ --index-document index.html -aws s3api put-bucket-acl --acl public-read --bucket $S3_LOG_BUCKET +aws s3 mb s3://$AWS_BUCKET --region $AWS_REGION +aws s3 website s3://$AWS_BUCKET/ --index-document index.html +aws s3api put-bucket-acl --acl public-read --bucket $AWS_BUCKET ``` To enable upload to the created bucket you need to set the following secrets: -- `S3_LOG_BUCKET`: name of your bucket in s3 (just the name, without `s3://`); create it before. +- `AWS_BUCKET`: name of your bucket in s3 (just the name, without `s3://`); create it before. - `AWS_ACCESS_KEY_ID`: your aws access key. - `AWS_SECRET_ACCESS_KEY`: your aws secret key. - `AWS_REGION`: important: the region where your bucket is. diff --git a/common/scala/Dockerfile b/common/scala/Dockerfile index 6db867ff240..c7ebd2b9e87 100644 --- a/common/scala/Dockerfile +++ b/common/scala/Dockerfile @@ -14,7 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # -FROM adoptopenjdk/openjdk11-openj9:x86_64-alpine-jdk-11.0.8_10_openj9-0.21.0 +# if you change version of openjsk, also update tools/github/setup.sh to download the corresponding jdk +FROM adoptopenjdk/openjdk11-openj9:x86_64-alpine-jdk-11.0.12_7_openj9-0.27.0 ENV LANG en_US.UTF-8 ENV LANGUAGE en_US:en diff --git a/tools/github/checkAndUploadLogs.sh b/tools/github/checkAndUploadLogs.sh index 74803b91a25..2e1144c09cc 100755 --- a/tools/github/checkAndUploadLogs.sh +++ b/tools/github/checkAndUploadLogs.sh @@ -17,7 +17,7 @@ # # check variables -for i in S3_LOG_BUCKET AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION +for i in AWS_BUCKET AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION do if test -z "${!i}" then echo "Required Environment Variable Missing: $i" ; exit 1 diff --git a/tools/github/debugAction.sh b/tools/github/debugAction.sh index e1691d13162..6ebe6e243d6 100755 --- a/tools/github/debugAction.sh +++ b/tools/github/debugAction.sh @@ -51,10 +51,18 @@ sleep 10 HAS_ERRORS=$(grep "command failed" < .ngrok.log) if [[ -z "$HAS_ERRORS" ]]; then + MSG="To connect: $(grep -o -E "tcp://(.+)" < .ngrok.log | sed "s/tcp:\/\//ssh $USER@/" | sed "s/:/ -p /")" echo "" echo "==========================================" - echo "To connect: $(grep -o -E "tcp://(.+)" < .ngrok.log | sed "s/tcp:\/\//ssh $USER@/" | sed "s/:/ -p /")" + echo "$MSG" echo "==========================================" + if test -n "$SLACK_WEBHOOK" + then + echo -n '{"text":' >/tmp/msg$$ + echo -n "$MSG" | jq -Rsa . >>/tmp/msg$$ + echo -n '}' >>/tmp/msg$$ + curl -X POST -H 'Content-type: application/json' --data "@/tmp/msg$$" "$SLACK_WEBHOOK" + fi else echo "$HAS_ERRORS" exit 1 diff --git a/tools/github/s3-upload.sh b/tools/github/s3-upload.sh index 6918f285950..270d657ac35 100755 --- a/tools/github/s3-upload.sh +++ b/tools/github/s3-upload.sh @@ -17,7 +17,7 @@ # # check variables -for i in S3_LOG_BUCKET AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY +for i in AWS_BUCKET AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY do if test -z "${!i}" then echo "Please set $i" ; exit 1 @@ -32,7 +32,7 @@ FROM="$1" TO="$2" BROWSER="https://raw.githubusercontent.com/qoomon/aws-s3-bucket-browser/master/index.html" -BUCKET_URL="https://$S3_LOG_BUCKET.s3.$AWS_REGION.amazonaws.com/" +BUCKET_URL="https://$AWS_BUCKET.s3.$AWS_REGION.amazonaws.com/" # install rclone if ! which rclone @@ -48,9 +48,9 @@ RCLONE="rclone --config /dev/null \ curl -s "$BROWSER" |\ sed -e 's!bucketUrl: undefined!bucketUrl: "'$BUCKET_URL'"!' |\ - $RCLONE rcat ":s3:$S3_LOG_BUCKET/index.html" + $RCLONE rcat ":s3:$AWS_BUCKET/index.html" -$RCLONE copyto "$FROM" ":s3:$S3_LOG_BUCKET/$TO/" +$RCLONE copyto "$FROM" ":s3:$AWS_BUCKET/$TO/" echo "logurl=${BUCKET_URL}index.html#$TO" >>"${GITHUB_OUTPUT:-/dev/stdout}" diff --git a/tools/github/setup.sh b/tools/github/setup.sh index 59151292126..b01371c9030 100755 --- a/tools/github/setup.sh +++ b/tools/github/setup.sh @@ -34,6 +34,14 @@ function retry() { fi } +# installing right version of jdk +JDK=https://github.com/ibmruntimes/semeru11-binaries/releases/download/jdk-11.0.12%2B7_openj9-0.27.0/ibm-semeru-open-jdk_x64_linux_11.0.12_7_openj9-0.27.0.tar.gz +curl -sL $JDK | tar xzvf - -C /usr/local +JAVA="$(which java)" +mv "$JAVA" "$JAVA".old +ln -sf /usr/local/jdk*/bin/java $JAVA +java -version + # Python python -m pip install --user couchdb From 72c5b19e485f9f82c29cf22175a8cb753ab1e6f4 Mon Sep 17 00:00:00 2001 From: Michele Sciabarra Date: Tue, 13 Dec 2022 07:36:26 +0000 Subject: [PATCH 20/39] more fixes - only one test failing against 1.0.0 --- .github/workflows/0-on-demand.yaml | 7 +++---- .github/workflows/1-unit.yaml | 2 +- .github/workflows/2-system.yaml | 2 +- .github/workflows/3-multi-runtime.yaml | 2 +- .github/workflows/4-standalone.yaml | 2 +- .github/workflows/5-scheduler.yaml | 2 +- .github/workflows/6-performance.yaml | 2 +- .../org/apache/openwhisk/core/admin/WskAdminTests.scala | 5 ++--- tools/github/runStandaloneTests.sh | 4 ++-- tools/github/setup.sh | 2 +- 10 files changed, 14 insertions(+), 16 deletions(-) diff --git a/.github/workflows/0-on-demand.yaml b/.github/workflows/0-on-demand.yaml index 77f6a7d8330..3e250086d95 100644 --- a/.github/workflows/0-on-demand.yaml +++ b/.github/workflows/0-on-demand.yaml @@ -34,11 +34,10 @@ on: - MultiRuntime - Standalone - Scheduler - ## TODO - #- Performance + - Performance ## Uncomment those to debug the build - - Dummy - - DummyKO + #- Dummy + #- DummyKO env: # openwhisk env diff --git a/.github/workflows/1-unit.yaml b/.github/workflows/1-unit.yaml index 93cab949650..d85ebc6bb60 100644 --- a/.github/workflows/1-unit.yaml +++ b/.github/workflows/1-unit.yaml @@ -20,7 +20,7 @@ name: Unit Tests on: # build on tagged push: - tags: '*' + tags: 'test-*' # build on pull requests pull_request: diff --git a/.github/workflows/2-system.yaml b/.github/workflows/2-system.yaml index 19ba441d776..6ca7f43fc7e 100644 --- a/.github/workflows/2-system.yaml +++ b/.github/workflows/2-system.yaml @@ -20,7 +20,7 @@ name: System Tests on: # build on tagged push: - tags: '*' + tags: 'test-*' # build on pull requests pull_request: diff --git a/.github/workflows/3-multi-runtime.yaml b/.github/workflows/3-multi-runtime.yaml index e81c0156c89..10ffb24db31 100644 --- a/.github/workflows/3-multi-runtime.yaml +++ b/.github/workflows/3-multi-runtime.yaml @@ -20,7 +20,7 @@ name: MultiRuntime Tests on: # build on tagged push: - tags: '*' + tags: 'test-*' # build on pull requests pull_request: diff --git a/.github/workflows/4-standalone.yaml b/.github/workflows/4-standalone.yaml index e2b9d3c317c..7fefe99c6a9 100644 --- a/.github/workflows/4-standalone.yaml +++ b/.github/workflows/4-standalone.yaml @@ -20,7 +20,7 @@ name: Standalone Tests on: # build on tagged push: - tags: '*' + tags: 'test-*' # build on pull requests pull_request: diff --git a/.github/workflows/5-scheduler.yaml b/.github/workflows/5-scheduler.yaml index d5b1b1e43f5..c7920e1f75f 100644 --- a/.github/workflows/5-scheduler.yaml +++ b/.github/workflows/5-scheduler.yaml @@ -20,7 +20,7 @@ name: Scheduler Tests on: # build on tagged push: - tags: '*' + tags: 'test-all-*' # build on pull requests pull_request: diff --git a/.github/workflows/6-performance.yaml b/.github/workflows/6-performance.yaml index b89ffad566c..9ad28c9d07b 100644 --- a/.github/workflows/6-performance.yaml +++ b/.github/workflows/6-performance.yaml @@ -20,7 +20,7 @@ name: Performance Tests on: # build on tagged push: - tags: '*' + tags: 'xtest-*' # build on pull requests pull_request: diff --git a/tests/src/test/scala/org/apache/openwhisk/core/admin/WskAdminTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/admin/WskAdminTests.scala index 64d5c82d9d8..84a73b53a9b 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/admin/WskAdminTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/admin/WskAdminTests.scala @@ -248,12 +248,11 @@ class WskAdminTests extends TestHelpers with WskActorSystem with Matchers with B val subject = Subject().asString try { // set some limits - wskadmin.cli(Seq("limits", "set", subject, "--allowedKinds", "nodejs:14", "blackbox")) + wskadmin.cli(Seq("limits", "set", subject, "--allowedKinds", "nodejs:10", "blackbox")) // check correctly set val lines = wskadmin.cli(Seq("limits", "get", subject)).stdout.linesIterator.toSeq lines should have size 1 - lines(0) should (be("allowedKinds = [u'nodejs:14', u'blackbox']") or be( - "allowedKinds = ['nodejs:14', 'blackbox']")) + lines(0) shouldBe "allowedKinds = ['nodejs:10', 'blackbox']" } finally { wskadmin.cli(Seq("limits", "delete", subject)).stdout should include("Limits deleted") } diff --git a/tools/github/runStandaloneTests.sh b/tools/github/runStandaloneTests.sh index ba10c5b4715..88dac0762a0 100755 --- a/tools/github/runStandaloneTests.sh +++ b/tools/github/runStandaloneTests.sh @@ -31,12 +31,12 @@ $ANSIBLE_CMD properties.yml -e manifest_file="/ansible/files/runtimes-nodeonly.j $ANSIBLE_CMD downloadcli-github.yml # Install kubectl -curl -Lo ./kubectl https://storage.googleapis.com/kubernetes-release/release/v1.16.1/bin/linux/amd64/kubectl +curl -Lo ./kubectl https://storage.googleapis.com/kubernetes-release/release/v1.25.3/bin/linux/amd64/kubectl chmod +x kubectl sudo cp kubectl /usr/local/bin/kubectl # Install kind -curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/v0.5.1/kind-linux-amd64 +curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/v0.17.0/kind-linux-amd64 chmod +x kind sudo cp kind /usr/local/bin/kind diff --git a/tools/github/setup.sh b/tools/github/setup.sh index b01371c9030..4c698158bbd 100755 --- a/tools/github/setup.sh +++ b/tools/github/setup.sh @@ -38,7 +38,7 @@ function retry() { JDK=https://github.com/ibmruntimes/semeru11-binaries/releases/download/jdk-11.0.12%2B7_openj9-0.27.0/ibm-semeru-open-jdk_x64_linux_11.0.12_7_openj9-0.27.0.tar.gz curl -sL $JDK | tar xzvf - -C /usr/local JAVA="$(which java)" -mv "$JAVA" "$JAVA".old +mv "$JAVA" "$JAVA"."$(date +%s)" ln -sf /usr/local/jdk*/bin/java $JAVA java -version From e699537cb09c255ee21193183e0a3acf4a239b45 Mon Sep 17 00:00:00 2001 From: Michele Sciabarra Date: Tue, 13 Dec 2022 20:37:35 +0000 Subject: [PATCH 21/39] finally passing all the tests on 1.0.0 --- .github/workflows/0-on-demand.yaml | 20 +-- .github/workflows/1-unit.yaml | 5 +- .github/workflows/2-system.yaml | 5 +- .github/workflows/3-multi-runtime.yaml | 5 +- .github/workflows/4-standalone.yaml | 5 +- .github/workflows/5-scheduler.yaml | 5 +- .github/workflows/6-performance.yaml | 5 +- ansible/group_vars/all | 143 +----------------- .../Unicode.cs | 2 +- tools/github/checkAndUploadLogs.sh | 10 +- tools/github/s3-upload.sh | 4 - tools/github/setup.sh | 26 +++- 12 files changed, 58 insertions(+), 177 deletions(-) diff --git a/.github/workflows/0-on-demand.yaml b/.github/workflows/0-on-demand.yaml index 3e250086d95..204f1cda132 100644 --- a/.github/workflows/0-on-demand.yaml +++ b/.github/workflows/0-on-demand.yaml @@ -28,7 +28,7 @@ on: test_suite: description: Select Test Suite to run type: choice - options: + options: - Unit - System - MultiRuntime @@ -36,7 +36,7 @@ on: - Scheduler - Performance ## Uncomment those to debug the build - #- Dummy + - Dummy #- DummyKO env: @@ -45,7 +45,6 @@ env: ANSIBLE_CMD: "ansible-playbook -i environments/local -e docker_image_prefix=testing" GRADLE_PROJS_SKIP: "" - ## secrets # (optional) slack incoming wehbook for notifications SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}} @@ -55,15 +54,15 @@ env: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.AWS_REGION }} + # github + GH_BUILD: ${{ github.run_id }}-${{ github.event_name }} + GH_BRANCH: ${{ github.head_ref || github.ref_name }} + # (optional) you need to add as secrets an ngrok token and a password to debug a build on demand NGROK_DEBUG: ${{ inputs.enable_ngrok_debug }} NGROK_TOKEN: ${{ secrets.NGROK_TOKEN }} NGROK_PASSWORD: ${{ secrets.NGROK_PASSWORD }} - # github - GH_BUILD: ${{ github.run_id }}-${{ github.event_name }} - GH_BRANCH: ${{ github.head_ref || github.ref_name }} - jobs: openwhisk: runs-on: ubuntu-22.04 @@ -80,11 +79,12 @@ jobs: name: Upload Logs run: ./tools/github/checkAndUploadLogs.sh ${{ env.TEST_SUITE }} - name: Slack Notification - run: > + run: > ./tools/github/writeOnSlack.sh "[$TEST_SUITE]" $GH_BUILD: ${{ steps.tests.outcome }} - $'\nbranch:' $GH_BRANCH - $'\nlogurl:' ${{ steps.logs.outputs.logurl }} + $'\nbranch:' $GH_BRANCH + $'\nlogs:' ${{ steps.logs.outputs.logs }} + $'\nreport:' ${{ steps.logs.outputs.report }} - name: Debug Action (if requested) run: ./tools/github/debugAction.sh - name: Wait for Debug (if requested) diff --git a/.github/workflows/1-unit.yaml b/.github/workflows/1-unit.yaml index d85ebc6bb60..4671e2ae967 100644 --- a/.github/workflows/1-unit.yaml +++ b/.github/workflows/1-unit.yaml @@ -20,7 +20,7 @@ name: Unit Tests on: # build on tagged push: - tags: 'test-*' + tags: '*-test*' # build on pull requests pull_request: @@ -64,6 +64,7 @@ jobs: ./tools/github/writeOnSlack.sh "[$TEST_SUITE]" $GH_BUILD: ${{ steps.tests.outcome }} $'\nbranch:' $GH_BRANCH - $'\nlogurl:' ${{ steps.logs.outputs.logurl }} + $'\nlogs:' ${{ steps.logs.outputs.logs }} + $'\nreport:' ${{ steps.logs.outputs.report }} - name: Results run: test "${{ steps.tests.outcome }}" == "success" diff --git a/.github/workflows/2-system.yaml b/.github/workflows/2-system.yaml index 6ca7f43fc7e..242b0954313 100644 --- a/.github/workflows/2-system.yaml +++ b/.github/workflows/2-system.yaml @@ -20,7 +20,7 @@ name: System Tests on: # build on tagged push: - tags: 'test-*' + tags: '*-test*' # build on pull requests pull_request: @@ -64,6 +64,7 @@ jobs: ./tools/github/writeOnSlack.sh "[$TEST_SUITE]" $GH_BUILD: ${{ steps.tests.outcome }} $'\nbranch:' $GH_BRANCH - $'\nlogurl:' ${{ steps.logs.outputs.logurl }} + $'\nlogs:' ${{ steps.logs.outputs.logs }} + $'\nreport:' ${{ steps.logs.outputs.report }} - name: Results run: test "${{ steps.tests.outcome }}" == "success" diff --git a/.github/workflows/3-multi-runtime.yaml b/.github/workflows/3-multi-runtime.yaml index 10ffb24db31..ab72d2cf399 100644 --- a/.github/workflows/3-multi-runtime.yaml +++ b/.github/workflows/3-multi-runtime.yaml @@ -20,7 +20,7 @@ name: MultiRuntime Tests on: # build on tagged push: - tags: 'test-*' + tags: '*-test*' # build on pull requests pull_request: @@ -64,6 +64,7 @@ jobs: ./tools/github/writeOnSlack.sh "[$TEST_SUITE]" $GH_BUILD: ${{ steps.tests.outcome }} $'\nbranch:' $GH_BRANCH - $'\nlogurl:' ${{ steps.logs.outputs.logurl }} + $'\nlogs:' ${{ steps.logs.outputs.logs }} + $'\nreport:' ${{ steps.logs.outputs.report }} - name: Results run: test "${{ steps.tests.outcome }}" == "success" diff --git a/.github/workflows/4-standalone.yaml b/.github/workflows/4-standalone.yaml index 7fefe99c6a9..4dd0d41ae5a 100644 --- a/.github/workflows/4-standalone.yaml +++ b/.github/workflows/4-standalone.yaml @@ -20,7 +20,7 @@ name: Standalone Tests on: # build on tagged push: - tags: 'test-*' + tags: '*-test*' # build on pull requests pull_request: @@ -64,6 +64,7 @@ jobs: ./tools/github/writeOnSlack.sh "[$TEST_SUITE]" $GH_BUILD: ${{ steps.tests.outcome }} $'\nbranch:' $GH_BRANCH - $'\nlogurl:' ${{ steps.logs.outputs.logurl }} + $'\nlogs:' ${{ steps.logs.outputs.logs }} + $'\nreport:' ${{ steps.logs.outputs.report }} - name: Results run: test "${{ steps.tests.outcome }}" == "success" diff --git a/.github/workflows/5-scheduler.yaml b/.github/workflows/5-scheduler.yaml index c7920e1f75f..efbb47e60d9 100644 --- a/.github/workflows/5-scheduler.yaml +++ b/.github/workflows/5-scheduler.yaml @@ -20,7 +20,7 @@ name: Scheduler Tests on: # build on tagged push: - tags: 'test-all-*' + tags: '*-test-all*' # build on pull requests pull_request: @@ -64,6 +64,7 @@ jobs: ./tools/github/writeOnSlack.sh "[$TEST_SUITE]" $GH_BUILD: ${{ steps.tests.outcome }} $'\nbranch:' $GH_BRANCH - $'\nlogurl:' ${{ steps.logs.outputs.logurl }} + $'\nlogs:' ${{ steps.logs.outputs.logs }} + $'\nreport:' ${{ steps.logs.outputs.report }} - name: Results run: test "${{ steps.tests.outcome }}" == "success" diff --git a/.github/workflows/6-performance.yaml b/.github/workflows/6-performance.yaml index 9ad28c9d07b..7e60042bb5e 100644 --- a/.github/workflows/6-performance.yaml +++ b/.github/workflows/6-performance.yaml @@ -20,7 +20,7 @@ name: Performance Tests on: # build on tagged push: - tags: 'xtest-*' + tags: '*-test*' # build on pull requests pull_request: @@ -73,6 +73,9 @@ jobs: continue-on-error: true - run: OPENWHISK_HOST="172.17.0.1" API_KEY="$(cat ansible/files/auth.guest)" CONNECTIONS="100" REQUESTS_PER_SEC="1" ASYNC="true" ./gradlew gatlingRun-org.apache.openwhisk.BlockingInvokeOneActionSimulation continue-on-error: true + # The following configuration does not make much sense. But we do not have enough users. But it's good to verify, that the test is still working. + - run: OPENWHISK_HOST="172.17.0.1" USERS="1" REQUESTS_PER_SEC="1" ./gradlew gatlingRun-org.apache.openwhisk.ColdBlockingInvokeSimulation + continue-on-error: true - name: Slack Notification run: > ./tools/github/writeOnSlack.sh "[$TEST_SUITE]" diff --git a/ansible/group_vars/all b/ansible/group_vars/all index bf61a4dc7bb..c7e1b65f609 100644 --- a/ansible/group_vars/all +++ b/ansible/group_vars/all @@ -51,12 +51,6 @@ whisk: feature_flags: require_api_key_annotation: "{{ require_api_key_annotation | default(true) | lower }}" require_response_payload: "{{ require_response_payload | default(true) | lower }}" - cluster_name: "{{ whisk_cluster_name | default('whisk') }}" - containerProxy: - timeouts: - idleContainer: "{{ containerProxy_timeouts_idleContainer | default('10 minutes') }}" - pauseGrace: "{{ containerProxy_timeouts_pauseGrace | default('10 seconds') }}" - keepingDuration: "{{ containerProxy_timeouts_keepingDuration | default('10 minutes') }}" ## # configuration parameters related to support runtimes (see org.apache.openwhisk.core.entity.ExecManifest for schema of the manifest). @@ -112,8 +106,6 @@ controller: authentication: spi: "{{ controller_authentication_spi | default('') }}" loglevel: "{{ controller_loglevel | default(whisk_loglevel) | default('INFO') }}" - username: "{{ controller_username | default('controller.user') }}" - password: "{{ controller_password | default('controller.pass') }}" entitlement: spi: "{{ controller_entitlement_spi | default('') }}" protocol: "{{ controller_protocol | default('https') }}" @@ -128,18 +120,12 @@ controller: password: "openwhisk" name: "{{ __controller_ssl_keyPrefix }}openwhisk-keystore.p12" extraEnv: "{{ controller_extraEnv | default({}) }}" - deployment: - ignore_error: "{{ controller_deployment_ignore_error | default('False') }}" - retries: "{{ controller_deployment_retries | default(180) }}" - delay: "{{ controller_deployment_delay | default(5) }}" jmx: basePortController: 15000 rmiBasePortController: 16000 basePortInvoker: 17000 rmiBasePortInvoker: 18000 - basePortScheduler: 21000 - rmiBasePortScheduler: 22000 user: "{{ jmxuser | default('jmxuser') }}" pass: "{{ jmxuser | default('jmxpass') }}" jvmCommonArgs: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.authenticate=true -Dcom.sun.management.jmxremote.password.file=/home/owuser/jmxremote.password -Dcom.sun.management.jmxremote.access.file=/home/owuser/jmxremote.access" @@ -152,8 +138,6 @@ registry: confdir: "{{ config_root_dir }}/registry" kafka: - topicsPrefix: "{{ kafka_topics_prefix | default('') }}" - topicsUserEventPrefix: "{{ kafka_topics_userEvent_prefix | default(kafka_topics_prefix) | default('') }}" ssl: client_authentication: required keystore: @@ -166,7 +150,7 @@ kafka: protocols: - TLSv1.2 protocol: "{{ kafka_protocol_for_setup }}" - version: "{{ kafka_version | default('2.13-2.7.0') }}" + version: 2.12-2.3.1 port: 9072 advertisedPort: 9093 ras: @@ -224,8 +208,6 @@ invoker: {% endif %}" extraEnv: "{{ invoker_extraEnv | default({}) }}" protocol: "{{ invoker_protocol | default('https') }}" - username: "{{ invoker_username | default('invoker.user') }}" - password: "{{ invoker_password | default('invoker.pass') }}" ssl: cn: "openwhisk-invokers" keyPrefix: "{{ __invoker_ssl_keyPrefix }}" @@ -236,14 +218,6 @@ invoker: keystore: password: "{{ invoker_keystore_password | default('openwhisk') }}" name: "{{ __invoker_ssl_keyPrefix }}openwhisk-keystore.p12" - container: - creationMaxPeek: "{{ container_creation_max_peek | default(500) }}" - reactiveSpi: "{{ invokerReactive_spi | default('') }}" - serverSpi: "{{ invokerServer_spi | default('') }}" - deployment: - ignore_error: "{{ invoker_deployment_ignore_error | default('False') }}" - retries: "{{ invoker_deployment_retries | default(180) }}" - delay: "{{ invoker_deployment_delay | default(5) }}" userLogs: spi: "{{ userLogs_spi | default('org.apache.openwhisk.core.containerpool.logging.DockerToActivationLogStoreProvider') }}" @@ -253,7 +227,7 @@ nginx: htmldir: "{{ ui_path | default(false) }}" dir: become: "{{ nginx_dir_become | default(false) }}" - version: "{{ nginx_version | default('1.21.1') }}" + version: "{{ nginx_version | default('1.19') }}" port: http: 80 https: 443 @@ -299,18 +273,13 @@ db: invoker: user: "{{ db_invoker_user | default(lookup('ini', 'db_username section=invoker file={{ playbook_dir }}/db_local.ini')) }}" pass: "{{ db_invoker_pass | default(lookup('ini', 'db_password section=invoker file={{ playbook_dir }}/db_local.ini')) }}" - scheduler: - user: "{{ db_scheduler_user | default(lookup('ini', 'db_username section=scheduler file={{ playbook_dir }}/db_local.ini')) }}" - pass: "{{ db_scheduler_pass | default(lookup('ini', 'db_password section=scheduler file={{ playbook_dir }}/db_local.ini')) }}" - artifact_store: - backend: "{{ db_artifact_backend | default('CouchDB') }}" activation_store: backend: "{{ db_activation_backend | default('CouchDB') }}" elasticsearch: protocol: "{{ elastic_protocol | default('http') }}" - port: "{{ elastic_port | default(9200) }}" + port: 9200 index_pattern: "{{ elastic_index_pattern | default('openwhisk-%s') }}" - base_transport_port: "{{ elastic_transport_port_base | default(9300) }}" + base_transport_port: 9300 confdir: "{{ config_root_dir }}/elasticsearch" dir: become: "{{ elastic_dir_become | default(false) }}" @@ -325,17 +294,12 @@ db: admin: username: "{{ elastic_username | default('admin') }}" password: "{{ elastic_password | default('admin') }}" - mongodb: - connect_string: "{{ mongodb_connect_string | default('mongodb://172.17.0.1:27017') }}" - database: "{{ mongodb_database | default('whisks') }}" - data_volume: "{{ mongodb_data_volume | default('mongo-data') }}" apigateway: port: api: 9000 mgmt: 9001 - # Default to 'nightly', which tracks the head revision of the master branch of apigateway's gitrepo - version: nightly + version: 1.0.0 redis: version: 4.0 @@ -353,11 +317,9 @@ elasticsearch: elasticsearch_connect_string: "{% set ret = [] %}\ {% for host in groups['elasticsearch'] %}\ - {{ ret.append( hostvars[host].ansible_host + ':' + ((db.elasticsearch.port|int+loop.index-1)|string) ) }}\ + {{ ret.append( hostvars[host].ansible_host + ':' + ((db.elasticsearch.port+loop.index-1)|string) ) }}\ {% endfor %}\ {{ ret | join(',') }}" -mongodb: - version: 4.4.0 docker: # The user to install docker for. Defaults to the ansible user if not set. This will be the user who is able to run @@ -458,96 +420,3 @@ metrics: port: "{{ metrics_kamon_statsd_port | default('8125') }}" user_events: "{{ user_events_enabled | default(false) | lower }}" - -zeroDowntimeDeployment: - enabled: "{{ zerodowntime_deployment_switch | default(false) }}" - -etcd: - version: "{{ etcd_version | default('v3.4.0') }}" - client: - port: 2379 - server: - port: 2480 - cluster: - token: "{{ etcd_cluster_token | default('openwhisk-etcd-token') }}" - dir: - data: "{{ etcd_data_dir | default('') }}" - lease: - timeout: "{{ etcd_lease_timeout | default(10) }}" - loglevel: "{{ etcd_log_level | default('info') }}" - quota_backend_bytes: "{{ etcd_quota_backend_bytes | default(2147483648) }}" # 2GB - snapshot_count: "{{ etcd_snapshot_count | default(10000) }}" - auto_compaction_retention: "{{ etcd_auto_compaction_retention | default('10m') }}" - auto_compaction_mode: "{{ etcd_auto_compaction_mode | default('periodic') }}" - pool_threads: "{{ etcd_pool_threads | default(10) }}" - -etcd_connect_string: "{% set ret = [] %}\ - {% for host in groups['etcd'] %}\ - {{ ret.append( hostvars[host].ansible_host + ':' + ((etcd.client.port+loop.index-1)|string) ) }}\ - {% endfor %}\ - {{ ret | join(',') }}" - - -__scheduler_blackbox_fraction: 0.10 - -watcher: - eventNotificationDelayMs: "{{ watcher_notification_delay | default('5000 ms') }}" - -durationChecker: - spi: "{{ duration_checker_spi | default('') }}" - timeWindow: "{{ duration_checker_time_window | default('1 d') }}" - -enable_scheduler: "{{ scheduler_enable | default(false) }}" - -scheduler: - protocol: "{{ scheduler_protocol | default('http') }}" - dir: - become: "{{ scheduler_dir_become | default(false) }}" - confdir: "{{ config_root_dir }}/scheduler" - basePort: 14001 - grpc: - basePort: 13001 - tls: "{{ scheduler_grpc_tls | default(false) }}" - maxPeek: "{{ scheduler_max_peek | default(128) }}" - heap: "{{ scheduler_heap | default('2g') }}" - arguments: "{{ scheduler_arguments | default('') }}" - instances: "{{ groups['schedulers'] | length }}" - username: "{{ scheduler_username | default('scheduler.user') }}" - password: "{{ scheduler_password | default('scheduler.pass') }}" - akka: - provider: cluster - cluster: - basePort: 25520 - host: "{{ groups['schedulers'] | map('extract', hostvars, 'ansible_host') | list }}" - bindPort: 3551 - # at this moment all schedulers are seed nodes - seedNodes: "{{ groups['schedulers'] | map('extract', hostvars, 'ansible_host') | list }}" - loglevel: "{{ scheduler_loglevel | default(whisk_loglevel) | default('INFO') }}" - extraEnv: "{{ scheduler_extraEnv | default({}) }}" - dataManagementService: - retryInterval: "{{ scheduler_dataManagementService_retryInterval | default('1 second') }}" - inProgressJobRetention: "{{ scheduler_inProgressJobRetention | default('20 seconds') }}" - blackboxMultiple: "{{ scheduler_blackboxMultiple | default(15) }}" - managedFraction: "{{ scheduler_managed_fraction | default(1.0 - (scheduler_blackbox_fraction | default(__scheduler_blackbox_fraction))) }}" - blackboxFraction: "{{ scheduler_blackbox_fraction | default(__scheduler_blackbox_fraction) }}" - scheduling: - staleThreshold: "{{ scheduler_scheduling_staleThreshold | default('100 milliseconds') }}" - checkInterval: "{{ scheduler_scheduling_checkInterval | default('100 milliseconds') }}" - dropInterval: "{{ scheduler_scheduling_dropInterval | default('10 seconds') }}" - queueManager: - maxSchedulingTime: "{{ scheduler_maxSchedulingTime | default('20 second') }}" - maxRetriesToGetQueue: "{{ scheduler_maxRetriesToGetQueue | default(13) }}" - queue: - # the queue's state Running timeout, e.g. if have no activation comes into queue when Running, the queue state will be changed from Running to Idle and delete the decision algorithm actor - idleGrace: "{{ scheduler_queue_idleGrace | default('10 minutes') }}" - # the queue's state Idle timeout, e.g. if have no activation comes into queue when Idle, the queue state will be changed from Idle to Removed - stopGrace: "{{ scheduler_queue_stopGrace | default('10 minutes') }}" - # the queue's state Paused timeout, e.g. if have no activation comes into queue when Paused, the queue state will be changed from Paused to Removed - flushGrace: "{{ scheduler_queue_flushGrace | default('60 seconds') }}" - gracefulShutdownTimeout: "{{ scheduler_queue_gracefulShutdownTimeout | default('5 seconds') }}" - maxRetentionSize: "{{ scheduler_queue_maxRetentionSize | default(10000) }}" - maxRetentionMs: "{{ scheduler_queue_maxRetentionMs | default(60000) }}" - maxBlackboxRetentionMs: "{{ scheduler_queue_maxBlackboxRetentionMs | default(300000) }}" - throttlingFraction: "{{ scheduler_queue_throttlingFraction | default(0.9) }}" - durationBufferSize: "{{ scheduler_queue_durationBufferSize | default(10) }}" - deployment_ignore_error: "{{ scheduler_deployment_ignore_error | default('False') }}" diff --git a/tests/dat/actions/unicode.tests/src/dotnet2.2/Apache.OpenWhisk.UnicodeTests.Dotnet/Unicode.cs b/tests/dat/actions/unicode.tests/src/dotnet2.2/Apache.OpenWhisk.UnicodeTests.Dotnet/Unicode.cs index 6963b869d85..fadd0909da2 100644 --- a/tests/dat/actions/unicode.tests/src/dotnet2.2/Apache.OpenWhisk.UnicodeTests.Dotnet/Unicode.cs +++ b/tests/dat/actions/unicode.tests/src/dotnet2.2/Apache.OpenWhisk.UnicodeTests.Dotnet/Unicode.cs @@ -32,4 +32,4 @@ public JObject Main(JObject args) return (message); } } -} \ No newline at end of file +} diff --git a/tools/github/checkAndUploadLogs.sh b/tools/github/checkAndUploadLogs.sh index 2e1144c09cc..912029ccbb7 100755 --- a/tools/github/checkAndUploadLogs.sh +++ b/tools/github/checkAndUploadLogs.sh @@ -42,13 +42,11 @@ TAGS="" [[ "$2" == "Unit" ]] && TAGS="db" LOG_DIR="$(date +%Y-%m-%d)/${LOG_NAME}-${GH_BUILD}-${GH_BRANCH}" +BUCKET_URL="https://$AWS_BUCKET.s3.$AWS_REGION.amazonaws.com" + +echo "logs=${BUCKET_URL}/index.html#${LOG_DIR}" >>${GITHUB_OUTPUT:-/dev/stdin} +echo "report=${BUCKET_URL}/${LOG_DIR}/test-reports/reports/tests/testCoverageLean/index.html" >>${GITHUB_OUTPUT:-/dev/stdin} -# Perf logs are typically about 20MB and thus rapidly fill our box account. -# Disable upload to reduce the interval at which we need to manually clean logs from box. -if [ "$LOG_NAME" == "Performance" ]; then - echo "Skipping upload of perf logs to conserve space" - exit 0 -fi ansible-playbook -i ansible/environments/local ansible/logs.yml diff --git a/tools/github/s3-upload.sh b/tools/github/s3-upload.sh index 270d657ac35..38c685a04db 100755 --- a/tools/github/s3-upload.sh +++ b/tools/github/s3-upload.sh @@ -51,7 +51,3 @@ curl -s "$BROWSER" |\ $RCLONE rcat ":s3:$AWS_BUCKET/index.html" $RCLONE copyto "$FROM" ":s3:$AWS_BUCKET/$TO/" -echo "logurl=${BUCKET_URL}index.html#$TO" >>"${GITHUB_OUTPUT:-/dev/stdout}" - - - diff --git a/tools/github/setup.sh b/tools/github/setup.sh index 4c698158bbd..e72b046051e 100755 --- a/tools/github/setup.sh +++ b/tools/github/setup.sh @@ -16,9 +16,9 @@ # limitations under the License. # -if [[ $TEST_SUITE =~ Dummy ]] -then echo skiping setup ; exit 0 -fi +#if [[ $TEST_SUITE =~ Dummy ]] +#then echo skipping setup ; exit 0 +#fi # retries a command for five times and exits with the non-zero exit if even after # the retries the command did not succeed. @@ -34,12 +34,18 @@ function retry() { fi } +# setup docker to listen in port 4243 +sudo systemctl stop docker +sudo sed -i -e 's!/usr/bin/dockerd -H fd://!/usr/bin/dockerd -H tcp://0.0.0.0:4243 -H fd://!' /lib/systemd/system/docker.service +sudo systemctl daemon-reload +sudo systemctl start docker + # installing right version of jdk JDK=https://github.com/ibmruntimes/semeru11-binaries/releases/download/jdk-11.0.12%2B7_openj9-0.27.0/ibm-semeru-open-jdk_x64_linux_11.0.12_7_openj9-0.27.0.tar.gz -curl -sL $JDK | tar xzvf - -C /usr/local +curl -sL $JDK | sudo tar xzvf - -C /usr/local JAVA="$(which java)" -mv "$JAVA" "$JAVA"."$(date +%s)" -ln -sf /usr/local/jdk*/bin/java $JAVA +sudo mv "$JAVA" "$JAVA"."$(date +%s)" +sudo ln -sf /usr/local/jdk*/bin/java $JAVA java -version # Python @@ -60,10 +66,14 @@ tools/github/scan.sh # Preload alpine 3.5 to avoid issues with depending images retry docker pull alpine:3.5 -# Basic check that all code compiles and depdendencies are downloaded correctly. +# exit if dummy test suite skipping the long compilation when debugging +if [[ $TEST_SUITE =~ Dummy ]] +then echo skiping setup ; exit 0 +fi + +# Basic check that all code compiles and dependencies are downloaded correctly. # Compiling the tests will compile all components as well. # # Downloads the gradle wrapper, dependencies and tries to compile the code. # Retried 5 times in case there are network hiccups. TERM=dumb retry ./gradlew :tests:compileTestScala - From 69be25900da8ff290ae919f8470492e5cd01a7ff Mon Sep 17 00:00:00 2001 From: Michele Sciabarra Date: Wed, 14 Dec 2022 07:52:32 +0000 Subject: [PATCH 22/39] restored master variables --- ansible/group_vars/all | 145 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 139 insertions(+), 6 deletions(-) diff --git a/ansible/group_vars/all b/ansible/group_vars/all index c7e1b65f609..072e4aebb15 100644 --- a/ansible/group_vars/all +++ b/ansible/group_vars/all @@ -51,6 +51,12 @@ whisk: feature_flags: require_api_key_annotation: "{{ require_api_key_annotation | default(true) | lower }}" require_response_payload: "{{ require_response_payload | default(true) | lower }}" + cluster_name: "{{ whisk_cluster_name | default('whisk') }}" + containerProxy: + timeouts: + idleContainer: "{{ containerProxy_timeouts_idleContainer | default('10 minutes') }}" + pauseGrace: "{{ containerProxy_timeouts_pauseGrace | default('10 seconds') }}" + keepingDuration: "{{ containerProxy_timeouts_keepingDuration | default('10 minutes') }}" ## # configuration parameters related to support runtimes (see org.apache.openwhisk.core.entity.ExecManifest for schema of the manifest). @@ -106,6 +112,8 @@ controller: authentication: spi: "{{ controller_authentication_spi | default('') }}" loglevel: "{{ controller_loglevel | default(whisk_loglevel) | default('INFO') }}" + username: "{{ controller_username | default('controller.user') }}" + password: "{{ controller_password | default('controller.pass') }}" entitlement: spi: "{{ controller_entitlement_spi | default('') }}" protocol: "{{ controller_protocol | default('https') }}" @@ -120,12 +128,18 @@ controller: password: "openwhisk" name: "{{ __controller_ssl_keyPrefix }}openwhisk-keystore.p12" extraEnv: "{{ controller_extraEnv | default({}) }}" + deployment: + ignore_error: "{{ controller_deployment_ignore_error | default('False') }}" + retries: "{{ controller_deployment_retries | default(180) }}" + delay: "{{ controller_deployment_delay | default(5) }}" jmx: basePortController: 15000 rmiBasePortController: 16000 basePortInvoker: 17000 rmiBasePortInvoker: 18000 + basePortScheduler: 21000 + rmiBasePortScheduler: 22000 user: "{{ jmxuser | default('jmxuser') }}" pass: "{{ jmxuser | default('jmxpass') }}" jvmCommonArgs: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.authenticate=true -Dcom.sun.management.jmxremote.password.file=/home/owuser/jmxremote.password -Dcom.sun.management.jmxremote.access.file=/home/owuser/jmxremote.access" @@ -138,6 +152,8 @@ registry: confdir: "{{ config_root_dir }}/registry" kafka: + topicsPrefix: "{{ kafka_topics_prefix | default('') }}" + topicsUserEventPrefix: "{{ kafka_topics_userEvent_prefix | default(kafka_topics_prefix) | default('') }}" ssl: client_authentication: required keystore: @@ -150,7 +166,7 @@ kafka: protocols: - TLSv1.2 protocol: "{{ kafka_protocol_for_setup }}" - version: 2.12-2.3.1 + version: "{{ kafka_version | default('2.13-2.7.0') }}" port: 9072 advertisedPort: 9093 ras: @@ -208,6 +224,8 @@ invoker: {% endif %}" extraEnv: "{{ invoker_extraEnv | default({}) }}" protocol: "{{ invoker_protocol | default('https') }}" + username: "{{ invoker_username | default('invoker.user') }}" + password: "{{ invoker_password | default('invoker.pass') }}" ssl: cn: "openwhisk-invokers" keyPrefix: "{{ __invoker_ssl_keyPrefix }}" @@ -218,6 +236,14 @@ invoker: keystore: password: "{{ invoker_keystore_password | default('openwhisk') }}" name: "{{ __invoker_ssl_keyPrefix }}openwhisk-keystore.p12" + container: + creationMaxPeek: "{{ container_creation_max_peek | default(500) }}" + reactiveSpi: "{{ invokerReactive_spi | default('') }}" + serverSpi: "{{ invokerServer_spi | default('') }}" + deployment: + ignore_error: "{{ invoker_deployment_ignore_error | default('False') }}" + retries: "{{ invoker_deployment_retries | default(180) }}" + delay: "{{ invoker_deployment_delay | default(5) }}" userLogs: spi: "{{ userLogs_spi | default('org.apache.openwhisk.core.containerpool.logging.DockerToActivationLogStoreProvider') }}" @@ -227,7 +253,7 @@ nginx: htmldir: "{{ ui_path | default(false) }}" dir: become: "{{ nginx_dir_become | default(false) }}" - version: "{{ nginx_version | default('1.19') }}" + version: "{{ nginx_version | default('1.21.1') }}" port: http: 80 https: 443 @@ -273,13 +299,18 @@ db: invoker: user: "{{ db_invoker_user | default(lookup('ini', 'db_username section=invoker file={{ playbook_dir }}/db_local.ini')) }}" pass: "{{ db_invoker_pass | default(lookup('ini', 'db_password section=invoker file={{ playbook_dir }}/db_local.ini')) }}" + scheduler: + user: "{{ db_scheduler_user | default(lookup('ini', 'db_username section=scheduler file={{ playbook_dir }}/db_local.ini')) }}" + pass: "{{ db_scheduler_pass | default(lookup('ini', 'db_password section=scheduler file={{ playbook_dir }}/db_local.ini')) }}" + artifact_store: + backend: "{{ db_artifact_backend | default('CouchDB') }}" activation_store: backend: "{{ db_activation_backend | default('CouchDB') }}" elasticsearch: protocol: "{{ elastic_protocol | default('http') }}" - port: 9200 + port: "{{ elastic_port | default(9200) }}" index_pattern: "{{ elastic_index_pattern | default('openwhisk-%s') }}" - base_transport_port: 9300 + base_transport_port: "{{ elastic_transport_port_base | default(9300) }}" confdir: "{{ config_root_dir }}/elasticsearch" dir: become: "{{ elastic_dir_become | default(false) }}" @@ -294,12 +325,17 @@ db: admin: username: "{{ elastic_username | default('admin') }}" password: "{{ elastic_password | default('admin') }}" + mongodb: + connect_string: "{{ mongodb_connect_string | default('mongodb://172.17.0.1:27017') }}" + database: "{{ mongodb_database | default('whisks') }}" + data_volume: "{{ mongodb_data_volume | default('mongo-data') }}" apigateway: port: api: 9000 mgmt: 9001 - version: 1.0.0 + # Default to 'nightly', which tracks the head revision of the master branch of apigateway's gitrepo + version: nightly redis: version: 4.0 @@ -317,9 +353,11 @@ elasticsearch: elasticsearch_connect_string: "{% set ret = [] %}\ {% for host in groups['elasticsearch'] %}\ - {{ ret.append( hostvars[host].ansible_host + ':' + ((db.elasticsearch.port+loop.index-1)|string) ) }}\ + {{ ret.append( hostvars[host].ansible_host + ':' + ((db.elasticsearch.port|int+loop.index-1)|string) ) }}\ {% endfor %}\ {{ ret | join(',') }}" +mongodb: + version: 4.4.0 docker: # The user to install docker for. Defaults to the ansible user if not set. This will be the user who is able to run @@ -420,3 +458,98 @@ metrics: port: "{{ metrics_kamon_statsd_port | default('8125') }}" user_events: "{{ user_events_enabled | default(false) | lower }}" + +zeroDowntimeDeployment: + enabled: "{{ zerodowntime_deployment_switch | default(false) }}" + +etcd: + version: "{{ etcd_version | default('v3.4.0') }}" + client: + port: 2379 + server: + port: 2480 + cluster: + token: "{{ etcd_cluster_token | default('openwhisk-etcd-token') }}" + dir: + data: "{{ etcd_data_dir | default('') }}" + lease: + timeout: "{{ etcd_lease_timeout | default(10) }}" + loglevel: "{{ etcd_log_level | default('info') }}" + quota_backend_bytes: "{{ etcd_quota_backend_bytes | default(2147483648) }}" # 2GB + snapshot_count: "{{ etcd_snapshot_count | default(10000) }}" + auto_compaction_retention: "{{ etcd_auto_compaction_retention | default('10m') }}" + auto_compaction_mode: "{{ etcd_auto_compaction_mode | default('periodic') }}" + pool_threads: "{{ etcd_pool_threads | default(10) }}" + +etcd_connect_string: "{% set ret = [] %}\ + {% for host in groups['etcd'] %}\ + {{ ret.append( hostvars[host].ansible_host + ':' + ((etcd.client.port+loop.index-1)|string) ) }}\ + {% endfor %}\ + {{ ret | join(',') }}" + + +__scheduler_blackbox_fraction: 0.10 + +watcher: + eventNotificationDelayMs: "{{ watcher_notification_delay | default('5000 ms') }}" + +durationChecker: + spi: "{{ duration_checker_spi | default('') }}" + timeWindow: "{{ duration_checker_time_window | default('1 d') }}" + +enable_scheduler: "{{ scheduler_enable | default(false) }}" + +scheduler: + protocol: "{{ scheduler_protocol | default('http') }}" + dir: + become: "{{ scheduler_dir_become | default(false) }}" + confdir: "{{ config_root_dir }}/scheduler" + basePort: 14001 + grpc: + basePort: 13001 + tls: "{{ scheduler_grpc_tls | default(false) }}" + maxPeek: "{{ scheduler_max_peek | default(128) }}" + heap: "{{ scheduler_heap | default('2g') }}" + arguments: "{{ scheduler_arguments | default('') }}" + instances: "{{ groups['schedulers'] | length }}" + username: "{{ scheduler_username | default('scheduler.user') }}" + password: "{{ scheduler_password | default('scheduler.pass') }}" + akka: + provider: cluster + cluster: + basePort: 25520 + host: "{{ groups['schedulers'] | map('extract', hostvars, 'ansible_host') | list }}" + bindPort: 3551 + # at this moment all schedulers are seed nodes + seedNodes: "{{ groups['schedulers'] | map('extract', hostvars, 'ansible_host') | list }}" + loglevel: "{{ scheduler_loglevel | default(whisk_loglevel) | default('INFO') }}" + extraEnv: "{{ scheduler_extraEnv | default({}) }}" + dataManagementService: + retryInterval: "{{ scheduler_dataManagementService_retryInterval | default('1 second') }}" + inProgressJobRetention: "{{ scheduler_inProgressJobRetention | default('20 seconds') }}" + blackboxMultiple: "{{ scheduler_blackboxMultiple | default(15) }}" + managedFraction: "{{ scheduler_managed_fraction | default(1.0 - (scheduler_blackbox_fraction | default(__scheduler_blackbox_fraction))) }}" + blackboxFraction: "{{ scheduler_blackbox_fraction | default(__scheduler_blackbox_fraction) }}" + scheduling: + staleThreshold: "{{ scheduler_scheduling_staleThreshold | default('100 milliseconds') }}" + checkInterval: "{{ scheduler_scheduling_checkInterval | default('100 milliseconds') }}" + dropInterval: "{{ scheduler_scheduling_dropInterval | default('10 seconds') }}" + queueManager: + maxSchedulingTime: "{{ scheduler_maxSchedulingTime | default('20 second') }}" + maxRetriesToGetQueue: "{{ scheduler_maxRetriesToGetQueue | default(13) }}" + queue: + # the queue's state Running timeout, e.g. if have no activation comes into queue when Running, the queue state will be changed from Running to Idle and delete the decision algorithm actor + idleGrace: "{{ scheduler_queue_idleGrace | default('10 minutes') }}" + # the queue's state Idle timeout, e.g. if have no activation comes into queue when Idle, the queue state will be changed from Idle to Removed + stopGrace: "{{ scheduler_queue_stopGrace | default('10 minutes') }}" + # the queue's state Paused timeout, e.g. if have no activation comes into queue when Paused, the queue state will be changed from Paused to Removed + flushGrace: "{{ scheduler_queue_flushGrace | default('60 seconds') }}" + gracefulShutdownTimeout: "{{ scheduler_queue_gracefulShutdownTimeout | default('5 seconds') }}" + maxRetentionSize: "{{ scheduler_queue_maxRetentionSize | default(10000) }}" + maxRetentionMs: "{{ scheduler_queue_maxRetentionMs | default(60000) }}" + maxBlackboxRetentionMs: "{{ scheduler_queue_maxBlackboxRetentionMs | default(300000) }}" + throttlingFraction: "{{ scheduler_queue_throttlingFraction | default(0.9) }}" + durationBufferSize: "{{ scheduler_queue_durationBufferSize | default(10) }}" + deployment_ignore_error: "{{ scheduler_deployment_ignore_error | default('False') }}" + dataManagementService: + retryInterval: "{{ scheduler_dataManagementService_retryInterval | default('1 second') }}" From e16ad251aea465854cd60db56ce4a155eed05e92 Mon Sep 17 00:00:00 2001 From: Michele Sciabarra Date: Wed, 14 Dec 2022 07:59:32 +0000 Subject: [PATCH 23/39] restored more master files --- .../Apache.OpenWhisk.UnicodeTests.Dotnet/Unicode.cs | 2 +- .../org/apache/openwhisk/core/admin/WskAdminTests.scala | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/dat/actions/unicode.tests/src/dotnet2.2/Apache.OpenWhisk.UnicodeTests.Dotnet/Unicode.cs b/tests/dat/actions/unicode.tests/src/dotnet2.2/Apache.OpenWhisk.UnicodeTests.Dotnet/Unicode.cs index fadd0909da2..6963b869d85 100644 --- a/tests/dat/actions/unicode.tests/src/dotnet2.2/Apache.OpenWhisk.UnicodeTests.Dotnet/Unicode.cs +++ b/tests/dat/actions/unicode.tests/src/dotnet2.2/Apache.OpenWhisk.UnicodeTests.Dotnet/Unicode.cs @@ -32,4 +32,4 @@ public JObject Main(JObject args) return (message); } } -} +} \ No newline at end of file diff --git a/tests/src/test/scala/org/apache/openwhisk/core/admin/WskAdminTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/admin/WskAdminTests.scala index 84a73b53a9b..64d5c82d9d8 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/admin/WskAdminTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/admin/WskAdminTests.scala @@ -248,11 +248,12 @@ class WskAdminTests extends TestHelpers with WskActorSystem with Matchers with B val subject = Subject().asString try { // set some limits - wskadmin.cli(Seq("limits", "set", subject, "--allowedKinds", "nodejs:10", "blackbox")) + wskadmin.cli(Seq("limits", "set", subject, "--allowedKinds", "nodejs:14", "blackbox")) // check correctly set val lines = wskadmin.cli(Seq("limits", "get", subject)).stdout.linesIterator.toSeq lines should have size 1 - lines(0) shouldBe "allowedKinds = ['nodejs:10', 'blackbox']" + lines(0) should (be("allowedKinds = [u'nodejs:14', u'blackbox']") or be( + "allowedKinds = ['nodejs:14', 'blackbox']")) } finally { wskadmin.cli(Seq("limits", "delete", subject)).stdout should include("Limits deleted") } From 268730f07e305cc37b50788f2ee9ee6c10b660f7 Mon Sep 17 00:00:00 2001 From: Michele Sciabarra Date: Wed, 14 Dec 2022 08:13:43 +0000 Subject: [PATCH 24/39] triggered on push and pull request --- .github/workflows/0-on-demand.yaml | 2 +- .github/workflows/2-system.yaml | 3 +-- .github/workflows/3-multi-runtime.yaml | 3 +-- .github/workflows/4-standalone.yaml | 3 +-- .github/workflows/5-scheduler.yaml | 3 +-- .github/workflows/6-performance.yaml | 3 +-- tools/macos/README.md | 6 +++--- tools/travis/setup.sh | 1 - 8 files changed, 9 insertions(+), 15 deletions(-) diff --git a/.github/workflows/0-on-demand.yaml b/.github/workflows/0-on-demand.yaml index 204f1cda132..f2ffce47238 100644 --- a/.github/workflows/0-on-demand.yaml +++ b/.github/workflows/0-on-demand.yaml @@ -36,7 +36,7 @@ on: - Scheduler - Performance ## Uncomment those to debug the build - - Dummy + #- Dummy #- DummyKO env: diff --git a/.github/workflows/2-system.yaml b/.github/workflows/2-system.yaml index 242b0954313..65af98ad53e 100644 --- a/.github/workflows/2-system.yaml +++ b/.github/workflows/2-system.yaml @@ -18,9 +18,8 @@ name: System Tests on: - # build on tagged + # build on push push: - tags: '*-test*' # build on pull requests pull_request: diff --git a/.github/workflows/3-multi-runtime.yaml b/.github/workflows/3-multi-runtime.yaml index ab72d2cf399..bc78c508645 100644 --- a/.github/workflows/3-multi-runtime.yaml +++ b/.github/workflows/3-multi-runtime.yaml @@ -18,9 +18,8 @@ name: MultiRuntime Tests on: - # build on tagged + # build on push push: - tags: '*-test*' # build on pull requests pull_request: diff --git a/.github/workflows/4-standalone.yaml b/.github/workflows/4-standalone.yaml index 4dd0d41ae5a..79c16acf65c 100644 --- a/.github/workflows/4-standalone.yaml +++ b/.github/workflows/4-standalone.yaml @@ -18,9 +18,8 @@ name: Standalone Tests on: - # build on tagged + # build on push push: - tags: '*-test*' # build on pull requests pull_request: diff --git a/.github/workflows/5-scheduler.yaml b/.github/workflows/5-scheduler.yaml index efbb47e60d9..e04db1aa494 100644 --- a/.github/workflows/5-scheduler.yaml +++ b/.github/workflows/5-scheduler.yaml @@ -18,9 +18,8 @@ name: Scheduler Tests on: - # build on tagged + # build on push push: - tags: '*-test-all*' # build on pull requests pull_request: diff --git a/.github/workflows/6-performance.yaml b/.github/workflows/6-performance.yaml index 7e60042bb5e..d9afd22b014 100644 --- a/.github/workflows/6-performance.yaml +++ b/.github/workflows/6-performance.yaml @@ -18,9 +18,8 @@ name: Performance Tests on: - # build on tagged + # build on push push: - tags: '*-test*' # build on pull requests pull_request: diff --git a/tools/macos/README.md b/tools/macos/README.md index 964bb949f73..8ff60fb266a 100644 --- a/tools/macos/README.md +++ b/tools/macos/README.md @@ -75,7 +75,7 @@ Follow instructions in [ansible/README.md](../../ansible/README.md) #### Using brew ```bash -brew install wsk +brew install wsk wsk property set --apihost https://localhost wsk property set --auth `cat ansible/files/auth.guest` ``` @@ -109,9 +109,9 @@ ansible-playbook wipe.yml ansible-playbook properties.yml ``` -To run the unit tests execute the command bellow from the project's root folder: +To run the unit tests execute the command bellow from the project's root folder: ```bash # go back to project's root folder cd ../ -./gradlew -PtestSetName="REQUIRE_ONLY_DB" :tests:testCoverageLean +./gradlew -PtestSetName="REQUIRE_ONLY_DB" :tests:testCoverageLean ``` diff --git a/tools/travis/setup.sh b/tools/travis/setup.sh index f6a3abf361b..6d67941e340 100755 --- a/tools/travis/setup.sh +++ b/tools/travis/setup.sh @@ -51,4 +51,3 @@ python -m pip install --user humanize requests # Downloads the gradle wrapper, dependencies and tries to compile the code. # Retried 5 times in case there are network hiccups. TERM=dumb retry ./gradlew :tests:compileTestScala - From bfb71a2a49f36b4cfd2352ebdaf78bca38bcbccb Mon Sep 17 00:00:00 2001 From: Michele Sciabarra Date: Wed, 14 Dec 2022 08:16:46 +0000 Subject: [PATCH 25/39] unit test too --- .github/workflows/1-unit.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/1-unit.yaml b/.github/workflows/1-unit.yaml index 4671e2ae967..4efbcc52011 100644 --- a/.github/workflows/1-unit.yaml +++ b/.github/workflows/1-unit.yaml @@ -18,9 +18,8 @@ name: Unit Tests on: - # build on tagged + # build on push push: - tags: '*-test*' # build on pull requests pull_request: From be2b928bae1a4ae65b94aac895d1eff7c11d4c46 Mon Sep 17 00:00:00 2001 From: Michele Sciabarra Date: Wed, 14 Dec 2022 11:57:43 +0000 Subject: [PATCH 26/39] wskadmin fixes --- tools/admin/wskadmin | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/admin/wskadmin b/tools/admin/wskadmin index 126e7847d57..82e67e1bd51 100755 --- a/tools/admin/wskadmin +++ b/tools/admin/wskadmin @@ -385,7 +385,6 @@ def getIdentitiesFromNamespace(args, props): 'protocol': protocol, 'host' : host, 'port' : port, - 'username': username, 'database': database, 'view' : args.view, 'ns' : args.namespace @@ -478,7 +477,6 @@ def whoisUserCmd(args, props): 'protocol': protocol, 'host' : host, 'port' : port, - 'username': username, 'database': database, 'view' : args.view, 'uuid' : uuid, From fcd7a845250eef9e80321dd2bd425b32398891e1 Mon Sep 17 00:00:00 2001 From: Michele Sciabarra Date: Wed, 14 Dec 2022 13:14:38 +0000 Subject: [PATCH 27/39] improve message --- .github/workflows/0-on-demand.yaml | 8 ++++---- .github/workflows/1-unit.yaml | 2 +- .github/workflows/2-system.yaml | 2 +- .github/workflows/3-multi-runtime.yaml | 2 +- .github/workflows/4-standalone.yaml | 2 +- .github/workflows/5-scheduler.yaml | 2 +- .github/workflows/6-performance.yaml | 2 +- .gitignore | 4 ++++ 8 files changed, 14 insertions(+), 10 deletions(-) diff --git a/.github/workflows/0-on-demand.yaml b/.github/workflows/0-on-demand.yaml index f2ffce47238..a6da6ad232d 100644 --- a/.github/workflows/0-on-demand.yaml +++ b/.github/workflows/0-on-demand.yaml @@ -55,7 +55,7 @@ env: AWS_REGION: ${{ secrets.AWS_REGION }} # github - GH_BUILD: ${{ github.run_id }}-${{ github.event_name }} + GH_BUILD: ${{ github.event_name }}-${{ github.sha }} GH_BRANCH: ${{ github.head_ref || github.ref_name }} # (optional) you need to add as secrets an ngrok token and a password to debug a build on demand @@ -80,9 +80,9 @@ jobs: run: ./tools/github/checkAndUploadLogs.sh ${{ env.TEST_SUITE }} - name: Slack Notification run: > - ./tools/github/writeOnSlack.sh "[$TEST_SUITE]" - $GH_BUILD: ${{ steps.tests.outcome }} - $'\nbranch:' $GH_BRANCH + ./tools/github/writeOnSlack.sh + "[$TEST_SUITE]" ${GH_BUILD}: ${{ steps.tests.outcome }} + $'\nbranch:' $GH_BRANCH ($(git log -1 --pretty '%s')) $'\nlogs:' ${{ steps.logs.outputs.logs }} $'\nreport:' ${{ steps.logs.outputs.report }} - name: Debug Action (if requested) diff --git a/.github/workflows/1-unit.yaml b/.github/workflows/1-unit.yaml index 4efbcc52011..b277d026601 100644 --- a/.github/workflows/1-unit.yaml +++ b/.github/workflows/1-unit.yaml @@ -19,7 +19,7 @@ name: Unit Tests on: # build on push - push: + #push: # build on pull requests pull_request: diff --git a/.github/workflows/2-system.yaml b/.github/workflows/2-system.yaml index 65af98ad53e..abc80b13019 100644 --- a/.github/workflows/2-system.yaml +++ b/.github/workflows/2-system.yaml @@ -19,7 +19,7 @@ name: System Tests on: # build on push - push: + #push: # build on pull requests pull_request: diff --git a/.github/workflows/3-multi-runtime.yaml b/.github/workflows/3-multi-runtime.yaml index bc78c508645..95b577732f6 100644 --- a/.github/workflows/3-multi-runtime.yaml +++ b/.github/workflows/3-multi-runtime.yaml @@ -19,7 +19,7 @@ name: MultiRuntime Tests on: # build on push - push: + #push: # build on pull requests pull_request: diff --git a/.github/workflows/4-standalone.yaml b/.github/workflows/4-standalone.yaml index 79c16acf65c..58af3c0323f 100644 --- a/.github/workflows/4-standalone.yaml +++ b/.github/workflows/4-standalone.yaml @@ -19,7 +19,7 @@ name: Standalone Tests on: # build on push - push: + #push: # build on pull requests pull_request: diff --git a/.github/workflows/5-scheduler.yaml b/.github/workflows/5-scheduler.yaml index e04db1aa494..e64a3b2505e 100644 --- a/.github/workflows/5-scheduler.yaml +++ b/.github/workflows/5-scheduler.yaml @@ -19,7 +19,7 @@ name: Scheduler Tests on: # build on push - push: + #push: # build on pull requests pull_request: diff --git a/.github/workflows/6-performance.yaml b/.github/workflows/6-performance.yaml index d9afd22b014..99c0ce5c2c1 100644 --- a/.github/workflows/6-performance.yaml +++ b/.github/workflows/6-performance.yaml @@ -19,7 +19,7 @@ name: Performance Tests on: # build on push - push: + #push: # build on pull requests pull_request: diff --git a/.gitignore b/.gitignore index 284e8d2254c..7a15a629f0d 100644 --- a/.gitignore +++ b/.gitignore @@ -96,3 +96,7 @@ core/routemgmt/deleteApi/utils.js core/routemgmt/getApi/apigw-utils.js core/routemgmt/getApi/package-lock.json core/routemgmt/getApi/utils.js + +# vscode metals +.bloop/ +.metals/ From 8914674995b587ae71950a9bb393966fbb96666a Mon Sep 17 00:00:00 2001 From: Michele Sciabarra Date: Wed, 14 Dec 2022 15:01:32 +0000 Subject: [PATCH 28/39] dummy enabled --- .github/workflows/0-on-demand.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/0-on-demand.yaml b/.github/workflows/0-on-demand.yaml index a6da6ad232d..163145acde6 100644 --- a/.github/workflows/0-on-demand.yaml +++ b/.github/workflows/0-on-demand.yaml @@ -36,7 +36,7 @@ on: - Scheduler - Performance ## Uncomment those to debug the build - #- Dummy + - Dummy #- DummyKO env: From 69a8043c4b82449b069a8ada655663b72385a4f0 Mon Sep 17 00:00:00 2001 From: Michele Sciabarra Date: Wed, 14 Dec 2022 15:06:40 +0000 Subject: [PATCH 29/39] better git message --- .github/workflows/0-on-demand.yaml | 3 ++- tools/github/checkAndUploadLogs.sh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/0-on-demand.yaml b/.github/workflows/0-on-demand.yaml index 163145acde6..71f26374948 100644 --- a/.github/workflows/0-on-demand.yaml +++ b/.github/workflows/0-on-demand.yaml @@ -82,7 +82,8 @@ jobs: run: > ./tools/github/writeOnSlack.sh "[$TEST_SUITE]" ${GH_BUILD}: ${{ steps.tests.outcome }} - $'\nbranch:' $GH_BRANCH ($(git log -1 --pretty '%s')) + $'\nbranch:' $GH_BRANCH + $'\nmessage:' "$(git log -1 --pretty '%s')" $'\nlogs:' ${{ steps.logs.outputs.logs }} $'\nreport:' ${{ steps.logs.outputs.report }} - name: Debug Action (if requested) diff --git a/tools/github/checkAndUploadLogs.sh b/tools/github/checkAndUploadLogs.sh index 912029ccbb7..74c0fc22ed8 100755 --- a/tools/github/checkAndUploadLogs.sh +++ b/tools/github/checkAndUploadLogs.sh @@ -44,7 +44,7 @@ TAGS="" LOG_DIR="$(date +%Y-%m-%d)/${LOG_NAME}-${GH_BUILD}-${GH_BRANCH}" BUCKET_URL="https://$AWS_BUCKET.s3.$AWS_REGION.amazonaws.com" -echo "logs=${BUCKET_URL}/index.html#${LOG_DIR}" >>${GITHUB_OUTPUT:-/dev/stdin} +echo "logs=${BUCKET_URL}/index.html#${LOG_DIR}/" >>${GITHUB_OUTPUT:-/dev/stdin} echo "report=${BUCKET_URL}/${LOG_DIR}/test-reports/reports/tests/testCoverageLean/index.html" >>${GITHUB_OUTPUT:-/dev/stdin} From 5e47a4d1e8c52dfe2ab95f849dc1e6a615a812f6 Mon Sep 17 00:00:00 2001 From: Michele Sciabarra Date: Wed, 14 Dec 2022 16:43:31 +0000 Subject: [PATCH 30/39] cleanup the log messages --- .github/workflows/0-on-demand.yaml | 2 +- .github/workflows/1-unit.yaml | 11 ++++++----- .github/workflows/2-system.yaml | 11 ++++++----- .github/workflows/3-multi-runtime.yaml | 11 ++++++----- .github/workflows/4-standalone.yaml | 11 ++++++----- .github/workflows/5-scheduler.yaml | 11 ++++++----- .github/workflows/6-performance.yaml | 10 ++++++---- 7 files changed, 37 insertions(+), 30 deletions(-) diff --git a/.github/workflows/0-on-demand.yaml b/.github/workflows/0-on-demand.yaml index 71f26374948..90272080029 100644 --- a/.github/workflows/0-on-demand.yaml +++ b/.github/workflows/0-on-demand.yaml @@ -83,7 +83,7 @@ jobs: ./tools/github/writeOnSlack.sh "[$TEST_SUITE]" ${GH_BUILD}: ${{ steps.tests.outcome }} $'\nbranch:' $GH_BRANCH - $'\nmessage:' "$(git log -1 --pretty '%s')" + $'\nmessage:' "$(git log -1 --oneline | cat)" $'\nlogs:' ${{ steps.logs.outputs.logs }} $'\nreport:' ${{ steps.logs.outputs.report }} - name: Debug Action (if requested) diff --git a/.github/workflows/1-unit.yaml b/.github/workflows/1-unit.yaml index b277d026601..f774aef0acb 100644 --- a/.github/workflows/1-unit.yaml +++ b/.github/workflows/1-unit.yaml @@ -19,7 +19,7 @@ name: Unit Tests on: # build on push - #push: + push: # build on pull requests pull_request: @@ -40,7 +40,7 @@ env: AWS_REGION: ${{ secrets.AWS_REGION }} # github - GH_BUILD: ${{ github.run_id }}-${{ github.event_name }} + GH_BUILD: ${{ github.event_name }}-${{ github.sha }} GH_BRANCH: ${{ github.head_ref || github.ref_name }} jobs: @@ -60,9 +60,10 @@ jobs: run: ./tools/github/checkAndUploadLogs.sh ${{ env.TEST_SUITE }} - name: Slack Notification run: > - ./tools/github/writeOnSlack.sh "[$TEST_SUITE]" - $GH_BUILD: ${{ steps.tests.outcome }} - $'\nbranch:' $GH_BRANCH + ./tools/github/writeOnSlack.sh + "[$TEST_SUITE]" ${GH_BUILD}: ${{ steps.tests.outcome }} + $'\nbranch:' $GH_BRANCH + $'\nmessage:' "$(git log -1 --oneline | cat)" $'\nlogs:' ${{ steps.logs.outputs.logs }} $'\nreport:' ${{ steps.logs.outputs.report }} - name: Results diff --git a/.github/workflows/2-system.yaml b/.github/workflows/2-system.yaml index abc80b13019..a47662e60a1 100644 --- a/.github/workflows/2-system.yaml +++ b/.github/workflows/2-system.yaml @@ -19,7 +19,7 @@ name: System Tests on: # build on push - #push: + push: # build on pull requests pull_request: @@ -40,7 +40,7 @@ env: AWS_REGION: ${{ secrets.AWS_REGION }} # github - GH_BUILD: ${{ github.run_id }}-${{ github.event_name }} + GH_BUILD: ${{ github.event_name }}-${{ github.sha }} GH_BRANCH: ${{ github.head_ref || github.ref_name }} jobs: @@ -60,9 +60,10 @@ jobs: run: ./tools/github/checkAndUploadLogs.sh ${{ env.TEST_SUITE }} - name: Slack Notification run: > - ./tools/github/writeOnSlack.sh "[$TEST_SUITE]" - $GH_BUILD: ${{ steps.tests.outcome }} - $'\nbranch:' $GH_BRANCH + ./tools/github/writeOnSlack.sh + "[$TEST_SUITE]" ${GH_BUILD}: ${{ steps.tests.outcome }} + $'\nbranch:' $GH_BRANCH + $'\nmessage:' "$(git log -1 --oneline | cat)" $'\nlogs:' ${{ steps.logs.outputs.logs }} $'\nreport:' ${{ steps.logs.outputs.report }} - name: Results diff --git a/.github/workflows/3-multi-runtime.yaml b/.github/workflows/3-multi-runtime.yaml index 95b577732f6..8409ea3ef20 100644 --- a/.github/workflows/3-multi-runtime.yaml +++ b/.github/workflows/3-multi-runtime.yaml @@ -19,7 +19,7 @@ name: MultiRuntime Tests on: # build on push - #push: + push: # build on pull requests pull_request: @@ -40,7 +40,7 @@ env: AWS_REGION: ${{ secrets.AWS_REGION }} # github - GH_BUILD: ${{ github.run_id }}-${{ github.event_name }} + GH_BUILD: ${{ github.event_name }}-${{ github.sha }} GH_BRANCH: ${{ github.head_ref || github.ref_name }} jobs: @@ -60,9 +60,10 @@ jobs: run: ./tools/github/checkAndUploadLogs.sh ${{ env.TEST_SUITE }} - name: Slack Notification run: > - ./tools/github/writeOnSlack.sh "[$TEST_SUITE]" - $GH_BUILD: ${{ steps.tests.outcome }} - $'\nbranch:' $GH_BRANCH + ./tools/github/writeOnSlack.sh + "[$TEST_SUITE]" ${GH_BUILD}: ${{ steps.tests.outcome }} + $'\nbranch:' $GH_BRANCH + $'\nmessage:' "$(git log -1 --oneline | cat)" $'\nlogs:' ${{ steps.logs.outputs.logs }} $'\nreport:' ${{ steps.logs.outputs.report }} - name: Results diff --git a/.github/workflows/4-standalone.yaml b/.github/workflows/4-standalone.yaml index 58af3c0323f..6c92a787581 100644 --- a/.github/workflows/4-standalone.yaml +++ b/.github/workflows/4-standalone.yaml @@ -19,7 +19,7 @@ name: Standalone Tests on: # build on push - #push: + push: # build on pull requests pull_request: @@ -40,7 +40,7 @@ env: AWS_REGION: ${{ secrets.AWS_REGION }} # github - GH_BUILD: ${{ github.run_id }}-${{ github.event_name }} + GH_BUILD: ${{ github.event_name }}-${{ github.sha }} GH_BRANCH: ${{ github.head_ref || github.ref_name }} jobs: @@ -60,9 +60,10 @@ jobs: run: ./tools/github/checkAndUploadLogs.sh ${{ env.TEST_SUITE }} - name: Slack Notification run: > - ./tools/github/writeOnSlack.sh "[$TEST_SUITE]" - $GH_BUILD: ${{ steps.tests.outcome }} - $'\nbranch:' $GH_BRANCH + ./tools/github/writeOnSlack.sh + "[$TEST_SUITE]" ${GH_BUILD}: ${{ steps.tests.outcome }} + $'\nbranch:' $GH_BRANCH + $'\nmessage:' "$(git log -1 --oneline | cat)" $'\nlogs:' ${{ steps.logs.outputs.logs }} $'\nreport:' ${{ steps.logs.outputs.report }} - name: Results diff --git a/.github/workflows/5-scheduler.yaml b/.github/workflows/5-scheduler.yaml index e64a3b2505e..2ae2a479de5 100644 --- a/.github/workflows/5-scheduler.yaml +++ b/.github/workflows/5-scheduler.yaml @@ -19,7 +19,7 @@ name: Scheduler Tests on: # build on push - #push: + push: # build on pull requests pull_request: @@ -40,7 +40,7 @@ env: AWS_REGION: ${{ secrets.AWS_REGION }} # github - GH_BUILD: ${{ github.run_id }}-${{ github.event_name }} + GH_BUILD: ${{ github.event_name }}-${{ github.sha }} GH_BRANCH: ${{ github.head_ref || github.ref_name }} jobs: @@ -60,9 +60,10 @@ jobs: run: ./tools/github/checkAndUploadLogs.sh ${{ env.TEST_SUITE }} - name: Slack Notification run: > - ./tools/github/writeOnSlack.sh "[$TEST_SUITE]" - $GH_BUILD: ${{ steps.tests.outcome }} - $'\nbranch:' $GH_BRANCH + ./tools/github/writeOnSlack.sh + "[$TEST_SUITE]" ${GH_BUILD}: ${{ steps.tests.outcome }} + $'\nbranch:' $GH_BRANCH + $'\nmessage:' "$(git log -1 --oneline | cat)" $'\nlogs:' ${{ steps.logs.outputs.logs }} $'\nreport:' ${{ steps.logs.outputs.report }} - name: Results diff --git a/.github/workflows/6-performance.yaml b/.github/workflows/6-performance.yaml index 99c0ce5c2c1..1a1eb6a46bd 100644 --- a/.github/workflows/6-performance.yaml +++ b/.github/workflows/6-performance.yaml @@ -19,7 +19,7 @@ name: Performance Tests on: # build on push - #push: + push: # build on pull requests pull_request: @@ -40,7 +40,7 @@ env: AWS_REGION: ${{ secrets.AWS_REGION }} # github - GH_BUILD: ${{ github.run_id }}-${{ github.event_name }} + GH_BUILD: ${{ github.event_name }}-${{ github.sha }} GH_BRANCH: ${{ github.head_ref || github.ref_name }} jobs: @@ -77,6 +77,8 @@ jobs: continue-on-error: true - name: Slack Notification run: > - ./tools/github/writeOnSlack.sh "[$TEST_SUITE]" + ./tools/github/writeOnSlack.sh + "[$TEST_SUITE]" ${GH_BUILD}: ${{ steps.tests.outcome }} $'\nbranch:' $GH_BRANCH - $'\nCheck GitHub logs for results' \ No newline at end of file + $'\nmessage:' "$(git log -1 --oneline | cat)" + $'\nCheck GitHub logs for results' From 9af08ef20f90415e2f0d7b2b21d7281fa3e2842e Mon Sep 17 00:00:00 2001 From: Michele Sciabarra Date: Thu, 15 Dec 2022 07:14:42 +0000 Subject: [PATCH 31/39] test to remove the docker special port --- .github/workflows/2-system.yaml | 3 +++ .github/workflows/3-multi-runtime.yaml | 3 +++ .github/workflows/4-standalone.yaml | 3 +++ .github/workflows/5-scheduler.yaml | 3 +++ .github/workflows/6-performance.yaml | 3 +++ tools/github/setup.sh | 8 ++++---- 6 files changed, 19 insertions(+), 4 deletions(-) diff --git a/.github/workflows/2-system.yaml b/.github/workflows/2-system.yaml index a47662e60a1..84234bf5820 100644 --- a/.github/workflows/2-system.yaml +++ b/.github/workflows/2-system.yaml @@ -21,6 +21,9 @@ on: # build on push push: # build on pull requests + tags: '*' + branches-ignore: '*' + pull_request: env: diff --git a/.github/workflows/3-multi-runtime.yaml b/.github/workflows/3-multi-runtime.yaml index 8409ea3ef20..4bce4e5822b 100644 --- a/.github/workflows/3-multi-runtime.yaml +++ b/.github/workflows/3-multi-runtime.yaml @@ -20,6 +20,9 @@ name: MultiRuntime Tests on: # build on push push: + tags: '*' + branches-ignore: '*' + # build on pull requests pull_request: diff --git a/.github/workflows/4-standalone.yaml b/.github/workflows/4-standalone.yaml index 6c92a787581..17b281f3236 100644 --- a/.github/workflows/4-standalone.yaml +++ b/.github/workflows/4-standalone.yaml @@ -21,6 +21,9 @@ on: # build on push push: # build on pull requests + tags: '*' + branches-ignore: '*' + pull_request: env: diff --git a/.github/workflows/5-scheduler.yaml b/.github/workflows/5-scheduler.yaml index 2ae2a479de5..0c53c8bd0af 100644 --- a/.github/workflows/5-scheduler.yaml +++ b/.github/workflows/5-scheduler.yaml @@ -21,6 +21,9 @@ on: # build on push push: # build on pull requests + tags: '*' + branches-ignore: '*' + pull_request: env: diff --git a/.github/workflows/6-performance.yaml b/.github/workflows/6-performance.yaml index 1a1eb6a46bd..4383a068cbd 100644 --- a/.github/workflows/6-performance.yaml +++ b/.github/workflows/6-performance.yaml @@ -21,6 +21,9 @@ on: # build on push push: # build on pull requests + tags: '*' + branches-ignore: '*' + pull_request: env: diff --git a/tools/github/setup.sh b/tools/github/setup.sh index e72b046051e..e839fb6d38f 100755 --- a/tools/github/setup.sh +++ b/tools/github/setup.sh @@ -35,10 +35,10 @@ function retry() { } # setup docker to listen in port 4243 -sudo systemctl stop docker -sudo sed -i -e 's!/usr/bin/dockerd -H fd://!/usr/bin/dockerd -H tcp://0.0.0.0:4243 -H fd://!' /lib/systemd/system/docker.service -sudo systemctl daemon-reload -sudo systemctl start docker +#sudo systemctl stop docker +#sudo sed -i -e 's!/usr/bin/dockerd -H fd://!/usr/bin/dockerd -H tcp://0.0.0.0:4243 -H fd://!' /lib/systemd/system/docker.service +#sudo systemctl daemon-reload +#sudo systemctl start docker # installing right version of jdk JDK=https://github.com/ibmruntimes/semeru11-binaries/releases/download/jdk-11.0.12%2B7_openj9-0.27.0/ibm-semeru-open-jdk_x64_linux_11.0.12_7_openj9-0.27.0.tar.gz From cdce26ffe7f5b6c910bd1ff4f5c684364b34248a Mon Sep 17 00:00:00 2001 From: Michele Sciabarra Date: Thu, 15 Dec 2022 07:20:11 +0000 Subject: [PATCH 32/39] ooops --- .github/workflows/1-unit.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/1-unit.yaml b/.github/workflows/1-unit.yaml index f774aef0acb..576c8de5fb9 100644 --- a/.github/workflows/1-unit.yaml +++ b/.github/workflows/1-unit.yaml @@ -20,6 +20,8 @@ name: Unit Tests on: # build on push push: + tags: '*' + branches-ignore: '*' # build on pull requests pull_request: From a9896b52715873eae542b20ac11dfa555ee71006 Mon Sep 17 00:00:00 2001 From: Michele Sciabarra Date: Thu, 15 Dec 2022 08:14:58 +0000 Subject: [PATCH 33/39] resuming port --- .github/workflows/0-on-demand.yaml | 2 +- .github/workflows/2-system.yaml | 2 +- .github/workflows/3-multi-runtime.yaml | 2 +- .github/workflows/4-standalone.yaml | 2 +- .github/workflows/5-scheduler.yaml | 2 +- .github/workflows/6-performance.yaml | 2 +- tools/github/setup.sh | 8 ++++---- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/0-on-demand.yaml b/.github/workflows/0-on-demand.yaml index 90272080029..acc5714f16e 100644 --- a/.github/workflows/0-on-demand.yaml +++ b/.github/workflows/0-on-demand.yaml @@ -81,7 +81,7 @@ jobs: - name: Slack Notification run: > ./tools/github/writeOnSlack.sh - "[$TEST_SUITE]" ${GH_BUILD}: ${{ steps.tests.outcome }} + "[$TEST_SUITE]" ${{ steps.tests.outcome }} on ${GH_BUILD} $'\nbranch:' $GH_BRANCH $'\nmessage:' "$(git log -1 --oneline | cat)" $'\nlogs:' ${{ steps.logs.outputs.logs }} diff --git a/.github/workflows/2-system.yaml b/.github/workflows/2-system.yaml index 84234bf5820..8ecf4bb2758 100644 --- a/.github/workflows/2-system.yaml +++ b/.github/workflows/2-system.yaml @@ -64,7 +64,7 @@ jobs: - name: Slack Notification run: > ./tools/github/writeOnSlack.sh - "[$TEST_SUITE]" ${GH_BUILD}: ${{ steps.tests.outcome }} + "[$TEST_SUITE]" ${{ steps.tests.outcome }} on ${GH_BUILD} $'\nbranch:' $GH_BRANCH $'\nmessage:' "$(git log -1 --oneline | cat)" $'\nlogs:' ${{ steps.logs.outputs.logs }} diff --git a/.github/workflows/3-multi-runtime.yaml b/.github/workflows/3-multi-runtime.yaml index 4bce4e5822b..fff541d5ef0 100644 --- a/.github/workflows/3-multi-runtime.yaml +++ b/.github/workflows/3-multi-runtime.yaml @@ -64,7 +64,7 @@ jobs: - name: Slack Notification run: > ./tools/github/writeOnSlack.sh - "[$TEST_SUITE]" ${GH_BUILD}: ${{ steps.tests.outcome }} + "[$TEST_SUITE]" ${{ steps.tests.outcome }} on ${GH_BUILD} $'\nbranch:' $GH_BRANCH $'\nmessage:' "$(git log -1 --oneline | cat)" $'\nlogs:' ${{ steps.logs.outputs.logs }} diff --git a/.github/workflows/4-standalone.yaml b/.github/workflows/4-standalone.yaml index 17b281f3236..961194d7666 100644 --- a/.github/workflows/4-standalone.yaml +++ b/.github/workflows/4-standalone.yaml @@ -64,7 +64,7 @@ jobs: - name: Slack Notification run: > ./tools/github/writeOnSlack.sh - "[$TEST_SUITE]" ${GH_BUILD}: ${{ steps.tests.outcome }} + "[$TEST_SUITE]" ${{ steps.tests.outcome }} on ${GH_BUILD} $'\nbranch:' $GH_BRANCH $'\nmessage:' "$(git log -1 --oneline | cat)" $'\nlogs:' ${{ steps.logs.outputs.logs }} diff --git a/.github/workflows/5-scheduler.yaml b/.github/workflows/5-scheduler.yaml index 0c53c8bd0af..033e154154a 100644 --- a/.github/workflows/5-scheduler.yaml +++ b/.github/workflows/5-scheduler.yaml @@ -64,7 +64,7 @@ jobs: - name: Slack Notification run: > ./tools/github/writeOnSlack.sh - "[$TEST_SUITE]" ${GH_BUILD}: ${{ steps.tests.outcome }} + "[$TEST_SUITE]" ${{ steps.tests.outcome }} on ${GH_BUILD} $'\nbranch:' $GH_BRANCH $'\nmessage:' "$(git log -1 --oneline | cat)" $'\nlogs:' ${{ steps.logs.outputs.logs }} diff --git a/.github/workflows/6-performance.yaml b/.github/workflows/6-performance.yaml index 4383a068cbd..2f0f0f4f1b3 100644 --- a/.github/workflows/6-performance.yaml +++ b/.github/workflows/6-performance.yaml @@ -81,7 +81,7 @@ jobs: - name: Slack Notification run: > ./tools/github/writeOnSlack.sh - "[$TEST_SUITE]" ${GH_BUILD}: ${{ steps.tests.outcome }} + "[$TEST_SUITE]" ${{ steps.tests.outcome }} on ${GH_BUILD} $'\nbranch:' $GH_BRANCH $'\nmessage:' "$(git log -1 --oneline | cat)" $'\nCheck GitHub logs for results' diff --git a/tools/github/setup.sh b/tools/github/setup.sh index e839fb6d38f..e72b046051e 100755 --- a/tools/github/setup.sh +++ b/tools/github/setup.sh @@ -35,10 +35,10 @@ function retry() { } # setup docker to listen in port 4243 -#sudo systemctl stop docker -#sudo sed -i -e 's!/usr/bin/dockerd -H fd://!/usr/bin/dockerd -H tcp://0.0.0.0:4243 -H fd://!' /lib/systemd/system/docker.service -#sudo systemctl daemon-reload -#sudo systemctl start docker +sudo systemctl stop docker +sudo sed -i -e 's!/usr/bin/dockerd -H fd://!/usr/bin/dockerd -H tcp://0.0.0.0:4243 -H fd://!' /lib/systemd/system/docker.service +sudo systemctl daemon-reload +sudo systemctl start docker # installing right version of jdk JDK=https://github.com/ibmruntimes/semeru11-binaries/releases/download/jdk-11.0.12%2B7_openj9-0.27.0/ibm-semeru-open-jdk_x64_linux_11.0.12_7_openj9-0.27.0.tar.gz From 79cc2f5ac1e0afe69a0a2543faad373b2b71cb5c Mon Sep 17 00:00:00 2001 From: Michele Sciabarra Date: Thu, 15 Dec 2022 09:08:36 +0000 Subject: [PATCH 34/39] fixed message --- .github/workflows/1-unit.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/1-unit.yaml b/.github/workflows/1-unit.yaml index 576c8de5fb9..4e27f72d98a 100644 --- a/.github/workflows/1-unit.yaml +++ b/.github/workflows/1-unit.yaml @@ -63,7 +63,7 @@ jobs: - name: Slack Notification run: > ./tools/github/writeOnSlack.sh - "[$TEST_SUITE]" ${GH_BUILD}: ${{ steps.tests.outcome }} + "[$TEST_SUITE]" ${{ steps.tests.outcome }} on ${GH_BUILD} $'\nbranch:' $GH_BRANCH $'\nmessage:' "$(git log -1 --oneline | cat)" $'\nlogs:' ${{ steps.logs.outputs.logs }} From 28dcf57da84fbc9f860592b4a8800ea897df66b8 Mon Sep 17 00:00:00 2001 From: Michele Sciabarra Date: Thu, 15 Dec 2022 10:26:36 +0000 Subject: [PATCH 35/39] show logs also in the build run --- .github/workflows/0-on-demand.yaml | 2 +- tools/github/checkAndUploadLogs.sh | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/0-on-demand.yaml b/.github/workflows/0-on-demand.yaml index acc5714f16e..851951a52be 100644 --- a/.github/workflows/0-on-demand.yaml +++ b/.github/workflows/0-on-demand.yaml @@ -37,7 +37,7 @@ on: - Performance ## Uncomment those to debug the build - Dummy - #- DummyKO + - DummyKO env: # openwhisk env diff --git a/tools/github/checkAndUploadLogs.sh b/tools/github/checkAndUploadLogs.sh index 74c0fc22ed8..79f169720b0 100755 --- a/tools/github/checkAndUploadLogs.sh +++ b/tools/github/checkAndUploadLogs.sh @@ -44,10 +44,12 @@ TAGS="" LOG_DIR="$(date +%Y-%m-%d)/${LOG_NAME}-${GH_BUILD}-${GH_BRANCH}" BUCKET_URL="https://$AWS_BUCKET.s3.$AWS_REGION.amazonaws.com" +echo "Logs: ${BUCKET_URL}/index.html#${LOG_DIR}/" +echo "Reports: ${BUCKET_URL}/${LOG_DIR}/test-reports/reports/tests/testCoverageLean/index.html" + echo "logs=${BUCKET_URL}/index.html#${LOG_DIR}/" >>${GITHUB_OUTPUT:-/dev/stdin} echo "report=${BUCKET_URL}/${LOG_DIR}/test-reports/reports/tests/testCoverageLean/index.html" >>${GITHUB_OUTPUT:-/dev/stdin} - ansible-playbook -i ansible/environments/local ansible/logs.yml ./tools/build/checkLogs.py logs "$TAGS" From 87c87de3185092c8316ea4cfbc8db163bcab2325 Mon Sep 17 00:00:00 2001 From: Michele Sciabarra Date: Thu, 15 Dec 2022 10:30:03 +0000 Subject: [PATCH 36/39] build on tag --- .github/workflows/1-unit.yaml | 4 ++-- .github/workflows/2-system.yaml | 4 ++-- .github/workflows/3-multi-runtime.yaml | 4 ++-- .github/workflows/4-standalone.yaml | 4 ++-- .github/workflows/5-scheduler.yaml | 4 ++-- .github/workflows/6-performance.yaml | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/1-unit.yaml b/.github/workflows/1-unit.yaml index 4e27f72d98a..1d0f1a1019d 100644 --- a/.github/workflows/1-unit.yaml +++ b/.github/workflows/1-unit.yaml @@ -20,8 +20,8 @@ name: Unit Tests on: # build on push push: - tags: '*' - branches-ignore: '*' + #tags: '*' + #branches-ignore: '*' # build on pull requests pull_request: diff --git a/.github/workflows/2-system.yaml b/.github/workflows/2-system.yaml index 8ecf4bb2758..dce7d9b6311 100644 --- a/.github/workflows/2-system.yaml +++ b/.github/workflows/2-system.yaml @@ -21,8 +21,8 @@ on: # build on push push: # build on pull requests - tags: '*' - branches-ignore: '*' + #tags: '*' + #branches-ignore: '*' pull_request: diff --git a/.github/workflows/3-multi-runtime.yaml b/.github/workflows/3-multi-runtime.yaml index fff541d5ef0..22a5dd3ef28 100644 --- a/.github/workflows/3-multi-runtime.yaml +++ b/.github/workflows/3-multi-runtime.yaml @@ -20,8 +20,8 @@ name: MultiRuntime Tests on: # build on push push: - tags: '*' - branches-ignore: '*' + #tags: '*' + #branches-ignore: '*' # build on pull requests pull_request: diff --git a/.github/workflows/4-standalone.yaml b/.github/workflows/4-standalone.yaml index 961194d7666..299d5cfd854 100644 --- a/.github/workflows/4-standalone.yaml +++ b/.github/workflows/4-standalone.yaml @@ -21,8 +21,8 @@ on: # build on push push: # build on pull requests - tags: '*' - branches-ignore: '*' + #tags: '*' + #branches-ignore: '*' pull_request: diff --git a/.github/workflows/5-scheduler.yaml b/.github/workflows/5-scheduler.yaml index 033e154154a..a74b0542146 100644 --- a/.github/workflows/5-scheduler.yaml +++ b/.github/workflows/5-scheduler.yaml @@ -21,8 +21,8 @@ on: # build on push push: # build on pull requests - tags: '*' - branches-ignore: '*' + #tags: '*' + #branches-ignore: '*' pull_request: diff --git a/.github/workflows/6-performance.yaml b/.github/workflows/6-performance.yaml index 2f0f0f4f1b3..54948207301 100644 --- a/.github/workflows/6-performance.yaml +++ b/.github/workflows/6-performance.yaml @@ -21,8 +21,8 @@ on: # build on push push: # build on pull requests - tags: '*' - branches-ignore: '*' + #tags: '*' + #ranches-ignore: '*' pull_request: From 04734c3b96a17d917d24c18db692f1aa08d09e4a Mon Sep 17 00:00:00 2001 From: Michele Sciabarra Date: Sat, 10 Feb 2024 11:58:58 +0000 Subject: [PATCH 37/39] nuvolaris fixes --- .gitignore | 1 + ansible/group_vars/all | 6 ++---- common/scala/Dockerfile | 18 +++++++++++++----- core/controller/Dockerfile | 22 ++++++++++++---------- core/cosmosdb/cache-invalidator/Dockerfile | 3 ++- core/invoker/Dockerfile | 10 ++++++---- core/monitoring/user-events/Dockerfile | 3 ++- core/scheduler/Dockerfile | 6 ++++-- core/standalone/Dockerfile | 3 ++- 9 files changed, 44 insertions(+), 28 deletions(-) diff --git a/.gitignore b/.gitignore index 7a15a629f0d..627244110ff 100644 --- a/.gitignore +++ b/.gitignore @@ -100,3 +100,4 @@ core/routemgmt/getApi/utils.js # vscode metals .bloop/ .metals/ +.git-hooks/ diff --git a/ansible/group_vars/all b/ansible/group_vars/all index 072e4aebb15..eee2f52efb3 100644 --- a/ansible/group_vars/all +++ b/ansible/group_vars/all @@ -166,7 +166,7 @@ kafka: protocols: - TLSv1.2 protocol: "{{ kafka_protocol_for_setup }}" - version: "{{ kafka_version | default('2.13-2.7.0') }}" + version: "{{ kafka_version | default('2.12-2.3.1') }}" port: 9072 advertisedPort: 9093 ras: @@ -186,7 +186,7 @@ kafka_connect_string: "{% set ret = [] %}\ kafka_protocol_for_setup: "{{ kafka_protocol | default('PLAINTEXT') }}" zookeeper: - version: 3.4 + version: 3.4.14 port: 2181 zookeeper_connect_string: "{% set ret = [] %}\ @@ -551,5 +551,3 @@ scheduler: throttlingFraction: "{{ scheduler_queue_throttlingFraction | default(0.9) }}" durationBufferSize: "{{ scheduler_queue_durationBufferSize | default(10) }}" deployment_ignore_error: "{{ scheduler_deployment_ignore_error | default('False') }}" - dataManagementService: - retryInterval: "{{ scheduler_dataManagementService_retryInterval | default('1 second') }}" diff --git a/common/scala/Dockerfile b/common/scala/Dockerfile index c7ebd2b9e87..fff30d22822 100644 --- a/common/scala/Dockerfile +++ b/common/scala/Dockerfile @@ -15,16 +15,24 @@ # limitations under the License. # # if you change version of openjsk, also update tools/github/setup.sh to download the corresponding jdk -FROM adoptopenjdk/openjdk11-openj9:x86_64-alpine-jdk-11.0.12_7_openj9-0.27.0 +FROM ubuntu:22.04 +RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections +RUN apt-get update && apt-get -y upgrade &&\ + apt-get -y install software-properties-common apt-utils \ + curl wget gpg unzip zip vim telnet \ + silversearcher-ag inetutils-ping locales &&\ + locale-gen en_US.UTF-8 + +# installing openjdk +RUN bash -c 'x86_64="https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.17%2B8/OpenJDK11U-jdk_x64_linux_hotspot_11.0.17_8.tar.gz" ;\ +aarch64="https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.17%2B8/OpenJDK11U-jdk_aarch64_linux_hotspot_11.0.17_8.tar.gz";\ +arch=$(arch) ; JDK=${!arch} ; curl -sL $JDK | tar xzvf - -C /usr/local ;\ +ln -sf /usr/local/jdk-*/bin/java /usr/bin/java' ENV LANG en_US.UTF-8 ENV LANGUAGE en_US:en ENV LC_ALL en_US.UTF-8 -# Switch to the HTTPS endpoint for the apk repositories as per https://github.com/gliderlabs/docker-alpine/issues/184 -RUN sed -i 's/http\:\/\/dl-cdn.alpinelinux.org/https\:\/\/alpine.global.ssl.fastly.net/g' /etc/apk/repositories -RUN apk add --update sed curl bash && apk update && apk upgrade - RUN mkdir /logs COPY transformEnvironment.sh / diff --git a/core/controller/Dockerfile b/core/controller/Dockerfile index 287935326f2..666647a61c9 100644 --- a/core/controller/Dockerfile +++ b/core/controller/Dockerfile @@ -15,7 +15,8 @@ # limitations under the License. # -FROM scala +ARG BASE=scala +FROM $BASE ENV UID=1001 \ NOT_ROOT_USER=owuser @@ -30,11 +31,11 @@ ENV SWAGGER_UI_DOWNLOAD_SHA256=3d7ef5ddc59e10f132fe99771498f0f1ba7a2cbfb9585f986 # If this cannot be guaranteed, set `invoker_use_runc: false` in the ansible env. ENV DOCKER_VERSION=18.06.3-ce -RUN apk add --update openssl +#RUN apk add --update openssl # Uncomment to fetch latest version of docker instead: RUN wget -qO- https://get.docker.com | sh # Install docker client -RUN curl -sSL -o docker-${DOCKER_VERSION}.tgz https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_VERSION}.tgz && \ +RUN curl -sSL -o docker-${DOCKER_VERSION}.tgz https://download.docker.com/linux/static/stable/$(arch)/docker-${DOCKER_VERSION}.tgz && \ tar --strip-components 1 -xvzf docker-${DOCKER_VERSION}.tgz -C /usr/bin docker/docker && \ tar --strip-components 1 -xvzf docker-${DOCKER_VERSION}.tgz -C /usr/bin docker/docker-runc && \ rm -f docker-${DOCKER_VERSION}.tgz && \ @@ -43,12 +44,12 @@ RUN curl -sSL -o docker-${DOCKER_VERSION}.tgz https://download.docker.com/linux/ ################################################################################################## # Install swagger-ui -RUN curl -sSL -o swagger-ui-v${SWAGGER_UI_VERSION}.tar.gz --no-verbose https://github.com/swagger-api/swagger-ui/archive/v${SWAGGER_UI_VERSION}.tar.gz && \ - echo "${SWAGGER_UI_DOWNLOAD_SHA256} swagger-ui-v${SWAGGER_UI_VERSION}.tar.gz" | sha256sum -c - && \ - mkdir swagger-ui && \ - tar zxf swagger-ui-v${SWAGGER_UI_VERSION}.tar.gz -C /swagger-ui --strip-components=2 swagger-ui-${SWAGGER_UI_VERSION}/dist && \ - rm swagger-ui-v${SWAGGER_UI_VERSION}.tar.gz && \ - sed -i s#http://petstore.swagger.io/v2/swagger.json#/api/v1/api-docs#g /swagger-ui/index.html +#RUN curl -sSL -o swagger-ui-v${SWAGGER_UI_VERSION}.tar.gz --no-verbose https://github.com/swagger-api/swagger-ui/archive/v${SWAGGER_UI_VERSION}.tar.gz && \ +# echo "${SWAGGER_UI_DOWNLOAD_SHA256} swagger-ui-v${SWAGGER_UI_VERSION}.tar.gz" | sha256sum -c - && \ +# mkdir swagger-ui && \ +# tar zxf swagger-ui-v${SWAGGER_UI_VERSION}.tar.gz -C /swagger-ui --strip-components=2 swagger-ui-${SWAGGER_UI_VERSION}/dist && \ +# rm swagger-ui-v${SWAGGER_UI_VERSION}.tar.gz && \ +# sed -i s#http://petstore.swagger.io/v2/swagger.json#/api/v1/api-docs#g /swagger-ui/index.html # Copy app jars ADD build/distributions/controller.tar / @@ -56,7 +57,8 @@ ADD build/distributions/controller.tar / COPY init.sh / RUN chmod +x init.sh -RUN adduser -D -u ${UID} -h /home/${NOT_ROOT_USER} -s /bin/bash ${NOT_ROOT_USER} +#RUN adduser -D -u ${UID} -h /home/${NOT_ROOT_USER} -s /bin/bash ${NOT_ROOT_USER} +RUN useradd -m -u ${UID} -d /home/${NOT_ROOT_USER} -s /bin/bash ${NOT_ROOT_USER} # It is possible to run as non root if you dont need invoker capabilities out of the controller today # When running it as a non-root user this has implications on the standard directory where runc stores its data. diff --git a/core/cosmosdb/cache-invalidator/Dockerfile b/core/cosmosdb/cache-invalidator/Dockerfile index e8df857541d..3cc1e2a8919 100644 --- a/core/cosmosdb/cache-invalidator/Dockerfile +++ b/core/cosmosdb/cache-invalidator/Dockerfile @@ -27,7 +27,8 @@ ADD build/distributions/cache-invalidator.tar / COPY init.sh / RUN chmod +x init.sh -RUN adduser -D -u ${UID} -h /home/${NOT_ROOT_USER} -s /bin/bash ${NOT_ROOT_USER} +#RUN adduser -D -u ${UID} -h /home/${NOT_ROOT_USER} -s /bin/bash ${NOT_ROOT_USER} +RUN useradd -m -u ${UID} -d /home/${NOT_ROOT_USER} -s /bin/bash ${NOT_ROOT_USER} USER ${NOT_ROOT_USER} EXPOSE 8080 diff --git a/core/invoker/Dockerfile b/core/invoker/Dockerfile index ca1561d7d2e..82a806f3c1e 100644 --- a/core/invoker/Dockerfile +++ b/core/invoker/Dockerfile @@ -15,7 +15,8 @@ # limitations under the License. # -FROM scala +ARG BASE=scala +FROM $BASE ENV UID=1001 \ NOT_ROOT_USER=owuser \ @@ -25,11 +26,11 @@ ENV UID=1001 \ # If this cannot be guaranteed, set `invoker_use_runc: false` in the ansible env. -RUN apk add --update openssl +#RUN apk add --update openssl # Uncomment to fetch latest version of docker instead: RUN wget -qO- https://get.docker.com | sh # Install docker client -RUN curl -sSL -o docker-${DOCKER_VERSION}.tgz https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_VERSION}.tgz && \ +RUN curl -sSL -o docker-${DOCKER_VERSION}.tgz https://download.docker.com/linux/static/stable/$(arch)/docker-${DOCKER_VERSION}.tgz && \ tar --strip-components 1 -xvzf docker-${DOCKER_VERSION}.tgz -C /usr/bin docker/docker && \ tar --strip-components 1 -xvzf docker-${DOCKER_VERSION}.tgz -C /usr/bin docker/docker-runc && \ rm -f docker-${DOCKER_VERSION}.tgz && \ @@ -43,7 +44,8 @@ RUN chmod +x init.sh # When running the invoker as a non-root user this has implications on the standard directory where runc stores its data. # The non-root user should have access on the directory and corresponding permission to make changes on it. -RUN adduser -D -u ${UID} -h /home/${NOT_ROOT_USER} -s /bin/bash ${NOT_ROOT_USER} +#RUN adduser -D -u ${UID} -h /home/${NOT_ROOT_USER} -s /bin/bash ${NOT_ROOT_USER} +RUN useradd -m -u ${UID} -d /home/${NOT_ROOT_USER} -s /bin/bash ${NOT_ROOT_USER} EXPOSE 8080 CMD ["./init.sh", "0"] diff --git a/core/monitoring/user-events/Dockerfile b/core/monitoring/user-events/Dockerfile index 95b06ae882b..6b2df680e17 100644 --- a/core/monitoring/user-events/Dockerfile +++ b/core/monitoring/user-events/Dockerfile @@ -26,7 +26,8 @@ ADD build/distributions/user-events.tar / COPY init.sh / RUN chmod +x init.sh -RUN adduser -D -u ${UID} -h /home/${NOT_ROOT_USER} -s /bin/bash ${NOT_ROOT_USER} +#RUN adduser -D -u ${UID} -h /home/${NOT_ROOT_USER} -s /bin/bash ${NOT_ROOT_USER} +RUN useradd -m -u ${UID} -d /home/${NOT_ROOT_USER} -s /bin/bash ${NOT_ROOT_USER} USER ${NOT_ROOT_USER} # Prometheus port diff --git a/core/scheduler/Dockerfile b/core/scheduler/Dockerfile index 244d80d7b05..9ad9808f216 100644 --- a/core/scheduler/Dockerfile +++ b/core/scheduler/Dockerfile @@ -15,7 +15,8 @@ # limitations under the License. # -FROM scala +ARG BASE=scala +FROM $BASE ENV UID=1001 \ NOT_ROOT_USER=owuser @@ -26,7 +27,8 @@ ADD build/distributions/scheduler.tar / COPY init.sh / RUN chmod +x init.sh -RUN adduser -D -u ${UID} -h /home/${NOT_ROOT_USER} -s /bin/bash ${NOT_ROOT_USER} +#RUN adduser -D -u ${UID} -h /home/${NOT_ROOT_USER} -s /bin/bash ${NOT_ROOT_USER} +RUN useradd -m -u ${UID} -d /home/${NOT_ROOT_USER} -s /bin/bash ${NOT_ROOT_USER} USER ${NOT_ROOT_USER} EXPOSE 8080 diff --git a/core/standalone/Dockerfile b/core/standalone/Dockerfile index 676b1eb1c98..875101727f0 100644 --- a/core/standalone/Dockerfile +++ b/core/standalone/Dockerfile @@ -14,7 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # -FROM scala +ARG BASE=scala +FROM $BASE ARG OPENWHISK_JAR ENV DOCKER_VERSION=18.06.3-ce ENV WSK_VERSION=1.0.0 From aebe041ea31195d8d5ad7af12120c9f7b3ba6aeb Mon Sep 17 00:00:00 2001 From: Michele Sciabarra Date: Sat, 10 Feb 2024 12:02:58 +0000 Subject: [PATCH 38/39] gitignore --- .gitignore | 1 + Taskfile.yml | 106 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+) create mode 100644 Taskfile.yml diff --git a/.gitignore b/.gitignore index 627244110ff..7f1712713c8 100644 --- a/.gitignore +++ b/.gitignore @@ -101,3 +101,4 @@ core/routemgmt/getApi/utils.js .bloop/ .metals/ .git-hooks/ +*.tar diff --git a/Taskfile.yml b/Taskfile.yml new file mode 100644 index 00000000000..8394a678bcb --- /dev/null +++ b/Taskfile.yml @@ -0,0 +1,106 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +version: '3' + +vars: + BASETAG: 0.3.0-morpheus + TAG: + sh: git describe --tags --abbrev=0 2>/dev/null || echo "no-tag" + TIMESTAMP: + sh: date +%y%m%d%H + S: "" + T: "all" + +tasks: + default: task --list-all + setup: {"silent": true} + + yaml: + silent: true + cmds: + - |- + A="{{.CLI_ARGS}}" + if test -z "$A" + then ls -1 *.yml | sed -e 's/.yml//' -e 's/^/- /' + echo "edit(@)" or run with '"task yaml -- [@]