Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Refactor some CI scripts #6276

Merged
merged 6 commits into from
Nov 14, 2024
Merged

Refactor some CI scripts #6276

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
3f6f2ee
launchdev.sh: make st2tests clone safer
cognifloyd Nov 6, 2024
a92df9f
launchdev.sh: add some safety checks
cognifloyd Nov 6, 2024
5a3e0dc
launchdev.sh: extract/consolidate PRE_SCRIPT definition
cognifloyd Nov 6, 2024
41fbccb
scripts/github/prepare-integration.sh: Drop conf modification
cognifloyd Nov 9, 2024
51bc606
ci: less verbose wget output
cognifloyd Nov 7, 2024
3e468ec
prepare-integration.sh: support alt VIRTUALENV_DIR
cognifloyd Nov 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion scripts/github/configure-rabbitmq.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ echo enabled RabbitMQ plugins:
# print plugins list to: (1) ease debugging, (2) pause till rabbitmq is really running
docker exec rabbitmq rabbitmq-plugins list -e
echo
sudo wget http://guest:guest@localhost:15672/cli/rabbitmqadmin -O /usr/local/bin/rabbitmqadmin
sudo wget --no-verbose http://guest:guest@localhost:15672/cli/rabbitmqadmin -O /usr/local/bin/rabbitmqadmin
sudo chmod +x /usr/local/bin/rabbitmqadmin
# print logs from stdout (RABBITMQ_LOGS=-)
docker logs --tail=100 rabbitmq
Expand Down
19 changes: 10 additions & 9 deletions scripts/github/prepare-integration.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,13 @@ if [ "$(whoami)" != 'root' ]; then
exit 2
fi

# Activate the virtualenv created during make requirements phase
# shellcheck disable=SC1091
source ./virtualenv/bin/activate
# ./virtualenv is the Makefile managed dir.
# The workflow should set this to use a pants exported or other venv instead.
VIRTUALENV_DIR="${VIRTUALENV_DIR:-./virtualenv}"

# Enable coordination backend to avoid race conditions with orquesta tests due
# to the lack of the coordination backend
sed -i "s#\#url = redis://localhost#url = redis://127.0.0.1#g" ./conf/st2.dev.conf
sed -i "s#\#url = redis://localhost#url = redis://127.0.0.1#g" ./conf/st2.ci.conf || true
# Activate the virtualenv created during make requirements phase
# shellcheck disable=SC1090,SC1091
source "${VIRTUALENV_DIR}/bin/activate"

echo "Used config for the tests"
echo ""
Expand All @@ -27,7 +26,9 @@ cat conf/st2.ci.conf || true
echo ""

# install st2 client
python ./st2client/setup.py develop
if [ ! -x "${VIRTUALENV_DIR}/bin/st2" ] || ! st2 --version >/dev/null 2>&1; then
python ./st2client/setup.py develop
fi
st2 --version

# Clean up old st2 log files
Expand Down Expand Up @@ -61,7 +62,7 @@ chmod 777 logs/*

# root needs to access write some lock files when creating virtualenvs
# o=other; X=only set execute bit if user execute bit is set (eg on dirs)
chmod -R o+rwX ./virtualenv/
chmod -R o+rwX "${VIRTUALENV_DIR}/"
# newer virtualenv versions are putting lock files under ~/.local
# as this script runs with sudo, HOME is actually the CI user's home
chmod -R o+rwX "${HOME}/.local/share/virtualenv"
53 changes: 33 additions & 20 deletions tools/launchdev.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,10 +110,17 @@ function init()
else
ST2_REPO=${CURRENT_DIR}/${COMMAND_PATH}/..
fi
ST2_REPO=$(readlink -f ${ST2_REPO})
ST2_LOGS="${ST2_REPO}/logs"
# ST2_REPO/virtualenv is the Makefile managed dir.
# The workflow should set this to use a pants exported or other venv instead.
VIRTUALENV=${VIRTUALENV_DIR:-${ST2_REPO}/virtualenv}
VIRTUALENV=$(readlink -f ${VIRTUALENV})
PY=${VIRTUALENV}/bin/python
if [ ! -f "${PY}" ]; then
eecho "${PY} does not exist"
exit 1
fi
PYTHON_VERSION=$(${PY} --version 2>&1)

echo -n "Using virtualenv: "; iecho "${VIRTUALENV}"
Expand Down Expand Up @@ -201,25 +208,26 @@ function st2start()
if [ "$copy_test_packs" = true ]; then
echo -n "Copying test packs examples and fixtures to "; iecho "$PACKS_BASE_DIR"
cp -Rp ./contrib/examples $PACKS_BASE_DIR
# Clone st2tests in /tmp directory.
pushd /tmp
# Clone st2tests in a tmp directory.
CLONE_TMP_DIR=$(mktemp -d)
pushd "${CLONE_TMP_DIR}"
echo Cloning https://github.com/StackStorm/st2tests.git
# -q = no progress reporting (better for CI). Errors will still print.
git clone -q https://github.com/StackStorm/st2tests.git
ret=$?
if [ ${ret} -eq 0 ]; then
cp -Rp ./st2tests/packs/fixtures $PACKS_BASE_DIR
rm -R st2tests/
else
eecho "Failed to clone st2tests repo"
fi
popd
rm -Rf "${CLONE_TMP_DIR}"
fi

# activate virtualenv to set PYTHONPATH
source ${VIRTUALENV}/bin/activate
source "${VIRTUALENV}/bin/activate"
# set configuration file location.
export ST2_CONFIG_PATH=${ST2_CONF};
export ST2_CONFIG_PATH="${ST2_CONF}"

# Kill existing st2 terminal multiplexor sessions
for tmux_session in $(tmux ls 2>/dev/null | awk -F: '/^st2-/ {print $1}')
Expand All @@ -228,22 +236,28 @@ function st2start()
tmux kill-session -t $tmux_session
done

local PRE_SCRIPT_VARS=()
PRE_SCRIPT_VARS+=("ST2_CONFIG_PATH=${ST2_CONF}")

# PRE_SCRIPT should not end with ';' so that using it is clear.
local PRE_SCRIPT="export ${PRE_SCRIPT_VARS[@]}; source ${VIRTUALENV}/bin/activate"

# Run the st2 API server
if [ "${use_gunicorn}" = true ]; then
echo 'Starting st2-api using gunicorn ...'
tmux new-session -d -s st2-api "export ST2_CONFIG_PATH=${ST2_CONF}; source ${VIRTUALENV}/bin/activate; ${VIRTUALENV}/bin/gunicorn st2api.wsgi:application -k eventlet -b $BINDING_ADDRESS:9101 --workers 1 2>&1 | tee -a ${ST2_LOGS}/st2-api.log"
tmux new-session -d -s st2-api "${PRE_SCRIPT}; ${VIRTUALENV}/bin/gunicorn st2api.wsgi:application -k eventlet -b $BINDING_ADDRESS:9101 --workers 1 2>&1 | tee -a ${ST2_LOGS}/st2-api.log"
else
echo 'Starting st2-api ...'
tmux new-session -d -s st2-api "export ST2_CONFIG_PATH=${ST2_CONF}; source ${VIRTUALENV}/bin/activate; ${VIRTUALENV}/bin/python ./st2api/bin/st2api --config-file $ST2_CONF 2>&1 | tee -a ${ST2_LOGS}/st2-api.log"
tmux new-session -d -s st2-api "${PRE_SCRIPT}; ${VIRTUALENV}/bin/python ./st2api/bin/st2api --config-file $ST2_CONF 2>&1 | tee -a ${ST2_LOGS}/st2-api.log"
fi

# Run st2stream API server
if [ "${use_gunicorn}" = true ]; then
echo 'Starting st2-stream using gunicorn ...'
tmux new-session -d -s st2-stream "export ST2_CONFIG_PATH=${ST2_CONF}; source ${VIRTUALENV}/bin/activate; ${VIRTUALENV}/bin/gunicorn st2stream.wsgi:application -k eventlet -b $BINDING_ADDRESS:9102 --workers 1 2>&1 | tee -a ${ST2_LOGS}/st2-stream.log"
tmux new-session -d -s st2-stream "${PRE_SCRIPT}; ${VIRTUALENV}/bin/gunicorn st2stream.wsgi:application -k eventlet -b $BINDING_ADDRESS:9102 --workers 1 2>&1 | tee -a ${ST2_LOGS}/st2-stream.log"
else
echo 'Starting st2-stream ...'
tmux new-session -d -s st2-stream "export ST2_CONFIG_PATH=${ST2_CONF}; source ${VIRTUALENV}/bin/activate; ${VIRTUALENV}/bin/python ./st2stream/bin/st2stream --config-file $ST2_CONF 2>&1 | tee -a ${ST2_LOGS}/st2-stream.log"
tmux new-session -d -s st2-stream "${PRE_SCRIPT}; ${VIRTUALENV}/bin/python ./st2stream/bin/st2stream --config-file $ST2_CONF 2>&1 | tee -a ${ST2_LOGS}/st2-stream.log"
fi

# give st2stream time to startup and load things into database
Expand All @@ -257,7 +271,7 @@ function st2start()
WORKFLOW_ENGINE_NAME=st2-workflow-$i
WORKFLOW_ENGINE_SESSIONS+=($WORKFLOW_ENGINE_NAME)
echo " $WORKFLOW_ENGINE_NAME ..."
tmux new-session -d -s $WORKFLOW_ENGINE_NAME "export ST2_CONFIG_PATH=${ST2_CONF}; source ${VIRTUALENV}/bin/activate; ${VIRTUALENV}/bin/python ./st2actions/bin/st2workflowengine --config-file $ST2_CONF 2>&1 | tee -a ${ST2_LOGS}/${WORKFLOW_ENGINE_NAME}.log"
tmux new-session -d -s $WORKFLOW_ENGINE_NAME "${PRE_SCRIPT}; ${VIRTUALENV}/bin/python ./st2actions/bin/st2workflowengine --config-file $ST2_CONF 2>&1 | tee -a ${ST2_LOGS}/${WORKFLOW_ENGINE_NAME}.log"
done

# Start a session for every runner
Expand All @@ -268,12 +282,12 @@ function st2start()
RUNNER_NAME=st2-actionrunner-$i
RUNNER_SESSIONS+=($RUNNER_NAME)
echo " $RUNNER_NAME ..."
tmux new-session -d -s $RUNNER_NAME "export ST2_CONFIG_PATH=${ST2_CONF}; source ${VIRTUALENV}/bin/activate; ${VIRTUALENV}/bin/python ./st2actions/bin/st2actionrunner --config-file $ST2_CONF 2>&1 | tee -a ${ST2_LOGS}/${RUNNER_NAME}.log"
tmux new-session -d -s $RUNNER_NAME "${PRE_SCRIPT}; ${VIRTUALENV}/bin/python ./st2actions/bin/st2actionrunner --config-file $ST2_CONF 2>&1 | tee -a ${ST2_LOGS}/${RUNNER_NAME}.log"
done

# Run the garbage collector service
echo 'Starting st2-garbagecollector ...'
tmux new-session -d -s st2-garbagecollector "export ST2_CONFIG_PATH=${ST2_CONF}; source ${VIRTUALENV}/bin/activate; ${VIRTUALENV}/bin/python ./st2reactor/bin/st2garbagecollector --config-file $ST2_CONF 2>&1 | tee -a ${ST2_LOGS}/st2-garbagecollector.log"
tmux new-session -d -s st2-garbagecollector "${PRE_SCRIPT}; ${VIRTUALENV}/bin/python ./st2reactor/bin/st2garbagecollector --config-file $ST2_CONF 2>&1 | tee -a ${ST2_LOGS}/st2-garbagecollector.log"

# Run the scheduler server
echo 'Starting st2-scheduler(s):'
Expand All @@ -283,33 +297,32 @@ function st2start()
SCHEDULER_NAME=st2-scheduler-$i
SCHEDULER_SESSIONS+=($SCHEDULER_NAME)
echo " $SCHEDULER_NAME ..."
tmux new-session -d -s $SCHEDULER_NAME "export ST2_CONFIG_PATH=${ST2_CONF}; source ${VIRTUALENV}/bin/activate; ${VIRTUALENV}/bin/python ./st2actions/bin/st2scheduler --config-file $ST2_CONF 2>&1 | tee -a ${ST2_LOGS}/${SCHEDULER_NAME}.log"
tmux new-session -d -s $SCHEDULER_NAME "${PRE_SCRIPT}; ${VIRTUALENV}/bin/python ./st2actions/bin/st2scheduler --config-file $ST2_CONF 2>&1 | tee -a ${ST2_LOGS}/${SCHEDULER_NAME}.log"
done

# Run the sensor container server
echo 'Starting st2-sensorcontainer ...'
tmux new-session -d -s st2-sensorcontainer "export ST2_CONFIG_PATH=${ST2_CONF}; source ${VIRTUALENV}/bin/activate; ${VIRTUALENV}/bin/python ./st2reactor/bin/st2sensorcontainer --config-file $ST2_CONF 2>&1 | tee -a ${ST2_LOGS}/st2-sensorcontainer.log"
tmux new-session -d -s st2-sensorcontainer "${PRE_SCRIPT}; ${VIRTUALENV}/bin/python ./st2reactor/bin/st2sensorcontainer --config-file $ST2_CONF 2>&1 | tee -a ${ST2_LOGS}/st2-sensorcontainer.log"

# Run the rules engine server
echo 'Starting st2-rulesengine ...'
tmux new-session -d -s st2-rulesengine "export ST2_CONFIG_PATH=${ST2_CONF}; source ${VIRTUALENV}/bin/activate; ${VIRTUALENV}/bin/python ./st2reactor/bin/st2rulesengine --config-file $ST2_CONF 2>&1 | tee -a ${ST2_LOGS}/st2-rulesengine.log"
tmux new-session -d -s st2-rulesengine "${PRE_SCRIPT}; ${VIRTUALENV}/bin/python ./st2reactor/bin/st2rulesengine --config-file $ST2_CONF 2>&1 | tee -a ${ST2_LOGS}/st2-rulesengine.log"

# Run the timer engine server
echo 'Starting st2-timersengine ...'
tmux new-session -d -s st2-timersengine "export ST2_CONFIG_PATH=${ST2_CONF}; source ${VIRTUALENV}/bin/activate; ${VIRTUALENV}/bin/python ./st2reactor/bin/st2timersengine --config-file $ST2_CONF 2>&1 | tee -a ${ST2_LOGS}/st2-timersengine.log"
tmux new-session -d -s st2-timersengine "${PRE_SCRIPT}; ${VIRTUALENV}/bin/python ./st2reactor/bin/st2timersengine --config-file $ST2_CONF 2>&1 | tee -a ${ST2_LOGS}/st2-timersengine.log"

# Run the actions notifier
echo 'Starting st2-notifier ...'
tmux new-session -d -s st2-notifier "export ST2_CONFIG_PATH=${ST2_CONF}; source ${VIRTUALENV}/bin/activate; ${VIRTUALENV}/bin/python ./st2actions/bin/st2notifier --config-file $ST2_CONF 2>&1 | tee -a ${ST2_LOGS}/st2-notifier.log"
tmux new-session -d -s st2-notifier "${PRE_SCRIPT}; ${VIRTUALENV}/bin/python ./st2actions/bin/st2notifier --config-file $ST2_CONF 2>&1 | tee -a ${ST2_LOGS}/st2-notifier.log"

# Run the auth API server
if [ "${use_gunicorn}" = true ]; then
echo 'Starting st2-auth using gunicorn ...'
export ST2_CONFIG_PATH=${ST2_CONF}
tmux new-session -d -s st2-auth "export ST2_CONFIG_PATH=${ST2_CONF}; source ${VIRTUALENV}/bin/activate; ${VIRTUALENV}/bin/gunicorn st2auth.wsgi:application -k eventlet -b $BINDING_ADDRESS:9100 --workers 1 2>&1 | tee -a ${ST2_LOGS}/st2-auth.log"
tmux new-session -d -s st2-auth "${PRE_SCRIPT}; ${VIRTUALENV}/bin/gunicorn st2auth.wsgi:application -k eventlet -b $BINDING_ADDRESS:9100 --workers 1 2>&1 | tee -a ${ST2_LOGS}/st2-auth.log"
else
echo 'Starting st2-auth ...'
tmux new-session -d -s st2-auth "export ST2_CONFIG_PATH=${ST2_CONF}; source ${VIRTUALENV}/bin/activate; ${VIRTUALENV}/bin/python ./st2auth/bin/st2auth --config-file $ST2_CONF 2>&1 | tee -a ${ST2_LOGS}/st2-auth.log"
tmux new-session -d -s st2-auth "${PRE_SCRIPT}; ${VIRTUALENV}/bin/python ./st2auth/bin/st2auth --config-file $ST2_CONF 2>&1 | tee -a ${ST2_LOGS}/st2-auth.log"
fi

# Check whether tmux sessions are started
Expand Down
Loading