feat: add --replace option to qshift #108

Summary
Jobs
- setup-idp
Run details
- Usage
- Workflow file

Workflow file for this run

.github/workflows/e2e-backstage.yml at 117cfe5

	name: Install-run-backstage-golden-path

	on:
	workflow_dispatch:
	push:
	branches:
	- main
	pull_request:
	# TODO: To be reviewed to figure out if we need it or not
	#pull_request_target:
	# branches: [ 'main' ]
	# # types: [opened, edited]

	permissions:
	contents: write
	pull-requests: write

	env:
	NODE_VERSION: 20.x

	IDPBUILDER_VERSION: v0.5.0

	TEKTON_VERSION: v0.60.1
	TEKTON_CLIENT_VERSION: 0.37.0

	# From idpbuilder project
	ARGOCD_VERSION: 2.10.7
	GITEA_VERSION: 1.22

	ARGOCD_CLIENT_VERSION: v2.11.4

	KUBEVIRT_VERSION: v1.2.1
	KUBEVIRT_CDI_VERSION: v1.59.0

	# The git repo name is also the name of the project's parameter of the template
	REPO_NAME: my-quarkus-app-job
	REPO_ORG: q-shift

	QUAY_ORG: qshift

	# This namespace will be used to build/deploy the generated quarkus app and
	# should be the same as the one defined within the ConfigMap of ArgocCD using: application.namespaces: "demo"
	KUBE_NAMESPACE: demo

	jobs:
	setup-idp:
	runs-on: ubuntu-latest
	steps:
	- name: Checkout
	uses: actions/checkout@v4

	- name: Install idpbuilder
	run: \|
	version=${IDPBUILDER_VERSION}
	curl -L -o ./idpbuilder.tar.gz "https://github.com/cnoe-io/idpbuilder/releases/download/${version}/idpbuilder-$(uname \| awk '{print tolower($0)}')-$(uname -m \| sed 's/x86_64/amd64/').tar.gz"
	tar xzf idpbuilder.tar.gz
	sudo mv ./idpbuilder /usr/local/bin/

	idpbuilder version

	- name: Install Argocd client
	run: \|
	curl -sSL -o argocd-linux-amd64 https://github.com/argoproj/argo-cd/releases/download/${ARGOCD_CLIENT_VERSION}/argocd-linux-amd64
	sudo install -m 555 argocd-linux-amd64 /usr/local/bin/argocd
	rm argocd-linux-amd64

	- name: Install tekton client
	run: \|
	curl -sSL "https://github.com/tektoncd/cli/releases/download/v{$TEKTON_CLIENT_VERSION}/tkn_${TEKTON_CLIENT_VERSION}_Linux_x86_64.tar.gz" -o tkn.tar.gz
	sudo tar xvzf tkn.tar.gz -C /usr/local/bin/ tkn
	tkn version

	- name: Create an IDP cluster and install the packages
	run: \|
	PACKAGES_DIR=manifest/idp/packages
	idpbuilder create \
	-p $PACKAGES_DIR/tekton \
	-p $PACKAGES_DIR/backstage \
	-c argocd:$PACKAGES_DIR/argocd/cm.yaml

	- name: Create the namespace where applications, resources should be deployed
	run: \|
	kubectl create ns ${KUBE_NAMESPACE}

	- name: Wait till IDP ArgoCD application is sync; ConfigMap patched
	run: \|
	SCRIPTS=$(pwd)/.github/scripts

	echo "Temporary workaround to refresh ArgoCD Application till https://github.com/cnoe-io/idpbuilder/pull/307 is released"
	kubectl annotate --overwrite applications -n argocd argocd argocd.argoproj.io/refresh='normal'

	if ! $SCRIPTS/waitFor.sh application argocd argocd Healthy; then
	echo "Failed to watch application argocd in namespace argocd"
	exit 1;
	fi

	echo "Wait till ConfigMap is patched with data: application.namespaces ..."
	until kubectl get -n argocd cm/argocd-cmd-params-cm -o json \| jq -e '.data \| has("application.namespaces")'; do
	echo "Still waiting ..."
	sleep 10s
	done

	echo "Rollout Argocd as resources changed ..."
	kubectl rollout restart -n argocd deployment argocd-server
	kubectl rollout restart -n argocd statefulset argocd-application-controller

	kubectl rollout status --watch statefulset/argocd-application-controller -n argocd --timeout=600s

	- name: Install Kubevirt
	run: \|
	function is_nested_virt_enabled() {
	kvm_nested="unknown"
	if [ -f "/sys/module/kvm_intel/parameters/nested" ]; then
	kvm_nested=$( cat /sys/module/kvm_intel/parameters/nested )
	elif [ -f "/sys/module/kvm_amd/parameters/nested" ]; then
	kvm_nested=$( cat /sys/module/kvm_amd/parameters/nested )
	fi
	[ "$kvm_nested" == "1" ] \|\| [ "$kvm_nested" == "Y" ] \|\| [ "$kvm_nested" == "y" ]
	}

	echo "Deploying KubeVirt"
	kubectl apply -f "https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_VERSION}/kubevirt-operator.yaml"
	kubectl apply -f "https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_VERSION}/kubevirt-cr.yaml"

	echo "Configuring Kubevirt to use emulation if needed"
	if ! is_nested_virt_enabled; then
	kubectl -n kubevirt patch kubevirt kubevirt --type=merge --patch '{"spec":{"configuration":{"developerConfiguration":{"useEmulation":true}}}}'
	fi

	echo "Deploying KubeVirt containerized-data-importer"
	kubectl apply -f "https://github.com/kubevirt/containerized-data-importer/releases/download/${KUBEVIRT_CDI_VERSION}/cdi-operator.yaml"
	kubectl apply -f "https://github.com/kubevirt/containerized-data-importer/releases/download/${KUBEVIRT_CDI_VERSION}/cdi-cr.yaml"

	echo "Waiting for KubeVirt to be ready"
	kubectl wait --for=condition=Available kubevirt kubevirt --namespace=kubevirt --timeout=5m

	- name: Give more RBAC to Virt resources and RW for PVC
	run: \|
	echo "Patch the StorageProfile to use the storageclass standard and give ReadWrite access"
	kubectl get StorageProfile
	kubectl patch --type merge -p '{"spec": {"claimPropertySets": [{"accessModes": ["ReadWriteOnce"], "volumeMode": "Filesystem"}]}}' StorageProfile standard

	kubectl create clusterrolebinding pod-kubevirt-viewer --clusterrole=kubevirt.io:view --serviceaccount=${KUBE_NAMESPACE}:default
	kubectl create clusterrolebinding cdi-kubevirt-viewer --clusterrole=cdi.kubevirt.io:view --serviceaccount=${KUBE_NAMESPACE}:default
	kubectl create clusterrolebinding quarkus-dev --clusterrole=admin --serviceaccount=${KUBE_NAMESPACE}:default

	# Give RBAC to the SA argocd-server of the namespace argocd to access Applications running in another namespaces
	kubectl create clusterrolebinding argocd-server-applications --clusterrole=argocd-applicationset-controller --serviceaccount=argocd:argocd-server

	- name: Install our Fedora podman image
	run: \|
	kubectl create ns vm-images
	kubectl apply -n vm-images -f manifest/installation/virt/quay-to-pvc-datavolume.yml
	kubectl wait datavolume -n vm-images podman-remote --for condition=Ready=True --timeout=360s

	- name: Create ssh key, secret and VM
	run: \|
	ssh-keygen -N "" -f id_rsa
	kubectl create secret generic quarkus-dev-ssh-key -n ${KUBE_NAMESPACE} --from-file=key=id_rsa.pub

	MANIFEST_PATH=./manifest/installation/virt
	kustomize build ${MANIFEST_PATH} \| kubectl apply -n ${KUBE_NAMESPACE} -f -
	kubectl wait --for=condition=Ready vm/quarkus-dev -n ${KUBE_NAMESPACE} --timeout=360s


	- name: Set idp env variables
	run: \|
	GITEA_HOST=$(kubectl get ingress/my-gitea -n gitea -ojson \| jq -r '.spec.rules[0].host')
	GITEA_USERNAME=$(idpbuilder get secrets -p gitea -ojson \| jq -r '.[].data.username')
	GITEA_PASSWORD=$(idpbuilder get secrets -p gitea -ojson \| jq -r '.[].data.password')

	echo "GITEA_HOST=$GITEA_HOST:8443" >> "$GITHUB_ENV"
	echo "GITEA_USERNAME=$GITEA_USERNAME" >> "$GITHUB_ENV"
	echo "GITEA_PASSWORD=$GITEA_PASSWORD" >> "$GITHUB_ENV"

	ARGOCD_USERNAME=$(idpbuilder get secrets -p argocd -ojson \| jq -r '.[].data.username')
	ARGOCD_PASSWORD=$(idpbuilder get secrets -p argocd -ojson \| jq -r '.[].data.password')
	ARGOCD_SERVER_HOST=$(kubectl get ingress/argocd-server-ingress -n argocd -ojson \| jq -r '.spec.rules[0].host')
	ARGOCD_SERVER_PORT=8443
	ARGOCD_SERVER_URL="https://$ARGOCD_SERVER_HOST:$ARGOCD_SERVER_PORT"

	echo "ARGOCD_SERVER_HOST=$ARGOCD_SERVER_HOST" >> "$GITHUB_ENV"
	echo "ARGOCD_SERVER_PORT=$ARGOCD_SERVER_PORT" >> "$GITHUB_ENV"
	echo "ARGOCD_SERVER_URL=$ARGOCD_SERVER_URL" >> "$GITHUB_ENV"
	echo "ARGOCD_USERNAME=$ARGOCD_USERNAME" >> "$GITHUB_ENV"
	echo "ARGOCD_PASSWORD=$ARGOCD_PASSWORD" >> "$GITHUB_ENV"

	KUBERNETES_TOKEN=$(kubectl get secret backstage-secret -n backstage -o json \| jq -r '.data.token' \| base64 -d)
	echo "KUBERNETES_TOKEN=$KUBERNETES_TOKEN" >> "$GITHUB_ENV"

	BACKSTAGE_AUTH_SECRET=$(node -p 'require("crypto").randomBytes(24).toString("base64")')
	echo "BACKSTAGE_AUTH_SECRET=$BACKSTAGE_AUTH_SECRET" >> "$GITHUB_ENV"

	echo "Get node IP and port to access it"
	CLUSTER_IP=$(docker inspect localdev-control-plane \| jq -r '.[].NetworkSettings.Ports."6443/tcp"[0].HostIp')
	CLUSTER_PORT=$(docker inspect localdev-control-plane \| jq -r '.[].NetworkSettings.Ports."6443/tcp"[0].HostPort')
	API_URL="https://$CLUSTER_IP:$CLUSTER_PORT"
	echo "API_URL=$API_URL" >> "$GITHUB_ENV"

	- name: Logon and configure ArgoCD server
	run: \|
	argocd --insecure login $ARGOCD_SERVER_HOST:$ARGOCD_SERVER_PORT --username $ARGOCD_USERNAME --password $ARGOCD_PASSWORD

	echo "Allow argocd to handle applications in all the namespaces"
	argocd proj add-source-namespace default '*'

	#- name: Create the secret containing the creds to write an image on Gitea
	# run: \|
	# GITEA_CREDS=$(echo -n "$GITEA_USERNAME:$GITEA_PASSWORD" \| base64 -w 0)
	#
	# cat <<EOF > config.json
	# {
	# "auths": {
	# "$GITEA_HOST": {
	# "auth": "$GITEA_CREDS"
	# }
	# }
	# }
	# EOF
	# kubectl create secret generic dockerconfig-secret -n demo --from-file=config.json

	#- name: Create the gitea organization
	# run: \|
	# echo "Creating the organization: $REPO_ORG on $GITEA_HOST"
	# curl -k -X POST \
	# "https://$GITEA_HOST/api/v1/orgs" \
	# -H 'accept: application/json' \
	# -H 'Content-Type: application/json' \
	# -u "$GITEA_USERNAME:$GITEA_PASSWORD" \
	# -d '{"username": "'"$REPO_ORG"'"}'

	- name: Create the secret containing the creds to write an image on Quay.io and org qshift
	env:
	QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
	QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}
	run: \|
	QUAY_CREDS=$(echo -n "$QUAY_USERNAME:$QUAY_ROBOT_TOKEN" \| base64 -w 0)

	cat <<EOF > config.json
	{
	"auths": {
	"quay.io/$QUAY_ORG": {
	"auth": "$QUAY_CREDS"
	}
	}
	}
	EOF
	kubectl create secret generic dockerconfig-secret -n demo --from-file=config.json

	- name: Rollout ArgoCD as we created a demo namespace where Applications will be deployed
	run: \|
	echo "Rollout Argocd to take into consideration the new namespace created ..."
	kubectl rollout restart -n argocd deployment argocd-server
	kubectl rollout restart -n argocd statefulset argocd-application-controller

	kubectl rollout status --watch statefulset/argocd-application-controller -n argocd --timeout=600s

	- uses: actions/setup-node@v4
	with:
	node-version: ${{ env.NODE_VERSION }}

	- run: corepack enable

	- name: Install & build QShift
	run: \|
	yarn --immutable
	yarn tsc
	yarn build:all

	- name: Configure app-config.local.yaml file
	env:
	GITHUB_PERSONAL_ACCESS_TOKEN: ${{ secrets.BACKSTAGE_GITHUB_TOKEN }}
	run: \|
	cat <<EOF > backstage_env_secret.env
	BACKSTAGE_APP_BASE_URL=http://localhost:3000
	BACKSTAGE_BACKEND_BASE_URL=http://localhost:7007
	BACKSTAGE_AUTH_SECRET=$BACKSTAGE_AUTH_SECRET
	#TEMPLATE_URL=https://github.com/q-shift/backstage-playground/blob/main/locations/root.yaml
	ARGOCD_SERVER=$ARGOCD_SERVER_URL
	ARGOCD_ADMIN_USER=$ARGOCD_USERNAME
	ARGOCD_ADMIN_PASSWORD=$ARGOCD_PASSWORD
	GITHUB_PERSONAL_ACCESS_TOKEN=$GITHUB_PERSONAL_ACCESS_TOKEN
	GITEA_HOST=$GITEA_HOST
	GITEA_USERNAME=$GITEA_USERNAME
	GITEA_PASSWORD="$GITEA_PASSWORD"
	KUBERNETES_API_URL=$API_URL
	KUBERNETES_TOKEN=$KUBERNETES_TOKEN
	EOF

	export $(grep -v '^#' backstage_env_secret.env \| xargs)
	envsubst < manifest/templates/app-config.qshift.tmpl > app-config.local.yaml

	cat app-config.local.yaml

	- name: Delete application repository
	# WARNING: Don't run jobs in // as both will create/push or delete content from same github repository
	run: \|
	if gh repo view $REPO_ORG/$REPO_NAME > /dev/null 2>&1; then
	echo "Repository '$REPO_ORG/$REPO_NAME' exists."
	gh repo delete $REPO_ORG/$REPO_NAME --yes
	fi
	env:
	GH_TOKEN: ${{ secrets.WORKFLOW_GITHUB_TOKEN }}

	- name: Start backstage
	# https://github.com/JarvusInnovations/background-action
	uses: JarvusInnovations/background-action@v1
	with:
	run: \|
	export NODE_OPTIONS=--no-node-snapshot
	export NODE_TLS_REJECT_UNAUTHORIZED=0
	yarn dev
	tail: true
	log-output: true
	wait-on: http://localhost:7007/settings

	- name: Check backstage resources
	run: \|
	get_entities_length() {
	curl -s -H "Authorization: Bearer $BACKSTAGE_AUTH_SECRET" http://localhost:7007/api/catalog/entities?filter=kind=location \| jq '. \| length'
	}

	until [[ $(get_entities_length) -gt 2 ]]; do
	echo "Wait till locations are generated ..."
	length=$(get_entities_length)
	echo "Current length: $length"
	sleep 5
	done

	echo "Show the locations ..."
	curl -s "http://localhost:7007/api/catalog/entities?filter=kind=location" \
	-H 'Content-Type: application/json' \
	-H "Authorization: Bearer $BACKSTAGE_AUTH_SECRET" \
	--compressed \
	\| jq -r

	- name : Scaffold a project and check task events
	run: \|
	DATA_TEST_PATH=.github/test-data
	RESPONSE=$(curl -s 'http://localhost:7007/api/scaffolder/v2/tasks' \
	-X POST \
	-H 'Content-Type: application/json' \
	-H "Authorization: Bearer $BACKSTAGE_AUTH_SECRET" \
	-d @${DATA_TEST_PATH}/quarkus-app-3.15-db.json)

	echo $RESPONSE

	retries=10
	until [[ $retries == 0 ]]; do
	TASK_ID=$(echo $RESPONSE \| jq -r '.id')
	EVENT_COMPLETION=$(curl -s -H 'Content-Type: application/json' -H "Authorization: Bearer $BACKSTAGE_AUTH_SECRET" "http://localhost:7007/api/scaffolder/v2/tasks/$TASK_ID/events" \| jq -r '.[] \| select(.type=="completion")')

	if [[ $EVENT_COMPLETION ]]; then
	if [[ $(echo $EVENT_COMPLETION \| jq -r '.body.message') == "failed" ]]; then
	echo "#### Scaffolding failed ! ####"
	curl -s -H 'Content-Type: application/json' -H "Authorization: Bearer $BACKSTAGE_AUTH_SECRET" "http://localhost:7007/api/scaffolder/v2/tasks/$TASK_ID/events" \| jq -r '.'
	exit 1;
	else
	echo "#### Scaffolding succeeded :-) ####"
	curl -s -H 'Content-Type: application/json' -H "Authorization: Bearer $BACKSTAGE_AUTH_SECRET" "http://localhost:7007/api/scaffolder/v2/tasks/$TASK_ID/events" \| jq -r '.'
	break
	fi
	fi

	sleep 15s
	retries=$((retries - 1))
	done

	- name: Check backstage components
	run: \|
	echo "Show the components ..."
	get_entities_length() {
	curl -s -H "Authorization: Bearer $BACKSTAGE_AUTH_SECRET" http://localhost:7007/api/catalog/entities?filter=kind=component \| jq '. \| length'
	}

	until [[ $(get_entities_length) -gt 0 ]]; do
	echo "Wait till component is created ..."
	echo "Component(s) length: $(get_entities_length)"
	sleep 10
	done

	curl -s "http://localhost:7007/api/catalog/entities?filter=kind=component" \
	-H 'Content-Type: application/json' \
	-H "Authorization: Bearer $BACKSTAGE_AUTH_SECRET" \
	--compressed \
	\| jq -r

	- name: Wait till the status of the Application parent is healthy
	run: \|
	SCRIPTS=$(pwd)/.github/scripts

	# echo "#########################################"
	# echo "Create manually the ArgoCD resources: project and application bootstrap ..."
	# echo "#########################################"
	# kubectl apply -f $(pwd)/.github/resources/argocd.yaml

	if ! $SCRIPTS/waitFor.sh application $REPO_NAME-bootstrap argocd Healthy; then
	echo "Failed to watch: application $REPO_NAME-bootstrap"
	exit 1;
	fi

	echo "#########################################"
	echo "Detail Application(s) information"
	echo "#########################################"
	kubectl get application -A

	- name: Wait till the children applications are synced too
	run: \|
	SCRIPTS=$(pwd)/.github/scripts

	echo "#########################################"
	echo "List the AppProject ..."
	echo "#########################################"
	kubectl get appproject -A

	echo "#########################################"
	echo "Show the AppProject ..."
	echo "#########################################"
	kubectl get appproject -A -oyaml

	echo "#########################################"
	echo "Log the Application YAML for $REPO_NAME-build-test-push ..."
	echo "#########################################"
	kubectl -n ${KUBE_NAMESPACE} get application $REPO_NAME-build-test-push -oyaml

	if ! $SCRIPTS/waitFor.sh application $REPO_NAME-build-test-push ${KUBE_NAMESPACE} Synced ; then
	$SCRIPTS/logApplication.sh $REPO_NAME build-test-push $KUBE_NAMESPACE
	exit 1;
	else
	echo "#########################################"
	echo "Show Application ${REPO_NAME}-build-test-push in namespace ${KUBE_NAMESPACE}"
	echo "#########################################"
	argocd app get ${KUBE_NAMESPACE}/${REPO_NAME}-build-test-push
	fi

	if ! $SCRIPTS/waitFor.sh application $REPO_NAME-db ${KUBE_NAMESPACE} Synced ; then
	$SCRIPTS/logApplication.sh $REPO_NAME db $KUBE_NAMESPACE
	exit 1;
	else
	echo "#########################################"
	echo "Show Application ${REPO_NAME}-db in namespace ${KUBE_NAMESPACE}"
	echo "#########################################"
	fi

	if ! $SCRIPTS/waitFor.sh application $REPO_NAME-deploy $KUBE_NAMESPACE Synced ; then
	$SCRIPTS/logApplication.sh $REPO_NAME deploy $KUBE_NAMESPACE
	exit 1;
	else
	echo "#########################################"
	echo "Show Application ${REPO_NAME}-deploy in namespace ${KUBE_NAMESPACE}"
	echo "#########################################"
	fi

	- name: Watch tekton resources ...
	run: \|
	get_pipelineruns_length() {
	tkn pipelinerun list -n ${KUBE_NAMESPACE} -o json \| jq -r '.items \| length'
	}

	echo "#########################################"
	echo "Pipelineruns starting ..."
	echo "#########################################"
	for i in `seq 30`; do
	if [[ $(get_pipelineruns_length) -gt 0 ]]; then
	tkn pipelinerun logs -n ${KUBE_NAMESPACE} $REPO_NAME-build-test-push -f
	else
	echo "#########################################"
	echo "Wait till the pipelineruns is running ..."
	echo "#########################################"
	kubectl get pods -n ${KUBE_NAMESPACE}

	sleep 30
	fi
	done

	- name: Wait till the application is running and log it
	run: \|
	SCRIPTS=$(pwd)/.github/scripts

	if ! $SCRIPTS/waitFor.sh deployment $REPO_NAME ${KUBE_NAMESPACE} READY; then
	echo "Failed to get deployment of $REPO_NAME in namespace ${KUBE_NAMESPACE}"
	exit 1;
	else
	echo "Show pods running within the namespace ${KUBE_NAMESPACE} ..."
	kubectl get pods -n ${KUBE_NAMESPACE}
	kubectl logs -n ${KUBE_NAMESPACE} -lapp.kubernetes.io/name=${REPO_NAME}
	fi

	- name: Execute if the job fails
	if: ${{ failure() }}
	run: \|
	SCRIPTS=$(pwd)/.github/scripts

	echo "#########################################"
	echo "Get pods ...."
	echo "#########################################"
	kubectl get pods -A

	echo "#########################################"
	echo "Show ArgoCD CM to verify if it includes parameters needed to work cross namespaces ..."
	echo "#########################################"
	kubectl get -n argocd cm/argocd-cmd-params-cm -oyaml

	$SCRIPTS/listShowDescribeResource.sh AppProject argocd

	$SCRIPTS/listShowDescribeResource.sh Applications ${KUBE_NAMESPACE}

	echo "#########################################"
	echo "Log ArgoCD application-controller ..."
	echo "#########################################"
	kubectl logs -n argocd -lapp.kubernetes.io/name=argocd-application-controller --tail=-1

	# $SCRIPTS/logTektonResources.sh

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

feat: add --replace option to qshift #108

Workflow file

feat: add --replace option to qshift #108

Jobs

Run details

Workflow file for this run