diff --git a/tests/ci/ci_test.sh b/tests/ci/ci_test.sh index fc2b4f1b45..b9b4164d38 100755 --- a/tests/ci/ci_test.sh +++ b/tests/ci/ci_test.sh @@ -1,96 +1,117 @@ #!/bin/bash -set -e -set -o pipefail +set -euo pipefail ARTIFACT_DIR="${ARTIFACT_DIR:-/tmp/artifacts}" -INFO="INFO" -log_info(){ - echo "$(date "+%Y-%m-%d %H:%M:%S") [$INFO] $1" +check_status() { + oc rollout status $1 -n $2 --timeout=5m || { + must_gather "$ARTIFACT_DIR" + exit 1 + } } -check_status(){ - res=$1 - namespace=$2 - log_info "Checking status of $res inside $namespace" - oc rollout status $res -n $namespace --timeout=5m || (must_gather "$ARTIFACT_DIR" && exit 1) -} -prereq(){ - log_info "Deploying prerequisites on cluster" - oc apply -f pre-requisites.yaml - oc create -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/master/bundle.yaml + +prereq() { + oc create -f pre-requisites.yaml + oc create -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/main/bundle.yaml oc process -f ../../resources/crds/observatorium-logs-crds-template.yaml | oc apply -f - } -minio(){ - log_info "Deploying resources inside minio namespace" + +minio() { oc create ns minio || true - sleep 5 - oc process -f ../minio-template.yaml -p MINIO_CPU_REQUEST=30m -p MINIO_CPU_LIMITS=50m -p MINIO_MEMORY_REQUEST=50Mi -p MINIO_MEMORY_LIMITS=100Mi --local -o yaml | sed -e 's/storage: [0-9].Gi/storage: 0.25Gi/g' | oc apply -n minio -f - - sleep 5 + oc wait --for=jsonpath='{.status.phase}=Active' namespace/minio --timeout=5s + oc process -f ../minio-template.yaml \ + -p MINIO_CPU_REQUEST=30m \ + -p MINIO_CPU_LIMITS=50m \ + -p MINIO_MEMORY_REQUEST=50Mi \ + -p MINIO_MEMORY_LIMITS=100Mi \ + --local -o yaml | \ + sed -e 's/storage: [0-9].Gi/storage: 0.25Gi/g' | \ + oc apply -n minio -f - check_status deployment/minio minio } -dex(){ - log_info "Deploying resources inside dex namespace" + +dex() { oc create ns dex || true - sleep 5 - oc process -f ../dex-template.yaml -p DEX_CPU_REQUEST=30m -p DEX_CPU_LIMITS=50m -p DEX_MEMORY_REQUEST=50Mi -p DEX_MEMORY_LIMITS=100Mi --local -o yaml | sed -e 's/storage: [0-9].Gi/storage: 0.25Gi/g' | oc apply -n dex -f - - sleep 5 + oc wait --for=jsonpath='{.status.phase}=Active' namespace/dex --timeout=5s + oc process -f ../dex-template.yaml \ + -p DEX_CPU_REQUEST=30m \ + -p DEX_CPU_LIMITS=50m \ + -p DEX_MEMORY_REQUEST=50Mi \ + -p DEX_MEMORY_LIMITS=100Mi \ + --local -o yaml | \ + sed -e 's/storage: [0-9].Gi/storage: 0.25Gi/g' | \ + oc apply -n dex -f - check_status deployment/dex dex } -observatorium_metrics(){ - log_info "Deploying resources inside observatorium-metrics namespace" + +observatorium_metrics() { oc create ns observatorium-metrics || true - sleep 5 - oc process -f ../observatorium-metrics-thanos-objectstorage-secret-template.yaml | oc apply --namespace observatorium-metrics -f - + oc wait --for=jsonpath='{.status.phase}=Active' namespace/observatorium-metrics --timeout=5s + oc process -f ../observatorium-metrics-thanos-objectstorage-secret-template.yaml | oc apply --namespace observatorium-metrics -f - oc apply -f ../observatorium-alertmanager-config-secret.yaml --namespace observatorium-metrics oc apply -f ../observatorium-cluster-role.yaml oc apply -f ../observatorium-cluster-role-binding.yaml oc apply --namespace observatorium-metrics -f ../observatorium-service-account.yaml - oc process --param-file=observatorium-metrics.ci.env -f ../../resources/services/observatorium-metrics-template.yaml | oc apply --namespace observatorium-metrics -f - - sleep 5 - resources=$(oc get statefulsets -o name -n observatorium-metrics ; oc get deployments -o name -n observatorium-metrics) - for res in $resources - do + oc process --param-file=observatorium-metrics.ci.env \ + -f ../../resources/services/observatorium-metrics-template.yaml | \ + oc apply --namespace observatorium-metrics -f - + resources=$( + oc get statefulsets -o name -n observatorium-metrics + oc get deployments -o name -n observatorium-metrics + ) + for res in $resources; do check_status $res observatorium-metrics done } -observatorium(){ - log_info "Deploying resources inside observatorium namespace" + +observatorium() { oc create ns observatorium || true - sleep 5 - oc apply -f ../observatorium-rules-objstore-secret.yaml --namespace observatorium - oc apply -f ../observatorium-rhobs-tenant-secret.yaml --namespace observatorium - oc process --param-file=observatorium.test.ci.env -f ../../resources/services/observatorium-template.yaml | oc apply --namespace observatorium -f - - sleep 5 - resources=$(oc get statefulsets -o name -n observatorium ; oc get deployments -o name -n observatorium) - for res in $resources - do + oc wait --for=jsonpath='{.status.phase}=Active' namespace/observatorium --timeout=5s + oc apply -f ../observatorium-rules-objstore-secret.yaml --namespace observatorium + oc apply -f ../observatorium-rhobs-tenant-secret.yaml --namespace observatorium + oc process --param-file=observatorium.test.ci.env \ + -f ../../resources/services/observatorium-template.yaml | \ + oc apply --namespace observatorium -f - + resources=$( + oc get statefulsets -o name -n observatorium + oc get deployments -o name -n observatorium + ) + for res in $resources; do check_status $res observatorium done } -telemeter(){ - log_info "Deploying resources inside telemeter namespace" + +telemeter() { oc create ns telemeter || true - sleep 5 - oc apply --namespace telemeter -f ../telemeter-token-refersher-oidc-secret.yaml - oc process --param-file=telemeter.ci.env -f ../../resources/services/telemeter-template.yaml | oc apply --namespace telemeter -f - - sleep 5 - resources=$(oc get statefulsets -o name -n telemeter ; oc get deployments -o name -n telemeter) - for res in $resources - do + oc wait --for=jsonpath='{.status.phase}=Active' namespace/telemeter --timeout=5s + oc apply --namespace telemeter -f ../telemeter-token-refersher-oidc-secret.yaml + oc process --param-file=telemeter.ci.env \ + -f ../../resources/services/telemeter-template.yaml | \ + oc apply --namespace telemeter -f - + resources=$( + oc get statefulsets -o name -n telemeter + oc get deployments -o name -n telemeter + ) + for res in $resources; do check_status $res telemeter done } -run_test(){ - log_info "Deploying observatorium-up for testing" + +run_test() { oc apply -n observatorium -f test-tenant.yaml oc apply -n observatorium -f rbac.yaml oc rollout restart deployment/observatorium-observatorium-api -n observatorium check_status deployment/observatorium-observatorium-api observatorium oc apply -n observatorium -f observatorium-up-metrics.yaml - oc wait --for=condition=complete --timeout=5m -n observatorium job/observatorium-up-metrics || (must_gather "$ARTIFACT_DIR" && exit 1) + oc wait --for=condition=complete --timeout=5m \ + -n observatorium job/observatorium-up-metrics || { + must_gather "$ARTIFACT_DIR" + exit 1 + } } + must_gather() { local artifact_dir="$1" @@ -118,8 +139,8 @@ must_gather() { oc get services --all-namespaces > "$artifact_dir/services" oc get endpoints --all-namespaces > "$artifact_dir/endpoints" } -case $1 in -metrics) + +ci.metrics() { prereq minio dex @@ -127,14 +148,46 @@ metrics) observatorium run_test telemeter - ;; -logs) +} + +ci.logs() { #TODO - ;; -traces) + : +} + +ci.traces(){ #TODO - ;; -*) - echo "usage: $(basename "$0") { metrics | logs | traces }" - ;; -esac + : +} + +ci.help() { + local fns=$(declare -F -p | cut -f3 -d ' ' | grep '^ci\.' | cut -f2- -d.) + read -d '^' -r docstring < + +task: +$(for fn in ${fns[@]};do printf " - %s\n" "${fn}";done) +^ +EOF_HELP + echo -e "$docstring" + exit 1 +} + +is_function() { + local fn=$1 + [[ $(type -t "$fn") == "function" ]] + return $? +} + +main() { + local fn=${1:-''} + local ci_fn="ci.$fn" + if ! is_function "$ci_fn"; then + ci.help + fi + $ci_fn + return $? +} +main "$@" + diff --git a/tests/ci/observatorium-metrics.ci.env b/tests/ci/observatorium-metrics.ci.env index ebaaec883d..bb1b760dc3 100644 --- a/tests/ci/observatorium-metrics.ci.env +++ b/tests/ci/observatorium-metrics.ci.env @@ -1,14 +1,7 @@ SERVICE_ACCOUNT_NAME=observatorium THANOS_S3_SECRET=thanos-test-s3 JAEGER_AGENT_IMAGE=jaegertracing/jaeger-agent - -THANOS_QUERIER_REPLICAS=1 -THANOS_QUERY_FRONTEND_REPLICAS=1 -THANOS_RECEIVE_REPLICAS=3 -THANOS_RULER_REPLICAS=1 -THANOS_STORE_BUCKET_CACHE_REPLICAS=1 -THANOS_STORE_INDEX_CACHE_REPLICAS=1 -THANOS_STORE_REPLICAS=1 +STORAGE_CLASS=kubevirt-hostpath-provisioner MEMCACHED_EXPORTER_MEMORY_REQUEST=200Mi MEMCACHED_CPU_LIMIT=50m @@ -23,6 +16,9 @@ THANOS_COMPACTOR_CPU_LIMIT=50m THANOS_COMPACTOR_CPU_REQUEST=50m THANOS_COMPACTOR_MEMORY_LIMIT=200Mi THANOS_COMPACTOR_MEMORY_REQUEST=200Mi +THANOS_COMPACTOR_PVC_REQUEST=0.25Gi +THANOS_QUERIER_REPLICAS=1 +THANOS_QUERY_FRONTEND_REPLICAS=1 THANOS_QUERIER_CPU_LIMIT=50m THANOS_QUERIER_CPU_REQUEST=50m THANOS_QUERIER_MEMORY_LIMIT=200Mi @@ -33,29 +29,36 @@ THANOS_QUERY_FRONTEND_MEMORY_LIMIT=200Mi THANOS_QUERY_FRONTEND_MEMORY_REQUEST=200Mi THANOS_QUERY_FRONTEND_QUERY_CACHE_MEMCACHED_CPU_LIMIT=50m THANOS_QUERY_FRONTEND_QUERY_CACHE_MEMCACHED_CPU_REQUEST=50m - THANOS_QUERY_FRONTEND_QUERY_CACHE_MEMCACHED_MEMORY_LIMIT=200Mi THANOS_QUERY_FRONTEND_QUERY_CACHE_MEMCACHED_MEMORY_REQUEST=200Mi -THANOS_RECEIVE_CPU_LIMIT=50m -THANOS_RECEIVE_CPU_REQUEST=50m -THANOS_RECEIVE_MEMORY_LIMIT=200Mi -THANOS_RECEIVE_MEMORY_REQUEST=200Mi -THANOS_RULER_CPU_LIMIT=50m -THANOS_RULER_CPU_REQUEST=50m -THANOS_RULER_MEMORY_LIMIT=200Mi -THANOS_RULER_MEMORY_REQUEST=200Mi +THANOS_STORE_INDEX_CACHE_MEMCACHED_CPU_LIMIT=50m +THANOS_STORE_INDEX_CACHE_MEMCACHED_CPU_REQUEST=50m +THANOS_STORE_INDEX_CACHE_MEMCACHED_MEMORY_LIMIT=200Mi +THANOS_STORE_INDEX_CACHE_MEMCACHED_MEMORY_REQUEST=200Mi THANOS_STORE_BUCKET_CACHE_MEMCACHED_CPU_LIMIT=50m THANOS_STORE_BUCKET_CACHE_MEMCACHED_CPU_REQUEST=50m THANOS_STORE_BUCKET_CACHE_MEMCACHED_MEMORY_LIMIT=200Mi THANOS_STORE_BUCKET_CACHE_MEMCACHED_MEMORY_REQUEST=200Mi +THANOS_STORE_BUCKET_CACHE_REPLICAS=1 +THANOS_STORE_INDEX_CACHE_REPLICAS=1 +THANOS_STORE_REPLICAS=1 THANOS_STORE_CPU_LIMIT=50m THANOS_STORE_CPU_REQUEST=50m -THANOS_STORE_INDEX_CACHE_MEMCACHED_CPU_LIMIT=50m -THANOS_STORE_INDEX_CACHE_MEMCACHED_CPU_REQUEST=50m -THANOS_STORE_INDEX_CACHE_MEMCACHED_MEMORY_LIMIT=200Mi -THANOS_STORE_INDEX_CACHE_MEMCACHED_MEMORY_REQUEST=200Mi THANOS_STORE_MEMORY_LIMIT=200Mi THANOS_STORE_MEMORY_REQUEST=200Mi +THANOS_STORE_PVC_STORAGE=0.25Gi +THANOS_RECEIVE_CPU_LIMIT=50m +THANOS_RECEIVE_CPU_REQUEST=50m +THANOS_RECEIVE_MEMORY_LIMIT=200Mi +THANOS_RECEIVE_MEMORY_REQUEST=200Mi +THANOS_RECEIVE_REPLICAS=3 +THANOS_RECEIVE_PVC_STORAGE=0.25Gi +THANOS_RULER_CPU_LIMIT=50m +THANOS_RULER_CPU_REQUEST=50m +THANOS_RULER_MEMORY_LIMIT=200Mi +THANOS_RULER_MEMORY_REQUEST=200Mi +THANOS_RULER_REPLICAS=1 +THANOS_RULER_PVC_REQUEST=0.25Gi THANOS_VOLCANO_CPU_LIMIT=50m THANOS_VOLCANO_CPU_REQUEST=50m THANOS_VOLCANO_MEMORY_LIMIT=200Mi @@ -64,9 +67,4 @@ OBSERVATORIUM_ALERTMANAGER_CPU_REQUEST=30m OBSERVATORIUM_ALERTMANAGER_MEMORY_REQUEST=50Mi OBSERVATORIUM_ALERTMANAGER_MEMORY_LIMIT=100Mi OBSERVATORIUM_ALERTMANAGER_CPU_LIMIT=50m -THANOS_RECEIVE_PVC_STORAGE=0.25Gi -THANOS_RULER_PVC_REQUEST=0.25Gi -THANOS_COMPACTOR_PVC_REQUEST=0.25Gi OBSERVATORIUM_ALERTMANAGER_PVC_STORAGE=0.25Gi -THANOS_STORE_PVC_STORAGE=0.25Gi -STORAGE_CLASS=kubevirt-hostpath-provisioner diff --git a/tests/ci/observatorium.test.ci.env b/tests/ci/observatorium.test.ci.env index 6e9f1f5a72..c9e968d3a5 100644 --- a/tests/ci/observatorium.test.ci.env +++ b/tests/ci/observatorium.test.ci.env @@ -1,18 +1,25 @@ SERVICE_ACCOUNT_NAME=default JAEGER_AGENT_IMAGE=jaegertracing/jaeger-agent -GUBERNATOR_IMAGE=thrawn01/gubernator - -GUBERNATOR_REPLICAS=1 -OBSERVATORIUM_API_REPLICAS=1 -UP_REPLICAS=1 RULES_OBJSTORE_S3_SECRET=rules-objstore-s3 MANAGED_TENANTS=rhobs OBSERVATORIUM_URL=http://observatorium-observatorium-api.observatorium.svc.cluster.local:8080 -LOG_RULES_ENABLED=false + +GUBERNATOR_IMAGE=thrawn01/gubernator +GUBERNATOR_REPLICAS=1 GUBERNATOR_CPU_LIMIT=50m GUBERNATOR_CPU_REQUEST=30m GUBERNATOR_MEMORY_LIMIT=100Mi GUBERNATOR_MEMORY_REQUEST=50Mi +OBSERVATORIUM_API_REPLICAS=1 +OBSERVATORIUM_API_CPU_LIMIT=50m +OBSERVATORIUM_API_CPU_REQUEST=30m +OBSERVATORIUM_API_MEMORY_LIMIT=100Mi +OBSERVATORIUM_API_MEMORY_REQUEST=50Mi +UP_REPLICAS=1 +UP_CPU_REQUEST=30m +UP_CPU_LIMIT=50m +UP_MEMORY_REQUEST=50Mi +UP_MEMORY_LIMIT=100Mi MEMCACHED_CPU_LIMIT=50m MEMCACHED_CPU_REQUEST=30m MEMCACHED_EXPORTER_CPU_LIMIT=50m @@ -24,15 +31,8 @@ OAUTH_PROXY_CPU_LIMITS=50m OAUTH_PROXY_CPU_REQUEST=30m OAUTH_PROXY_MEMORY_LIMITS=100Mi OAUTH_PROXY_MEMORY_REQUEST=50Mi -OBSERVATORIUM_API_CPU_LIMIT=50m -OBSERVATORIUM_API_CPU_REQUEST=30m -OBSERVATORIUM_API_MEMORY_LIMIT=100Mi -OBSERVATORIUM_API_MEMORY_REQUEST=50Mi OPA_AMS_CPU_LIMIT=50m OPA_AMS_CPU_REQUEST=30m OPA_AMS_MEMORY_LIMIT=100Mi OPA_AMS_MEMORY_REQUEST=50Mi -UP_CPU_REQUEST=30m -UP_CPU_LIMIT=50m -UP_MEMORY_REQUEST=50Mi -UP_MEMORY_LIMIT=100Mi + diff --git a/tests/ci/telemeter.ci.env b/tests/ci/telemeter.ci.env index 97872f8023..8aae1c70d2 100644 --- a/tests/ci/telemeter.ci.env +++ b/tests/ci/telemeter.ci.env @@ -1,6 +1,6 @@ SERVICE_ACCOUNT_NAME=default - TELEMETER_FORWARD_URL=http://observatorium-observatorium-api.observatorium.svc.cluster.local:8080/api/metrics/v1/telemeter/api/v1/receive + MEMCACHED_CPU_LIMIT=1 REPLICAS=1 MEMCACHED_CPU_LIMIT=50m