Enable gitops and targeted upgrades in development environment

WIP WIP fix varaible fix env vars Update e2e.sh (#1355) Update e2e.sh Improve logging in dev scripts (#1354) Update GitHub handle in OWNERS file (#1353) * Update GitHub handle Co-authored-by: Moritz Clasmeier <[email protected]> remove RHACS_STANDALONE var Update e2e.sh Fix precommit for flag fitops (#1358) Fix pre-commit Fix operator versions and pray that it works Fix gitops and fail early. Please pass Try to fix e2e tests push changes Fix tests Fix tests Skip upgrade test Add global image pull secrets create image pull secret WIP Increase memory limits explicit config for canary e2e & derease polling WIP
stackrox · Oct 20, 2023 · a601e84 · a601e84
1 parent 39cdb8d
commit a601e84
Show file tree

Hide file tree

Showing 26 changed files with 352 additions and 208 deletions.
diff --git a/.openshift-ci/tests/e2e.sh b/.openshift-ci/tests/e2e.sh
@@ -9,8 +9,10 @@ export GITROOT
 # shellcheck source=/dev/null
 source "${GITROOT}/dev/env/scripts/lib.sh"
 
-RUN_AUTH_E2E_DEFAULT="false"
-RUN_CENTRAL_E2E_DEFAULT="true"
+export RUN_AUTH_E2E_DEFAULT="false"
+export RUN_CENTRAL_E2E_DEFAULT="true"
+export RHACS_GITOPS_ENABLED="true"
+export RHACS_TARGETED_OPERATOR_UPGRADES="true"
 
 if [[ "${OPENSHIFT_CI:-}" == "true" ]]; then
     # We are running in an OpenShift CI context, configure accordingly.
@@ -34,10 +36,11 @@ if [[ "${OPENSHIFT_CI:-}" == "true" ]]; then
     export STATIC_TOKEN="${FLEET_STATIC_TOKEN:-}"
     export STATIC_TOKEN_ADMIN="${FLEET_STATIC_TOKEN_ADMIN:-}"
     export CLUSTER_TYPE="openshift-ci"
-    export GOARGS="-mod=mod" # For some reason we need this in the offical base images.
+    export GOARGS="-mod=mod" # For some reason we need this in the official base images.
     export GINKGO_FLAGS="--no-color -v"
     # When running in OpenShift CI, ensure we also run the auth E2E tests.
     RUN_AUTH_E2E_DEFAULT="true"
+    export INHERIT_IMAGEPULLSECRETS="true" # pragma: allowlist secret
 else
     log "Executing in local context"
 fi
@@ -158,6 +161,7 @@ if [[ "$DUMP_LOGS" == "true" ]]; then
 
     log "** BEGIN OPERATOR STATE **"
     $KUBECTL -n "$STACKROX_OPERATOR_NAMESPACE" get pods || true
+    $KUBECTL -n "$STACKROX_OPERATOR_NAMESPACE" get pods -o yaml || true
     $KUBECTL -n "$STACKROX_OPERATOR_NAMESPACE" describe pods || true
     $KUBECTL -n "$STACKROX_OPERATOR_NAMESPACE" get subscriptions || true
     $KUBECTL -n "$STACKROX_OPERATOR_NAMESPACE" describe subscriptions || true

diff --git a/dev/env/defaults/00-defaults.env b/dev/env/defaults/00-defaults.env
@@ -21,7 +21,7 @@ export IGNORE_REPOSITORY_DIRTINESS_DEFAULT="false"
 export ENABLE_DB_PORT_FORWARDING_DEFAULT="false"
 export ENABLE_FM_PORT_FORWARDING_DEFAULT="false"
 export OPENSHIFT_MARKETPLACE_DEFAULT="false"
-export INSTALL_OPERATOR_DEFAULT="true"
+export INSTALL_OPERATOR_DEFAULT="false"
 export INSTALL_OPENSHIFT_ROUTER_DEFAULT="true"
 
 export DATABASE_HOST_DEFAULT="db"
@@ -49,7 +49,7 @@ export IMAGE_PULL_DOCKER_CONFIG_DEFAULT=""
 export SPAWN_LOGGER_DEFAULT="false"
 export DUMP_LOGS_DEFAULT="false"
 export OPERATOR_SOURCE_DEFAULT=""
-export INSTALL_OLM_DEFAULT="true"
+export INSTALL_OLM_DEFAULT="false"
 export FINAL_TEAR_DOWN_DEFAULT="false"
 export DOCKER_CONFIG_DEFAULT="${GITROOT}/.docker"
 export SKIP_TESTS_DEFAULT="false"
@@ -63,6 +63,5 @@ export RHACS_OPERATOR_RESOURCES_DEFAULTS='{"requests":{"cpu":"200m","memory":"30
 
 export ENABLE_EXTERNAL_CONFIG_DEFAULT="true"
 export AWS_AUTH_HELPER_DEFAULT=""
-export RHACS_TARGETED_OPERATOR_UPGRADES_DEFAULT="false"
-export RHACS_STANDALONE_MODE_DEFAULT="false"
+export RHACS_TARGETED_OPERATOR_UPGRADES_DEFAULT="true"
 export RHACS_GITOPS_ENABLED_DEFAULT="true"
diff --git a/dev/env/defaults/10-operator-source.env b/dev/env/defaults/10-operator-source.env
@@ -5,6 +5,6 @@ if is_openshift_cluster "${CLUSTER_TYPE:-}"; then
 else
     export OPERATOR_SOURCE_DEFAULT="quay"
     if [[ "${INSTALL_OPERATOR:-$INSTALL_OPERATOR_DEFAULT}" == "true" ]]; then
-        export INSTALL_OLM_DEFAULT="true"
+        export INSTALL_OLM_DEFAULT="false"
     fi
 fi
diff --git a/dev/env/defaults/cluster-type-colima/env b/dev/env/defaults/cluster-type-colima/env
@@ -2,7 +2,7 @@ export ENABLE_DB_PORT_FORWARDING_DEFAULT="true"
 export ENABLE_FM_PORT_FORWARDING_DEFAULT="true"
 export OPERATOR_SOURCE_DEFAULT="quay"
 export INHERIT_IMAGEPULLSECRETS_DEFAULT="true" # pragma: allowlist secret
-export INSTALL_OLM_DEFAULT="true"
+export INSTALL_OLM_DEFAULT="false"
 
 if grep -q "runtime: docker" <(colima status 2>&1); then
   export DOCKER_HOST="unix://$HOME/.colima/docker.sock"

diff --git a/dev/env/defaults/cluster-type-docker/env b/dev/env/defaults/cluster-type-docker/env
@@ -2,5 +2,6 @@ export ENABLE_DB_PORT_FORWARDING_DEFAULT="true"
 export ENABLE_FM_PORT_FORWARDING_DEFAULT="true"
 export OPERATOR_SOURCE_DEFAULT="quay"
 export INHERIT_IMAGEPULLSECRETS_DEFAULT="true" # pragma: allowlist secret
-export INSTALL_OLM_DEFAULT="true"
+export INSTALL_OLM_DEFAULT="false"
+export ENABLE_EXTERNAL_CONFIG_DEFAULT="true"
 export AWS_AUTH_HELPER_DEFAULT="aws-saml"
diff --git a/dev/env/defaults/cluster-type-kind/env b/dev/env/defaults/cluster-type-kind/env
@@ -3,5 +3,6 @@ export ENABLE_DB_PORT_FORWARDING_DEFAULT="true"
 export ENABLE_FM_PORT_FORWARDING_DEFAULT="true"
 export OPERATOR_SOURCE_DEFAULT="quay"
 export INHERIT_IMAGEPULLSECRETS_DEFAULT="true" # pragma: allowlist secret
-export INSTALL_OLM_DEFAULT="true"
+export INSTALL_OLM_DEFAULT="false"
+export ENABLE_EXTERNAL_CONFIG_DEFAULT="true"
 export AWS_AUTH_HELPER_DEFAULT="aws-saml"
diff --git a/dev/env/defaults/cluster-type-minikube/env b/dev/env/defaults/cluster-type-minikube/env
@@ -3,5 +3,6 @@ export ENABLE_DB_PORT_FORWARDING_DEFAULT="true"
 export ENABLE_FM_PORT_FORWARDING_DEFAULT="true"
 export OPERATOR_SOURCE_DEFAULT="quay"
 export INHERIT_IMAGEPULLSECRETS_DEFAULT="true" # pragma: allowlist secret
-export INSTALL_OLM_DEFAULT="true"
+export INSTALL_OLM_DEFAULT="false"
+export ENABLE_EXTERNAL_CONFIG_DEFAULT="true"
 export AWS_AUTH_HELPER_DEFAULT="aws-saml"
diff --git a/dev/env/defaults/cluster-type-rancher-desktop/env b/dev/env/defaults/cluster-type-rancher-desktop/env
@@ -3,7 +3,7 @@ export ENABLE_DB_PORT_FORWARDING_DEFAULT="true"
 export ENABLE_FM_PORT_FORWARDING_DEFAULT="true"
 export OPERATOR_SOURCE_DEFAULT="quay"
 export INHERIT_IMAGEPULLSECRETS_DEFAULT="true" # pragma: allowlist secret
-export INSTALL_OLM_DEFAULT="true"
+export INSTALL_OLM_DEFAULT="false"
 export RANCHER_DESKTOP_BIN=${RANCHER_DESKTOP_BIN:-"${HOME}/.rd/bin"}
 export KUBECTL_DEFAULT="${RANCHER_DESKTOP_BIN}/kubectl"
 export DOCKER_DEFAULT="${RANCHER_DESKTOP_BIN}/docker"

diff --git a/dev/env/manifests/fleet-manager/02-fleet-manager-deployment.yaml b/dev/env/manifests/fleet-manager/02-fleet-manager-deployment.yaml
@@ -43,36 +43,10 @@ spec:
             - containerPort: 8083
             - containerPort: 8080
           env:
-          - name: CENTRAL_CPU_REQUEST
-            value: 200m
-          - name: CENTRAL_MEMORY_REQUEST
-            value: 300M
-          - name: CENTRAL_CPU_LIMIT
-            value: 200m
-          - name: CENTRAL_MEMORY_LIMIT
-            value: 300M
-          - name: SCANNER_ANALYZER_CPU_REQUEST
-            value: 200m
-          - name: SCANNER_ANALYZER_MEMORY_REQUEST
-            value: 300M
-          - name: SCANNER_ANALYZER_CPU_LIMIT
-            value: 200m
-          - name: SCANNER_ANALYZER_MEMORY_LIMIT
-            value: 300M
-          - name: SCANNER_ANALYZER_AUTOSCALING
-            value: Disabled
-          - name: SCANNER_ANALYZER_REPLICAS
-            value: "1"
-          - name: SCANNER_DB_CPU_REQUEST
-            value: 200m
-          - name: SCANNER_DB_MEMORY_REQUEST
-            value: 200M
-          - name: SCANNER_DB_CPU_LIMIT
-            value: 200m
-          - name: SCANNER_DB_MEMORY_LIMIT
-            value: 200M
           - name: RHACS_GITOPS_ENABLED
             value: "$RHACS_GITOPS_ENABLED"
+          - name: RHACS_TARGETED_OPERATOR_UPGRADES
+            value: "true"
           readinessProbe:
             httpGet:
               path: /healthcheck

diff --git a/dev/env/manifests/fleet-manager/04-gitops-config.yaml b/dev/env/manifests/fleet-manager/04-gitops-config.yaml
@@ -13,32 +13,25 @@ data:
 
       operators:
 
-        - deploymentName: "rhacs-operator-4.2.1-rc.3"
-          image: "quay.io/rhacs-eng/stackrox-operator:4.2.1-rc.3"
-          centralLabelSelector: "rhacs.redhat.com/version-selector=4.2.1-rc.3"
+        - deploymentName: "rhacs-operator-4.2.0-366-g069902f3f9"
+          image: "quay.io/rhacs-eng/stackrox-operator:4.2.0-366-g069902f3f9"
+          centralLabelSelector: "rhacs.redhat.com/version-selector=4.2.0-366-g069902f3f9"
           securedClusterReconcilerEnabled: false
-          resources:
-            requests:
-              cpu: 100m
-              memory: 200Mi
 
         - deploymentName: "rhacs-operator-4.2.2-rc.0"
           image: "quay.io/rhacs-eng/stackrox-operator:4.2.2-rc.0"
           centralLabelSelector: "rhacs.redhat.com/version-selector=4.2.2-rc.0"
           securedClusterReconcilerEnabled: false
-          resources:
-            requests:
-              cpu: 100m
-              memory: 200Mi
+
     centrals:
       overrides:
         - instanceIds:
             - "*"
           patch: |
-            # Set label for all centrals to 4.2.1-rc.3
+            # Set label for all centrals to 4.2.2-rc.0
             metadata:
               labels:
-                rhacs.redhat.com/version-selector: "4.2.1-rc.3"
+                rhacs.redhat.com/version-selector: "4.2.2-rc.0"
             # Adjust centrals for development environment
             spec:
               central:
@@ -47,7 +40,7 @@ data:
                     enabled: false
                 resources:
                   limits:
-                    cpu: null
+                    cpu: 500m
                     memory: 1Gi
                   requests:
                     cpu: 100m
@@ -56,16 +49,19 @@ data:
                 analyzer:
                   resources:
                     limits:
-                      cpu: null
+                      cpu: 500m
                       memory: 1Gi
                     requests:
-                      cpu: 100m
-                      memory: 500Mi
+                      cpu: 200m
+                      memory: 300Mi
+                  scaling:
+                    autoScaling: "Disabled"
+                    replicas: 1
                 db:
                   resources:
                     limits:
-                      cpu: null
+                      cpu: 500m
                       memory: 1Gi
                     requests:
-                      cpu: 100m
-                      memory: 500Mi
+                      cpu: 200m
+                      memory: 200Mi
diff --git a/dev/env/manifests/fleetshard-sync/02-fleetshard-sync-deployment.yaml b/dev/env/manifests/fleetshard-sync/02-fleetshard-sync-deployment.yaml
@@ -42,7 +42,7 @@ spec:
                   key: "rhsso-service-account-client-secret"
                   optional: false
             - name: RUNTIME_POLL_PERIOD
-              value: 10s
+              value: 2s
             - name: AUDIT_LOG_ENABLED
               value: "$AUDIT_LOG_ENABLED"
             - name: MANAGED_DB_ENABLED
@@ -67,6 +67,8 @@ spec:
                 secretKeyRef:
                   name: fleetshard-sync
                   key: "aws-secret-access-key"
+            - name: RHACS_TARGETED_OPERATOR_UPGRADES
+              value: "true"
             - name: TENANT_IMAGE_PULL_SECRET
               valueFrom:
                 secretKeyRef:

diff --git a/dev/env/scripts/bootstrap.sh b/dev/env/scripts/bootstrap.sh
@@ -30,6 +30,7 @@ Output:
 ${kc_output:-(no output)}"
 fi
 
+# Create Namespaces.
 apply "${MANIFESTS_DIR}/shared"
 wait_for_default_service_account "$ACSCS_NAMESPACE"
 
@@ -61,11 +62,6 @@ else
     apply "${MANIFESTS_DIR}/monitoring"
 fi
 
-if [[ "$RHACS_STANDALONE_MODE" == "true" ]]; then
-    log "Updating operator configmap to enable standalone mode"
-    apply "${MANIFESTS_DIR}/rhacs-operator/03-operators-config.yaml"
-fi
-
 if is_local_cluster "$CLUSTER_TYPE"; then
     if [[ ("$INSTALL_OPERATOR" == "true" && "$OPERATOR_SOURCE" == "quay") || "$FLEET_MANAGER_IMAGE" =~ ^quay.io/ ]]; then
         if docker_logged_in "quay.io"; then

diff --git a/dev/env/scripts/create-imagepullsecrets b/dev/env/scripts/create-imagepullsecrets
@@ -102,3 +102,15 @@ type: kubernetes.io/dockerconfigjson
 EOF
 )
 echo "$res" | $KUBECTL -n "$ACSCS_NAMESPACE" apply -f -
+
+if [[ "${OPENSHIFT_CI:-}" == "true" ]]; then
+    dir=$(mktemp -d)
+    global_pull_secret="$dir/global_pull_secret"
+
+    echo "Adding global image pull secret for upstream images..."
+    oc get secret/pull-secret -n openshift-config --template='{{index .data ".dockerconfigjson" | base64decode}}' > "$global_pull_secret"
+    oc registry login --registry="quay.io" --auth-basic="$username:$password" --to="$global_pull_secret"
+    oc set data secret/pull-secret -n openshift-config --from-file=.dockerconfigjson="$global_pull_secret"
+
+    rm "$global_pull_secret"
+fi
diff --git a/dev/env/scripts/docker.sh b/dev/env/scripts/docker.sh
@@ -115,5 +115,11 @@ preload_dependency_images() {
         docker_pull "${IMAGE_REGISTRY}/main:${CENTRAL_VERSION}"
         docker_pull "${IMAGE_REGISTRY}/central-db:${CENTRAL_VERSION}"
     fi
+
+    if [[ "$CLUSTER_TYPE"  == "kind" ]]; then
+        log "Ensuring operator images exist from dev GitOps config"
+        ensure_operator_image_exists.sh
+    fi
+
     log "Images preloaded"
 }
diff --git a/dev/env/scripts/ensure_operator_image_exists.sh b/dev/env/scripts/ensure_operator_image_exists.sh
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+operator_images=$(yq e '.data."config.yaml"' dev/env/manifests/fleet-manager/04-gitops-config.yaml | yq -o json -M | jq -r '.rhacsOperators.operators[].image')
+
+for operator_image in $operator_images; do
+    docker pull "$operator_image"
+    if [[ "$CLUSTER_TYPE" == "kind" ]]; then
+            kind load docker-image "$operator_image"
+    fi
+done
diff --git a/dev/env/scripts/install_operator.sh b/dev/env/scripts/install_operator.sh
@@ -7,6 +7,12 @@ source "${GITROOT}/dev/env/scripts/lib.sh"
 # shellcheck source=/dev/null
 source "${GITROOT}/dev/env/scripts/docker.sh"
 
+init
+
+if [[ "$RHACS_TARGETED_OPERATOR_UPGRADES" == "true" ]]; then
+  exit 0
+fi
+
 if [[ "$INSTALL_OLM" == "true" ]]; then
     if ! command -v operator-sdk >/dev/null 2>&1; then
         die "Error: Unable to install OLM, operator-sdk executable is not found"

diff --git a/dev/env/scripts/lib.sh b/dev/env/scripts/lib.sh
@@ -143,7 +143,6 @@ init() {
     export FLEET_MANAGER_IMAGE=${FLEET_MANAGER_IMAGE:-$FLEET_MANAGER_IMAGE_DEFAULT}
     export IGNORE_REPOSITORY_DIRTINESS=${IGNORE_REPOSITORY_DIRTINESS:-$IGNORE_REPOSITORY_DIRTINESS_DEFAULT}
     export RHACS_TARGETED_OPERATOR_UPGRADES=${RHACS_TARGETED_OPERATOR_UPGRADES:-$RHACS_TARGETED_OPERATOR_UPGRADES_DEFAULT}
-    export RHACS_STANDALONE_MODE=${RHACS_STANDALONE_MODE:-$RHACS_STANDALONE_MODE_DEFAULT}
     export RHACS_GITOPS_ENABLED=${RHACS_GITOPS_ENABLED:-$RHACS_GITOPS_ENABLED_DEFAULT}
 
     local fleet_manager_command="/usr/local/bin/fleet-manager serve --force-leader --api-server-bindaddress=0.0.0.0:8000 --health-check-server-bindaddress=0.0.0.0:8083 --kubeconfig=/secrets/kubeconfig --enable-central-external-certificate=$ENABLE_CENTRAL_EXTERNAL_CERTIFICATE --central-domain-name='$CENTRAL_DOMAIN_NAME'"

diff --git a/dev/env/scripts/reset b/dev/env/scripts/reset
@@ -22,4 +22,6 @@ apply "${GITROOT}/dev/env/manifests/rhacs-operator/00-namespace.yaml"
 log "Apply default gitops..."
 apply "${GITROOT}/dev/env/manifests/fleet-manager/04-gitops-config.yaml"
 
+$KUBECTL -n acscs delete pod -l application=fleetshard-sync
+
 log "Resetting complete."
diff --git a/dev/env/scripts/up.sh b/dev/env/scripts/up.sh
@@ -69,11 +69,6 @@ if [[ "$SPAWN_LOGGER" == "true" && -n "${LOG_DIR:-}" ]]; then
     $KUBECTL -n "$ACSCS_NAMESPACE" logs -l application=fleet-manager --all-containers --pod-running-timeout=1m --since=1m --tail=100 -f >"${LOG_DIR}/pod-logs_fleet-manager.txt" 2>&1 &
 fi
 
-if [[ "$RHACS_STANDALONE_MODE" == "true" ]]; then
-    log "Updating operator configmap"
-    apply "${MANIFESTS_DIR}/rhacs-operator/03-operators-config.yaml"
-fi
-
 log "Deploying fleetshard-sync"
 TENANT_IMAGE_PULL_SECRET=""
 if [[ "$INHERIT_IMAGEPULLSECRETS" == "true" ]]; then # pragma: allowlist secret