sync: rc-2023-11-16.1 to stage (#1474)

sync-branches: New code has just landed in rc-2023-11-16.1, so let's
bring stage up to speed!

.github/dependabot.yaml

@@ -21,3 +21,7 @@ updates:
     directory: "/probe"
     schedule:
       interval: "weekly"
+  - package-ecosystem: "docker"
+    directory: "/dp-terraform/helm"
+    schedule:
+      interval: "weekly"

.github/workflows/ci.yaml

@@ -1,5 +1,4 @@
-# This runs checks to verify if code is properly formatted and that tests (unit and integration against a mocked environment) are passing.
-name: Verify & Test
+name: CI
 
 on:
   push:
@@ -34,32 +33,46 @@ on:
       - 'pkg/api/openapi/docs/**'
       - 'pkg/api/openapi/.openapi-generator-ignore'
 
-# TODO make sure that the secrets are configured for your repository
-env:
-  # set ocm env to integration
-  OCM_ENV: integration
-  # Dummy SSO variables
-  SSO_CLIENT_ID: ${{ secrets.SSO_CLIENT_ID }}
-  SSO_CLIENT_SECRET: ${{ secrets.SSO_CLIENT_SECRET }}
-  OSD_IDP_SSO_CLIENT_ID: ${{ secrets.OSD_IDP_SSO_CLIENT_ID }}
-  OSD_IDP_SSO_CLIENT_SECRET: ${{ secrets.OSD_IDP_SSO_CLIENT_SECRET }}
-  # Dummy AWS credentials
-  AWS_ACCOUNT_ID: aws_accountid
-  AWS_ACCESS_KEY: aws_accesskey
-  AWS_SECRET_ACCESS_KEY: aws_secretaccesskey # pragma: allowlist secret - dummy value
-  ROUTE53_ACCESS_KEY: aws_route53_access_key # pragma: allowlist secret - dummy value
-  ROUTE53_SECRET_ACCESS_KEY: aws_route53_secret_access_key # pragma: allowlist secret - dummy value
-  # Dummy Central TLS env variables
-  CENTRAL_TLS_CERT: central_tls_cert # pragma: allowlist secret - dummy value
-  CENTRAL_TLS_KEY: central_tls_key # pragma: allowlist secret - dummy value
-  # So that OCM secrets are initialised
-  DOCKER_PR_CHECK: true
-  TEST_TIMEOUT: 30m
-
 jobs:
+  # This runs all pre-commit hooks defined within .pre-commit-config.yaml.
+  pre-commit:
+    name: "Run pre-commit hooks"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v3
+      - uses: actions/setup-go@v3
+        with:
+          go-version: "1.20"
+      - uses: pre-commit/[email protected]
+      - uses: pre-commit/[email protected]
+        name: Verify generated files are up-to-date
+        with:
+          extra_args: --hook-stage=manual --all-files
+
   verify-test:
     name: "Verify & Test"
     runs-on: ubuntu-latest
+    env:
+      # TODO make sure that the secrets are configured for your repository
+      OCM_ENV: integration
+      # Dummy SSO variables
+      SSO_CLIENT_ID: ${{ secrets.SSO_CLIENT_ID }}
+      SSO_CLIENT_SECRET: ${{ secrets.SSO_CLIENT_SECRET }}
+      OSD_IDP_SSO_CLIENT_ID: ${{ secrets.OSD_IDP_SSO_CLIENT_ID }}
+      OSD_IDP_SSO_CLIENT_SECRET: ${{ secrets.OSD_IDP_SSO_CLIENT_SECRET }}
+      # Dummy AWS credentials
+      AWS_ACCOUNT_ID: aws_accountid
+      AWS_ACCESS_KEY: aws_accesskey
+      AWS_SECRET_ACCESS_KEY: aws_secretaccesskey # pragma: allowlist secret - dummy value
+      ROUTE53_ACCESS_KEY: aws_route53_access_key # pragma: allowlist secret - dummy value
+      ROUTE53_SECRET_ACCESS_KEY: aws_route53_secret_access_key # pragma: allowlist secret - dummy value
+      # Dummy Central TLS env variables
+      CENTRAL_TLS_CERT: central_tls_cert # pragma: allowlist secret - dummy value
+      CENTRAL_TLS_KEY: central_tls_key # pragma: allowlist secret - dummy value
+      # So that OCM secrets are initialised
+      DOCKER_PR_CHECK: true
+      TEST_TIMEOUT: 30m
     services:
       postgres:
         image: postgres:11
@@ -106,21 +119,30 @@ jobs:
           export PATH=${PATH}:$GOPATH/bin
           make verify binary test test/integration
         timeout-minutes: 14
-      - name: Build and publish fleet-manager-tools image to quay.io
-        if: github.event_name == 'push'
-        env:
-          QUAY_USER: ${{ secrets.QUAY_RHACS_ENG_FM_RW_USERNAME }}
-          QUAY_TOKEN: ${{ secrets.QUAY_RHACS_ENG_FM_RW_PASSWORD }}
-          QUAY_IMAGE_REPOSITORY: rhacs-eng/fleet-manager-tools
-        run: |
-          chmod +x ./build_push_fleet_manager_tools.sh
-          ./build_push_fleet_manager_tools.sh
-      - name: Build and publish fleet* image to quay.io
+  build-push-images:
+    name: "Build and push fleet* images to quay.io"
+    runs-on: ubuntu-latest
+    needs: [pre-commit, verify-test]
+    # Skip for external contributions.
+    if: |
+      github.event_name == 'push' || !github.event.pull_request.head.repo.fork
+    steps:
+      - name: Login to Quay.io
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_RHACS_ENG_FM_RW_USERNAME }}
+          password: ${{ secrets.QUAY_RHACS_ENG_FM_RW_PASSWORD }}
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # Critical for correct image detection in Makefile
+      - name: Build and push fleet-manager-tools image to quay.io
         if: github.event_name == 'push'
         env:
-          QUAY_USER: ${{ secrets.QUAY_RHACS_ENG_FM_RW_USERNAME }}
-          QUAY_TOKEN: ${{ secrets.QUAY_RHACS_ENG_FM_RW_PASSWORD }}
-          QUAY_IMAGE_REPOSITORY: rhacs-eng/fleet-manager
-        run: |
-          chmod +x ./build_push_fleet_manager.sh
-          ./build_push_fleet_manager.sh
+          TAG: ${{ github.ref_name }}
+        run: make image/push/fleet-manager-tools
+      - name: Build and push fleetshard-operator image to quay.io
+        run: make image/push/fleetshard-operator
+      - name: Build and push fleet-manager image to quay.io
+        run: make image/push/fleet-manager

.secrets.baseline

@@ -322,7 +322,7 @@
         "filename": "dp-terraform/helm/rhacs-terraform/charts/observability/templates/01-operator-06-cr.yaml",
         "hashed_secret": "3e513f12b341ed3327bea645a728401b5d0f9ddb",
         "is_verified": false,
-        "line_number": 15
+        "line_number": 21
       }
     ],
     "dp-terraform/helm/rhacs-terraform/charts/secured-cluster/init-bundle.yaml": [
@@ -564,5 +564,5 @@
       }
     ]
   },
-  "generated_at": "2023-11-06T14:09:00Z"
+  "generated_at": "2023-11-13T13:31:40Z"
 }

Makefile

@@ -10,18 +10,10 @@ SHELL = bash
 binary:=fleet-manager
 
 # The image tag for building and pushing comes from TAG environment variable by default.
-# If there is no TAG env than CI_TAG is used instead.
-# Otherwise image tag is generated based on git tags.
+# Otherwise image tag is generated based on current commit hash.
+# The version should be a 7-char hash from git. This is what the deployment process in app-interface expects.
 ifeq ($(TAG),)
-ifeq (,$(wildcard CI_TAG))
-ifeq ($(IGNORE_REPOSITORY_DIRTINESS),true)
-TAG=$(shell git describe --tags --abbrev=10 --long)
-else
-TAG=$(shell git describe --tags --abbrev=10 --dirty --long)
-endif
-else
-TAG=$(shell cat CI_TAG)
-endif
+TAG=$(shell git rev-parse --short=7 HEAD)
 endif
 image_tag = $(TAG)
 
@@ -50,9 +42,6 @@ probe_image_repository:=$(PROBE_IMAGE_NAME)
 external_image_registry:= $(IMAGE_REGISTRY)
 internal_image_registry:=image-registry.openshift-image-registry.svc:5000
 
-# Test image name that will be used for PR checks
-test_image:=test/$(IMAGE_NAME)
-
 DOCKER ?= docker
 DOCKER_CONFIG ?= "${HOME}/.docker"
 
@@ -535,17 +524,17 @@ image/build/probe:
 .PHONY: image/build/probe
 
 image/build/fleet-manager-tools: GOOS=linux
-image/build/fleet-manager-tools: IMAGE_REF="$(external_image_registry)/rhacs-eng/fleet-manager-tools:$(image_tag)"
+image/build/fleet-manager-tools: IMAGE_REF="$(external_image_registry)/fleet-manager-tools:$(image_tag)"
 image/build/fleet-manager-tools: fleet-manager fleetshard-sync acsfleetctl
 	DOCKER_CONFIG=${DOCKER_CONFIG} $(DOCKER) build -t $(IMAGE_REF) -f Dockerfile.tools .
 	DOCKER_CONFIG=${DOCKER_CONFIG} $(DOCKER) tag $(IMAGE_REF) fleet-manager-tools:$(image_tag)
 .PHONY: image/build/multi-target/fleet-manager-tools
 
-image/push/fleet-manager-tools: IMAGE_REF="$(external_image_registry)/rhacs-eng/fleet-manager-tools:$(image_tag)"
+image/push/fleet-manager-tools: IMAGE_REF="$(external_image_registry)/fleet-manager-tools:$(image_tag)"
 image/push/fleet-manager-tools: image/build/fleet-manager-tools
 	DOCKER_CONFIG=${DOCKER_CONFIG} $(DOCKER) push $(IMAGE_REF)
 	@echo
-	@echo "Image fleet-manager tools was pushed as $(IMAGE_REF)."
+	@echo "Image fleet-manager-tools was pushed as $(IMAGE_REF)."
 .PHONY: image/push/fleet-manager-tools
 
 # Build and push the image
@@ -574,6 +563,16 @@ image/push/internal: docker/login/internal
 	$(DOCKER) push "$(shell oc get route default-route -n openshift-image-registry -o jsonpath="{.spec.host}")/$(probe_image_repository):$(IMAGE_TAG)"
 .PHONY: image/push/internal
 
+image/build/fleetshard-operator: IMAGE_REF="$(external_image_registry)/fleetshard-operator:$(image_tag)"
+image/build/fleetshard-operator:
+	$(DOCKER) build -t $(IMAGE_REF) ${PROJECT_PATH}/dp-terraform/helm
+.PHONY: image/build/fleetshard-operator
+
+image/push/fleetshard-operator: IMAGE_REF="$(external_image_registry)/fleetshard-operator:$(image_tag)"
+image/push/fleetshard-operator: image/build/fleetshard-operator
+	$(DOCKER) push $(IMAGE_REF)
+.PHONY: image/push/fleetshard-operator
+
 # Run the probe based e2e test in container
 test/e2e/probe/run: image/build/probe
 test/e2e/probe/run: IMAGE_REF="$(external_image_registry)/$(probe_image_repository):$(image_tag)"

build_push_fleet_manager.sh

@@ -42,7 +42,6 @@ make \
   DOCKER_CONFIG="${DOCKER_CONFIG}" \
   QUAY_USER="${QUAY_USER}" \
   QUAY_TOKEN="${QUAY_TOKEN}" \
-  TAG="${VERSION}" \
   external_image_registry="quay.io" \
   internal_image_registry="quay.io" \
   image_repository="${IMAGE_REPOSITORY}" \

build_push_probe.sh

@@ -39,7 +39,6 @@ make \
   DOCKER_CONFIG="${DOCKER_CONFIG}" \
   QUAY_PROBE_USER="${QUAY_USER}" \
   QUAY_PROBE_TOKEN="${QUAY_TOKEN}" \
-  TAG="${VERSION}" \
   external_image_registry="quay.io" \
   internal_image_registry="quay.io" \
   probe_image_repository="${IMAGE_REPOSITORY}" \

cmd/fleet-manager/main_test.go

@@ -38,15 +38,15 @@ func TestInjections(t *testing.T) {
 
 	var bootList []environments.BootService
 	env.MustResolve(&bootList)
-	Expect(len(bootList)).To(Equal(6))
+	Expect(len(bootList)).To(Equal(7))
 
 	_, ok := bootList[0].(*server.APIServer)
 	Expect(ok).To(Equal(true))
 	_, ok = bootList[1].(*server.MetricsServer)
 	Expect(ok).To(Equal(true))
-	_, ok = bootList[2].(*server.HealthCheckServer)
+	_, ok = bootList[3].(*server.HealthCheckServer)
 	Expect(ok).To(Equal(true))
-	_, ok = bootList[3].(*workers.LeaderElectionManager)
+	_, ok = bootList[4].(*workers.LeaderElectionManager)
 	Expect(ok).To(Equal(true))
 
 	var workerList []workers.Worker

deploy/helm/probe/deploy.sh

@@ -2,11 +2,12 @@
 set -euo pipefail
 
 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+ROOT_DIR="$SCRIPT_DIR/../../.."
 
 # shellcheck source=scripts/lib/external_config.sh
-source "$SCRIPT_DIR/../../../scripts/lib/external_config.sh"
+source "$ROOT_DIR/scripts/lib/external_config.sh"
 # shellcheck source=scripts/lib/helm.sh
-source "$SCRIPT_DIR/../../../scripts/lib/helm.sh"
+source "$ROOT_DIR/scripts/lib/helm.sh"
 
 if [[ $# -ne 2 ]]; then
     echo "Usage: $0 [environment] [cluster]" >&2
@@ -19,8 +20,7 @@ ENVIRONMENT=$1
 CLUSTER_NAME=$2
 PROBE_IMAGE_ORG="rhacs-eng"
 PROBE_IMAGE_NAME="blackbox-monitoring-probe-service"
-# Get HEAD for both main and production. This is the latest merged commit.
-PROBE_IMAGE_TAG="$(git rev-parse --short=7 HEAD)"
+PROBE_IMAGE_TAG="$(make --quiet --no-print-directory -C "${ROOT_DIR}" tag)"
 PROBE_IMAGE="quay.io/${PROBE_IMAGE_ORG}/${PROBE_IMAGE_NAME}:${PROBE_IMAGE_TAG}"
 
 init_chamber
@@ -57,9 +57,9 @@ if [[ $CLUSTER_ENVIRONMENT != "$ENVIRONMENT" ]]; then
 fi
 
 if [[ "${HELM_DRY_RUN:-}" == "true" ]]; then
-    "${SCRIPT_DIR}/../../../scripts/check_image_exists.sh" "${PROBE_IMAGE_ORG}" "${PROBE_IMAGE_NAME}" "${PROBE_IMAGE_TAG}" 0 || echo >&2 "Ignoring failed image check in dry-run mode."
+    "${ROOT_DIR}/scripts/check_image_exists.sh" "${PROBE_IMAGE_ORG}" "${PROBE_IMAGE_NAME}" "${PROBE_IMAGE_TAG}" 0 || echo >&2 "Ignoring failed image check in dry-run mode."
 else
-    "${SCRIPT_DIR}/../../../scripts/check_image_exists.sh" "${PROBE_IMAGE_ORG}" "${PROBE_IMAGE_NAME}" "${PROBE_IMAGE_TAG}"
+    "${ROOT_DIR}/scripts/check_image_exists.sh" "${PROBE_IMAGE_ORG}" "${PROBE_IMAGE_NAME}" "${PROBE_IMAGE_TAG}"
 fi
 
 load_external_config "cluster-${CLUSTER_NAME}" CLUSTER_

dev/config/dataplane-cluster-configuration-crc.yaml

@@ -11,7 +11,7 @@ clusters:
    region: standalone
    schedulable: true
    status: ready
-   central_instance_limit: 5
+   central_instance_limit: 9999
    provider_type: standalone
    supported_instance_type: "eval,standard"
    cluster_dns: apps-crc.testing

dev/config/dataplane-cluster-configuration-dockerdesktop.yaml

@@ -14,4 +14,4 @@ clusters:
    provider_type: standalone
    supported_instance_type: "eval,standard"
    cluster_dns: kubernetes.docker.internal
-   central_instance_limit: 5
+   central_instance_limit: 9999

dev/config/dataplane-cluster-configuration-kind.yaml

@@ -14,4 +14,4 @@ clusters:
    provider_type: standalone
    supported_instance_type: "eval,standard"
    cluster_dns: kubernetes.docker.internal
-   central_instance_limit: 5
+   central_instance_limit: 99999

dev/config/dataplane-cluster-configuration-minikube.yaml

@@ -24,7 +24,7 @@ clusters:
    region: standalone
    schedulable: true
    status: ready
-   central_instance_limit: 5
+   central_instance_limit: 9999
    provider_type: standalone
    supported_instance_type: "eval,standard"
    cluster_dns: cluster.local