Fix the development environment for linux, darwin and M1 arch (#1041)

* Fix the development environment for linux, darwin and M1 arch * Update the kube-apiserver binaries * Update golangci lint version * Fix the clusterstatus based on last operatorAction
cockroachdb · Nov 14, 2024 · 339275d · 339275d
1 parent 6b957dd
commit 339275d
Show file tree

Hide file tree

Showing 28 changed files with 183 additions and 95 deletions.
diff --git a/.bazelrc b/.bazelrc
@@ -1,2 +1,2 @@
 build --workspace_status_command hack/build/print-workspace-status.sh
-test --test_output=errors --test_timeout=-1,-1,-1,2400
+test --test_output=all --test_timeout=-1,-1,-1,2400
diff --git a/.github/workflows/e2e-kind-upgrades.yaml b/.github/workflows/e2e-kind-upgrades.yaml
@@ -26,7 +26,7 @@ on:
 
 jobs:
   e2e-kind-upgrades:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-4-core
     strategy:
       fail-fast: false
     steps:

diff --git a/DEVELOPER.md b/DEVELOPER.md
@@ -44,6 +44,13 @@ since there's no need to set up a remote GKE/OpenShift cluster.
 
 **make dev/up**
 
+Pre-requisite: The command sets up a k3d registry, and to push an image to it, the registry name must resolve to 
+localhost or 127.0.0.1. Please add the following line to your /etc/hosts file:
+
+```shell
+127.0.0.1 registry.localhost
+```
+
 This command will get everything set up for you to begin testing out the operator. Specifically it will:
 
 * Start a k3d cluster named test (context=k3d-test) with a managed docker registry
@@ -56,6 +63,18 @@ This command will get everything set up for you to begin testing out the operato
 
 Tears down the k3d cluster.
 
+[Existing Cluster & Repo]:
+
+export DEV_REGISTRY="Your_Registry"
+**make k8s/apply**
+
+This command will deploy the operator to an existing cluster. It will build the operator image and push it to the DEV_REGISTRY.
+This newly built image will be used to deploy the operator.
+
+**make k8s/delete**
+
+Tears down the cockroach operator deployment applied on the cluster.
+
 ## Testing CR Database
 
 Notes on how to test an existing CR Database.
@@ -71,6 +90,9 @@ SELECT * FROM bank.accounts;
 
 ## Developer Install Instructions
 
+Pre-requisite: 
+1. Install kustomize
+
 These instructions are for developers only. If you want to try the alpha please use the instructions in the next
 section.
 
@@ -79,13 +101,18 @@ Install the operator
 ```console
 $ git clone https://github.com/cockroachdb/cockroach-operator.git
 $ export CLUSTER=test
+$ export APP_VERSION=v$(cat version.txt)
+$ export DEV_REGISTRY=us.gcr.io/$(gcloud config get-value project)
 # create a gke cluster
 $ ./hack/create-gke-cluster.sh -c $CLUSTER
 
-$ DEV_REGISTRY=us.gcr.io/$(gcloud config get-value project) \
+$ bazel run //hack/crdbversions:crdbversions -- -operator-image ${DEV_REGISTRY}/cockroach-operator \
+-operator-version ${APP_VERSION} -crdb-versions $(PWD)/crdb-versions.yaml -repo-root $(PWD)
+ 
+$ DEV_REGISTRY=${DEV_REGISTRY} \
 K8S_CLUSTER=$(kubectl config view --minify -o=jsonpath='{.contexts[0].context.cluster}') \
 bazel run --stamp --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 \
-                 //manifests:install_operator.apply
+                 --define APP_VERSION=${APP_VERSION} //config/default:install.apply
 ```
 
 There are various examples that can be installed. The files are located in the examples directory.

diff --git a/Makefile b/Makefile
@@ -59,7 +59,7 @@ test/pkg:
 # takes a bit of time.
 .PHONY: test/verify
 test/verify:
-	bazel test //hack/...
+	bazel test --test_output=all //hack/...
 
 .PHONY: test/lint
 test/lint:
@@ -94,7 +94,7 @@ test/e2e-short:
 test/e2e/testrunner-k3d-%: PACKAGE=$*
 test/e2e/testrunner-k3d-%:
 	bazel run //hack/k8s:k8s -- -type k3d
-	bazel test --stamp //e2e/$(PACKAGE)/... --test_arg=-test.v --test_arg=-test.parallel=4 --test_arg=parallel=true
+	bazel test --test_output=all --stamp //e2e/$(PACKAGE)/... --test_arg=-test.v --test_arg=-test.parallel=4 --test_arg=parallel=true
 
 # Use this target to run e2e tests using a k3d k8s cluster.
 # This target uses k3d to start a k8s cluster  and runs the e2e tests
@@ -221,7 +221,7 @@ test/preflight-%: release/generate-bundle
 #
 .PHONY: dev/build
 dev/build: dev/syncdeps
-	bazel build //...
+	bazel build //... --define APP_VERSION=$(APP_VERSION)
 
 .PHONY: dev/fmt
 dev/fmt:
@@ -280,17 +280,15 @@ dev/down:
 #
 .PHONY: k8s/apply
 k8s/apply:
-	K8S_CLUSTER=gke_$(GCP_PROJECT)_$(GCP_ZONE)_$(CLUSTER_NAME) \
 	DEV_REGISTRY=$(DEV_REGISTRY) \
 	bazel run --stamp --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 \
 		//config/default:install.apply \
 		--define APP_VERSION=$(APP_VERSION)
 
 .PHONY: k8s/delete
 k8s/delete:
-	K8S_CLUSTER=gke_$(GCP_PROJECT)_$(GCP_ZONE)_$(CLUSTER_NAME) \
 	DEV_REGISTRY=$(DEV_REGISTRY) \
-	bazel run --stamp --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 \
+	bazel run --stamp --incompatible_use_cc_configure_from_rules_cc \
 		//config/default:install.delete \
 		--define APP_VERSION=$(APP_VERSION)
 

diff --git a/WORKSPACE b/WORKSPACE
@@ -69,9 +69,9 @@ go_dependencies()
 ######################################
 http_archive(
     name = "io_bazel_rules_docker",
-    sha256 = "1f4e59843b61981a96835dc4ac377ad4da9f8c334ebe5e0bb3f58f80c09735f4",
-    strip_prefix = "rules_docker-0.19.0",
-    urls = ["https://github.com/bazelbuild/rules_docker/releases/download/v0.19.0/rules_docker-v0.19.0.tar.gz"],
+    sha256 = "92779d3445e7bdc79b961030b996cb0c91820ade7ffa7edca69273f404b085d5",
+    strip_prefix = "rules_docker-0.20.0",
+    urls = ["https://github.com/bazelbuild/rules_docker/releases/download/v0.20.0/rules_docker-v0.20.0.tar.gz"],
 )
 
 load(
@@ -103,6 +103,19 @@ container_pull(
     tag = "latest",
 )
 
+http_archive(
+    name = "rules_pkg",
+    sha256 = "8f9ee2dc10c1ae514ee599a8b42ed99fa262b757058f65ad3c384289ff70c4b8",
+    urls = [
+        "https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.9.1/rules_pkg-0.9.1.tar.gz",
+        "https://github.com/bazelbuild/rules_pkg/releases/download/0.9.1/rules_pkg-0.9.1.tar.gz",
+    ],
+)
+
+load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies")
+
+rules_pkg_dependencies()
+
 ################################
 # Load rules_k8s and configure #
 ################################

diff --git a/cmd/cockroach-operator/BUILD.bazel b/cmd/cockroach-operator/BUILD.bazel
@@ -1,7 +1,7 @@
 load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
 load("@io_bazel_rules_docker//container:container.bzl", "container_image")
 load("@io_bazel_rules_docker//go:image.bzl", "go_image")
-load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar")
+load("@rules_pkg//:pkg.bzl", "pkg_tar")
 load("@io_bazel_rules_docker//docker/util:run.bzl", "container_run_and_commit_layer")
 
 go_library(
@@ -55,6 +55,7 @@ container_run_and_commit_layer(
 
 container_image(
     name = "ubi_base_image",
+    architecture = "amd64",
     # References container_pull from WORKSPACE
     base = "@redhat_ubi_minimal//image",
     labels = {

diff --git a/config/default/BUILD.bazel b/config/default/BUILD.bazel
@@ -32,7 +32,7 @@ k8s_deploy(
     name = "image",
     images = {
         # when running locally, use the image from the local codebase
-        "cockroachdb/cockroach-operator:$(APP_VERSION)": "//cmd/cockroach-operator:operator_image",
+        "cockroach-operator:$(APP_VERSION)": "//cmd/cockroach-operator:operator_image",
     },
     resolver_args = ["--allow_unused_images"],
     template = ":manifest",

diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml
@@ -148,9 +148,9 @@ rules:
   resources:
   - persistentvolumeclaims
   verbs:
+  - delete
   - list
   - update
-  - delete
 - apiGroups:
   - ""
   resources:

diff --git a/config/templates/deployment_image.yaml.in b/config/templates/deployment_image.yaml.in
@@ -24,7 +24,7 @@ spec:
     spec:
       containers:
         - name: cockroach-operator
-          image: cockroachdb/cockroach-operator:{{.OperatorVersion}}
+          image: {{.OperatorImage}}:{{.OperatorVersion}}
           env:
 {{- range .CrdbVersions}}
             {{- /*

diff --git a/e2e/e2e.go b/e2e/e2e.go
@@ -28,10 +28,14 @@ var (
 
 // Some common values used in e2e test suites.
 const (
-	MinorVersion1      = "cockroachdb/cockroach:v24.1.0"
-	MinorVersion2      = "cockroachdb/cockroach:v24.1.2"
-	MajorVersion       = "cockroachdb/cockroach:v24.2.2"
-	NonExistentVersion = "cockroachdb/cockroach-non-existent:v21.1.999"
-	SkipFeatureVersion = "cockroachdb/cockroach:v20.1.0"
-	InvalidImage       = "nginx:latest"
+	MinorVersion1        = "cockroachdb/cockroach:v24.1.0"
+	MinorVersion2        = "cockroachdb/cockroach:v24.1.2"
+	MajorVersion         = "cockroachdb/cockroach:v24.2.2"
+	NonExistentVersion   = "cockroachdb/cockroach-non-existent:v21.1.999"
+	SkipFeatureVersion   = "cockroachdb/cockroach:v20.1.0"
+	InvalidImage         = "nginx:latest"
+	DefaultCPULimit      = "800m"
+	DefaultMemoryLimit   = "3Gi"
+	DefaultCPURequest    = "500m"
+	DefaultMemoryRequest = "1Gi"
 )
diff --git a/e2e/upgrades/BUILD.bazel b/e2e/upgrades/BUILD.bazel
@@ -11,6 +11,8 @@ go_test(
         "//pkg/testutil/env:go_default_library",
         "@com_github_go_logr_zapr//:go_default_library",
         "@com_github_stretchr_testify//require:go_default_library",
+        "@io_k8s_api//core/v1:go_default_library",
+        "@io_k8s_apimachinery//pkg/api/resource:go_default_library",
         "@io_k8s_sigs_controller_runtime//pkg/client:go_default_library",
         "@org_uber_go_zap//zaptest:go_default_library",
     ],

diff --git a/e2e/upgrades/upgrades_test.go b/e2e/upgrades/upgrades_test.go
@@ -28,6 +28,21 @@ import (
 	"github.com/go-logr/zapr"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/zap/zaptest"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+)
+
+var (
+	resRequirements = corev1.ResourceRequirements{
+		Limits: corev1.ResourceList{
+			corev1.ResourceCPU:    resource.MustParse(e2e.DefaultCPULimit),
+			corev1.ResourceMemory: resource.MustParse(e2e.DefaultMemoryLimit),
+		},
+		Requests: corev1.ResourceList{
+			corev1.ResourceCPU:    resource.MustParse(e2e.DefaultCPURequest),
+			corev1.ResourceMemory: resource.MustParse(e2e.DefaultMemoryRequest),
+		},
+	}
 )
 
 // TestUpgradesMinorVersion tests a minor version bump
@@ -52,7 +67,7 @@ func TestUpgradesMinorVersion(t *testing.T) {
 
 	builder := testutil.NewBuilder("crdb").WithNodeCount(3).WithTLS().
 		WithImage(e2e.MinorVersion1).
-		WithPVDataStore("1Gi")
+		WithPVDataStore("1Gi").WithResources(resRequirements)
 
 	steps := testutil.Steps{
 		{
@@ -103,7 +118,7 @@ func TestUpgradesMajorVersion20to21(t *testing.T) {
 
 	builder := testutil.NewBuilder("crdb").WithNodeCount(3).WithTLS().
 		WithImage(e2e.MinorVersion2).
-		WithPVDataStore("1Gi")
+		WithPVDataStore("1Gi").WithResources(resRequirements)
 
 	steps := testutil.Steps{
 		{
@@ -152,7 +167,7 @@ func TestUpgradesMajorVersion20_1To20_2(t *testing.T) {
 
 	builder := testutil.NewBuilder("crdb").WithNodeCount(3).WithTLS().
 		WithImage("cockroachdb/cockroach:v20.1.16").
-		WithPVDataStore("1Gi")
+		WithPVDataStore("1Gi").WithResources(resRequirements)
 
 	steps := testutil.Steps{
 		{
@@ -209,7 +224,8 @@ func TestUpgradesMinorVersionThenRollback(t *testing.T) {
 		WithNodeCount(3).
 		WithTLS().
 		WithImage(e2e.MinorVersion1).
-		WithPVDataStore("1Gi")
+		WithPVDataStore("1Gi").
+		WithResources(resRequirements)
 
 	steps := testutil.Steps{
 		{
@@ -274,7 +290,8 @@ func TestUpgradeWithInvalidVersion(t *testing.T) {
 
 	builder := testutil.NewBuilder("crdb").WithNodeCount(3).WithTLS().
 		WithImage(e2e.MinorVersion1).
-		WithPVDataStore("1Gi")
+		WithPVDataStore("1Gi").
+		WithResources(resRequirements)
 
 	steps := testutil.Steps{
 		{
@@ -324,7 +341,8 @@ func TestUpgradeWithInvalidImage(t *testing.T) {
 
 	builder := testutil.NewBuilder("crdb").WithNodeCount(3).WithTLS().
 		WithImage(e2e.MinorVersion1).
-		WithPVDataStore("1Gi")
+		WithPVDataStore("1Gi").
+		WithResources(resRequirements)
 
 	steps := testutil.Steps{
 		{
@@ -374,7 +392,7 @@ func TestUpgradeWithMajorVersionExcludingMajorFeature(t *testing.T) {
 
 	builder := testutil.NewBuilder("crdb").WithNodeCount(3).WithTLS().
 		WithImage(e2e.SkipFeatureVersion).
-		WithPVDataStore("1Gi")
+		WithPVDataStore("1Gi").WithResources(resRequirements)
 
 	steps := testutil.Steps{
 		{

diff --git a/hack/BUILD.bazel b/hack/BUILD.bazel
@@ -106,7 +106,7 @@ sh_test(
     args = [
         "run",
         "--config .golangci.yaml",
-        "--timeout 5m",
+        "--timeout 10m",
     ],
     data = ["@//:all-srcs"],
     env = {

diff --git a/hack/bin/BUILD.bazel b/hack/bin/BUILD.bazel
@@ -38,10 +38,14 @@ genrule(
     visibility = ["//visibility:public"],
 )
 
+# fetch preflight has only linux supported binary and only used for releasing the operator to openshift
+# Hence, fetching same binary for all platforms to allow bazel sync on other platform.
 genrule(
     name = "fetch_preflight",
     srcs = select({
         ":k8": ["@preflight_linux//file"],
+        ":m1": ["@preflight_linux//file"],
+        ":darwin": ["@preflight_linux//file"],
     }),
     outs = ["preflight"],
     cmd = "cp $(SRCS) $@",
@@ -75,7 +79,7 @@ genrule(
 genrule(
     name = "io_kubernetes_kube-apiserver",
     srcs = select({
-        ":m1": ["@kube-apiserver_darwin_amd64//file"],
+        ":m1": ["@kube-apiserver_darwin_arm64//file"],
         ":darwin": ["@kube-apiserver_darwin_amd64//file"],
         ":k8": ["@kube-apiserver_linux_amd64//file"],
     }),