From 413fb38d3dce4d632e1449e9e52b18456abc5e88 Mon Sep 17 00:00:00 2001 From: prafull01 Date: Tue, 27 Aug 2024 21:48:02 +0530 Subject: [PATCH] Fix the development environment for linux, darwin and M1 arch --- DEVELOPER.md | 19 ++++++++++++++-- Makefile | 2 +- WORKSPACE | 19 +++++++++++++--- cmd/cockroach-operator/BUILD.bazel | 3 ++- config/default/BUILD.bazel | 2 +- config/templates/deployment_image.yaml.in | 2 +- hack/bin/BUILD.bazel | 6 +++-- hack/build/repos.bzl | 4 ++-- hack/common.sh | 27 ++++++++++++++++++----- hack/crdbversions/main.go | 7 ++++-- hack/crdbversions/main_test.go | 2 +- hack/create-gke-cluster.sh | 4 +++- hack/dev.sh | 5 ++++- 13 files changed, 78 insertions(+), 24 deletions(-) diff --git a/DEVELOPER.md b/DEVELOPER.md index 4557768ef..5e3a1caf0 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -44,6 +44,13 @@ since there's no need to set up a remote GKE/OpenShift cluster. **make dev/up** +Pre-requisite: The command sets up a k3d registry, and to push an image to it, the registry name must resolve to +localhost or 127.0.0.1. Please add the following line to your /etc/hosts file: + +```shell +127.0.0.1 registry.localhost +``` + This command will get everything set up for you to begin testing out the operator. Specifically it will: * Start a k3d cluster named test (context=k3d-test) with a managed docker registry @@ -71,6 +78,9 @@ SELECT * FROM bank.accounts; ## Developer Install Instructions +Pre-requisite: +1. Install kustomize + These instructions are for developers only. If you want to try the alpha please use the instructions in the next section. @@ -79,13 +89,18 @@ Install the operator ```console $ git clone https://github.com/cockroachdb/cockroach-operator.git $ export CLUSTER=test +$ export APP_VERSION=v$(cat version.txt) +$ export DEV_REGISTRY=us.gcr.io/$(gcloud config get-value project) # create a gke cluster $ ./hack/create-gke-cluster.sh -c $CLUSTER -$ DEV_REGISTRY=us.gcr.io/$(gcloud config get-value project) \ +$ bazel run //hack/crdbversions:crdbversions -- -operator-image ${DEV_REGISTRY}/cockroach-operator \ +-operator-version ${APP_VERSION} -crdb-versions $(PWD)/crdb-versions.yaml -repo-root $(PWD) + +$ DEV_REGISTRY=${DEV_REGISTRY} \ K8S_CLUSTER=$(kubectl config view --minify -o=jsonpath='{.contexts[0].context.cluster}') \ bazel run --stamp --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 \ - //manifests:install_operator.apply + --define APP_VERSION=${APP_VERSION} //config/default:install.apply ``` There are various examples that can be installed. The files are located in the examples directory. diff --git a/Makefile b/Makefile index e105a017c..6c0d17099 100644 --- a/Makefile +++ b/Makefile @@ -221,7 +221,7 @@ test/preflight-%: release/generate-bundle # .PHONY: dev/build dev/build: dev/syncdeps - bazel build //... + bazel build //... --define APP_VERSION=$(APP_VERSION) .PHONY: dev/fmt dev/fmt: diff --git a/WORKSPACE b/WORKSPACE index ecc104fdf..a02bd6486 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -69,9 +69,9 @@ go_dependencies() ###################################### http_archive( name = "io_bazel_rules_docker", - sha256 = "1f4e59843b61981a96835dc4ac377ad4da9f8c334ebe5e0bb3f58f80c09735f4", - strip_prefix = "rules_docker-0.19.0", - urls = ["https://github.com/bazelbuild/rules_docker/releases/download/v0.19.0/rules_docker-v0.19.0.tar.gz"], + sha256 = "92779d3445e7bdc79b961030b996cb0c91820ade7ffa7edca69273f404b085d5", + strip_prefix = "rules_docker-0.20.0", + urls = ["https://github.com/bazelbuild/rules_docker/releases/download/v0.20.0/rules_docker-v0.20.0.tar.gz"], ) load( @@ -103,6 +103,19 @@ container_pull( tag = "latest", ) +http_archive( + name = "rules_pkg", + sha256 = "8f9ee2dc10c1ae514ee599a8b42ed99fa262b757058f65ad3c384289ff70c4b8", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.9.1/rules_pkg-0.9.1.tar.gz", + "https://github.com/bazelbuild/rules_pkg/releases/download/0.9.1/rules_pkg-0.9.1.tar.gz", + ], +) + +load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies") + +rules_pkg_dependencies() + ################################ # Load rules_k8s and configure # ################################ diff --git a/cmd/cockroach-operator/BUILD.bazel b/cmd/cockroach-operator/BUILD.bazel index fabf7c688..d8264b6bb 100644 --- a/cmd/cockroach-operator/BUILD.bazel +++ b/cmd/cockroach-operator/BUILD.bazel @@ -1,7 +1,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") load("@io_bazel_rules_docker//container:container.bzl", "container_image") load("@io_bazel_rules_docker//go:image.bzl", "go_image") -load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar") +load("@rules_pkg//:pkg.bzl", "pkg_tar") load("@io_bazel_rules_docker//docker/util:run.bzl", "container_run_and_commit_layer") go_library( @@ -55,6 +55,7 @@ container_run_and_commit_layer( container_image( name = "ubi_base_image", + architecture = "amd64", # References container_pull from WORKSPACE base = "@redhat_ubi_minimal//image", labels = { diff --git a/config/default/BUILD.bazel b/config/default/BUILD.bazel index aa5e08a4b..95c973a6c 100644 --- a/config/default/BUILD.bazel +++ b/config/default/BUILD.bazel @@ -32,7 +32,7 @@ k8s_deploy( name = "image", images = { # when running locally, use the image from the local codebase - "cockroachdb/cockroach-operator:$(APP_VERSION)": "//cmd/cockroach-operator:operator_image", + "cockroach-operator:$(APP_VERSION)": "//cmd/cockroach-operator:operator_image", }, resolver_args = ["--allow_unused_images"], template = ":manifest", diff --git a/config/templates/deployment_image.yaml.in b/config/templates/deployment_image.yaml.in index f63c2a972..de84448cf 100644 --- a/config/templates/deployment_image.yaml.in +++ b/config/templates/deployment_image.yaml.in @@ -24,7 +24,7 @@ spec: spec: containers: - name: cockroach-operator - image: cockroachdb/cockroach-operator:{{.OperatorVersion}} + image: {{.OperatorImage}}:{{.OperatorVersion}} env: {{- range .CrdbVersions}} {{- /* diff --git a/hack/bin/BUILD.bazel b/hack/bin/BUILD.bazel index d912063d1..9033aea90 100644 --- a/hack/bin/BUILD.bazel +++ b/hack/bin/BUILD.bazel @@ -42,6 +42,8 @@ genrule( name = "fetch_preflight", srcs = select({ ":k8": ["@preflight_linux//file"], + ":m1": ["@preflight_linux//file"], + ":darwin": ["@preflight_linux//file"], }), outs = ["preflight"], cmd = "cp $(SRCS) $@", @@ -75,8 +77,8 @@ genrule( genrule( name = "io_kubernetes_kube-apiserver", srcs = select({ - ":m1": ["@kube-apiserver_darwin_amd64//file"], - ":darwin": ["@kube-apiserver_darwin_amd64//file"], + ":m1": ["@kube-apiserver_linux_amd64//file"], + ":darwin": ["@kube-apiserver_linux_amd64//file"], ":k8": ["@kube-apiserver_linux_amd64//file"], }), outs = ["kube-apiserver"], diff --git a/hack/build/repos.bzl b/hack/build/repos.bzl index d9b12d409..fb74638f0 100644 --- a/hack/build/repos.bzl +++ b/hack/build/repos.bzl @@ -5336,8 +5336,8 @@ def _go_dependencies(): build_file_generation = "on", build_file_proto_mode = "disable", importpath = "gopkg.in/yaml.v3", - sum = "h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=", - version = "v3.0.0-20210107192922-496545a6307b", + sum = "h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA=", + version = "v3.0.0", ) go_repository( name = "io_etcd_go_bbolt", diff --git a/hack/common.sh b/hack/common.sh index eaae1c181..52a04dabf 100755 --- a/hack/common.sh +++ b/hack/common.sh @@ -28,9 +28,9 @@ command -v kubectl >/dev/null 2>&1 || { \ command -v kustomize >/dev/null 2>&1 || { \ echo >&2 "I require kustomize but it's not installed. Aborting."; exit 1; } -usage() { echo "Usage: $0 [-c ]" 1>&2; exit 1; } +usage() { echo "Usage: $0 [-c -r -z ]" 1>&2; exit 1; } # parse -c flag for the CLUSTER_NAME using getopts -while getopts ":c:i:" opt; do +while getopts ":c:i:z:r:" opt; do case ${opt} in c) CLUSTER_NAME=$OPTARG @@ -38,6 +38,12 @@ while getopts ":c:i:" opt; do i) IMAGE_NAME=$OPTARG ;; + z) + ZONE=$OPTARG + ;; + r) + REGION=$OPTARG + ;; \?) echo "Invalid flag on command line: $OPTARG" 1>&2 ;; @@ -55,20 +61,29 @@ if [ -z "${CLUSTER_NAME}" ]; then usage fi -# Get the default zone and use it or die -ZONE=$(gcloud config get-value compute/zone) +# If user did not pass in -z flag then get the default zone and use it or die if [ -z "${ZONE}" ]; then + # Get the default zone and use it or die + ZONE=$(gcloud config get-value compute/zone) + if [ -z "${ZONE}" ]; then echo "gcloud cli must be configured with a default zone." 1>&2 echo "run 'gcloud config set compute/zone ZONE'." 1>&2 echo "replace 'ZONE' with the zone name like us-west1-a." 1>&2 + echo "Or provide 'ZONE' as input to script as follows:" 1>&2 + usage exit 1; + fi fi -#Get the default region and use it or die -REGION=$(gcloud config get-value compute/region) +#If user did not pass in -r flag then get the default region and use it or die if [ -z "${REGION}" ]; then + REGION=$(gcloud config get-value compute/region) + if [ -z "${REGION}" ]; then echo "gcloud cli must be configured with a default region." 1>&2 echo "run 'gcloud config set compute/region REGION'." 1>&2 echo "replace 'REGION' with the region name like us-west1." 1>&2 + echo "Or provide 'REGION' as input to script as follows:" 1>&2 + usage exit 1; + fi fi diff --git a/hack/crdbversions/main.go b/hack/crdbversions/main.go index ebb573ce0..52ffb6d69 100644 --- a/hack/crdbversions/main.go +++ b/hack/crdbversions/main.go @@ -59,6 +59,7 @@ type crdbVersion struct { type templateData struct { CrdbVersions []crdbVersion LatestStableCrdbVersion string + OperatorImage string OperatorVersion string GeneratedWarning string Year string @@ -68,6 +69,7 @@ func main() { log.SetFlags(0) crdbVersionsFile := flag.String("crdb-versions", "", "YAML file with CRDB versions") operatorVersion := flag.String("operator-version", "", "Current operator version") + operatorImage := flag.String("operator-image", "cockroachdb/cockroach-operator", "Current operator image") repoRoot := flag.String("repo-root", "", "Git repository root") flag.Parse() @@ -87,7 +89,7 @@ func main() { log.Fatalf("Cannot read versions file: %s", err) } - data, err := generateTemplateData(vs, *operatorVersion) + data, err := generateTemplateData(vs, *operatorVersion, *operatorImage) if err != nil { log.Fatalf("Cannot generate template data: %s", err) } @@ -139,10 +141,11 @@ func readCrdbVersions(r io.Reader) ([]crdbVersion, error) { return versions.CrdbVersions, nil } -func generateTemplateData(crdbVersions []crdbVersion, operatorVersion string) (templateData, error) { +func generateTemplateData(crdbVersions []crdbVersion, operatorVersion, operatorImage string) (templateData, error) { var data templateData data.Year = fmt.Sprint(time.Now().Year()) data.OperatorVersion = operatorVersion + data.OperatorImage = operatorImage data.CrdbVersions = crdbVersions latestStable := crdbVersions[len(crdbVersions)-1].Tag diff --git a/hack/crdbversions/main_test.go b/hack/crdbversions/main_test.go index 57ba65da4..599621038 100644 --- a/hack/crdbversions/main_test.go +++ b/hack/crdbversions/main_test.go @@ -65,7 +65,7 @@ func TestGenerateTemplateData(t *testing.T) { crdbVersions = append(crdbVersions, crdbVersion{Tag: r}) } - data, err := generateTemplateData(crdbVersions, "2.0.1") + data, err := generateTemplateData(crdbVersions, "2.0.1", "cockroachdb/cockroach-operator") require.NoError(t, err) require.Len(t, data.CrdbVersions, len(crdbVersions)) require.Equal(t, "1.2.3", data.LatestStableCrdbVersion) diff --git a/hack/create-gke-cluster.sh b/hack/create-gke-cluster.sh index 6804ace66..afe2bae42 100755 --- a/hack/create-gke-cluster.sh +++ b/hack/create-gke-cluster.sh @@ -29,6 +29,8 @@ CLUSTER_NAME="" # TODO maybe set default zone? ZONE="" +REGION="" + # shellcheck disable=SC1090 source "$ROOT"/common.sh # shellcheck disable=SC1090 @@ -39,7 +41,7 @@ enable-service compute.googleapis.com enable-service container.googleapis.com GKE_VERSION=$(gcloud container get-server-config \ - --format="value(validMasterVersions[0])") + --format="value(validMasterVersions[0])" --region=$REGION) # Get a comma separated list of zones from the default region ZONESINREGION="" diff --git a/hack/dev.sh b/hack/dev.sh index e0f334806..569a51544 100755 --- a/hack/dev.sh +++ b/hack/dev.sh @@ -22,6 +22,7 @@ CLUSTER_NAME="dev" NODE_IMAGE="rancher/k3s:v1.23.3-k3s1" REGISTRY_NAME="registry.localhost" REGISTRY_PORT=5000 +DEV_REGISTRY="${REGISTRY_NAME}:${REGISTRY_PORT}" main() { case "${1:-}" in @@ -45,8 +46,10 @@ install_operator() { # Can't seem to figure out how to leverage the stamp variables here. So for # now I've added a defined make variable which can be used for substitution # in //config/default/BUILD.bazel. + # ${REGISTRY_NAME} must be mapped to localhost or 127.0.0.1 to push the image to the k3d registry. + bazel run //hack/crdbversions:crdbversions -- -operator-image ${DEV_REGISTRY}/cockroach-operator -operator-version ${APP_VERSION} -crdb-versions $(PWD)/crdb-versions.yaml -repo-root $(PWD) K8S_CLUSTER="k3d-${CLUSTER_NAME}" \ - DEV_REGISTRY="${REGISTRY_NAME}:${REGISTRY_PORT}" \ + DEV_REGISTRY="${DEV_REGISTRY}" \ bazel run \ --stamp \ --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 \