Merge branch 'main' of github.com:linode/provider-ceph into traces

.github/workflows/ci.yml

@@ -10,7 +10,7 @@ on:
 
 env:
   # Common versions
-  GO_VERSION: '1.20'
+  GO_VERSION: '1.21'
   GOLANGCI_VERSION: 'v1.52.2'
   DOCKER_BUILDX_VERSION: 'v0.9.1'
 

.github/workflows/kuttl-e2e-test-1.25.yaml

@@ -7,7 +7,6 @@ jobs:
     runs-on: ubuntu-latest
     env:
       KUTTL: /usr/local/bin/kubectl-kuttl
-      COMPOSE: /usr/local/bin/docker-compose
     steps:
       - name: Cancel Previous Runs
         uses: styfle/[email protected]
@@ -16,13 +15,11 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2
         with:
-          go-version: '1.20'
+          go-version: '1.21'
       - name: Install dependencies
         run: |
           sudo curl -Lo $KUTTL https://github.com/kudobuilder/kuttl/releases/download/v0.13.0/kubectl-kuttl_0.13.0_linux_x86_64
           sudo chmod +x $KUTTL
-          sudo curl -Lo $COMPOSE https://github.com/docker/compose/releases/download/v2.17.3/docker-compose-linux-x86_64
-          sudo chmod +x $COMPOSE
       - name: Build
         run: make submodules build
       - name: Create test environment

.github/workflows/kuttl-e2e-test-1.26.yaml

@@ -7,7 +7,6 @@ jobs:
     runs-on: ubuntu-latest
     env:
       KUTTL: /usr/local/bin/kubectl-kuttl
-      COMPOSE: /usr/local/bin/docker-compose
     steps:
       - name: Cancel Previous Runs
         uses: styfle/[email protected]
@@ -16,13 +15,11 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2
         with:
-          go-version: '1.20'
+          go-version: '1.21'
       - name: Install dependencies
         run: |
           sudo curl -Lo $KUTTL https://github.com/kudobuilder/kuttl/releases/download/v0.13.0/kubectl-kuttl_0.13.0_linux_x86_64
           sudo chmod +x $KUTTL
-          sudo curl -Lo $COMPOSE https://github.com/docker/compose/releases/download/v2.17.3/docker-compose-linux-x86_64
-          sudo chmod +x $COMPOSE
       - name: Build
         run: make submodules build
       - name: Create test environment

.github/workflows/kuttl-e2e-test-1.27.yaml

@@ -7,7 +7,6 @@ jobs:
     runs-on: ubuntu-latest
     env:
       KUTTL: /usr/local/bin/kubectl-kuttl
-      COMPOSE: /usr/local/bin/docker-compose
     steps:
       - name: Cancel Previous Runs
         uses: styfle/[email protected]
@@ -16,13 +15,11 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2
         with:
-          go-version: '1.20'
+          go-version: '1.21'
       - name: Install dependencies
         run: |
           sudo curl -Lo $KUTTL https://github.com/kudobuilder/kuttl/releases/download/v0.13.0/kubectl-kuttl_0.13.0_linux_x86_64
           sudo chmod +x $KUTTL
-          sudo curl -Lo $COMPOSE https://github.com/docker/compose/releases/download/v2.17.3/docker-compose-linux-x86_64
-          sudo chmod +x $COMPOSE
       - name: Build
         run: make submodules build
       - name: Create test environment

.github/workflows/kuttl-e2e-test-1.28.yaml

@@ -0,0 +1,37 @@
+# This file was auto-generated by hack/generate-tests.sh
+name: kuttl e2e test 1.28
+on: [push]
+jobs:
+  test:
+    name: kuttl e2e test 1.28
+    runs-on: ubuntu-latest
+    env:
+      KUTTL: /usr/local/bin/kubectl-kuttl
+    steps:
+      - name: Cancel Previous Runs
+        uses: styfle/[email protected]
+        with:
+          access_token: ${{ github.token }}
+      - uses: actions/checkout@v2
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '1.20'
+      - name: Install dependencies
+        run: |
+          sudo curl -Lo $KUTTL https://github.com/kudobuilder/kuttl/releases/download/v0.13.0/kubectl-kuttl_0.13.0_linux_x86_64
+          sudo chmod +x $KUTTL
+      - name: Build
+        run: make submodules build
+      - name: Create test environment
+        run: make crossplane-cluster load-package
+      - name: Run kuttl tests 1.28
+        run: kubectl-kuttl test --config e2e/kuttl/stable/provider-ceph-1.28.yaml
+        env:
+          AWS_ACCESS_KEY_ID: 'Dummy'
+          AWS_SECRET_ACCESS_KEY: 'Dummy'
+          AWS_DEFAULT_REGION: 'us-east-1'
+      - uses: actions/upload-artifact@v3
+        if: ${{ always() }}
+        with:
+          name: kind-logs
+          path: kind-logs-*

.golangci.yml

@@ -81,10 +81,6 @@ linters-settings:
     # with golangci-lint call it on a directory with the changed file.
     check-exported: false
 
-  nakedret:
-    # make an issue if func has more lines of code than this setting and it has naked returns; default is 30
-    max-func-lines: 30
-
   prealloc:
     # XXX: we don't recommend using this linter before doing performance profiling.
     # For most programs usage of prealloc will be a premature optimization.
@@ -171,7 +167,6 @@ linters:
     - nlreturn
     - noctx
     - nolintlint
-    - nakedret
     - paralleltest
     - prealloc
     - predeclared

.mirrord/README.MD

@@ -0,0 +1,4 @@
+Currently outgoing network and filesystem are disabled, because we compile static binary and mirrord is not able to catch file system requests.
+Because it is't able to read resolver config properly, it is not able to resolve in-cluster host names.
+
+For more info please follow: https://github.com/metalbear-co/mirrord/issues/1922

.mirrord/mirrord.json

@@ -0,0 +1,22 @@
+{
+    "target": {
+        "path": "deploy/provider-ceph-provider-cep",
+        "namespace": "crossplane-system"
+    },
+    "agent": {
+      "log_level": "info",
+      "namespace": "crossplane-system",
+      "privileged": true
+    },
+    "feature": {
+        "network": {
+            "incoming": "steal",
+            "outgoing": false
+        },
+        "fs": "local",
+        "env": true
+    },
+    "operator": false,
+    "pause": true,
+    "accept_invalid_certificates": true
+}

.mirrord/mirrord.mk

@@ -0,0 +1,10 @@
+MIRRORD_VERSION ?= 3.70.0
+MIRRORD := $(TOOLS_HOST_DIR)/mirrord-$(MIRRORD_VERSION)
+
+# mirrord download and install
+$(MIRRORD):
+	@$(INFO) installing mirrord $(MIRRORD_VERSION)
+	@mkdir -p $(TOOLS_HOST_DIR) || $(FAIL)
+	@curl -fsSLo $(MIRRORD) https://github.com/metalbear-co/mirrord/releases/download/$(MIRRORD_VERSION)/mirrord_$(HOST_PLATFORM:-=_) || $(FAIL)
+	@chmod +x $(MIRRORD)
+	@$(OK) installing mirrord $(MIRRORD_VERSION)

Makefile

@@ -9,9 +9,10 @@ PLATFORMS ?= linux_amd64 linux_arm64
 # Generate kuttl e2e tests for the following kind node versions
 # TEST_KIND_NODES is not intended to be updated manually.
 # Please edit LATEST_KIND_NODE instead and run 'make update-kind-nodes'.
-TEST_KIND_NODES ?= 1.25.0,1.26.0,1.27.0
+TEST_KIND_NODES ?= 1.25.11,1.26.6,1.27.3,1.28.0
 
-LATEST_KIND_NODE ?= 1.27.0
+LATEST_KUBE_VERSION ?= 1.28
+LATEST_KIND_NODE ?= 1.28.0
 REPO ?= provider-ceph
 
 # ====================================================================================
@@ -36,6 +37,9 @@ GO111MODULE = on
 # Husky git hook manager tasks.
 -include .husky/husky.mk
 
+# Mirrord local dev tasks.
+-include .mirrord/mirrord.mk
+
 # ====================================================================================
 # Setup Images
 
@@ -114,16 +118,10 @@ run: go.build
 # Spin up a Kind cluster and localstack.
 # Create k8s service to allows pods to communicate with
 # localstack.
-cluster: $(KIND) $(KUBECTL) $(COMPOSE) cluster-clean
-	@$(INFO) Creating localstack
-	@$(COMPOSE) -f e2e/localstack/docker-compose.yml up -d
-	@$(OK) Creating localstack
+cluster: $(KIND) $(KUBECTL) cluster-clean
 	@$(INFO) Creating kind cluster
-	@$(KIND) create cluster --name=$(KIND_CLUSTER_NAME)
+	@$(KIND) create cluster --name=$(KIND_CLUSTER_NAME) --config e2e/kind/kind-config-$(LATEST_KUBE_VERSION).yaml
 	@$(OK) Creating kind cluster
-	@$(INFO) Creating Localstack Service
-	@$(KUBECTL) apply -R -f e2e/localstack/service.yaml
-	@$(OK) Creating Localstack Service
 
 # Spin up a Kind cluster and localstack and install Crossplane via Helm.
 crossplane-cluster: $(HELM3) cluster
@@ -143,7 +141,7 @@ load-package: $(KIND) build
 	@$(MAKE) local.xpkg.sync
 	@$(INFO) deploying provider package $(PROJECT_NAME)
 	@$(KIND) load docker-image $(BUILD_REGISTRY)/$(PROJECT_NAME)-$(ARCH) -n $(KIND_CLUSTER_NAME)
-	@echo '{"apiVersion":"pkg.crossplane.io/v1alpha1","kind":"ControllerConfig","metadata":{"name":"config"},"spec":{"args":["--zap-devel","--enable-validation-webhooks", "--kube-client-rate=100000", "--reconcile-timeout=2s", "--max-reconcile-rate=5000", "--reconcile-concurrency=100", "--poll=30m", "--sync=1h"],"image":"$(BUILD_REGISTRY)/$(PROJECT_NAME)-$(ARCH)"}}' | $(KUBECTL) apply -f -
+	@echo '{"apiVersion":"pkg.crossplane.io/v1alpha1","kind":"ControllerConfig","metadata":{"name":"config"},"spec":{"args":["--zap-devel", "--enable-validation-webhooks", "--kube-client-rate=80000", "--reconcile-timeout=5s", "--max-reconcile-rate=600", "--reconcile-concurrency=160", "--poll=30m", "--sync=1h"],"image":"$(BUILD_REGISTRY)/$(PROJECT_NAME)-$(ARCH)"}}' | $(KUBECTL) apply -f -
 	@echo '{"apiVersion":"pkg.crossplane.io/v1","kind":"Provider","metadata":{"name":"$(PROJECT_NAME)"},"spec":{"package":"$(PROJECT_NAME)-$(VERSION).gz","packagePullPolicy":"Never","controllerConfigRef":{"name":"config"}}}' | $(KUBECTL) apply -f -
 	@$(OK) deploying provider package $(PROJECT_NAME) $(VERSION)
 
@@ -155,40 +153,45 @@ load-package: $(KIND) build
 # Destroy Kind and localstack.
 kuttl: $(KUTTL) crossplane-cluster load-package
 	@$(INFO) Running kuttl test suite
-	@$(KUTTL) test --config e2e/kuttl/stable/provider-ceph-1.27.yaml
+	@$(KUTTL) test --config e2e/kuttl/stable/provider-ceph-$(LATEST_KUBE_VERSION).yaml
 	@$(OK) Running kuttl test suite
 	@$(MAKE) cluster-clean
 
 ceph-kuttl: $(KIND) $(KUTTL) $(HELM3) cluster-clean
 	@$(INFO) Creating kind cluster
 	@$(KIND) create cluster --name=$(KIND_CLUSTER_NAME)
 	@$(OK) Creating kind cluster
-	@$(KUTTL) test --config e2e/kuttl/ceph/provider-ceph-1.27.yaml
+	@$(KUTTL) test --config e2e/kuttl/ceph/provider-ceph-$(LATEST_KUBE_VERSION).yaml
 
 # Spin up a Kind cluster and localstack and install Crossplane CRDs (not
 # containerised Crossplane componenets).
 # Install local provider-ceph CRDs.
 # Create ProviderConfig CR representing localstack.
 dev-cluster: $(KUBECTL) cluster
-	@$(INFO) Installing CRDs and ProviderConfig
+	@$(INFO) Installing CRDs, ProviderConfig and Localstack
 	@$(KUBECTL) apply -k https://github.com/crossplane/crossplane//cluster?ref=master
 	@$(KUBECTL) apply -R -f package/crds
 	@# TODO: apply package/webhookconfigurations when webhooks can be enabled locally.
-	@$(KUBECTL) apply -R -f e2e/localstack/localstack-provider-cfg.yaml
+	@$(KUBECTL) apply -R -f e2e/localstack/localstack-deployment.yaml
+	@$(KUBECTL) apply -R -f e2e/localstack/localstack-provider-cfg-host.yaml
 	@$(OK) Installing CRDs and ProviderConfig
 
 # Best for development - locally run provider-ceph controller.
 # Removes need for Crossplane install via Helm.
 dev: dev-cluster run
 
+# Best for development - locally run provider-ceph controller.
+mirrord: dev-cluster crossplane-cluster load-package
+
+mirrord-run:
+	@$(INFO) Starting mirrord on deployment
+	$(MIRRORD) exec -f .mirrord/mirrord.json make run
+
 # Destroy Kind cluster and localstack.
-cluster-clean: $(KIND) $(KUBECTL) $(COMPOSE)
+cluster-clean: $(KIND) $(KUBECTL)
 	@$(INFO) Deleting kind cluster
 	@$(KIND) delete cluster --name=$(KIND_CLUSTER_NAME)
 	@$(OK) Deleting kind cluster
-	@$(INFO) Tearing down localstack
-	@$(COMPOSE) -f e2e/localstack/docker-compose.yml stop
-	@$(OK) Tearing down localstack
 
 .PHONY: submodules fallthrough test-integration run cluster dev-cluster dev cluster-clean
 
@@ -249,14 +252,6 @@ help-special: crossplane.help
 
 .PHONY: crossplane.help help-special aws
 
-# Install Docker Compose to run localstack.
-COMPOSE ?= $PWD/bin/docker-compose
-compose:
-ifeq (,$(wildcard $(COMPOSE)))
-	curl -sL https://github.com/docker/compose/releases/download/v2.17.3/docker-compose-linux-x86_64 -o $(COMPOSE)
-	chmod +x $(COMPOSE)
-endif
-
 # Install aws cli for testing.
 AWS ?= /usr/local/bin/aws
 aws:

README.md

@@ -36,7 +36,7 @@ guide may also be of use.
 Install the provider by using the Upbound CLI after changing the image tag to the latest release:
 
 ```
-up ctp provider install linode/provider-ceph:v0.0.16
+up ctp provider install linode/provider-ceph:v0.0.21
 ```
 
 Alternatively, you can use declarative installation:
@@ -47,7 +47,7 @@ kind: Provider
 metadata:
   name: linode-provider-ceph
 spec:
-  package: xpkg.upbound.io/linode/provider-ceph:v0.0.16
+  package: xpkg.upbound.io/linode/provider-ceph:v0.0.21
 EOF
 ```
 See [WEBHOOKS.md](docs/WEBHOOKS.md) for instructions on how to enable webhooks.

apis/provider-ceph/v1alpha1/utils.go

@@ -19,8 +19,11 @@ package v1alpha1
 const (
 	HealthCheckLabelKey = "provider-ceph.crossplane.io"
 	HealthCheckLabelVal = "health-check-bucket"
+	BackendLabelPrefix  = "provider-ceph.backends."
 )
 
+// Deprecation warning: This function exists for compatibility reasons,
+// and would be removed soon.
 func IsHealthCheckBucket(bucket *Bucket) bool {
 	if val, ok := bucket.GetLabels()[HealthCheckLabelKey]; ok {
 		if val == HealthCheckLabelVal {

apis/v1alpha1/providerconfig_types.go

@@ -41,7 +41,7 @@ type ProviderConfigSpec struct {
 
 	DisableHealthCheck bool `json:"disableHealthCheck,omitempty"`
 
-	// +kubebuilder:validation:Minimum:=0
+	// +kubebuilder:validation:Minimum:=2
 	// +kubebuilder:default:=30
 	HealthCheckIntervalSeconds int32 `json:"healthCheckIntervalSeconds,omitempty"`
 }
@@ -67,8 +67,9 @@ type ProviderConfigStatus struct {
 
 	// Health of the s3 backend represented by the ProviderConfig determined
 	// by periodic health check.
-	//+kubebuilder:validation:Enum=Healthy;Unhealthy;Unknown
+	// +kubebuilder:validation:Enum=Healthy;Unhealthy;Unknown
 	Health                    HealthStatus `json:"health,omitempty"`
+	Reason                    string       `json:"reason,omitempty"`
 	xpv1.ProviderConfigStatus `json:",inline"`
 }
 

cmd/provider/main.go

@@ -50,6 +50,7 @@ import (
 	ceph "github.com/linode/provider-ceph/internal/controller"
 	"github.com/linode/provider-ceph/internal/controller/bucket"
 	"github.com/linode/provider-ceph/internal/features"
+	"github.com/linode/provider-ceph/internal/s3/cache"
 )
 
 var defaultZapConfig = map[string]string{
@@ -66,9 +67,11 @@ func main() {
 
 		syncInterval          = app.Flag("sync", "How often all resources will be double-checked for drift from the desired state.").Short('s').Default("1h").Duration()
 		pollInterval          = app.Flag("poll", "How often individual resources will be checked for drift from the desired state").Short('p').Default("30m").Duration()
+		bucketExistsCache     = app.Flag("bucket-exists-cache", "How long the provider caches bucket exists result").Short('c').Default("5s").Duration()
 		reconcileConcurrency  = app.Flag("reconcile-concurrency", "Set number of reconciliation loops.").Default("100").Int()
 		maxReconcileRate      = app.Flag("max-reconcile-rate", "The global maximum rate per second at which resources may checked for drift from the desired state.").Default("1000").Int()
-		reconcileTimeout      = app.Flag("reconcile-timeout", "Object reconciliation timeout").Short('t').Default("1s").Duration()
+		reconcileTimeout      = app.Flag("reconcile-timeout", "Object reconciliation timeout").Short('t').Default("3s").Duration()
+		creationGracePeriod   = app.Flag("creation-grace-period", "Duration to wait for the external API to report that a newly created external resource exists.").Default("10s").Duration()
 		metricsExportTimeout  = app.Flag("metrics-export-timeout", "Timeout when exporting metrics").Default("2s").Duration()
 		metricsExportInterval = app.Flag("metrics-export-interval", "Interval at which metrics are exported").Default("5s").Duration()
 		metricsExportAddress  = app.Flag("metrics-export-address", "Address of otel collector").Default("opentelemetry-collector.opentelemetry:4317").String()
@@ -170,6 +173,8 @@ func main() {
 
 	cfg = ratelimiter.LimitRESTConfig(cfg, *kubeClientRate)
 
+	cache.BucketExistsCacheTTL = *bucketExistsCache
+
 	const oneDotTwo = 1.2
 	const two = 2
 
@@ -232,6 +237,6 @@ func main() {
 			Complete(), "Cannot setup bucket validating webhook")
 	}
 
-	kingpin.FatalIfError(ceph.Setup(mgr, o, backendStore, *autoPauseBucket, *pollInterval, *reconcileTimeout), "Cannot setup Ceph controllers")
+	kingpin.FatalIfError(ceph.Setup(mgr, o, backendStore, *autoPauseBucket, *pollInterval, *reconcileTimeout, *creationGracePeriod), "Cannot setup Ceph controllers")
 	kingpin.FatalIfError(mgr.Start(ctrl.SetupSignalHandler()), "Cannot start controller manager")
 }

docs/WEBHOOKS.md

@@ -2,7 +2,7 @@
 
 ## Enable Webhooks
 - Webhooks are enabled in Crossplane by default from `v1.13` onwards. For previous versions of Crossplane, include the flag `--set webhooks.enabled=true` when [installing Crossplane via Helm](https://docs.crossplane.io/v1.11/software/install/#install-the-crossplane-helm-chart).
-- To enable webhooks in Provider Ceph, set the `--enable-webhooks` flag for the Provider Ceph controller. See example below using a controller configuration:
+- To enable webhooks in Provider Ceph, set the `--enable-validation-webhooks` flag for the Provider Ceph controller. See example below using a controller configuration:
 
 `Provider` with reference to a `ControllerConfig` (**Note:** package version is omitted):
 ```