diff --git a/src/cloud-api-adaptor/docs/addnewprovider.md b/src/cloud-api-adaptor/docs/addnewprovider.md index f2da7b9e6..a48ba3c0d 100644 --- a/src/cloud-api-adaptor/docs/addnewprovider.md +++ b/src/cloud-api-adaptor/docs/addnewprovider.md @@ -1,4 +1,4 @@ -# :memo: Adding support for a new provider +# :memo: Adding support for a new built-in provider ### Step 1: Initialize and register the cloud provider manager @@ -53,10 +53,391 @@ To include your provider you need reference it from the main package. Go build t ``` Note the comment at the top of the file, when building ensure `-tags=` is set to include your new provider. See the [Makefile](../../cloud-api-adaptor/Makefile#L26) for more context and usage. -#### Step 3: Add documentation on how to build a Pod VM image +### Step 3: Add documentation on how to build a Pod VM image For using the provider, a pod VM image needs to be created in order to create the peer pod instances. Add the instructions for building the peer pod VM image at the root directory similar to the other providers. -#### Step 4: Add E2E tests for the new provider +### Step 4: Add E2E tests for the new provider For more information, please refer to the section on [adding support for a new cloud provider](../test/e2e/README.md#adding-support-for-a-new-cloud-provider) in the E2E testing documentation. + + +# :memo: Adding support for a new external provider +External plugins are loaded dynamically and you don't need to recompile `cloud-api-adaptor` and `peerpod-ctrl` for adding external plugins. + +The following section describes building and using an external `libvirt` plugin. + +Assume you are using the `cloud-api-adaptor` and `peerpod-ctrl` image with the external plugin function support. + +And you are currently located in the root folder of the cloud-api-adaptor on your development machine. + +### Step 1: Initialize and register the cloud provider manager + +```bash +mkdir -p src/libvirt/build +cd src/libvirt + +cat > manager.go < **Note:** The the package name must be "main" for external plugin, all other required methods are same as built-in plugin. + +### Step 2: Add provider specific code +```bash +cat > provider.go < go.mod < ../cloud-providers + +EOF + +go mod tidy +``` + +### Step 4: build the external cloud provider plugin file via docker +```bash +cat > Dockerfile < **Note:** the external cloud provider plugin need to be built using the same golang and package versions that was used to build cloud-api-adaptor. + +The built out external plugin file is "src/libvirt/build/libvirt.so" + +### Step 5: Prepare the test libvirt peerpod env by following this [document](../libvirt/README.md) + +### Step 6: Update the "peer-pods-cm" configmap to enable cloud provider external `libvirt` plugin +- Calculate the SHA256 checksum of the built external plugin +```bash +sha256sum src/libvirt/build/libvirt.so +60e5cdbcb910c6331c796ce66dfa32e50bf083689ffdf18ee136d91a9da5ddab src/libvirt/build/libvirt.so +``` +- Update "peer-pods-cm" configmap +```bash +kubectl edit cm peer-pods-cm -n confidential-containers-system +... + CLOUD_PROVIDER: libvirt + ENABLE_CLOUD_PROVIDER_EXTERNAL_PLUGIN: "true" + CLOUD_PROVIDER_EXTERNAL_PLUGIN_HASH: 60e5cdbcb910c6331c796ce66dfa32e50bf083689ffdf18ee136d91a9da5ddab + CLOUD_PROVIDER_EXTERNAL_PLUGIN_PATH: /cloud-providers/libvirt.so +... +``` +> **Note:** CLOUD_PROVIDER_EXTERNAL_PLUGIN_HASH is sha256sum of the built out external cloud provider plugin + +### Step 7: Actions on the worker node +- Copy the external plugin file to worker node `/opt/cloud-api-adaptor/plugins` folder +```bash +ssh root@worker-ip 'mkdir -p /opt/cloud-api-adaptor/plugins && chmod +x /opt/cloud-api-adaptor/plugins' +scp src/libvirt/build/libvirt.so root@worker-ip:/opt/cloud-api-adaptor/plugins +``` +- Prepare `entrypoint.sh` for the external `libvirt` plugin +```bash +cat <<'EOF' > src/libvirt/build/entrypoint.sh +#!/bin/bash + +CLOUD_PROVIDER=${1:-$CLOUD_PROVIDER} +# Enabling dynamically loaded cloud provider external plugin feature, disabled by default +ENABLE_CLOUD_PROVIDER_EXTERNAL_PLUGIN=${ENABLE_CLOUD_PROVIDER_EXTERNAL_PLUGIN:-false} + +CRI_RUNTIME_ENDPOINT=${CRI_RUNTIME_ENDPOINT:-/run/cri-runtime.sock} +optionals+="" + +# Ensure you add a space before the closing quote (") when updating the optionals +# example: +# following is the correct method: optionals+="-option val " +# following is the incorrect method: optionals+="-option val" + +[[ -S ${CRI_RUNTIME_ENDPOINT} ]] && optionals+="-cri-runtime-endpoint ${CRI_RUNTIME_ENDPOINT} " +[[ "${PAUSE_IMAGE}" ]] && optionals+="-pause-image ${PAUSE_IMAGE} " +[[ "${VXLAN_PORT}" ]] && optionals+="-vxlan-port ${VXLAN_PORT} " +[[ "${CACERT_FILE}" ]] && optionals+="-ca-cert-file ${CACERT_FILE} " +[[ "${CERT_FILE}" ]] && [[ "${CERT_KEY}" ]] && optionals+="-cert-file ${CERT_FILE} -cert-key ${CERT_KEY} " +[[ "${TLS_SKIP_VERIFY}" ]] && optionals+="-tls-skip-verify " +[[ "${PROXY_TIMEOUT}" ]] && optionals+="-proxy-timeout ${PROXY_TIMEOUT} " +[[ "${AA_KBC_PARAMS}" ]] && optionals+="-aa-kbc-params ${AA_KBC_PARAMS} " +[[ "${FORWARDER_PORT}" ]] && optionals+="-forwarder-port ${FORWARDER_PORT} " +[[ "${CLOUD_CONFIG_VERIFY}" == "true" ]] && optionals+="-cloud-config-verify " + +test_vars() { + for i in "$@"; do + [ -z "${!i}" ] && echo "\$$i is NOT set" && EXT=1 + done + [[ -n $EXT ]] && exit 1 +} + +one_of() { + for i in "$@"; do + [ -n "${!i}" ] && echo "\$$i is SET" && EXIST=1 + done + [[ -z $EXIST ]] && echo "At least one of these must be SET: $*" && exit 1 +} + +libvirt() { + test_vars LIBVIRT_URI + + [[ "${DISABLECVM}" = "true" ]] && optionals+="-disable-cvm " + set -x + exec cloud-api-adaptor libvirt \ + -uri "${LIBVIRT_URI}" \ + -data-dir /opt/data-dir \ + -pods-dir /run/peerpod/pods \ + -network-name "${LIBVIRT_NET:-default}" \ + -pool-name "${LIBVIRT_POOL:-default}" \ + ${optionals} \ + -socket /run/peerpod/hypervisor.sock +} + +help_msg() { + cat <<'HELP_EOF' +Usage: + CLOUD_PROVIDER=libvirt $0 +or + $0 libvirt +in addition all cloud provider specific env variables must be set and valid +(CLOUD_PROVIDER is currently set to "$CLOUD_PROVIDER") +HELP_EOF +} + +if [[ "$CLOUD_PROVIDER" == "libvirt" ]]; then + libvirt +else + help_msg +fi +EOF +chmod +x src/libvirt/build/entrypoint.sh +``` +- Copy the external plugin file to worker node `/opt/cloud-api-adaptor/plugins` folder +```bash +scp src/libvirt/build/entrypoint.sh root@worker-ip:/opt/cloud-api-adaptor/plugins +``` +### Step 8: Update cloud-api-adaptor damonset to use the external `libvirt` plugin +- Run the `kubectl edit` command to update cloud-api-adaptor damonset +```bash +kubectl edit ds cloud-api-adaptor-daemonset -n confidential-containers-system +``` +- Overwrite the command +```yaml + spec: + containers: + - command: + - /cloud-providers/entrypoint.sh +``` +- Mount `/opt/cloud-api-adaptor/plugins/` from worker node to the `cloud-api-adaptor-con` container +```yaml +... + volumeMounts: + - mountPath: /cloud-providers + name: provider-dir +... + volumes: + - hostPath: + path: /opt/cloud-api-adaptor/plugins + type: Directory + name: provider-dir +... +``` +### Step 9: Update peerpod-ctrl deployment to use the external `libvirt` plugin +- Run the edit command to update peerpod-ctrl deployment +```bash +kubectl edit deployment peerpod-ctrl-controller-manager -n confidential-containers-system +``` +- Mount `/opt/cloud-api-adaptor/plugins` from worker node to the `manager` container +```yaml +... + volumeMounts: + - mountPath: /cloud-providers + name: provider-dir +... + volumes: + - hostPath: + path: /opt/cloud-api-adaptor/plugins + type: Directory + name: provider-dir +... +``` +### Step 10: Verify cloud-api-adaptor/peerpod-ctrl pod is running without error +```bash +kubectl logs -n confidential-containers-system ds/cloud-api-adaptor-daemonset + ++ exec cloud-api-adaptor libvirt -uri 'qemu+ssh://root@192.168.122.1/system?no_verify=1' -data-dir /opt/data-dir -pods-dir /run/peerpod/pods -network-name default -pool-name default -disable-cvm -socket /run/peerpod/hypervisor.sock +2024/04/17 04:34:56 [adaptor/cloud] Loading external plugin libvirt from /cloud-providers/libvirt.so +2024/04/17 04:34:56 [adaptor/cloud] Successfully opened the external plugin /cloud-providers/libvirt.so +cloud-api-adaptor version v0.8.2-dev + commit: a8b81333ccb6b0e0adf71c8eda675da97d24d649 + go: go1.21.9 +cloud-api-adaptor: starting Cloud API Adaptor daemon for "libvirt" +2024/04/17 04:34:56 [adaptor/cloud/libvirt] libvirt config: &libvirt.Config{URI:"qemu+ssh://root@192.168.122.1/system?no_verify=1", PoolName:"default", NetworkName:"default", DataDir:"/opt/data-dir", DisableCVM:true, VolName:"podvm-base.qcow2", LaunchSecurity:"", Firmware:"/usr/share/edk2/ovmf/OVMF_CODE.fd"} +2024/04/17 04:34:56 [adaptor/cloud/libvirt] Created libvirt connection +2024/04/17 04:34:56 [adaptor] server config: &adaptor.ServerConfig{TLSConfig:(*tlsutil.TLSConfig)(0xc0000d4080), SocketPath:"/run/peerpod/hypervisor.sock", CriSocketPath:"", PauseImage:"", PodsDir:"/run/peerpod/pods", ForwarderPort:"15150", ProxyTimeout:300000000000, AAKBCParams:"", EnableCloudConfigVerify:false} +2024/04/17 04:34:56 [util/k8sops] initialized PeerPodService +2024/04/17 04:34:56 [probe/probe] Using port: 8000 +2024/04/17 04:34:56 [adaptor] server started +2024/04/17 04:35:35 [probe/probe] nodeName: peer-pods-worker-0 +2024/04/17 04:35:35 [probe/probe] Selected pods count: 10 +2024/04/17 04:35:35 [probe/probe] Ignored standard pod: cc-operator-controller-manager-857f844f7d-7xmjr +2024/04/17 04:35:35 [probe/probe] Ignored standard pod: cc-operator-daemon-install-ms96p +2024/04/17 04:35:35 [probe/probe] Ignored standard pod: cc-operator-pre-install-daemon-wfnfl +2024/04/17 04:35:35 [probe/probe] Ignored standard pod: cloud-api-adaptor-daemonset-66srd +2024/04/17 04:35:35 [probe/probe] Ignored standard pod: peerpod-ctrl-controller-manager-865cb874d-mknth +2024/04/17 04:35:35 [probe/probe] Ignored standard pod: ingress-nginx-admission-create-lxlx4 +2024/04/17 04:35:35 [probe/probe] Ignored standard pod: ingress-nginx-admission-patch-j2xqb +2024/04/17 04:35:35 [probe/probe] Ignored standard pod: ingress-nginx-controller-7bf7bc78dc-c4tzl +2024/04/17 04:35:35 [probe/probe] Ignored standard pod: kube-flannel-ds-rk56t +2024/04/17 04:35:35 [probe/probe] Ignored standard pod: kube-proxy-464dj +2024/04/17 04:35:35 [probe/probe] All PeerPods standup. we do not check the PeerPods status any more. +... + +kubectl logs -n confidential-containers-system $(kubectl get po -A | grep peerpod-ctrl-controller-manager | awk '{print $2}') + +2024-04-17T04:35:20Z INFO controller-runtime.metrics Metrics server is starting to listen {"addr": "127.0.0.1:8080"} +2024/04/17 04:35:20 [adaptor/cloud] Loading external plugin libvirt from /cloud-providers/libvirt.so +2024/04/17 04:35:20 [adaptor/cloud] Successfully opened the external plugin /cloud-providers/libvirt.so +2024/04/17 04:35:20 [adaptor/cloud/libvirt] libvirt config: &libvirt.Config{URI:"qemu+ssh://root@192.168.122.1/system?no_verify=1", PoolName:"default", NetworkName:"default", DataDir:"", DisableCVM:false, VolName:"podvm-base.qcow2", LaunchSecurity:"", Firmware:"/usr/share/edk2/ovmf/OVMF_CODE.fd"} +2024/04/17 04:35:20 [adaptor/cloud/libvirt] Created libvirt connection +2024-04-17T04:35:20Z INFO setup starting manager +2024-04-17T04:35:20Z INFO Starting server {"path": "/metrics", "kind": "metrics", "addr": "127.0.0.1:8080"} +I0417 04:35:20.876671 1 leaderelection.go:248] attempting to acquire leader lease confidential-containers-system/33f6c5d6.confidentialcontainers.org... +2024-04-17T04:35:20Z INFO Starting server {"kind": "health probe", "addr": "[::]:8081"} +I0417 04:35:37.265021 1 leaderelection.go:258] successfully acquired lease confidential-containers-system/33f6c5d6.confidentialcontainers.org +2024-04-17T04:35:37Z DEBUG events peerpod-ctrl-controller-manager-865cb874d-mknth_da8a80e2-4984-4720-828e-3d3b3ff53b2a became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"confidential-containers-system","name":"33f6c5d6.confidentialcontainers.org","uid":"3e18f493-803b-490d-b9e0-b23104bac54e","apiVersion":"coordination.k8s.io/v1","resourceVersion":"17873"}, "reason": "LeaderElection"} +2024-04-17T04:35:37Z INFO Starting EventSource {"controller": "peerpod", "controllerGroup": "confidentialcontainers.org", "controllerKind": "PeerPod", "source": "kind source: *v1alpha1.PeerPod"} +2024-04-17T04:35:37Z INFO Starting Controller {"controller": "peerpod", "controllerGroup": "confidentialcontainers.org", "controllerKind": "PeerPod"} +2024-04-17T04:35:37Z INFO Starting workers {"controller": "peerpod", "controllerGroup": "confidentialcontainers.org", "controllerKind": "PeerPod", "worker count": 1} +... +``` + +#### Troubleshooting +- "failed to map segment from shared object" from CAA/Peerpod-ctrl log +> - Please make sure `CLOUD_PROVIDER_EXTERNAL_PLUGIN_PATH` on worker node have execute permissions, `chmod +x $CLOUD_PROVIDER_EXTERNAL_PLUGIN_PATH` +- "plugin was built with a different version of package XXX" from CAA/Peerpod-ctrl log +> - Please check the go.mod of CAA and plugins project, the CAA and plugins should be built with same version of issue package XXX +> - Please make sure use same golang env to build CAA, Peerpod-ctrl and cloud-provider plugins diff --git a/src/cloud-api-adaptor/entrypoint.sh b/src/cloud-api-adaptor/entrypoint.sh index 6a443baad..9c0e326cb 100755 --- a/src/cloud-api-adaptor/entrypoint.sh +++ b/src/cloud-api-adaptor/entrypoint.sh @@ -1,6 +1,9 @@ #!/bin/bash CLOUD_PROVIDER=${1:-$CLOUD_PROVIDER} +# Enabling dynamically loaded cloud provider external plugin feature, disabled by default +ENABLE_CLOUD_PROVIDER_EXTERNAL_PLUGIN=${ENABLE_CLOUD_PROVIDER_EXTERNAL_PLUGIN:-false} + CRI_RUNTIME_ENDPOINT=${CRI_RUNTIME_ENDPOINT:-/run/cri-runtime.sock} optionals+="" diff --git a/src/cloud-api-adaptor/install/overlays/aws/kustomization.yaml b/src/cloud-api-adaptor/install/overlays/aws/kustomization.yaml index f48d5ebbb..de52aba1a 100644 --- a/src/cloud-api-adaptor/install/overlays/aws/kustomization.yaml +++ b/src/cloud-api-adaptor/install/overlays/aws/kustomization.yaml @@ -17,6 +17,7 @@ configMapGenerator: namespace: confidential-containers-system literals: - CLOUD_PROVIDER="aws" + - ENABLE_CLOUD_PROVIDER_EXTERNAL_PLUGIN="false" # flag to enable/disable dynamically load cloud provider external plugin feature - CLOUD_CONFIG_VERIFY="false" # It's better set as true to enable could config verify in production env #- PAUSE_IMAGE="" # Uncomment and set if you want to use a specific pause image #- VXLAN_PORT="" # Uncomment and set if you want to use a specific vxlan port. Defaults to 4789 diff --git a/src/cloud-api-adaptor/install/overlays/azure/kustomization.yaml b/src/cloud-api-adaptor/install/overlays/azure/kustomization.yaml index 931ceb572..e2a6d51a5 100644 --- a/src/cloud-api-adaptor/install/overlays/azure/kustomization.yaml +++ b/src/cloud-api-adaptor/install/overlays/azure/kustomization.yaml @@ -17,6 +17,7 @@ configMapGenerator: namespace: confidential-containers-system literals: - CLOUD_PROVIDER="azure" + - ENABLE_CLOUD_PROVIDER_EXTERNAL_PLUGIN="false" # flag to enable/disable dynamically load cloud provider external plugin feature - CLOUD_CONFIG_VERIFY="false" # It's better set as true to enable could config verify in production env - AZURE_SUBSCRIPTION_ID="" #set - AZURE_REGION="eastus" #set diff --git a/src/cloud-api-adaptor/install/overlays/ibmcloud-powervs/kustomization.yaml b/src/cloud-api-adaptor/install/overlays/ibmcloud-powervs/kustomization.yaml index b2f993089..6d66f66f4 100644 --- a/src/cloud-api-adaptor/install/overlays/ibmcloud-powervs/kustomization.yaml +++ b/src/cloud-api-adaptor/install/overlays/ibmcloud-powervs/kustomization.yaml @@ -17,6 +17,7 @@ configMapGenerator: namespace: confidential-containers-system literals: - CLOUD_PROVIDER="ibmcloud-powervs" + - ENABLE_CLOUD_PROVIDER_EXTERNAL_PLUGIN="false" # flag to enable/disable dynamically load cloud provider external plugin feature - CLOUD_CONFIG_VERIFY="false" # It's better set as true to enable could config verify in production env - POWERVS_SERVICE_INSTANCE_ID="" #set - POWERVS_NETWORK_ID="" #set diff --git a/src/cloud-api-adaptor/install/overlays/ibmcloud/kustomization.yaml b/src/cloud-api-adaptor/install/overlays/ibmcloud/kustomization.yaml index d3ac6ca8b..078804bfd 100644 --- a/src/cloud-api-adaptor/install/overlays/ibmcloud/kustomization.yaml +++ b/src/cloud-api-adaptor/install/overlays/ibmcloud/kustomization.yaml @@ -17,6 +17,7 @@ configMapGenerator: namespace: confidential-containers-system literals: - CLOUD_PROVIDER="ibmcloud" + - ENABLE_CLOUD_PROVIDER_EXTERNAL_PLUGIN="false" # flag to enable/disable dynamically load cloud provider external plugin feature - CLOUD_CONFIG_VERIFY="false" # It's better set as true to enable could config verify in production env - IBMCLOUD_VPC_ENDPOINT="" #set - IBMCLOUD_RESOURCE_GROUP_ID="" #set diff --git a/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml b/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml index 78015593e..2acfc4cc8 100644 --- a/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml +++ b/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml @@ -17,6 +17,7 @@ configMapGenerator: namespace: confidential-containers-system literals: - CLOUD_PROVIDER="libvirt" + - ENABLE_CLOUD_PROVIDER_EXTERNAL_PLUGIN="false" # flag to enable/disable dynamically load cloud provider external plugin feature - CLOUD_CONFIG_VERIFY="false" # It's better set as true to enable could config verify in production env - LIBVIRT_URI="qemu+ssh://root@192.168.122.1/system?no_verify=1" #set - LIBVIRT_NET="default" # set diff --git a/src/cloud-api-adaptor/install/overlays/vsphere/kustomization.yaml b/src/cloud-api-adaptor/install/overlays/vsphere/kustomization.yaml index 312f72930..f6ef80807 100644 --- a/src/cloud-api-adaptor/install/overlays/vsphere/kustomization.yaml +++ b/src/cloud-api-adaptor/install/overlays/vsphere/kustomization.yaml @@ -17,6 +17,7 @@ configMapGenerator: namespace: confidential-containers-system literals: - CLOUD_PROVIDER="vsphere" + - ENABLE_CLOUD_PROVIDER_EXTERNAL_PLUGIN="false" # flag to enable/disable dynamically load cloud provider external plugin feature - CLOUD_CONFIG_VERIFY="false" # It's better set as true to enable could config verify in production env - GOVC_URL="" # Setting the vCenter URL is required. - GOVC_DATACENTER="" # Setting the vCenter datacenter is required. diff --git a/src/cloud-api-adaptor/install/yamls/caa-pod.yaml b/src/cloud-api-adaptor/install/yamls/caa-pod.yaml index db2faf79c..59d9ee3cb 100644 --- a/src/cloud-api-adaptor/install/yamls/caa-pod.yaml +++ b/src/cloud-api-adaptor/install/yamls/caa-pod.yaml @@ -59,6 +59,10 @@ spec: - mountPath: /run/netns mountPropagation: HostToContainer name: netns + # # setting for cloud provider external plugin + # - mountPath: /cloud-providers + # name: provider-dir + # # setting for cloud provider external plugin hostNetwork: true dnsPolicy: ClusterFirstWithHostNet nodeSelector: @@ -79,3 +83,9 @@ spec: - hostPath: path: /run/netns name: netns + # # setting for cloud provider external plugin + # - hostPath: + # path: /opt/cloud-api-adaptor/plugins + # type: Directory + # name: provider-dir + # # setting for cloud provider external plugin diff --git a/src/cloud-providers/go.mod b/src/cloud-providers/go.mod index 503207739..4a623919c 100644 --- a/src/cloud-providers/go.mod +++ b/src/cloud-providers/go.mod @@ -11,20 +11,20 @@ require ( github.com/IBM/go-sdk-core/v5 v5.13.1 github.com/IBM/platform-services-go-sdk v0.36.0 github.com/IBM/vpc-go-sdk v0.35.0 - github.com/avast/retry-go/v4 v4.3.3 + github.com/avast/retry-go/v4 v4.5.1 github.com/aws/aws-sdk-go-v2 v1.21.0 github.com/aws/aws-sdk-go-v2/config v1.15.11 github.com/aws/aws-sdk-go-v2/credentials v1.12.6 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.6 github.com/aws/aws-sdk-go-v2/service/ec2 v1.117.0 - github.com/kdomanski/iso9660 v0.3.5 + github.com/kdomanski/iso9660 v0.4.0 github.com/stretchr/testify v1.8.4 - github.com/vmware/govmomi v0.29.0 + github.com/vmware/govmomi v0.33.1 gopkg.in/yaml.v2 v2.4.0 k8s.io/apimachinery v0.26.0 k8s.io/client-go v0.26.0 - libvirt.org/go/libvirt v1.8002.0 - libvirt.org/go/libvirtxml v1.9004.0 + libvirt.org/go/libvirt v1.9008.0 + libvirt.org/go/libvirtxml v1.9007.0 ) require ( diff --git a/src/cloud-providers/go.sum b/src/cloud-providers/go.sum index f8179eab6..4b9fb2405 100644 --- a/src/cloud-providers/go.sum +++ b/src/cloud-providers/go.sum @@ -31,8 +31,8 @@ github.com/apparentlymart/go-cidr v1.1.0 h1:2mAhrMoF+nhXqxTzSZMUzDHkLjmIHC+Zzn4t github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= -github.com/avast/retry-go/v4 v4.3.3 h1:G56Bp6mU0b5HE1SkaoVjscZjlQb0oy4mezwY/cGH19w= -github.com/avast/retry-go/v4 v4.3.3/go.mod h1:rg6XFaiuFYII0Xu3RDbZQkxCofFwruZKW8oEF1jpWiU= +github.com/avast/retry-go/v4 v4.5.1 h1:AxIx0HGi4VZ3I02jr78j5lZ3M6x1E0Ivxa6b0pUUh7o= +github.com/avast/retry-go/v4 v4.5.1/go.mod h1:/sipNsvNB3RRuT5iNcb6h73nw3IBmXJ/H3XrCQYSOpc= github.com/aws/aws-sdk-go-v2 v1.16.5/go.mod h1:Wh7MEsmEApyL5hrWzpDkba4gwAPc5/piwLVLFnCxp48= github.com/aws/aws-sdk-go-v2 v1.21.0 h1:gMT0IW+03wtYJhRqTVYn0wLzwdnK9sRMcxmtfGzRdJc= github.com/aws/aws-sdk-go-v2 v1.21.0/go.mod h1:/RfNgGmRxI+iFOB1OeJUyxiU+9s88k3pfHvDagGEp0M= @@ -213,8 +213,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= -github.com/kdomanski/iso9660 v0.3.5 h1:LO1n75zPjLeDQkz0Pyk1eZ7JGinjKjk2C174GSABVwY= -github.com/kdomanski/iso9660 v0.3.5/go.mod h1:K+UlIGxKgtrdAWyoigPnFbeQLVs/Xudz4iztWFThBwo= +github.com/kdomanski/iso9660 v0.4.0 h1:BPKKdcINz3m0MdjIMwS0wx1nofsOjxOq8TOr45WGHFg= +github.com/kdomanski/iso9660 v0.4.0/go.mod h1:OxUSupHsO9ceI8lBLPJKWBTphLemjrCQY8LPXM7qSzU= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= @@ -307,8 +307,8 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/vmware/govmomi v0.29.0 h1:SHJQ7DUc4fltFZv16znJNGHR1/XhiDK5iKxm2OqwkuU= -github.com/vmware/govmomi v0.29.0/go.mod h1:F7adsVewLNHsW/IIm7ziFURaXDaHEwcc+ym4r3INMdY= +github.com/vmware/govmomi v0.33.1 h1:qS2VpEBd/WLbzLO5McI6h5o5zaKsrezUxRY5r9jkW8A= +github.com/vmware/govmomi v0.33.1/go.mod h1:QuzWGiEMA/FYlu5JXKjytiORQoxv2hTHdS2lWnIqKMM= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= @@ -474,10 +474,10 @@ k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+O k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 h1:KTgPnR10d5zhztWptI952TNtt/4u5h3IzDXkdIMuo2Y= k8s.io/utils v0.0.0-20221128185143-99ec85e7a448/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -libvirt.org/go/libvirt v1.8002.0 h1:X8gz2Sa1ek4S5FznpDpeRz6JpNb7NdkfzTii5GMIwDY= -libvirt.org/go/libvirt v1.8002.0/go.mod h1:1WiFE8EjZfq+FCVog+rvr1yatKbKZ9FaFMZgEqxEJqQ= -libvirt.org/go/libvirtxml v1.9004.0 h1:h+nhEZCABCnK4go0GLRN2WZhIhRrLAqsz84t553oiM4= -libvirt.org/go/libvirtxml v1.9004.0/go.mod h1:7Oq2BLDstLr/XtoQD8Fr3mfDNrzlI3utYKySXF2xkng= +libvirt.org/go/libvirt v1.9008.0 h1:LLpjuSQm9gChnx7I/44SLLg/eyvTnJpcMAFmKot65Zc= +libvirt.org/go/libvirt v1.9008.0/go.mod h1:1WiFE8EjZfq+FCVog+rvr1yatKbKZ9FaFMZgEqxEJqQ= +libvirt.org/go/libvirtxml v1.9007.0 h1:NjFBpv5aDutbtuem7VP9s8ZW8XEuCKIO7kkvx+sildQ= +libvirt.org/go/libvirtxml v1.9007.0/go.mod h1:7Oq2BLDstLr/XtoQD8Fr3mfDNrzlI3utYKySXF2xkng= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= diff --git a/src/cloud-providers/table.go b/src/cloud-providers/table.go index 5b535d421..00dfd6361 100644 --- a/src/cloud-providers/table.go +++ b/src/cloud-providers/table.go @@ -1,7 +1,14 @@ package provider import ( + "crypto/sha256" "flag" + "fmt" + "io" + "os" + "path/filepath" + "plugin" + "strings" ) type CloudProvider interface { @@ -12,7 +19,93 @@ type CloudProvider interface { var providerTable map[string]CloudProvider = make(map[string]CloudProvider) +func getFileNameAndSha256sum(providerPath string) (string, string, error) { + file, err := os.Open(providerPath) + if err != nil { + return "", "", err + } + defer file.Close() + + // Get the base filename without the directory path + filename := filepath.Base(providerPath) + + hash := sha256.New() + if _, err := io.Copy(hash, file); err != nil { + return filename, "", err + } + sum := hash.Sum(nil) + return filename, fmt.Sprintf("%x", sum), nil +} + +func hasExecutePermission(providerPath string) (bool, error) { + // Get the parent directory of the specified file path + dir := filepath.Dir(providerPath) + + // Stat the directory to get its file info + dirInfo, err := os.Stat(dir) + if err != nil { + return false, err + } + + // Check if the directory has execute permission for the current user + mode := dirInfo.Mode() + executePermission := mode&os.ModeDir != 0 && mode&0100 != 0 + + return executePermission, nil +} + +// LoadCloudProvider loads cloud provider external plugin from the given path CLOUD_PROVIDER_EXTERNAL_PLUGIN_PATH +// The values of 1) ${CLOUD_PROVIDER}, 2) the filename of the cloud provider external plugin +// and 3) the provider defined within the external plugin must all match +func LoadCloudProvider(name string) { + if os.Getenv("ENABLE_CLOUD_PROVIDER_EXTERNAL_PLUGIN") != "true" { + logger.Printf("Cloud provider external plugin loading is disabled, skipping plugin loading") + return + } + externalPluginPath := os.Getenv("CLOUD_PROVIDER_EXTERNAL_PLUGIN_PATH") + executePermission, err := hasExecutePermission(externalPluginPath) + if err != nil { + logger.Printf("Failed to retrieve file information for the parent directory of CLOUD_PROVIDER_EXTERNAL_PLUGIN_PATH %s", err) + return + } + if !executePermission { + logger.Printf("The parent directory of the external plugin %s lacks execute permissions", filepath.Dir(externalPluginPath)) + return + } + cloudProviderPluginHash := os.Getenv("CLOUD_PROVIDER_EXTERNAL_PLUGIN_HASH") + if externalPluginPath == "" { + logger.Printf("Env CLOUD_PROVIDER_EXTERNAL_PLUGIN_PATH is not set") + return + } + if cloudProviderPluginHash == "" { + logger.Printf("Env CLOUD_PROVIDER_EXTERNAL_PLUGIN_HASH is not set") + return + } + filename, realPluginHash, err := getFileNameAndSha256sum(externalPluginPath) + if !strings.EqualFold(filename, name+".so") { + logger.Printf("Filename of the external plugin: %s, is not match with CLOUD_PROVIDER: %s", filename, name) + return + } + logger.Printf("Loading external plugin %s from %s", name, externalPluginPath) + + if err != nil { + logger.Printf("Failed to calculate the SHA256 checksum of the external plugin %s", err) + return + } + if cloudProviderPluginHash != realPluginHash { + logger.Printf("The sha256sum of the external plugin: %s doesn't match the one from configmap: %s", realPluginHash, cloudProviderPluginHash) + return + } + _, err = plugin.Open(externalPluginPath) + if err != nil { + logger.Printf("Failed to open the external plugin %s", err) + } else { + logger.Printf("Successfully opened the external plugin %s", externalPluginPath) + } +} + func Get(name string) CloudProvider { + LoadCloudProvider(name) return providerTable[name] } diff --git a/src/peerpod-ctrl/Dockerfile b/src/peerpod-ctrl/Dockerfile index df5c33a0f..0ed8980ee 100644 --- a/src/peerpod-ctrl/Dockerfile +++ b/src/peerpod-ctrl/Dockerfile @@ -1,11 +1,11 @@ # Build the manager binary -FROM --platform=$TARGETPLATFORM quay.io/confidential-containers/golang-fedora:1.20.8-38 as builder +FROM --platform=$TARGETPLATFORM quay.io/confidential-containers/golang-fedora:1.21.9-38 as builder ARG TARGETOS ARG TARGETARCH ARG CGO_ENABLED=1 ARG GOFLAGS -WORKDIR /workspace/ +WORKDIR /work RUN if [ "$CGO_ENABLED" = 1 ] ; then dnf install -y libvirt-devel && dnf clean all; fi # Copy the Go Modules manifests COPY peerpod-ctrl/go.mod peerpod-ctrl/go.mod @@ -13,7 +13,7 @@ COPY peerpod-ctrl/go.sum peerpod-ctrl/go.sum COPY cloud-providers cloud-providers # cache deps before building and copying source so that we don't need to re-download as much # and so that source changes don't invalidate our downloaded layer -WORKDIR /workspace/peerpod-ctrl/ +WORKDIR /work/peerpod-ctrl/ RUN go mod download # Copy the go source @@ -35,6 +35,6 @@ ARG CGO_ENABLED=1 RUN if [ "$CGO_ENABLED" = 1 ] ; then dnf install -y libvirt-libs openssh-clients && dnf clean all; fi WORKDIR / -COPY --from=builder /workspace/peerpod-ctrl/manager . +COPY --from=builder /work/peerpod-ctrl/manager . ENTRYPOINT ["/manager"] diff --git a/src/peerpod-ctrl/config/manager/manager.yaml b/src/peerpod-ctrl/config/manager/manager.yaml index 69d126380..ce8ba3b5a 100644 --- a/src/peerpod-ctrl/config/manager/manager.yaml +++ b/src/peerpod-ctrl/config/manager/manager.yaml @@ -62,6 +62,12 @@ spec: defaultMode: 384 optional: true secretName: ssh-key-secret + # setting for cloud provider external plugin + # - hostPath: + # path: /opt/cloud-api-adaptor/plugins + # type: Directory + # name: provider-dir + # setting for cloud provider external plugin securityContext: runAsNonRoot: false # TODO(user): For common cases that do not require escalating privileges @@ -80,6 +86,10 @@ spec: - mountPath: /root/.ssh/ name: ssh readOnly: true + # # setting for cloud provider external plugin + # - mountPath: /cloud-providers + # name: provider-dir + # # setting for cloud provider external plugin envFrom: - secretRef: name: peer-pods-secret diff --git a/src/peerpodconfig-ctrl/go.mod b/src/peerpodconfig-ctrl/go.mod index 1cc0acc5c..01254fb1e 100644 --- a/src/peerpodconfig-ctrl/go.mod +++ b/src/peerpodconfig-ctrl/go.mod @@ -3,7 +3,7 @@ module github.com/confidential-containers/cloud-api-adaptor/src/peerpodconfig-ct go 1.20 require ( - github.com/go-logr/logr v1.2.3 + github.com/go-logr/logr v1.3.0 github.com/onsi/ginkgo/v2 v2.6.1 github.com/onsi/gomega v1.24.2 k8s.io/api v0.26.0 @@ -16,24 +16,24 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/emicklei/go-restful/v3 v3.9.0 // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-logr/zapr v1.2.3 // indirect - github.com/go-openapi/jsonpointer v0.19.5 // indirect - github.com/go-openapi/jsonreference v0.20.0 // indirect - github.com/go-openapi/swag v0.19.14 // indirect + github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.22.4 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/google/gnostic v0.5.7-v3refs // indirect - github.com/google/go-cmp v0.5.9 // indirect - github.com/google/gofuzz v1.1.0 // indirect - github.com/google/uuid v1.1.2 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/uuid v1.3.1 // indirect github.com/imdario/mergo v0.3.6 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/mailru/easyjson v0.7.6 // indirect + github.com/mailru/easyjson v0.7.7 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect @@ -61,11 +61,11 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.26.0 // indirect k8s.io/component-base v0.26.0 // indirect - k8s.io/klog/v2 v2.80.1 // indirect + k8s.io/klog/v2 v2.110.1 // indirect k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 // indirect - sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) diff --git a/src/peerpodconfig-ctrl/go.sum b/src/peerpodconfig-ctrl/go.sum index 55d970a65..5d404fcdc 100644 --- a/src/peerpodconfig-ctrl/go.sum +++ b/src/peerpodconfig-ctrl/go.sum @@ -50,8 +50,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= -github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -67,20 +67,18 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= -github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A= github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4= -github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= -github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA= -github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= -github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= +github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -110,8 +108,9 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= @@ -125,11 +124,12 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= -github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -140,8 +140,8 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -162,14 +162,14 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= -github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2 h1:hAHbPm5IJGijwng3PWk09JkG9WeqChjprR5s9bBZ+OM= github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= @@ -181,8 +181,6 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/onsi/ginkgo/v2 v2.6.1 h1:1xQPCjcqYw/J5LchOcp4/2q/jzJFjiAOc25chhnDw+Q= github.com/onsi/ginkgo/v2 v2.6.1/go.mod h1:yjiuMwPokqY1XauOgju45q3sJt6VzQ/Fict1LFVcsAo= github.com/onsi/gomega v1.24.2 h1:J/tulyYK6JwBldPViHJReihxxZ+22FHs0piGjQAvoUE= @@ -207,12 +205,16 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -496,8 +498,8 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= @@ -527,8 +529,8 @@ k8s.io/client-go v0.26.0 h1:lT1D3OfO+wIi9UFolCrifbjUUgu7CpLca0AD8ghRLI8= k8s.io/client-go v0.26.0/go.mod h1:I2Sh57A79EQsDmn7F7ASpmru1cceh3ocVT9KlX2jEZg= k8s.io/component-base v0.26.0 h1:0IkChOCohtDHttmKuz+EP3j3+qKmV55rM9gIFTXA7Vs= k8s.io/component-base v0.26.0/go.mod h1:lqHwlfV1/haa14F/Z5Zizk5QmzaVf23nQzCwVOQpfC8= -k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= -k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= +k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+OGxg8HsuBr/5f6tVAjDu6E= k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 h1:KTgPnR10d5zhztWptI952TNtt/4u5h3IzDXkdIMuo2Y= @@ -538,9 +540,9 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/controller-runtime v0.14.0 h1:ju2xsov5Ara6FoQuddg+az+rAxsUsTYn2IYyEKCTyDc= sigs.k8s.io/controller-runtime v0.14.0/go.mod h1:GaRkrY8a7UZF0kqFFbUKG7n9ICiTY5T55P1RiE3UZlU= -sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= -sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=