From 99825f2663cd8513691cf7156d50e7e9e9a291e5 Mon Sep 17 00:00:00 2001 From: leigh capili Date: Mon, 8 Feb 2021 01:14:53 -0700 Subject: [PATCH 1/2] Add registry cred Deployments/CronJobs for aws/gcp/azure via kustomize Signed-off-by: leigh capili --- .../_base/kustomization.yaml | 20 +++ .../_base/kustomizeconfig.yaml | 3 + .../registry-credentials-sync/_base/sync.yaml | 125 ++++++++++++++++++ .../_cronjobs/_base/kustomization.yaml | 20 +++ .../_cronjobs/_base/kustomizeconfig.yaml | 3 + .../_cronjobs/_base/sync.yaml | 101 ++++++++++++++ .../_cronjobs/aws/config-patches.yaml | 52 ++++++++ .../_cronjobs/aws/kubectl-patch.yaml | 30 +++++ .../_cronjobs/aws/kustomization.yaml | 26 ++++ .../_cronjobs/aws/reconcile-patch.yaml | 29 ++++ .../_cronjobs/azure/az-identity.yaml | 7 + .../_cronjobs/azure/config-patches.yaml | 41 ++++++ .../_cronjobs/azure/kubectl-patch.yaml | 30 +++++ .../_cronjobs/azure/kustomization.yaml | 28 ++++ .../_cronjobs/azure/kustomizeconfig.yaml | 3 + .../_cronjobs/azure/reconcile-patch.yaml | 37 ++++++ .../_cronjobs/gcp/config-patches.yaml | 28 ++++ .../_cronjobs/gcp/kustomization.yaml | 15 +++ .../_cronjobs/gcp/reconcile-patch.yaml | 29 ++++ .../aws/config-patches.yaml | 42 ++++++ .../aws/kubectl-patch.yaml | 28 ++++ .../aws/kustomization.yaml | 26 ++++ .../aws/reconcile-patch.yaml | 28 ++++ .../azure/az-identity.yaml | 7 + .../azure/config-patches.yaml | 39 ++++++ .../azure/kubectl-patch.yaml | 28 ++++ .../azure/kustomization.yaml | 28 ++++ .../azure/kustomizeconfig.yaml | 3 + .../azure/reconcile-patch.yaml | 30 +++++ .../gcp/config-patches.yaml | 20 +++ .../gcp/kustomization.yaml | 15 +++ .../gcp/reconcile-patch.yaml | 28 ++++ 32 files changed, 949 insertions(+) create mode 100644 manifests/integrations/registry-credentials-sync/_base/kustomization.yaml create mode 100644 manifests/integrations/registry-credentials-sync/_base/kustomizeconfig.yaml create mode 100644 manifests/integrations/registry-credentials-sync/_base/sync.yaml create mode 100644 manifests/integrations/registry-credentials-sync/_cronjobs/_base/kustomization.yaml create mode 100644 manifests/integrations/registry-credentials-sync/_cronjobs/_base/kustomizeconfig.yaml create mode 100644 manifests/integrations/registry-credentials-sync/_cronjobs/_base/sync.yaml create mode 100644 manifests/integrations/registry-credentials-sync/_cronjobs/aws/config-patches.yaml create mode 100644 manifests/integrations/registry-credentials-sync/_cronjobs/aws/kubectl-patch.yaml create mode 100644 manifests/integrations/registry-credentials-sync/_cronjobs/aws/kustomization.yaml create mode 100644 manifests/integrations/registry-credentials-sync/_cronjobs/aws/reconcile-patch.yaml create mode 100644 manifests/integrations/registry-credentials-sync/_cronjobs/azure/az-identity.yaml create mode 100644 manifests/integrations/registry-credentials-sync/_cronjobs/azure/config-patches.yaml create mode 100644 manifests/integrations/registry-credentials-sync/_cronjobs/azure/kubectl-patch.yaml create mode 100644 manifests/integrations/registry-credentials-sync/_cronjobs/azure/kustomization.yaml create mode 100644 manifests/integrations/registry-credentials-sync/_cronjobs/azure/kustomizeconfig.yaml create mode 100644 manifests/integrations/registry-credentials-sync/_cronjobs/azure/reconcile-patch.yaml create mode 100644 manifests/integrations/registry-credentials-sync/_cronjobs/gcp/config-patches.yaml create mode 100644 manifests/integrations/registry-credentials-sync/_cronjobs/gcp/kustomization.yaml create mode 100644 manifests/integrations/registry-credentials-sync/_cronjobs/gcp/reconcile-patch.yaml create mode 100644 manifests/integrations/registry-credentials-sync/aws/config-patches.yaml create mode 100644 manifests/integrations/registry-credentials-sync/aws/kubectl-patch.yaml create mode 100644 manifests/integrations/registry-credentials-sync/aws/kustomization.yaml create mode 100644 manifests/integrations/registry-credentials-sync/aws/reconcile-patch.yaml create mode 100644 manifests/integrations/registry-credentials-sync/azure/az-identity.yaml create mode 100644 manifests/integrations/registry-credentials-sync/azure/config-patches.yaml create mode 100644 manifests/integrations/registry-credentials-sync/azure/kubectl-patch.yaml create mode 100644 manifests/integrations/registry-credentials-sync/azure/kustomization.yaml create mode 100644 manifests/integrations/registry-credentials-sync/azure/kustomizeconfig.yaml create mode 100644 manifests/integrations/registry-credentials-sync/azure/reconcile-patch.yaml create mode 100644 manifests/integrations/registry-credentials-sync/gcp/config-patches.yaml create mode 100644 manifests/integrations/registry-credentials-sync/gcp/kustomization.yaml create mode 100644 manifests/integrations/registry-credentials-sync/gcp/reconcile-patch.yaml diff --git a/manifests/integrations/registry-credentials-sync/_base/kustomization.yaml b/manifests/integrations/registry-credentials-sync/_base/kustomization.yaml new file mode 100644 index 0000000000..c26a2c0ab0 --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/_base/kustomization.yaml @@ -0,0 +1,20 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +commonLabels: + app: credentials-sync + +resources: +- sync.yaml + +vars: +- name: KUBE_SECRET + objref: + kind: ConfigMap + name: credentials-sync + apiVersion: v1 + fieldref: + fieldpath: data.KUBE_SECRET + +configurations: +- kustomizeconfig.yaml diff --git a/manifests/integrations/registry-credentials-sync/_base/kustomizeconfig.yaml b/manifests/integrations/registry-credentials-sync/_base/kustomizeconfig.yaml new file mode 100644 index 0000000000..61edffd433 --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/_base/kustomizeconfig.yaml @@ -0,0 +1,3 @@ +varReference: +- path: rules/resourceNames + kind: Role diff --git a/manifests/integrations/registry-credentials-sync/_base/sync.yaml b/manifests/integrations/registry-credentials-sync/_base/sync.yaml new file mode 100644 index 0000000000..4efb25f0fe --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/_base/sync.yaml @@ -0,0 +1,125 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: credentials-sync +data: + # Patch this ConfigMap with additional values needed for your cloud + KUBE_SECRET: my-registry-token # does not yet exist -- will be created in the same Namespace + SYNC_PERIOD: "3600" # tokens expire; refresh faster than that + +--- +# This Deployment frequently fetches registry tokens and applies them as an imagePullSecret. +# It's done as a 1-replica Deployment rather than a CronJob, because CronJob scheduling can +# block cluster bootstraps and cold-reboots from obtaining registry tokens for a considerable time. +# This deployment will immediately fetch a token, which reduces latency for working image updates. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: credentials-sync + namespace: flux-system +spec: + replicas: 1 + strategy: + type: Recreate + template: + spec: + serviceAccount: credentials-sync + containers: + - image: busybox # override this with a cloud-specific image + name: sync + envFrom: + - configMapRef: + name: credentials-sync + env: + - name: RECONCILE_SH # override this env var with a shell function in a kustomize patch + value: |- + reconcile() { + echo reconciling... + } + command: + - bash + - -ceu + - |- + # template reconcile() into the script + # env var is expanded by k8s before the pod starts + $(RECONCILE_SH) + + apply-secret() { + /kbin/kubectl create secret docker-registry "${1}" \ + --docker-server="${2}" \ + --docker-username="${3}" \ + --docker-password="${4}" \ + --dry-run=client -o=yaml \ + | grep -v "creationTimestamp:" \ + | /kbin/kubectl apply -f - + } + + pause_loop() { + sleep "${SYNC_PERIOD:-3600}" || true + } + + graceful_exit() { + echo "Trapped signal -- $(date)" + job_ids="$( + jobs \ + | grep "pause_loop" \ + | cut -d] -f1 \ + | tr [ % + )" + # shellcheck disable=SC2086 + if [ "${job_ids}" ]; then + kill ${job_ids} + fi + wait + echo "Graceful exit -- $(date)" + } + + trap graceful_exit INT TERM + + echo "Loop started (period: ${SYNC_PERIOD} s) -- $(date)" + while true; do + reconcile & wait $! + pause_loop & wait $! + done + resources: {} + + +# RBAC necessary for our Deployment to apply our imagePullSecret +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: credentials-sync + namespace: flux-system +rules: +- apiGroups: [""] + resources: + - secrets + verbs: + - get + - create + - update + - patch + # # Lock this down to the specific Secret name (Optional) + resourceNames: + - $(KUBE_SECRET) # templated from kustomize vars referencing ConfigMap, also see kustomizeconfig.yaml +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: credentials-sync + namespace: flux-system +subjects: +- kind: ServiceAccount + name: credentials-sync +roleRef: + kind: Role + name: credentials-sync + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: credentials-sync + namespace: flux-system diff --git a/manifests/integrations/registry-credentials-sync/_cronjobs/_base/kustomization.yaml b/manifests/integrations/registry-credentials-sync/_cronjobs/_base/kustomization.yaml new file mode 100644 index 0000000000..c26a2c0ab0 --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/_cronjobs/_base/kustomization.yaml @@ -0,0 +1,20 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +commonLabels: + app: credentials-sync + +resources: +- sync.yaml + +vars: +- name: KUBE_SECRET + objref: + kind: ConfigMap + name: credentials-sync + apiVersion: v1 + fieldref: + fieldpath: data.KUBE_SECRET + +configurations: +- kustomizeconfig.yaml diff --git a/manifests/integrations/registry-credentials-sync/_cronjobs/_base/kustomizeconfig.yaml b/manifests/integrations/registry-credentials-sync/_cronjobs/_base/kustomizeconfig.yaml new file mode 100644 index 0000000000..61edffd433 --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/_cronjobs/_base/kustomizeconfig.yaml @@ -0,0 +1,3 @@ +varReference: +- path: rules/resourceNames + kind: Role diff --git a/manifests/integrations/registry-credentials-sync/_cronjobs/_base/sync.yaml b/manifests/integrations/registry-credentials-sync/_cronjobs/_base/sync.yaml new file mode 100644 index 0000000000..863301709c --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/_cronjobs/_base/sync.yaml @@ -0,0 +1,101 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: credentials-sync +data: + # Patch this ConfigMap with additional values needed for your cloud + KUBE_SECRET: my-registry-token # does not yet exist -- will be created in the same Namespace + +--- +# This CronJob frequently fetches registry tokens and applies them as an imagePullSecret. +# note: CronJob scheduling can block cluster bootstraps and cold-reboots from obtaining registry tokens for a considerable time. +# To run the job immediately, do `kubectl create job --from=cronjob/credentials-sync -n flux-system credentials-sync-init` +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: credentials-sync + namespace: flux-system +spec: + suspend: false + schedule: 0 */6 * * * + failedJobsHistoryLimit: 1 + successfulJobsHistoryLimit: 1 + jobTemplate: + spec: + template: + spec: + serviceAccountName: credentials-sync + restartPolicy: Never + containers: + - image: busybox # override this with a cloud-specific image + name: sync + envFrom: + - configMapRef: + name: credentials-sync + env: + - name: RECONCILE_SH # override this env var with a shell function in a kustomize patch + value: |- + reconcile() { + echo reconciling... + } + command: + - bash + - -ceu + - |- + # template reconcile() into the script + # env var is expanded by k8s before the pod starts + $(RECONCILE_SH) + + apply-secret() { + /kbin/kubectl create secret docker-registry "${1}" \ + --docker-server="${2}" \ + --docker-username="${3}" \ + --docker-password="${4}" \ + --dry-run=client -o=yaml \ + | grep -v "creationTimestamp:" \ + | /kbin/kubectl apply -f - + } + + reconcile + resources: {} + + +# RBAC necessary for our Deployment to apply our imagePullSecret +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: credentials-sync + namespace: flux-system +rules: +- apiGroups: [""] + resources: + - secrets + verbs: + - get + - create + - update + - patch + # # Lock this down to the specific Secret name (Optional) + resourceNames: + - $(KUBE_SECRET) # templated from kustomize vars referencing ConfigMap, also see kustomizeconfig.yaml +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: credentials-sync + namespace: flux-system +subjects: +- kind: ServiceAccount + name: credentials-sync +roleRef: + kind: Role + name: credentials-sync + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: credentials-sync + namespace: flux-system diff --git a/manifests/integrations/registry-credentials-sync/_cronjobs/aws/config-patches.yaml b/manifests/integrations/registry-credentials-sync/_cronjobs/aws/config-patches.yaml new file mode 100644 index 0000000000..fabaead4f8 --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/_cronjobs/aws/config-patches.yaml @@ -0,0 +1,52 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: credentials-sync +data: + ECR_REGION: us-east-1 # set the region + ECR_REGISTRY: .dkr.ecr..amazonaws.com # fill in the account id and region + KUBE_SECRET: ecr-credentials # does not yet exist -- will be created in the same Namespace + + +# Bind IRSA for the ServiceAccount +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: credentials-sync + namespace: flux-system + annotations: + eks.amazonaws.com/role-arn: # set the ARN for your role + + +# Set the reconcile period +--- +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: credentials-sync + namespace: flux-system +spec: + schedule: 0 */6 * * * # every 6hrs -- ECR tokens expire every 12 hours; refresh faster than that + + +## If not using IRSA, set the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables +## Store these values in a Secret and load them in the container using envFrom. +## For managing this secret via GitOps, consider using SOPS or SealedSecrets and add that manifest in a resource file for this kustomize build. +## https://toolkit.fluxcd.io/guides/mozilla-sops/ +## https://toolkit.fluxcd.io/guides/sealed-secrets/ +# --- +# apiVersion: apps/v1 +# kind: Deployment +# metadata: +# name: credentials-sync +# namespace: flux-system +# spec: +# template: +# spec: +# containers: +# - name: sync +# envFrom: +# secretRef: +# name: $(ECR_SECRET_NAME) # uncomment the var for this in kustomization.yaml diff --git a/manifests/integrations/registry-credentials-sync/_cronjobs/aws/kubectl-patch.yaml b/manifests/integrations/registry-credentials-sync/_cronjobs/aws/kubectl-patch.yaml new file mode 100644 index 0000000000..b4d83e225d --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/_cronjobs/aws/kubectl-patch.yaml @@ -0,0 +1,30 @@ +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: credentials-sync + namespace: flux-system +spec: + jobTemplate: + spec: + template: + spec: + initContainers: + - image: bitnami/kubectl + name: copy-kubectl + # it's okay to do this because kubectl is a statically linked binary + command: + - sh + - -ceu + - cp $(which kubectl) /kbin/ + resources: {} + volumeMounts: + - name: kbin + mountPath: /kbin + containers: + - name: sync + volumeMounts: + - name: kbin + mountPath: /kbin + volumes: + - name: kbin + emptyDir: {} diff --git a/manifests/integrations/registry-credentials-sync/_cronjobs/aws/kustomization.yaml b/manifests/integrations/registry-credentials-sync/_cronjobs/aws/kustomization.yaml new file mode 100644 index 0000000000..11eea1b462 --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/_cronjobs/aws/kustomization.yaml @@ -0,0 +1,26 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namePrefix: ecr- +commonLabels: + app: ecr-credentials-sync + +namespace: flux-system + +bases: +- ../_base +## If not using IRSA, consider creating the following file via SOPS or SealedSecrets +# - encrypted-secret.yaml + +patchesStrategicMerge: +- config-patches.yaml +- kubectl-patch.yaml +- reconcile-patch.yaml + +## uncomment if using encrypted-secret.yaml +# vars: +# - name: ECR_SECRET_NAME +# objref: +# kind: Secret +# name: credentials-sync +# apiVersion: v1 diff --git a/manifests/integrations/registry-credentials-sync/_cronjobs/aws/reconcile-patch.yaml b/manifests/integrations/registry-credentials-sync/_cronjobs/aws/reconcile-patch.yaml new file mode 100644 index 0000000000..98264fedf4 --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/_cronjobs/aws/reconcile-patch.yaml @@ -0,0 +1,29 @@ +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: credentials-sync + namespace: flux-system +spec: + jobTemplate: + spec: + template: + spec: + containers: + - name: sync + image: mcr.microsoft.com/azure-cli + env: + - name: RECONCILE_SH + value: |- + reconcile() { + echo "Starting ECR token sync -- $(date)" + echo "Logging into ECR: ${ECR_REGION} -- ${ECR_REGISTRY}" + token="$(aws ecr get-login-password --region "${ECR_REGION}")" + user="AWS" + server="${ECR_REGISTRY}" + + echo "Creating secret: ${KUBE_SECRET}" + apply-secret "${KUBE_SECRET}" "${token}" "${user}" "${server}" + + echo "Finished ECR token sync -- $(date)" + echo + } diff --git a/manifests/integrations/registry-credentials-sync/_cronjobs/azure/az-identity.yaml b/manifests/integrations/registry-credentials-sync/_cronjobs/azure/az-identity.yaml new file mode 100644 index 0000000000..c3c6be81e3 --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/_cronjobs/azure/az-identity.yaml @@ -0,0 +1,7 @@ +# This is a stub resource patched by config-patches.yaml, so that all config is visible in one file +--- +apiVersion: aadpodidentity.k8s.io/v1 +kind: AzureIdentity +metadata: + name: credentials-sync # if this is changed, also change in config-patches.yaml + namespace: flux-system diff --git a/manifests/integrations/registry-credentials-sync/_cronjobs/azure/config-patches.yaml b/manifests/integrations/registry-credentials-sync/_cronjobs/azure/config-patches.yaml new file mode 100644 index 0000000000..a642886016 --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/_cronjobs/azure/config-patches.yaml @@ -0,0 +1,41 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: credentials-sync +data: + ACR_NAME: my-registry + KUBE_SECRET: acr-my-registry # does not yet exist -- will be created in the same Namespace + +# Create an identity in Azure and assign it a role to pull from ACR (note: the identity's resourceGroup should match the desired ACR): +# az identity create -n acr-sync +# az role assignment create --role AcrPull --assignee-object-id "$(az identity show -n acr-sync -o tsv --query principalId)" +# Fetch the clientID and resourceID to configure the AzureIdentity spec below: +# az identity show -n acr-sync -otsv --query clientId +# az identity show -n acr-sync -otsv --query resourceId +--- +apiVersion: aadpodidentity.k8s.io/v1 +kind: AzureIdentity +metadata: + name: credentials-sync # name must match the stub-resource in az-identity.yaml + namespace: flux-system +spec: + clientID: 82d01fb0-7799-4d9d-92c7-21e7632c0000 + resourceID: /subscriptions/873c7e7f-76cd-4805-ae86-b923850b0000/resourcegroups/stealthybox/providers/Microsoft.ManagedIdentity/userAssignedIdentities/acr-sync + type: 0 # user-managed identity + +# Set the reconcile period + specify the pod-identity via the aadpodidbinding label +--- +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: credentials-sync + namespace: flux-system +spec: + schedule: 0 * * * * # ACR tokens expire every 3 hours; refresh faster than that + jobTemplate: + spec: + template: + metadata: + labels: + aadpodidbinding: $(AZ_IDENTITY_NAME) # match the AzureIdentity name diff --git a/manifests/integrations/registry-credentials-sync/_cronjobs/azure/kubectl-patch.yaml b/manifests/integrations/registry-credentials-sync/_cronjobs/azure/kubectl-patch.yaml new file mode 100644 index 0000000000..b4d83e225d --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/_cronjobs/azure/kubectl-patch.yaml @@ -0,0 +1,30 @@ +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: credentials-sync + namespace: flux-system +spec: + jobTemplate: + spec: + template: + spec: + initContainers: + - image: bitnami/kubectl + name: copy-kubectl + # it's okay to do this because kubectl is a statically linked binary + command: + - sh + - -ceu + - cp $(which kubectl) /kbin/ + resources: {} + volumeMounts: + - name: kbin + mountPath: /kbin + containers: + - name: sync + volumeMounts: + - name: kbin + mountPath: /kbin + volumes: + - name: kbin + emptyDir: {} diff --git a/manifests/integrations/registry-credentials-sync/_cronjobs/azure/kustomization.yaml b/manifests/integrations/registry-credentials-sync/_cronjobs/azure/kustomization.yaml new file mode 100644 index 0000000000..1dd497e054 --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/_cronjobs/azure/kustomization.yaml @@ -0,0 +1,28 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namePrefix: acr- +commonLabels: + app: acr-credentials-sync + +namespace: flux-system + +bases: +- ../_base +resources: +- az-identity.yaml + +patchesStrategicMerge: +- config-patches.yaml +- kubectl-patch.yaml +- reconcile-patch.yaml + +vars: +- name: AZ_IDENTITY_NAME + objref: + kind: AzureIdentity + name: credentials-sync + apiVersion: aadpodidentity.k8s.io/v1 + +configurations: +- kustomizeconfig.yaml diff --git a/manifests/integrations/registry-credentials-sync/_cronjobs/azure/kustomizeconfig.yaml b/manifests/integrations/registry-credentials-sync/_cronjobs/azure/kustomizeconfig.yaml new file mode 100644 index 0000000000..22524c1d75 --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/_cronjobs/azure/kustomizeconfig.yaml @@ -0,0 +1,3 @@ +varReference: +- path: spec/jobTemplate/spec/template/metadata/labels + kind: Deployment diff --git a/manifests/integrations/registry-credentials-sync/_cronjobs/azure/reconcile-patch.yaml b/manifests/integrations/registry-credentials-sync/_cronjobs/azure/reconcile-patch.yaml new file mode 100644 index 0000000000..b36fa90c49 --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/_cronjobs/azure/reconcile-patch.yaml @@ -0,0 +1,37 @@ +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: credentials-sync + namespace: flux-system +spec: + jobTemplate: + spec: + template: + spec: + containers: + - name: sync + image: mcr.microsoft.com/azure-cli + env: + - name: RECONCILE_SH + value: |- + reconcile() { + echo "Starting ACR token sync -- $(date)" + echo "Logging into Azure" + az login --identity + echo "Logging into ACR: ${ACR_NAME}" + output="$(az acr login --expose-token -o=tsv -n "${ACR_NAME}")" + read token server <<< "${output}" + user="00000000-0000-0000-0000-000000000000" + + echo "Creating secret: ${KUBE_SECRET}" + /kbin/kubectl create secret docker-registry "${KUBE_SECRET}" \ + --docker-server="${server}" \ + --docker-username="00000000-0000-0000-0000-000000000000" \ + --docker-password="${token}" \ + --dry-run=client -o=yaml \ + | grep -v "creationTimestamp:" \ + | /kbin/kubectl apply -f - + + echo "Finished ACR token sync -- $(date)" + echo + } diff --git a/manifests/integrations/registry-credentials-sync/_cronjobs/gcp/config-patches.yaml b/manifests/integrations/registry-credentials-sync/_cronjobs/gcp/config-patches.yaml new file mode 100644 index 0000000000..fdbb39d151 --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/_cronjobs/gcp/config-patches.yaml @@ -0,0 +1,28 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: credentials-sync +data: + GCR_REGISTRY: gcr.io # set the registry + KUBE_SECRET: gcr-credentials # does not yet exist -- will be created in the same Namespace + +# Bind to the GCP service-account +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: credentials-sync + namespace: flux-system + annotations: + iam.gke.io/gcp-service-account: @.iam.gserviceaccount.com # set the GCP service-account + +# Set the reconcile period +--- +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: credentials-sync + namespace: flux-system +spec: + schedule: 0,30 * * * * # 30m interval -- GCR tokens expire every hour; refresh faster than that diff --git a/manifests/integrations/registry-credentials-sync/_cronjobs/gcp/kustomization.yaml b/manifests/integrations/registry-credentials-sync/_cronjobs/gcp/kustomization.yaml new file mode 100644 index 0000000000..ea28e0b609 --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/_cronjobs/gcp/kustomization.yaml @@ -0,0 +1,15 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namePrefix: gcr- +commonLabels: + app: gcr-credentials-sync + +namespace: flux-system + +bases: +- ../_base + +patchesStrategicMerge: +- config-patches.yaml +- reconcile-patch.yaml diff --git a/manifests/integrations/registry-credentials-sync/_cronjobs/gcp/reconcile-patch.yaml b/manifests/integrations/registry-credentials-sync/_cronjobs/gcp/reconcile-patch.yaml new file mode 100644 index 0000000000..5b5ced3f79 --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/_cronjobs/gcp/reconcile-patch.yaml @@ -0,0 +1,29 @@ +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: credentials-sync + namespace: flux-system +spec: + jobTemplate: + spec: + template: + spec: + containers: + - name: sync + image: aws/aws-cli + env: + - name: RECONCILE_SH + value: |- + reconcile() { + echo "Starting GCR token sync -- $(date)" + echo "Logging into ECR: ${ECR_REGION} -- ${ECR_REGISTRY}" + token="$(gcloud auth print-access-token)" + user="oauth2accesstoken " + server="${GCR_REGISTRY}" + + echo "Creating secret: ${KUBE_SECRET}" + apply-secret "${KUBE_SECRET}" "${token}" "${user}" "${server}" + + echo "Finished GCR token sync -- $(date)" + echo + } diff --git a/manifests/integrations/registry-credentials-sync/aws/config-patches.yaml b/manifests/integrations/registry-credentials-sync/aws/config-patches.yaml new file mode 100644 index 0000000000..69f12363ed --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/aws/config-patches.yaml @@ -0,0 +1,42 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: credentials-sync +data: + ECR_REGION: us-east-1 # set the region + ECR_REGISTRY: .dkr.ecr..amazonaws.com # fill in the account id and region + KUBE_SECRET: ecr-credentials # does not yet exist -- will be created in the same Namespace + SYNC_PERIOD: "21600" # 6hrs -- ECR tokens expire every 12 hours; refresh faster than that + + +# Bind IRSA for the ServiceAccount +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: credentials-sync + namespace: flux-system + annotations: + eks.amazonaws.com/role-arn: # set the ARN for your role + + +## If not using IRSA, set the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables +## Store these values in a Secret and load them in the container using envFrom. +## For managing this secret via GitOps, consider using SOPS or SealedSecrets and add that manifest in a resource file for this kustomize build. +## https://toolkit.fluxcd.io/guides/mozilla-sops/ +## https://toolkit.fluxcd.io/guides/sealed-secrets/ +# --- +# apiVersion: apps/v1 +# kind: Deployment +# metadata: +# name: credentials-sync +# namespace: flux-system +# spec: +# template: +# spec: +# containers: +# - name: sync +# envFrom: +# secretRef: +# name: $(ECR_SECRET_NAME) # uncomment the var for this in kustomization.yaml diff --git a/manifests/integrations/registry-credentials-sync/aws/kubectl-patch.yaml b/manifests/integrations/registry-credentials-sync/aws/kubectl-patch.yaml new file mode 100644 index 0000000000..b054d7ce5b --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/aws/kubectl-patch.yaml @@ -0,0 +1,28 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: credentials-sync + namespace: flux-system +spec: + template: + spec: + initContainers: + - image: bitnami/kubectl + name: copy-kubectl + # it's okay to do this because kubectl is a statically linked binary + command: + - sh + - -ceu + - cp $(which kubectl) /kbin/ + resources: {} + volumeMounts: + - name: kbin + mountPath: /kbin + containers: + - name: sync + volumeMounts: + - name: kbin + mountPath: /kbin + volumes: + - name: kbin + emptyDir: {} diff --git a/manifests/integrations/registry-credentials-sync/aws/kustomization.yaml b/manifests/integrations/registry-credentials-sync/aws/kustomization.yaml new file mode 100644 index 0000000000..11eea1b462 --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/aws/kustomization.yaml @@ -0,0 +1,26 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namePrefix: ecr- +commonLabels: + app: ecr-credentials-sync + +namespace: flux-system + +bases: +- ../_base +## If not using IRSA, consider creating the following file via SOPS or SealedSecrets +# - encrypted-secret.yaml + +patchesStrategicMerge: +- config-patches.yaml +- kubectl-patch.yaml +- reconcile-patch.yaml + +## uncomment if using encrypted-secret.yaml +# vars: +# - name: ECR_SECRET_NAME +# objref: +# kind: Secret +# name: credentials-sync +# apiVersion: v1 diff --git a/manifests/integrations/registry-credentials-sync/aws/reconcile-patch.yaml b/manifests/integrations/registry-credentials-sync/aws/reconcile-patch.yaml new file mode 100644 index 0000000000..edac3a1154 --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/aws/reconcile-patch.yaml @@ -0,0 +1,28 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: credentials-sync + namespace: flux-system +spec: + template: + spec: + containers: + - name: sync + image: aws/aws-cli + env: + - name: RECONCILE_SH + value: |- + reconcile() { + echo "Starting ECR token sync -- $(date)" + echo "Logging into ECR: ${ECR_REGION} -- ${ECR_REGISTRY}" + token="$(aws ecr get-login-password --region "${ECR_REGION}")" + user="AWS" + server="${ECR_REGISTRY}" + + echo "Creating secret: ${KUBE_SECRET}" + apply-secret "${KUBE_SECRET}" "${token}" "${user}" "${server}" + + echo "Finished ECR token sync -- $(date)" + echo + } diff --git a/manifests/integrations/registry-credentials-sync/azure/az-identity.yaml b/manifests/integrations/registry-credentials-sync/azure/az-identity.yaml new file mode 100644 index 0000000000..c3c6be81e3 --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/azure/az-identity.yaml @@ -0,0 +1,7 @@ +# This is a stub resource patched by config-patches.yaml, so that all config is visible in one file +--- +apiVersion: aadpodidentity.k8s.io/v1 +kind: AzureIdentity +metadata: + name: credentials-sync # if this is changed, also change in config-patches.yaml + namespace: flux-system diff --git a/manifests/integrations/registry-credentials-sync/azure/config-patches.yaml b/manifests/integrations/registry-credentials-sync/azure/config-patches.yaml new file mode 100644 index 0000000000..d386a497d2 --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/azure/config-patches.yaml @@ -0,0 +1,39 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: credentials-sync +data: + ACR_NAME: my-registry + KUBE_SECRET: acr-my-registry # does not yet exist -- will be created in the same Namespace + SYNC_PERIOD: "3600" # ACR tokens expire every 3 hours; refresh faster than that + +# Create an identity in Azure and assign it a role to pull from ACR (note: the identity's resourceGroup should match the desired ACR): +# az identity create -n acr-sync +# az role assignment create --role AcrPull --assignee-object-id "$(az identity show -n acr-sync -o tsv --query principalId)" +# Fetch the clientID and resourceID to configure the AzureIdentity spec below: +# az identity show -n acr-sync -otsv --query clientId +# az identity show -n acr-sync -otsv --query resourceId +--- +apiVersion: aadpodidentity.k8s.io/v1 +kind: AzureIdentity +metadata: + name: credentials-sync # name must match the stub-resource in az-identity.yaml + namespace: flux-system +spec: + clientID: 82d01fb0-7799-4d9d-92c7-21e7632c0000 + resourceID: /subscriptions/873c7e7f-76cd-4805-ae86-b923850b0000/resourcegroups/stealthybox/providers/Microsoft.ManagedIdentity/userAssignedIdentities/acr-sync + type: 0 # user-managed identity + +# Specify the pod-identity via the aadpodidbinding label +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: credentials-sync + namespace: flux-system +spec: + template: + metadata: + labels: + aadpodidbinding: $(AZ_IDENTITY_NAME) # match the AzureIdentity name diff --git a/manifests/integrations/registry-credentials-sync/azure/kubectl-patch.yaml b/manifests/integrations/registry-credentials-sync/azure/kubectl-patch.yaml new file mode 100644 index 0000000000..b054d7ce5b --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/azure/kubectl-patch.yaml @@ -0,0 +1,28 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: credentials-sync + namespace: flux-system +spec: + template: + spec: + initContainers: + - image: bitnami/kubectl + name: copy-kubectl + # it's okay to do this because kubectl is a statically linked binary + command: + - sh + - -ceu + - cp $(which kubectl) /kbin/ + resources: {} + volumeMounts: + - name: kbin + mountPath: /kbin + containers: + - name: sync + volumeMounts: + - name: kbin + mountPath: /kbin + volumes: + - name: kbin + emptyDir: {} diff --git a/manifests/integrations/registry-credentials-sync/azure/kustomization.yaml b/manifests/integrations/registry-credentials-sync/azure/kustomization.yaml new file mode 100644 index 0000000000..1dd497e054 --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/azure/kustomization.yaml @@ -0,0 +1,28 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namePrefix: acr- +commonLabels: + app: acr-credentials-sync + +namespace: flux-system + +bases: +- ../_base +resources: +- az-identity.yaml + +patchesStrategicMerge: +- config-patches.yaml +- kubectl-patch.yaml +- reconcile-patch.yaml + +vars: +- name: AZ_IDENTITY_NAME + objref: + kind: AzureIdentity + name: credentials-sync + apiVersion: aadpodidentity.k8s.io/v1 + +configurations: +- kustomizeconfig.yaml diff --git a/manifests/integrations/registry-credentials-sync/azure/kustomizeconfig.yaml b/manifests/integrations/registry-credentials-sync/azure/kustomizeconfig.yaml new file mode 100644 index 0000000000..afd68fe5de --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/azure/kustomizeconfig.yaml @@ -0,0 +1,3 @@ +varReference: +- path: spec/template/metadata/labels + kind: Deployment diff --git a/manifests/integrations/registry-credentials-sync/azure/reconcile-patch.yaml b/manifests/integrations/registry-credentials-sync/azure/reconcile-patch.yaml new file mode 100644 index 0000000000..c6e1691da9 --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/azure/reconcile-patch.yaml @@ -0,0 +1,30 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: credentials-sync + namespace: flux-system +spec: + template: + spec: + containers: + - name: sync + image: mcr.microsoft.com/azure-cli + env: + - name: RECONCILE_SH + value: |- + reconcile() { + echo "Starting ACR token sync -- $(date)" + echo "Logging into Azure" + az login --identity + echo "Logging into ACR: ${ACR_NAME}" + output="$(az acr login --expose-token -o=tsv -n "${ACR_NAME}")" + read token server <<< "${output}" + user="00000000-0000-0000-0000-000000000000" + + echo "Creating secret: ${KUBE_SECRET}" + apply-secret "${KUBE_SECRET}" "${token}" "${user}" "${server}" + + echo "Finished ECR token sync -- $(date)" + echo + } diff --git a/manifests/integrations/registry-credentials-sync/gcp/config-patches.yaml b/manifests/integrations/registry-credentials-sync/gcp/config-patches.yaml new file mode 100644 index 0000000000..dda354ce1b --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/gcp/config-patches.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: credentials-sync +data: + GCR_REGISTRY: gcr.io # set the registry + KUBE_SECRET: gcr-credentials # does not yet exist -- will be created in the same Namespace + SYNC_PERIOD: "1800" # 30m -- GCR tokens expire every hour; refresh faster than that + + +# Bind to the GCP service-account +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: credentials-sync + namespace: flux-system + annotations: + iam.gke.io/gcp-service-account: @.iam.gserviceaccount.com # set the GCP service-account diff --git a/manifests/integrations/registry-credentials-sync/gcp/kustomization.yaml b/manifests/integrations/registry-credentials-sync/gcp/kustomization.yaml new file mode 100644 index 0000000000..ea28e0b609 --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/gcp/kustomization.yaml @@ -0,0 +1,15 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namePrefix: gcr- +commonLabels: + app: gcr-credentials-sync + +namespace: flux-system + +bases: +- ../_base + +patchesStrategicMerge: +- config-patches.yaml +- reconcile-patch.yaml diff --git a/manifests/integrations/registry-credentials-sync/gcp/reconcile-patch.yaml b/manifests/integrations/registry-credentials-sync/gcp/reconcile-patch.yaml new file mode 100644 index 0000000000..9c78e4f4db --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/gcp/reconcile-patch.yaml @@ -0,0 +1,28 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: credentials-sync + namespace: flux-system +spec: + template: + spec: + containers: + - name: sync + image: aws/aws-cli + env: + - name: RECONCILE_SH + value: |- + reconcile() { + echo "Starting GCR token sync -- $(date)" + echo "Logging into ECR: ${ECR_REGION} -- ${ECR_REGISTRY}" + token="$(gcloud auth print-access-token)" + user="oauth2accesstoken " + server="${GCR_REGISTRY}" + + echo "Creating secret: ${KUBE_SECRET}" + apply-secret "${KUBE_SECRET}" "${token}" "${user}" "${server}" + + echo "Finished GCR token sync -- $(date)" + echo + } From e99b1c3ed86e49f6fa882645134a03144081161d Mon Sep 17 00:00:00 2001 From: leigh capili Date: Mon, 8 Feb 2021 01:15:31 -0700 Subject: [PATCH 2/2] Document ACR / AKS Image Update Considerations Signed-off-by: leigh capili --- docs/guides/image-update.md | 87 ++++++++++++++++++++++++++++++++++++- 1 file changed, 85 insertions(+), 2 deletions(-) diff --git a/docs/guides/image-update.md b/docs/guides/image-update.md index 4f0628a1ef..929deca1e2 100644 --- a/docs/guides/image-update.md +++ b/docs/guides/image-update.md @@ -657,7 +657,7 @@ spec: #### Using access token [short-lived] -!!!note "Workload Identity" +!!! note "Workload Identity" Please ensure that you enable workload identity for your cluster, create a GCP service account that has access to the container registry and create an IAM policy binding between the GCP service account and the Kubernetes service account so that the pods created by the cronjob can access GCP APIs and get the token. @@ -792,4 +792,87 @@ or [Sealed Secrets](sealed-secrets.md) , commit and push the encypted file to gi ### Azure Container Registry -TODO +AKS clusters are not able to pull and run images from ACR by default. +Read [Integrating AKS /w ACR](https://docs.microsoft.com/en-us/azure/aks/cluster-container-registry-integration) as a potential pre-requisite +before integrating Flux `ImageRepositories` with ACR. + +Note that the resulting ImagePullSecret for Flux could also be specified by Pods within the same Namespace to pull and run ACR images as well. + +#### Generating Tokens for Managed Identities [short-lived] + +With [AAD Pod-Identity](https://azure.github.io/aad-pod-identity/docs/), we can create Pods that have their own +cloud credentials for accessing Azure services like ACR. + +Your cluster should have `--enable-managed-identity` configured. +This software can be [installed via Helm](https://azure.github.io/aad-pod-identity/docs/getting-started/installation/) not managed by Azure. +Use Flux's `HelmRepository` and `HelmRelease` object to manage the aad-pod-identity installation from a bootstrap repository. + +!!! As an alternative to Helm, the `--enable-aad-pod-identity` flag for the `az aks create` is currently in Preview. + Follow the Azure guide for [Creating an AKS cluster with AAD Pod Identity](https://docs.microsoft.com/en-us/azure/aks/use-azure-ad-pod-identity) if you would like to enable this feature with the Azure CLI. + +Once we have AAD Pod Identity installed, we can create a Deployment that frequently refreshes an image pull secret into +our desired Namespace. + +Create a directory in your control repository and save this `kustomization.yaml`: +```yaml +# kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- https://github.com/fluxcd/flux2/archive/main.zip//manifests/integrations/registry-credentials-sync/azure +patchesStrategicMerge: +- config-patches.yaml +``` +Save and configure the following patch -- note the instructional comments for configuring matching Azure resources: +```yaml +# config-patches.yaml +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: credentials-sync +data: + ACR_NAME: my-registry + KUBE_SECRET: my-registry # does not yet exist -- will be created in the same Namespace + SYNC_PERIOD: "3600" # ACR tokens expire every 3 hours; refresh faster than that + +# Create an identity in Azure and assign it a role to pull from ACR (note: the identity's resourceGroup should match the desired ACR): +# az identity create -n acr-sync +# az role assignment create --role AcrPull --assignee-object-id "$(az identity show -n acr-sync -o tsv --query principalId)" +# Fetch the clientID and resourceID to configure the AzureIdentity spec below: +# az identity show -n acr-sync -otsv --query clientId +# az identity show -n acr-sync -otsv --query resourceId +--- +apiVersion: aadpodidentity.k8s.io/v1 +kind: AzureIdentity +metadata: + name: credentials-sync # name must match the stub-resource in az-identity.yaml + namespace: flux-system +spec: + clientID: 4ceaa448-d7b9-4a80-8f32-497eaf3d3287 + resourceID: /subscriptions/8c69185e-55f9-4d00-8e71-a1b1bb1386a1/resourcegroups/stealthybox/providers/Microsoft.ManagedIdentity/userAssignedIdentities/acr-sync + type: 0 # user-managed identity +``` + +Verify that `kustomize build .` works, then commit the directory to you control repo. +Flux will apply the Deployment and it will use the AAD managed identity for that Pod to regularly fetch ACR tokens into your configured `KUBE_SECRET` name. +Reference the `KUBE_SECRET` value from any `ImageRepository` objects for that ACR registry. + +This example uses the `fluxcd/flux2` github archive as a remote base, but you may copy the [./manifests/integrations/registry-credentials-sync/azure](github.com/fluxcd/flux2/tree/main/manifests/integrations/registry-credentials-sync/azure) +folder into your own repository or use a git submodule to vendor it if preferred. + +#### Using Static Credentials [long-lived] + +!!! Using a static credential requires a Secrets management solution compatible with your GitOps workflow. + +Follow the official Azure documentation for [Creating an Image Pull Secret for ACR](https://docs.microsoft.com/en-us/azure/container-registry/container-registry-auth-kubernetes). + +Instead of creating the Secret directly into your Kubernetes cluster, encrypt it using [Mozilla SOPS](mozilla-sops.md) +or [Sealed Secrets](sealed-secrets.md), then commit and push the encypted file to git. + +This Secret should be in the same Namespace as your flux `ImageRepository` object. +Update the `ImageRepository.spec.secretRef` to point to it. + +It is also possible to create [Repository Scoped Tokens](https://docs.microsoft.com/en-us/azure/container-registry/container-registry-repository-scoped-permissions). + +!!! Note that this feature is in preview and does have limitations.