From c715da594bf3fceb1384659585c77a12de606feb Mon Sep 17 00:00:00 2001 From: dystewart Date: Fri, 18 Nov 2022 10:41:50 -0500 Subject: [PATCH 1/5] Change ingressController to use routeSelector To allow for more granularity, we are opting to go with a route selector as opposed to a namespace selector for the ingress controller. The idea here is that we're filtering the traffic serviced by the ingress controller at the route level as opposed to the entire namespace. --- .../ingresscontrollers/external-apps-ingress-controller.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-scope/overlays/nerc-ocp-prod/ingresscontrollers/external-apps-ingress-controller.yaml b/cluster-scope/overlays/nerc-ocp-prod/ingresscontrollers/external-apps-ingress-controller.yaml index 563e2085..715ec185 100644 --- a/cluster-scope/overlays/nerc-ocp-prod/ingresscontrollers/external-apps-ingress-controller.yaml +++ b/cluster-scope/overlays/nerc-ocp-prod/ingresscontrollers/external-apps-ingress-controller.yaml @@ -15,6 +15,6 @@ spec: nodeSelector: matchLabels: nerc.mghpcc.org/external-ingress: 'true' - namespaceSelector: + routeSelector: matchLabels: type: external From c9729e14e3bd5b5bb6582094c3328814858ecd79 Mon Sep 17 00:00:00 2001 From: dystewart Date: Fri, 18 Nov 2022 11:07:57 -0500 Subject: [PATCH 2/5] Configure address for external ingress service This service is used with the external ingressController and connects with our public metallb loadBalancer The important fields to pay attention to here are the: metadata.annotations metallb.universe.tf/address-pool: public spec.loadBalancerIP: 199.94.61.6 This is the ip where we are servicing external traffic --- .../external-apps-ingress-controller.yaml | 2 +- .../overlays/nerc-ocp-prod/kustomization.yaml | 1 + ...uter-external-apps-ingress-controller.yaml | 33 +++++++++++++++++++ 3 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 cluster-scope/overlays/nerc-ocp-prod/services/router-external-apps-ingress-controller.yaml diff --git a/cluster-scope/overlays/nerc-ocp-prod/ingresscontrollers/external-apps-ingress-controller.yaml b/cluster-scope/overlays/nerc-ocp-prod/ingresscontrollers/external-apps-ingress-controller.yaml index 715ec185..b9d53e84 100644 --- a/cluster-scope/overlays/nerc-ocp-prod/ingresscontrollers/external-apps-ingress-controller.yaml +++ b/cluster-scope/overlays/nerc-ocp-prod/ingresscontrollers/external-apps-ingress-controller.yaml @@ -17,4 +17,4 @@ spec: nerc.mghpcc.org/external-ingress: 'true' routeSelector: matchLabels: - type: external + nerc.mghpcc.org/external-ingress: 'true' diff --git a/cluster-scope/overlays/nerc-ocp-prod/kustomization.yaml b/cluster-scope/overlays/nerc-ocp-prod/kustomization.yaml index c86a6ae2..c1df838b 100644 --- a/cluster-scope/overlays/nerc-ocp-prod/kustomization.yaml +++ b/cluster-scope/overlays/nerc-ocp-prod/kustomization.yaml @@ -20,6 +20,7 @@ resources: - nodenetworkconfigurationpolicies - metallb - clusterversion.yaml +- services/router-external-apps-ingress-controller.yaml patches: - path: ingresscontrollers/default_patch.yaml diff --git a/cluster-scope/overlays/nerc-ocp-prod/services/router-external-apps-ingress-controller.yaml b/cluster-scope/overlays/nerc-ocp-prod/services/router-external-apps-ingress-controller.yaml new file mode 100644 index 00000000..d9438a96 --- /dev/null +++ b/cluster-scope/overlays/nerc-ocp-prod/services/router-external-apps-ingress-controller.yaml @@ -0,0 +1,33 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + metallb.universe.tf/address-pool: public + traffic-policy.network.alpha.openshift.io/local-with-fallback: "" + labels: + app: router + ingresscontroller.operator.openshift.io/owning-ingresscontroller: external-apps-ingress-controller + router: router-external-apps-ingress-controller + name: router-external-apps-ingress-controller + namespace: openshift-ingress +spec: + allocateLoadBalancerNodePorts: true + externalTrafficPolicy: Local + healthCheckNodePort: 32573 + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + loadBalancerIP: 199.94.61.6 + ports: + - name: http + nodePort: 31731 + port: 80 + targetPort: http + - name: https + nodePort: 31651 + port: 443 + targetPort: https + selector: + ingresscontroller.operator.openshift.io/deployment-ingresscontroller: external-apps-ingress-controller + type: LoadBalancer From 294d93ed0f46f58b61a2729363ece479c4018fe9 Mon Sep 17 00:00:00 2001 From: Lars Kellogg-Stedman Date: Mon, 21 Nov 2022 16:57:12 -0500 Subject: [PATCH 3/5] Rename external-apps-ingress-controller to external-apps We already know it's an ingress controller. --- ...al-apps-ingress-controller.yaml => external-apps.yaml} | 5 ++++- cluster-scope/overlays/nerc-ocp-prod/kustomization.yaml | 4 ++-- ...-ingress-controller.yaml => router-external-apps.yaml} | 8 ++++---- 3 files changed, 10 insertions(+), 7 deletions(-) rename cluster-scope/overlays/nerc-ocp-prod/ingresscontrollers/{external-apps-ingress-controller.yaml => external-apps.yaml} (73%) rename cluster-scope/overlays/nerc-ocp-prod/services/{router-external-apps-ingress-controller.yaml => router-external-apps.yaml} (80%) diff --git a/cluster-scope/overlays/nerc-ocp-prod/ingresscontrollers/external-apps-ingress-controller.yaml b/cluster-scope/overlays/nerc-ocp-prod/ingresscontrollers/external-apps.yaml similarity index 73% rename from cluster-scope/overlays/nerc-ocp-prod/ingresscontrollers/external-apps-ingress-controller.yaml rename to cluster-scope/overlays/nerc-ocp-prod/ingresscontrollers/external-apps.yaml index b9d53e84..de74a080 100644 --- a/cluster-scope/overlays/nerc-ocp-prod/ingresscontrollers/external-apps-ingress-controller.yaml +++ b/cluster-scope/overlays/nerc-ocp-prod/ingresscontrollers/external-apps.yaml @@ -1,9 +1,12 @@ apiVersion: operator.openshift.io/v1 kind: IngressController metadata: - name: external-apps-ingress-controller + name: external-apps namespace: openshift-ingress-operator spec: + # The "domain" setting does not automatically apply to routes as you might + # expect; see https://github.com/OCP-on-NERC/operations/issues/41 for + # details. domain: apps.shift.nerc.mghpcc.org defaultCertificate: name: external-apps-ingress-certificate diff --git a/cluster-scope/overlays/nerc-ocp-prod/kustomization.yaml b/cluster-scope/overlays/nerc-ocp-prod/kustomization.yaml index c1df838b..352db285 100644 --- a/cluster-scope/overlays/nerc-ocp-prod/kustomization.yaml +++ b/cluster-scope/overlays/nerc-ocp-prod/kustomization.yaml @@ -10,7 +10,7 @@ resources: - ../../bundles/xdmod-reader - feature/odf - ../../base/core/namespaces/openshift-gitops -- ingresscontrollers/external-apps-ingress-controller.yaml +- ingresscontrollers/external-apps.yaml - externalsecrets - apiserver/cluster.yaml - secretstores @@ -20,7 +20,7 @@ resources: - nodenetworkconfigurationpolicies - metallb - clusterversion.yaml -- services/router-external-apps-ingress-controller.yaml +- services/router-external-apps.yaml patches: - path: ingresscontrollers/default_patch.yaml diff --git a/cluster-scope/overlays/nerc-ocp-prod/services/router-external-apps-ingress-controller.yaml b/cluster-scope/overlays/nerc-ocp-prod/services/router-external-apps.yaml similarity index 80% rename from cluster-scope/overlays/nerc-ocp-prod/services/router-external-apps-ingress-controller.yaml rename to cluster-scope/overlays/nerc-ocp-prod/services/router-external-apps.yaml index d9438a96..e97fe8d9 100644 --- a/cluster-scope/overlays/nerc-ocp-prod/services/router-external-apps-ingress-controller.yaml +++ b/cluster-scope/overlays/nerc-ocp-prod/services/router-external-apps.yaml @@ -6,9 +6,9 @@ metadata: traffic-policy.network.alpha.openshift.io/local-with-fallback: "" labels: app: router - ingresscontroller.operator.openshift.io/owning-ingresscontroller: external-apps-ingress-controller - router: router-external-apps-ingress-controller - name: router-external-apps-ingress-controller + ingresscontroller.operator.openshift.io/owning-ingresscontroller: external-apps + router: router-external-apps + name: router-external-apps namespace: openshift-ingress spec: allocateLoadBalancerNodePorts: true @@ -29,5 +29,5 @@ spec: port: 443 targetPort: https selector: - ingresscontroller.operator.openshift.io/deployment-ingresscontroller: external-apps-ingress-controller + ingresscontroller.operator.openshift.io/deployment-ingresscontroller: external-apps type: LoadBalancer From a08390d24e20aea7061f472de4cfbfb72738b882 Mon Sep 17 00:00:00 2001 From: Lars Kellogg-Stedman Date: Mon, 21 Nov 2022 21:53:05 -0500 Subject: [PATCH 4/5] Relocate external ingress related resources to feature/external-ingress Keep all the external-ingress resources together so they're easier to manage. --- .../external-apps-ingress-certificate.yaml | 16 +++++++++ .../ingresscontrollers/external-apps.yaml | 0 .../external-ingress/kustomization.yaml | 8 +++++ .../overlays/nerc-ocp-prod/kustomization.yaml | 3 +- .../services/router-external-apps.yaml | 33 ------------------- 5 files changed, 25 insertions(+), 35 deletions(-) create mode 100644 cluster-scope/overlays/nerc-ocp-prod/feature/external-ingress/externalsecrets/external-apps-ingress-certificate.yaml rename cluster-scope/overlays/nerc-ocp-prod/{ => feature/external-ingress}/ingresscontrollers/external-apps.yaml (100%) create mode 100644 cluster-scope/overlays/nerc-ocp-prod/feature/external-ingress/kustomization.yaml delete mode 100644 cluster-scope/overlays/nerc-ocp-prod/services/router-external-apps.yaml diff --git a/cluster-scope/overlays/nerc-ocp-prod/feature/external-ingress/externalsecrets/external-apps-ingress-certificate.yaml b/cluster-scope/overlays/nerc-ocp-prod/feature/external-ingress/externalsecrets/external-apps-ingress-certificate.yaml new file mode 100644 index 00000000..48f73ecf --- /dev/null +++ b/cluster-scope/overlays/nerc-ocp-prod/feature/external-ingress/externalsecrets/external-apps-ingress-certificate.yaml @@ -0,0 +1,16 @@ +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: external-apps-ingress-certificate + namespace: openshift-ingress-operator +spec: + secretStoreRef: + name: nerc-secret-store + kind: SecretStore + target: + name: external-apps-ingress-certificate + template: + type: kubernetes.io/tls + dataFrom: + - extract: + key: nerc/nerc-ocp-prod/openshift-ingress/external-apps-ingress-certificate diff --git a/cluster-scope/overlays/nerc-ocp-prod/ingresscontrollers/external-apps.yaml b/cluster-scope/overlays/nerc-ocp-prod/feature/external-ingress/ingresscontrollers/external-apps.yaml similarity index 100% rename from cluster-scope/overlays/nerc-ocp-prod/ingresscontrollers/external-apps.yaml rename to cluster-scope/overlays/nerc-ocp-prod/feature/external-ingress/ingresscontrollers/external-apps.yaml diff --git a/cluster-scope/overlays/nerc-ocp-prod/feature/external-ingress/kustomization.yaml b/cluster-scope/overlays/nerc-ocp-prod/feature/external-ingress/kustomization.yaml new file mode 100644 index 00000000..aaf44b27 --- /dev/null +++ b/cluster-scope/overlays/nerc-ocp-prod/feature/external-ingress/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +commonLabels: + nerg.mghpcc.org/feature: external-ingress + +resources: +- ingresscontrollers/external-apps.yaml +- externalsecrets/external-apps-ingress-certificate.yaml diff --git a/cluster-scope/overlays/nerc-ocp-prod/kustomization.yaml b/cluster-scope/overlays/nerc-ocp-prod/kustomization.yaml index 352db285..d12a7938 100644 --- a/cluster-scope/overlays/nerc-ocp-prod/kustomization.yaml +++ b/cluster-scope/overlays/nerc-ocp-prod/kustomization.yaml @@ -9,8 +9,8 @@ resources: - ../../bundles/metallb - ../../bundles/xdmod-reader - feature/odf +- feature/external-ingress - ../../base/core/namespaces/openshift-gitops -- ingresscontrollers/external-apps.yaml - externalsecrets - apiserver/cluster.yaml - secretstores @@ -20,7 +20,6 @@ resources: - nodenetworkconfigurationpolicies - metallb - clusterversion.yaml -- services/router-external-apps.yaml patches: - path: ingresscontrollers/default_patch.yaml diff --git a/cluster-scope/overlays/nerc-ocp-prod/services/router-external-apps.yaml b/cluster-scope/overlays/nerc-ocp-prod/services/router-external-apps.yaml deleted file mode 100644 index e97fe8d9..00000000 --- a/cluster-scope/overlays/nerc-ocp-prod/services/router-external-apps.yaml +++ /dev/null @@ -1,33 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - annotations: - metallb.universe.tf/address-pool: public - traffic-policy.network.alpha.openshift.io/local-with-fallback: "" - labels: - app: router - ingresscontroller.operator.openshift.io/owning-ingresscontroller: external-apps - router: router-external-apps - name: router-external-apps - namespace: openshift-ingress -spec: - allocateLoadBalancerNodePorts: true - externalTrafficPolicy: Local - healthCheckNodePort: 32573 - internalTrafficPolicy: Cluster - ipFamilies: - - IPv4 - ipFamilyPolicy: SingleStack - loadBalancerIP: 199.94.61.6 - ports: - - name: http - nodePort: 31731 - port: 80 - targetPort: http - - name: https - nodePort: 31651 - port: 443 - targetPort: https - selector: - ingresscontroller.operator.openshift.io/deployment-ingresscontroller: external-apps - type: LoadBalancer From 3bb77c26d0f1ed39f86aac7a77346c2c2433235b Mon Sep 17 00:00:00 2001 From: Lars Kellogg-Stedman Date: Mon, 21 Nov 2022 20:20:32 -0500 Subject: [PATCH 5/5] Patch external ingress service via argocd post-sync hook When we create an IngressController, the IngressController creates a Service. We need to modify that service resource to add both an annotation and a loadBalancerIP, but ArgoCD doesn't allow us to patch existing resources. The best we can do is apply patches using an ArgoCD post-sync hook [1]. This commit adds a post-sync Job that takes care of patching the Service resource. This job will run after every sync operation. [1]: https://argo-cd.readthedocs.io/en/stable/user-guide/resource_hooks/ --- .../external-ingress/kustomization.yaml | 1 + .../external-ingress/post-sync-hook/job.yaml | 34 ++++++++++++++ .../post-sync-hook/kustomization.yaml | 21 +++++++++ .../patches/router-external-apps.patch.yaml | 8 ++++ .../external-ingress/post-sync-hook/role.yaml | 11 +++++ .../post-sync-hook/rolebinding.yaml | 11 +++++ .../post-sync-hook/scripts/apply-patches.sh | 47 +++++++++++++++++++ .../post-sync-hook/serviceaccount.yaml | 4 ++ 8 files changed, 137 insertions(+) create mode 100644 cluster-scope/overlays/nerc-ocp-prod/feature/external-ingress/post-sync-hook/job.yaml create mode 100644 cluster-scope/overlays/nerc-ocp-prod/feature/external-ingress/post-sync-hook/kustomization.yaml create mode 100644 cluster-scope/overlays/nerc-ocp-prod/feature/external-ingress/post-sync-hook/patches/router-external-apps.patch.yaml create mode 100644 cluster-scope/overlays/nerc-ocp-prod/feature/external-ingress/post-sync-hook/role.yaml create mode 100644 cluster-scope/overlays/nerc-ocp-prod/feature/external-ingress/post-sync-hook/rolebinding.yaml create mode 100644 cluster-scope/overlays/nerc-ocp-prod/feature/external-ingress/post-sync-hook/scripts/apply-patches.sh create mode 100644 cluster-scope/overlays/nerc-ocp-prod/feature/external-ingress/post-sync-hook/serviceaccount.yaml diff --git a/cluster-scope/overlays/nerc-ocp-prod/feature/external-ingress/kustomization.yaml b/cluster-scope/overlays/nerc-ocp-prod/feature/external-ingress/kustomization.yaml index aaf44b27..92f9f0d0 100644 --- a/cluster-scope/overlays/nerc-ocp-prod/feature/external-ingress/kustomization.yaml +++ b/cluster-scope/overlays/nerc-ocp-prod/feature/external-ingress/kustomization.yaml @@ -6,3 +6,4 @@ commonLabels: resources: - ingresscontrollers/external-apps.yaml - externalsecrets/external-apps-ingress-certificate.yaml +- post-sync-hook diff --git a/cluster-scope/overlays/nerc-ocp-prod/feature/external-ingress/post-sync-hook/job.yaml b/cluster-scope/overlays/nerc-ocp-prod/feature/external-ingress/post-sync-hook/job.yaml new file mode 100644 index 00000000..3c474db8 --- /dev/null +++ b/cluster-scope/overlays/nerc-ocp-prod/feature/external-ingress/post-sync-hook/job.yaml @@ -0,0 +1,34 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: patch-external-ingress-service + annotations: + argocd.argoproj.io/hook: PostSync + argocd.argoproj.io/hook-delete-policy: BeforeHookCreation +spec: + template: + spec: + restartPolicy: OnFailure + serviceAccountName: patcher + containers: + - name: patcher + # This is 4.10.42 + image: quay.io/openshift/origin-cli:4.10 + command: + - bash + - /scripts/apply-patches.sh + - /patches + volumeMounts: + - name: patch-scripts + mountPath: /scripts + - name: patches + mountPath: /patches + + volumes: + - name: patch-scripts + configMap: + name: patch-scripts + + - name: patches + configMap: + name: patches diff --git a/cluster-scope/overlays/nerc-ocp-prod/feature/external-ingress/post-sync-hook/kustomization.yaml b/cluster-scope/overlays/nerc-ocp-prod/feature/external-ingress/post-sync-hook/kustomization.yaml new file mode 100644 index 00000000..30d1e93d --- /dev/null +++ b/cluster-scope/overlays/nerc-ocp-prod/feature/external-ingress/post-sync-hook/kustomization.yaml @@ -0,0 +1,21 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: openshift-ingress + +commonLabels: + app: patch-external-ingress-service + +resources: + - job.yaml + - serviceaccount.yaml + - role.yaml + - rolebinding.yaml + +configMapGenerator: + - name: patch-scripts + files: + - scripts/apply-patches.sh + + - name: patches + files: + - patches/router-external-apps.patch.yaml diff --git a/cluster-scope/overlays/nerc-ocp-prod/feature/external-ingress/post-sync-hook/patches/router-external-apps.patch.yaml b/cluster-scope/overlays/nerc-ocp-prod/feature/external-ingress/post-sync-hook/patches/router-external-apps.patch.yaml new file mode 100644 index 00000000..8a198ee9 --- /dev/null +++ b/cluster-scope/overlays/nerc-ocp-prod/feature/external-ingress/post-sync-hook/patches/router-external-apps.patch.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + metallb.universe.tf/address-pool: public + name: router-external-apps +spec: + loadBalancerIP: 199.94.61.6 diff --git a/cluster-scope/overlays/nerc-ocp-prod/feature/external-ingress/post-sync-hook/role.yaml b/cluster-scope/overlays/nerc-ocp-prod/feature/external-ingress/post-sync-hook/role.yaml new file mode 100644 index 00000000..d77125b9 --- /dev/null +++ b/cluster-scope/overlays/nerc-ocp-prod/feature/external-ingress/post-sync-hook/role.yaml @@ -0,0 +1,11 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: patcher-allow-patching +rules: + - apiGroups: + - '' + resources: + - services + verbs: + - patch diff --git a/cluster-scope/overlays/nerc-ocp-prod/feature/external-ingress/post-sync-hook/rolebinding.yaml b/cluster-scope/overlays/nerc-ocp-prod/feature/external-ingress/post-sync-hook/rolebinding.yaml new file mode 100644 index 00000000..ba21e3e8 --- /dev/null +++ b/cluster-scope/overlays/nerc-ocp-prod/feature/external-ingress/post-sync-hook/rolebinding.yaml @@ -0,0 +1,11 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: patcher-allow-patching +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: patcher-allow-patching +subjects: +- kind: ServiceAccount + name: patcher diff --git a/cluster-scope/overlays/nerc-ocp-prod/feature/external-ingress/post-sync-hook/scripts/apply-patches.sh b/cluster-scope/overlays/nerc-ocp-prod/feature/external-ingress/post-sync-hook/scripts/apply-patches.sh new file mode 100644 index 00000000..0cc0bcb8 --- /dev/null +++ b/cluster-scope/overlays/nerc-ocp-prod/feature/external-ingress/post-sync-hook/scripts/apply-patches.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +PATCH_DIR="$1" + +shopt -s nullglob + +mapfile -t patches < <(printf "%s\n" \ + "$PATCH_DIR"/*.patch.yaml \ + "$PATCH_DIR"/*.jsonpatch.yaml \ + "$PATCH_DIR"/*.jsonmerge.yaml | sort) + +for patch in "${patches[@]}"; do + + # For strategic merge patches it's possible to infer the target of the patch + # from the patch itself, but other patch types -- such as JSONPatch patches + # -- require us to provide an explicit target. + # + # The following code replaces the patch type in the filename with "target" + # (so that "something.patch.yaml" becomes "something.target.yaml"), and + # if the resulting filename exists it will be used to determine the + # target of the patch. + targetname=$(awk -vOFS=. -F. '{$(NF-1) = "target"; print}' <<<"$patch") + + if [[ -f "$targetname" ]]; then + target=$targetname + else + target=$patch + fi + + case $patch in + *.patch.yaml) + patch_type=strategic;; + *.jsonpatch.yaml) + patch_type=json;; + *.mergepatch.yaml) + patch_type=merge;; + + *) echo "ERROR: $patch: unknown patch type" >&2 + continue + ;; + esac + + echo "Applying $patch" + if ! kubectl patch -f "$target" --patch-file "$patch" --type "$patch_type"; then + echo "ERROR: $patch: failed to apply" >&2 + fi +done diff --git a/cluster-scope/overlays/nerc-ocp-prod/feature/external-ingress/post-sync-hook/serviceaccount.yaml b/cluster-scope/overlays/nerc-ocp-prod/feature/external-ingress/post-sync-hook/serviceaccount.yaml new file mode 100644 index 00000000..012eacc4 --- /dev/null +++ b/cluster-scope/overlays/nerc-ocp-prod/feature/external-ingress/post-sync-hook/serviceaccount.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: patcher