From 7387e342f1f0d44cc5304a6a7a404e9461daa141 Mon Sep 17 00:00:00 2001 From: Brant Knudson Date: Thu, 22 Apr 2021 14:06:26 -0500 Subject: [PATCH] CASMPET-4080: Updates for Istio 1.7.8 The charts are updated for Istio 1.7.8. Here's a summary of the changes. * cray-istio-operator I updated the charts/istio-operator subchart from the 1.7.8 distribution. I got rid of the wait-jobs hook because this isn't necessary since Helm takes care of making sure the CRD is ready. On a related note, I found that when I upgraded from Istio 1.6.13 that Helm deleted the IstioOperator CRD. To work around this, there's an upgrade hook that recreates the IstioOperator CRD if it doens't exist. * cray-istio-deploy Just changed the default image tags and removed things from the README that weren't accurate. * cray-istio I updated the charts/istio and charts/ingressgatewayhmn subcharts with the latest version from the 1.7.8 distribution. I removed the transfer encoding workaround ( CASMPET-3079 ). The upstream bug is fixed: https://github.com/envoyproxy/envoy/issues/10041 I removed the tcp-stats-filter-1.6 memory leak workaround ( CASMPET-4026 ). Ths upstream bug is fixed: https://github.com/istio/istio/issues/24720 The istio-ingressgateway now needs `runAsRoot: true`, see https://istio.io/latest/news/releases/1.7.x/announcing-1.7/upgrade-notes/#gateways-run-as-non-root . --- kubernetes/cray-istio-deploy/README.md | 14 -- kubernetes/cray-istio-deploy/values.yaml | 4 +- kubernetes/cray-istio-operator/Chart.yaml | 2 +- kubernetes/cray-istio-operator/README.md | 13 +- .../charts/istio-operator/Chart.yaml | 6 +- .../istio-operator/crds/crd-operator.yaml | 74 +++++++ .../istio-operator/templates/clusterrole.yaml | 18 +- .../templates/clusterrole_binding.yaml | 6 +- .../templates/crd-operator.yaml | 46 ---- .../charts/istio-operator/templates/crds.yaml | 6 + .../istio-operator/templates/deployment.yaml | 31 ++- .../istio-operator/templates/service.yaml | 2 +- .../templates/service_account.yaml | 2 +- .../charts/istio-operator/values.yaml | 25 ++- .../files/crd-operator.yaml | 74 +++++++ .../_recreate-iop-crd-upgrade-script.yaml | 18 ++ .../templates/recreate-iop-crd-upgrade.yaml | 65 ++++++ .../templates/wait-jobs.yaml | 30 --- kubernetes/cray-istio-operator/values.yaml | 2 +- kubernetes/cray-istio/README.md | 28 +-- .../charts/ingressgatewayhmn/Chart.yaml | 4 +- .../ingressgatewayhmn/templates/_affinity.tpl | 4 +- .../templates/deployment.yaml | 25 ++- .../templates/meshexpansion.yaml | 1 - .../templates/poddisruptionbudget.yaml | 1 - .../templates/preconfigured.yaml | 13 +- .../charts/ingressgatewayhmn/values.yaml | 200 +++++++++++++++++- kubernetes/cray-istio/charts/istio/Chart.yaml | 4 +- .../charts/istio/templates/_affinity.tpl | 4 +- .../charts/istio/templates/deployment.yaml | 25 ++- .../charts/istio/templates/meshexpansion.yaml | 1 - .../istio/templates/poddisruptionbudget.yaml | 1 - .../charts/istio/templates/preconfigured.yaml | 13 +- .../cray-istio/charts/istio/values.yaml | 200 +++++++++++++++++- kubernetes/cray-istio/files/modify_runtime.sh | 18 -- .../templates/proxy-mem-leak-workaround.yaml | 55 ----- .../templates/te-bug-workaround.yaml | 78 ------- kubernetes/cray-istio/values.yaml | 3 +- 38 files changed, 779 insertions(+), 337 deletions(-) create mode 100644 kubernetes/cray-istio-operator/charts/istio-operator/crds/crd-operator.yaml delete mode 100644 kubernetes/cray-istio-operator/charts/istio-operator/templates/crd-operator.yaml create mode 100644 kubernetes/cray-istio-operator/charts/istio-operator/templates/crds.yaml create mode 100644 kubernetes/cray-istio-operator/files/crd-operator.yaml create mode 100644 kubernetes/cray-istio-operator/templates/_recreate-iop-crd-upgrade-script.yaml create mode 100644 kubernetes/cray-istio-operator/templates/recreate-iop-crd-upgrade.yaml delete mode 100644 kubernetes/cray-istio-operator/templates/wait-jobs.yaml delete mode 100644 kubernetes/cray-istio/files/modify_runtime.sh delete mode 100644 kubernetes/cray-istio/templates/proxy-mem-leak-workaround.yaml delete mode 100644 kubernetes/cray-istio/templates/te-bug-workaround.yaml diff --git a/kubernetes/cray-istio-deploy/README.md b/kubernetes/cray-istio-deploy/README.md index 71840f1..e501f16 100644 --- a/kubernetes/cray-istio-deploy/README.md +++ b/kubernetes/cray-istio-deploy/README.md @@ -1,17 +1,3 @@ This chart creates the IstioOperator object for the Cray system. It will be run after the cray-istio-operator chart starts the Istio Operator which defines the IstioOperator custom resource. - -Istio is currently configured using the Helm passthrough method, see the info box on -https://archive.istio.io/v1.5/docs/setup/install/istioctl/#customizing-the-configuration . -The Helm passthrough method is used because we used Helm previously and this -makes for minimal changes to the config. - -Note that this chart is configured to disable the ingress gateway component. -This is because Istio 1.5 has a bug/limitation where it always deploys a Gateway -but it doesn't provide the options that we're already using for that Gateway. -The ingress-gateway is deployed by the cray-istio chart which runs after this -chart. -See https://github.com/istio/istio/issues/21577 for a discussion of the bug. -According to this issue the Gateway doesn't get created in 1.6. If that's the -case we can change this chart to enable the ingress-gateway component. diff --git a/kubernetes/cray-istio-deploy/values.yaml b/kubernetes/cray-istio-deploy/values.yaml index 9c7aa40..ba5ae05 100644 --- a/kubernetes/cray-istio-deploy/values.yaml +++ b/kubernetes/cray-istio-deploy/values.yaml @@ -1,6 +1,6 @@ hub: dtr.dev.cray.com/cray -tag: 1.6.13-cray1-20210202183713_549528c5da +tag: 1.7.8-cray1 kubectl: image: @@ -9,7 +9,7 @@ kubectl: pilot: hub: dtr.dev.cray.com/cray - tag: 1.6.13-cray1-20210202183713_549528c5da + tag: 1.7.8-cray1 meshConfig: accessLogFile: /dev/stdout diff --git a/kubernetes/cray-istio-operator/Chart.yaml b/kubernetes/cray-istio-operator/Chart.yaml index c7c9dd3..6106dff 100644 --- a/kubernetes/cray-istio-operator/Chart.yaml +++ b/kubernetes/cray-istio-operator/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v1 -appVersion: 1.6.13 +appVersion: 1.7.8 name: cray-istio-operator description: Deploys the istio operator for Cray systems. version: 1.18.3 diff --git a/kubernetes/cray-istio-operator/README.md b/kubernetes/cray-istio-operator/README.md index 8e467ea..f95f561 100644 --- a/kubernetes/cray-istio-operator/README.md +++ b/kubernetes/cray-istio-operator/README.md @@ -1,8 +1,17 @@ This deploys the Istio operator. There are instructions here: -https://archive.istio.io/v1.5/docs/setup/install/standalone-operator/ +https://istio.io/v1.7/docs/setup/install/operator/ The istio-operator chart in the charts/ directory was copied from the istio release which is available for download at https://github.com/istio/istio/releases/ . -The chart is in `install/kubernetes/operator/charts/istio-operator`. +The chart is in `manifests/charts/istio-operator`. + +# Recreate IstioOperator CRD on upgrade + +When upgrading from the 1.6 istio-operator chart to 1.7 Helm winds up deleting +the IstioOperator CRD. The workaround is to have a post-upgrade hook that +checks for the condition and runs kubectl apply to recreate the +IstioOperator CRD. + +This can probably be removed in the next Istio upgrade. diff --git a/kubernetes/cray-istio-operator/charts/istio-operator/Chart.yaml b/kubernetes/cray-istio-operator/charts/istio-operator/Chart.yaml index 4f4977f..807caae 100644 --- a/kubernetes/cray-istio-operator/charts/istio-operator/Chart.yaml +++ b/kubernetes/cray-istio-operator/charts/istio-operator/Chart.yaml @@ -1,12 +1,12 @@ apiVersion: v1 name: istio-operator -version: 1.6.0 +version: 1.7.0 tillerVersion: ">=2.7.2" description: Helm chart for deploying Istio operator keywords: - istio - operator sources: - - http://github.com/istio/istio/operator + - https://github.com/istio/istio/tree/master/operator engine: gotpl -icon: https://istio.io/favicons/android-192x192.png +icon: https://istio.io/latest/favicons/android-192x192.png diff --git a/kubernetes/cray-istio-operator/charts/istio-operator/crds/crd-operator.yaml b/kubernetes/cray-istio-operator/charts/istio-operator/crds/crd-operator.yaml new file mode 100644 index 0000000..05fb86a --- /dev/null +++ b/kubernetes/cray-istio-operator/charts/istio-operator/crds/crd-operator.yaml @@ -0,0 +1,74 @@ +# SYNC WITH manifests/charts/base/files +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: istiooperators.install.istio.io + labels: + release: istio +spec: + group: install.istio.io + names: + kind: IstioOperator + plural: istiooperators + singular: istiooperator + shortNames: + - iop + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Istio control plane revision + jsonPath: .spec.revision + name: Revision + type: string + - description: IOP current state + jsonPath: .status.status + type: string + name: Status + - jsonPath: .metadata.creationTimestamp + description: + "CreationTimestamp is a timestamp representing the server time when + this object was created. It is not guaranteed to be set in happens-before order + across separate operations. Clients may not set this value. It is represented + in RFC3339 form and is in UTC. Populated by the system. Read-only. Null for + lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata" + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: + "APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. + More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#resources" + type: string + kind: + description: + "Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. + More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#types-kinds" + type: string + spec: + description: + "Specification of the desired state of the istio control plane resource. + More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status" + x-kubernetes-preserve-unknown-fields: true + type: object + status: + description: + "Status describes each of istio control plane component status at the current time. + 0 means NONE, 1 means UPDATING, 2 means HEALTHY, 3 means ERROR, 4 means RECONCILING. + More info: https://github.com/istio/api/blob/master/operator/v1alpha1/istio.operator.v1alpha1.pb.html & + https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status" + x-kubernetes-preserve-unknown-fields: true + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- + diff --git a/kubernetes/cray-istio-operator/charts/istio-operator/templates/clusterrole.yaml b/kubernetes/cray-istio-operator/charts/istio-operator/templates/clusterrole.yaml index 53be806..7f64a2e 100644 --- a/kubernetes/cray-istio-operator/charts/istio-operator/templates/clusterrole.yaml +++ b/kubernetes/cray-istio-operator/charts/istio-operator/templates/clusterrole.yaml @@ -2,7 +2,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: creationTimestamp: null - name: istio-operator + name: istio-operator{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }} rules: # istio groups - apiGroups: @@ -29,12 +29,6 @@ rules: - '*' verbs: - '*' -- apiGroups: - - rbac.istio.io - resources: - - '*' - verbs: - - '*' - apiGroups: - security.istio.io resources: @@ -81,6 +75,7 @@ rules: verbs: - get - create + - update - apiGroups: - policy resources: @@ -96,6 +91,14 @@ rules: - rolebindings verbs: - '*' +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update - apiGroups: - "" resources: @@ -104,6 +107,7 @@ rules: - events - namespaces - pods + - pods/proxy - persistentvolumeclaims - secrets - services diff --git a/kubernetes/cray-istio-operator/charts/istio-operator/templates/clusterrole_binding.yaml b/kubernetes/cray-istio-operator/charts/istio-operator/templates/clusterrole_binding.yaml index 39e2663..9b9df7d 100644 --- a/kubernetes/cray-istio-operator/charts/istio-operator/templates/clusterrole_binding.yaml +++ b/kubernetes/cray-istio-operator/charts/istio-operator/templates/clusterrole_binding.yaml @@ -1,13 +1,13 @@ kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: istio-operator + name: istio-operator{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }} subjects: - kind: ServiceAccount - name: istio-operator + name: istio-operator{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }} namespace: {{.Values.operatorNamespace}} roleRef: kind: ClusterRole - name: istio-operator + name: istio-operator{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }} apiGroup: rbac.authorization.k8s.io --- diff --git a/kubernetes/cray-istio-operator/charts/istio-operator/templates/crd-operator.yaml b/kubernetes/cray-istio-operator/charts/istio-operator/templates/crd-operator.yaml deleted file mode 100644 index 6b85c37..0000000 --- a/kubernetes/cray-istio-operator/charts/istio-operator/templates/crd-operator.yaml +++ /dev/null @@ -1,46 +0,0 @@ -# SYNC WITH manifests/charts/base/files -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: istiooperators.install.istio.io -spec: - group: install.istio.io - names: - kind: IstioOperator - plural: istiooperators - singular: istiooperator - shortNames: - - iop - scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. - More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. - More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - spec: - description: 'Specification of the desired state of the istio control plane resource. - More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status' - type: object - status: - description: 'Status describes each of istio control plane component status at the current time. - 0 means NONE, 1 means UPDATING, 2 means HEALTHY, 3 means ERROR, 4 means RECONCILING. - More info: https://github.com/istio/api/blob/master/operator/v1alpha1/istio.operator.v1alpha1.pb.html & - https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status' - type: object - versions: - - name: v1alpha1 - served: true - storage: true ---- diff --git a/kubernetes/cray-istio-operator/charts/istio-operator/templates/crds.yaml b/kubernetes/cray-istio-operator/charts/istio-operator/templates/crds.yaml new file mode 100644 index 0000000..a370365 --- /dev/null +++ b/kubernetes/cray-istio-operator/charts/istio-operator/templates/crds.yaml @@ -0,0 +1,6 @@ +{{- if .Values.enableCRDTemplates -}} +{{- range $path, $bytes := .Files.Glob "crds/*.yaml" -}} +--- +{{ $.Files.Get $path }} +{{- end -}} +{{- end -}} diff --git a/kubernetes/cray-istio-operator/charts/istio-operator/templates/deployment.yaml b/kubernetes/cray-istio-operator/charts/istio-operator/templates/deployment.yaml index 5ef7848..1baaa8d 100644 --- a/kubernetes/cray-istio-operator/charts/istio-operator/templates/deployment.yaml +++ b/kubernetes/cray-istio-operator/charts/istio-operator/templates/deployment.yaml @@ -2,7 +2,7 @@ apiVersion: apps/v1 kind: Deployment metadata: namespace: {{.Values.operatorNamespace}} - name: istio-operator + name: istio-operator{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }} spec: replicas: 1 selector: @@ -13,30 +13,39 @@ spec: labels: name: istio-operator spec: - serviceAccountName: istio-operator + serviceAccountName: istio-operator{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }} containers: - name: istio-operator image: {{.Values.hub}}/operator:{{.Values.tag}} command: - operator - server + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + runAsGroup: 1337 + runAsUser: 1337 + runAsNonRoot: true imagePullPolicy: IfNotPresent resources: - limits: - cpu: 200m - memory: 256Mi - requests: - cpu: 50m - memory: 128Mi +{{ toYaml .Values.operator.resources | trim | indent 12 }} env: - name: WATCH_NAMESPACE - value: {{.Values.istioNamespace}} + value: {{.Values.watchedNamespaces | quote}} - name: LEADER_ELECTION_NAMESPACE - value: {{.Values.operatorNamespace}} + value: {{.Values.operatorNamespace | quote}} - name: POD_NAME valueFrom: fieldRef: fieldPath: metadata.name - name: OPERATOR_NAME - value: {{.Values.operatorNamespace}} + value: {{.Values.operatorNamespace | quote}} + - name: WAIT_FOR_RESOURCES_TIMEOUT + value: {{.Values.waitForResourcesTimeout | quote}} + - name: REVISION + value: {{.Values.revision | quote}} --- diff --git a/kubernetes/cray-istio-operator/charts/istio-operator/templates/service.yaml b/kubernetes/cray-istio-operator/charts/istio-operator/templates/service.yaml index 93e2638..e7967a7 100644 --- a/kubernetes/cray-istio-operator/charts/istio-operator/templates/service.yaml +++ b/kubernetes/cray-istio-operator/charts/istio-operator/templates/service.yaml @@ -4,7 +4,7 @@ metadata: namespace: {{.Values.operatorNamespace}} labels: name: istio-operator - name: istio-operator + name: istio-operator{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }} spec: ports: - name: http-metrics diff --git a/kubernetes/cray-istio-operator/charts/istio-operator/templates/service_account.yaml b/kubernetes/cray-istio-operator/charts/istio-operator/templates/service_account.yaml index 32307f3..cb708ee 100644 --- a/kubernetes/cray-istio-operator/charts/istio-operator/templates/service_account.yaml +++ b/kubernetes/cray-istio-operator/charts/istio-operator/templates/service_account.yaml @@ -2,5 +2,5 @@ apiVersion: v1 kind: ServiceAccount metadata: namespace: {{.Values.operatorNamespace}} - name: istio-operator + name: istio-operator{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }} --- diff --git a/kubernetes/cray-istio-operator/charts/istio-operator/values.yaml b/kubernetes/cray-istio-operator/charts/istio-operator/values.yaml index 6a653f5..45c5c96 100644 --- a/kubernetes/cray-istio-operator/charts/istio-operator/values.yaml +++ b/kubernetes/cray-istio-operator/charts/istio-operator/values.yaml @@ -1,4 +1,25 @@ hub: gcr.io/istio-testing -tag: 1.6-dev +tag: latest + operatorNamespace: istio-operator -istioNamespace: istio-system + +# Used to replace istioNamespace to support operator watch multiple namespaces. +watchedNamespaces: istio-system +waitForResourcesTimeout: 300s + +# Used for helm2 to add the CRDs to templates. +enableCRDTemplates: false + +# revision for the operator resources +revision: "" + +# Operator resource defaults +operator: + resources: + limits: + cpu: 200m + memory: 256Mi + requests: + cpu: 50m + memory: 128Mi + diff --git a/kubernetes/cray-istio-operator/files/crd-operator.yaml b/kubernetes/cray-istio-operator/files/crd-operator.yaml new file mode 100644 index 0000000..05fb86a --- /dev/null +++ b/kubernetes/cray-istio-operator/files/crd-operator.yaml @@ -0,0 +1,74 @@ +# SYNC WITH manifests/charts/base/files +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: istiooperators.install.istio.io + labels: + release: istio +spec: + group: install.istio.io + names: + kind: IstioOperator + plural: istiooperators + singular: istiooperator + shortNames: + - iop + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Istio control plane revision + jsonPath: .spec.revision + name: Revision + type: string + - description: IOP current state + jsonPath: .status.status + type: string + name: Status + - jsonPath: .metadata.creationTimestamp + description: + "CreationTimestamp is a timestamp representing the server time when + this object was created. It is not guaranteed to be set in happens-before order + across separate operations. Clients may not set this value. It is represented + in RFC3339 form and is in UTC. Populated by the system. Read-only. Null for + lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata" + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: + "APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. + More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#resources" + type: string + kind: + description: + "Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. + More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#types-kinds" + type: string + spec: + description: + "Specification of the desired state of the istio control plane resource. + More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status" + x-kubernetes-preserve-unknown-fields: true + type: object + status: + description: + "Status describes each of istio control plane component status at the current time. + 0 means NONE, 1 means UPDATING, 2 means HEALTHY, 3 means ERROR, 4 means RECONCILING. + More info: https://github.com/istio/api/blob/master/operator/v1alpha1/istio.operator.v1alpha1.pb.html & + https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status" + x-kubernetes-preserve-unknown-fields: true + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- + diff --git a/kubernetes/cray-istio-operator/templates/_recreate-iop-crd-upgrade-script.yaml b/kubernetes/cray-istio-operator/templates/_recreate-iop-crd-upgrade-script.yaml new file mode 100644 index 0000000..1a44aa8 --- /dev/null +++ b/kubernetes/cray-istio-operator/templates/_recreate-iop-crd-upgrade-script.yaml @@ -0,0 +1,18 @@ +{{- /* +Copyright 2021 Hewlett Packard Enterprise Development LP +*/ -}} +{{ define "cray-istio-operator.recreate-iop-crd-upgrade-script" }} + +sleep 30 # Delay to give the istiooperators that upgrade from 1.6 deletes a chance to go away. + +if [[ "$(kubectl get crds | grep istiooperators.install.istio.io | wc -l)" != "1" ]]; then + +kubectl apply -f - <=2.7.2" description: Helm chart for deploying Istio gateways keywords: @@ -10,4 +10,4 @@ keywords: sources: - http://github.com/istio/istio engine: gotpl -icon: https://istio.io/favicons/android-192x192.png +icon: https://istio.io/latest/favicons/android-192x192.png diff --git a/kubernetes/cray-istio/charts/ingressgatewayhmn/templates/_affinity.tpl b/kubernetes/cray-istio/charts/ingressgatewayhmn/templates/_affinity.tpl index 5335b59..400ff54 100644 --- a/kubernetes/cray-istio/charts/ingressgatewayhmn/templates/_affinity.tpl +++ b/kubernetes/cray-istio/charts/ingressgatewayhmn/templates/_affinity.tpl @@ -11,7 +11,7 @@ {{- define "nodeAffinityRequiredDuringScheduling" }} nodeSelectorTerms: - matchExpressions: - - key: beta.kubernetes.io/arch + - key: kubernetes.io/arch operator: In values: {{- range $key, $val := .global.arch }} @@ -34,7 +34,7 @@ - weight: {{ $val | int }} preference: matchExpressions: - - key: beta.kubernetes.io/arch + - key: kubernetes.io/arch operator: In values: - {{ $key | quote }} diff --git a/kubernetes/cray-istio/charts/ingressgatewayhmn/templates/deployment.yaml b/kubernetes/cray-istio/charts/ingressgatewayhmn/templates/deployment.yaml index 2531df1..ad387df 100644 --- a/kubernetes/cray-istio/charts/ingressgatewayhmn/templates/deployment.yaml +++ b/kubernetes/cray-istio/charts/ingressgatewayhmn/templates/deployment.yaml @@ -36,7 +36,7 @@ spec: service.istio.io/canonical-revision: latest {{- end }} annotations: - {{- if .Values.global.meshConfig.enablePrometheusMerge }} + {{- if .Values.meshConfig.enablePrometheusMerge }} prometheus.io/port: "15090" prometheus.io/scrape: "true" prometheus.io/path: "/stats/prometheus" @@ -72,7 +72,7 @@ spec: - /bin/sh args: - -c - - sysctl -w kernel.core_pattern=/var/lib/istio/core.proxy && ulimit -c unlimited + - sysctl -w kernel.core_pattern=/var/lib/istio/data/core.proxy && ulimit -c unlimited securityContext: runAsUser: 0 runAsGroup: 0 @@ -93,8 +93,10 @@ spec: {{- range $key, $val := $gateway.ports }} - containerPort: {{ $val.targetPort | default $val.port }} {{- end }} + {{- if $.Values.global.meshExpansion.enabled }} {{- range $key, $val := $gateway.meshExpansionPorts }} - - containerPort: {{ $val.port }} + - containerPort: {{ $val.targetPort | default $val.port }} + {{- end }} {{- end }} - containerPort: 15090 protocol: TCP @@ -212,7 +214,7 @@ spec: - name: {{ $key }} value: {{ $val }} {{- end }} - {{- range $key, $value := .Values.global.meshConfig.defaultConfig.proxyMetadata }} + {{- range $key, $value := .Values.meshConfig.defaultConfig.proxyMetadata }} - name: {{ $key }} value: "{{ $value }}" {{- end }} @@ -242,7 +244,7 @@ spec: mountPath: /var/run/secrets/tokens readOnly: true {{- end }} - - name: ingressgatewaysdsudspath + - name: gatewaysdsudspath mountPath: /var/run/ingress_gateway {{- if .Values.global.mountMtlsCerts }} # Use the key and cert mounted to /etc/certs/ for the in-cluster mTLS communications. @@ -250,6 +252,8 @@ spec: mountPath: /etc/certs readOnly: true {{- end }} + - mountPath: /var/lib/istio/data + name: istio-data - name: podinfo mountPath: /etc/istio/pod {{- range $gateway.secretVolumes }} @@ -257,6 +261,13 @@ spec: mountPath: {{ .mountPath | quote }} readOnly: true {{- end }} + {{- range $gateway.configVolumes }} + {{- if .mountPath }} + - name: {{ .name }} + mountPath: {{ .mountPath | quote }} + readOnly: true + {{- end }} + {{- end }} {{- if $gateway.additionalContainers }} {{ toYaml $gateway.additionalContainers | indent 8 }} {{- end }} @@ -277,7 +288,9 @@ spec: fieldPath: metadata.annotations - name: istio-envoy emptyDir: {} - - name: ingressgatewaysdsudspath + - name: gatewaysdsudspath + emptyDir: {} + - name: istio-data emptyDir: {} {{- if eq .Values.global.jwtPolicy "third-party-jwt" }} - name: istio-token diff --git a/kubernetes/cray-istio/charts/ingressgatewayhmn/templates/meshexpansion.yaml b/kubernetes/cray-istio/charts/ingressgatewayhmn/templates/meshexpansion.yaml index c4b382a..4160aa5 100644 --- a/kubernetes/cray-istio/charts/ingressgatewayhmn/templates/meshexpansion.yaml +++ b/kubernetes/cray-istio/charts/ingressgatewayhmn/templates/meshexpansion.yaml @@ -74,4 +74,3 @@ spec: mode: DISABLE {{- end }} - diff --git a/kubernetes/cray-istio/charts/ingressgatewayhmn/templates/poddisruptionbudget.yaml b/kubernetes/cray-istio/charts/ingressgatewayhmn/templates/poddisruptionbudget.yaml index 9100fc1..a0346d7 100644 --- a/kubernetes/cray-istio/charts/ingressgatewayhmn/templates/poddisruptionbudget.yaml +++ b/kubernetes/cray-istio/charts/ingressgatewayhmn/templates/poddisruptionbudget.yaml @@ -13,5 +13,4 @@ spec: selector: matchLabels: {{ $gateway.labels | toYaml | trim | indent 6 }} - release: {{ .Release.Name }} {{- end }} diff --git a/kubernetes/cray-istio/charts/ingressgatewayhmn/templates/preconfigured.yaml b/kubernetes/cray-istio/charts/ingressgatewayhmn/templates/preconfigured.yaml index 35de3f9..0c95aa8 100644 --- a/kubernetes/cray-istio/charts/ingressgatewayhmn/templates/preconfigured.yaml +++ b/kubernetes/cray-istio/charts/ingressgatewayhmn/templates/preconfigured.yaml @@ -13,7 +13,7 @@ spec: istio: ingressgateway servers: - hosts: - - "*.global" + - "*.{{ .Values.global.multiCluster.globalDomainSuffix | trim }}" port: name: tls number: 15443 @@ -21,6 +21,7 @@ spec: tls: mode: AUTO_PASSTHROUGH --- +{{- if .Values.global.multiCluster.includeEnvoyFilter }} apiVersion: networking.istio.io/v1alpha3 kind: EnvoyFilter metadata: @@ -46,11 +47,13 @@ spec: operation: INSERT_AFTER value: name: "envoy.filters.network.tcp_cluster_rewrite" - config: - cluster_pattern: "\\.global$" + typed_config: + "@type": "type.googleapis.com/istio.envoy.config.filter.network.tcp_cluster_rewrite.v2alpha1.TcpClusterRewrite" + cluster_pattern: "\\.{{ .Values.global.multiCluster.globalDomainSuffix | trim }}$" cluster_replacement: ".svc.{{ .Values.global.proxy.clusterDomain }}" --- -## To ensure all traffic to *.global is using mTLS +{{- end }} +## To ensure all traffic to globalDomainSuffix is using mTLS apiVersion: networking.istio.io/v1alpha3 kind: DestinationRule metadata: @@ -60,7 +63,7 @@ metadata: {{ $gateway.labels | toYaml | indent 4 }} release: {{ .Release.Name }} spec: - host: "*.global" + host: "*.{{ .Values.global.multiCluster.globalDomainSuffix | trim }}" {{- if .Values.global.defaultConfigVisibilitySettings }} exportTo: - '*' diff --git a/kubernetes/cray-istio/charts/ingressgatewayhmn/values.yaml b/kubernetes/cray-istio/charts/ingressgatewayhmn/values.yaml index 83fb5c6..bb54874 100644 --- a/kubernetes/cray-istio/charts/ingressgatewayhmn/values.yaml +++ b/kubernetes/cray-istio/charts/ingressgatewayhmn/values.yaml @@ -134,8 +134,206 @@ gateways: podAntiAffinityTermLabelSelector: [] # whether to run the gateway in a privileged container - runAsRoot: true + runAsRoot: false # Revision is set as 'version' label and part of the resource names when installing multiple control planes. revision: "" +global: + # set the default set of namespaces to which services, service entries, virtual services, destination + # rules should be exported to. Currently only one value can be provided in this list. This value + # should be one of the following two options: + # * implies these objects are visible to all namespaces, enabling any sidecar to talk to any other sidecar. + # . implies these objects are visible to only to sidecars in the same namespace, or if imported as a Sidecar.egress.host + defaultConfigVisibilitySettings: [] + + # enable pod disruption budget for the control plane, which is used to + # ensure Istio control plane components are gradually upgraded or recovered. + defaultPodDisruptionBudget: + enabled: true + + # A minimal set of requested resources to applied to all deployments so that + # Horizontal Pod Autoscaler will be able to function (if set). + # Each component can overwrite these default values by adding its own resources + # block in the relevant section below and setting the desired resources values. + defaultResources: + requests: + cpu: 10m + # memory: 128Mi + # limits: + # cpu: 100m + # memory: 128Mi + + # Default node tolerations to be applied to all deployments so that all pods can be + # scheduled to a particular nodes with matching taints. Each component can overwrite + # these default values by adding its tolerations block in the relevant section below + # and setting the desired values. + # Configure this field in case that all pods of Istio control plane are expected to + # be scheduled to particular nodes with specified taints. + defaultTolerations: [] + + # Default hub for Istio images. + # Releases are published to docker hub under 'istio' project. + # Dev builds from prow are on gcr.io + hub: gcr.io/istio-testing + + # Default tag for Istio images. + tag: latest + + # Specify image pull policy if default behavior isn't desired. + # Default behavior: latest images will be Always else IfNotPresent. + imagePullPolicy: "" + + # ImagePullSecrets for all ServiceAccount, list of secrets in the same namespace + # to use for pulling any images in pods that reference this ServiceAccount. + # For components that don't use ServiceAccounts (i.e. grafana, servicegraph, tracing) + # ImagePullSecrets will be added to the corresponding Deployment(StatefulSet) objects. + # Must be set for any cluster configured with private docker registry. + imagePullSecrets: [] + # - private-registry-key + + # To output all istio components logs in json format by adding --log_as_json argument to each container argument + logAsJson: false + + # Comma-separated minimum per-scope logging level of messages to output, in the form of :,: + # The control plane has different scopes depending on component, but can configure default log level across all components + # If empty, default scope and level will be used as configured in code + logging: + level: "default:info" + + # If set to true, the pilot and citadel mtls will be exposed on the + # ingress gateway + meshExpansion: + enabled: false + # If set to true, the pilot and citadel mtls and the plain text pilot ports + # will be exposed on an internal gateway + useILB: false + + # Kubernetes >=v1.11.0 will create two PriorityClass, including system-cluster-critical and + # system-node-critical, it is better to configure this in order to make sure your Istio pods + # will not be killed because of low priority class. + # Refer to https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass + # for more detail. + priorityClassName: "" + + proxy: + image: proxyv2 + + # CAUTION: It is important to ensure that all Istio helm charts specify the same clusterDomain value + # cluster domain. Default value is "cluster.local". + clusterDomain: "cluster.local" + + # Per Component log level for proxy, applies to gateways and sidecars. If a component level is + # not set, then the global "logLevel" will be used. + componentLogLevel: "misc:error" + + # If set, newly injected sidecars will have core dumps enabled. + enableCoreDump: false + + # Log level for proxy, applies to gateways and sidecars. + # Expected values are: trace|debug|info|warning|error|critical|off + logLevel: warning + + ############################################################################################## + # The following values are found in other charts. To effectively modify these values, make # + # make sure they are consistent across your Istio helm charts # + ############################################################################################## + + # The customized CA address to retrieve certificates for the pods in the cluster. + # CSR clients such as the Istio Agent and ingress gateways can use this to specify the CA endpoint. + caAddress: "" + + # Used to locate istio-pilot. + # Default is to install pilot in a dedicated namespace, istio-pilot11. You can use multiple namespaces, but + # for each 'profile' you need to match the control plane namespace and the value of istioNamespace + # It is assumed that istio-system is running either 1.0 or an upgraded version of 1.1, but only security components are + # used (citadel generating the secrets). + istioNamespace: istio-system + + # Configure the policy for validating JWT. + # Currently, two options are supported: "third-party-jwt" and "first-party-jwt". + jwtPolicy: "third-party-jwt" + + # Mesh ID means Mesh Identifier. It should be unique within the scope where + # meshes will interact with each other, but it is not required to be + # globally/universally unique. For example, if any of the following are true, + # then two meshes must have different Mesh IDs: + # - Meshes will have their telemetry aggregated in one place + # - Meshes will be federated together + # - Policy will be written referencing one mesh from the other + # + # If an administrator expects that any of these conditions may become true in + # the future, they should ensure their meshes have different Mesh IDs + # assigned. + # + # Within a multicluster mesh, each cluster must be (manually or auto) + # configured to have the same Mesh ID value. If an existing cluster 'joins' a + # multicluster mesh, it will need to be migrated to the new mesh ID. Details + # of migration TBD, and it may be a disruptive operation to change the Mesh + # ID post-install. + # + # If the mesh admin does not specify a value, Istio will use the value of the + # mesh's Trust Domain. The best practice is to select a proper Trust Domain + # value. + meshID: "" + + # Use the user-specified, secret volume mounted key and certs for Pilot and workloads. + mountMtlsCerts: false + + multiCluster: + # Set to true to connect two kubernetes clusters via their respective + # ingressgateway services when pods in each cluster cannot directly + # talk to one another. All clusters should be using Istio mTLS and must + # have a shared root CA for this model to work. + enabled: false + # Should be set to the name of the cluster this installation will run in. This is required for sidecar injection + # to properly label proxies + clusterName: "" + # The suffix for global service names + globalDomainSuffix: "global" + # Enable envoy filter to translate `globalDomainSuffix` to cluster local suffix for cross cluster communication + includeEnvoyFilter: true + + # Network defines the network this cluster belong to. This name + # corresponds to the networks in the map of mesh networks. + network: "" + + # Configure the certificate provider for control plane communication. + # Currently, two providers are supported: "kubernetes" and "istiod". + # As some platforms may not have kubernetes signing APIs, + # Istiod is the default + pilotCertProvider: istiod + + sds: + # The JWT token for SDS and the aud field of such JWT. See RFC 7519, section 4.1.3. + # When a CSR is sent from Citadel Agent to the CA (e.g. Citadel), this aud is to make sure the + # JWT is intended for the CA. + token: + aud: istio-ca + + sts: + # The service port used by Security Token Service (STS) server to handle token exchange requests. + # Setting this port to a non-zero value enables STS server. + servicePort: 0 + + # The trust domain corresponds to the trust root of a system + # Refer to https://github.com/spiffe/spiffe/blob/master/standards/SPIFFE-ID.md#21-trust-domain + # Indicate the domain used in SPIFFE identity URL + # The default depends on the environment. + # kubernetes: cluster.local + # else: default dns domain + trustDomain: "cluster.local" + +meshConfig: + enablePrometheusMerge: true + defaultConfig: + proxyMetadata: {} + tracing: + # tlsSettings: + # mode: DISABLE # DISABLE, SIMPLE, MUTUAL, ISTIO_MUTUAL + # clientCertificate: # example: /etc/istio/tracer/cert-chain.pem + # privateKey: # example: /etc/istio/tracer/key.pem + # caCertificates: # example: /etc/istio/tracer/root-cert.pem + # sni: # example: tracer.somedomain + # subjectAltNames: [] + # - tracer.somedomain diff --git a/kubernetes/cray-istio/charts/istio/Chart.yaml b/kubernetes/cray-istio/charts/istio/Chart.yaml index ee5bc24..334bf7b 100644 --- a/kubernetes/cray-istio/charts/istio/Chart.yaml +++ b/kubernetes/cray-istio/charts/istio/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 name: istio -version: 1.2.0 +version: 1.1.0 tillerVersion: ">=2.7.2" description: Helm chart for deploying Istio gateways keywords: @@ -10,4 +10,4 @@ keywords: sources: - http://github.com/istio/istio engine: gotpl -icon: https://istio.io/favicons/android-192x192.png +icon: https://istio.io/latest/favicons/android-192x192.png diff --git a/kubernetes/cray-istio/charts/istio/templates/_affinity.tpl b/kubernetes/cray-istio/charts/istio/templates/_affinity.tpl index 5335b59..400ff54 100644 --- a/kubernetes/cray-istio/charts/istio/templates/_affinity.tpl +++ b/kubernetes/cray-istio/charts/istio/templates/_affinity.tpl @@ -11,7 +11,7 @@ {{- define "nodeAffinityRequiredDuringScheduling" }} nodeSelectorTerms: - matchExpressions: - - key: beta.kubernetes.io/arch + - key: kubernetes.io/arch operator: In values: {{- range $key, $val := .global.arch }} @@ -34,7 +34,7 @@ - weight: {{ $val | int }} preference: matchExpressions: - - key: beta.kubernetes.io/arch + - key: kubernetes.io/arch operator: In values: - {{ $key | quote }} diff --git a/kubernetes/cray-istio/charts/istio/templates/deployment.yaml b/kubernetes/cray-istio/charts/istio/templates/deployment.yaml index f6421eb..c1f5bd5 100644 --- a/kubernetes/cray-istio/charts/istio/templates/deployment.yaml +++ b/kubernetes/cray-istio/charts/istio/templates/deployment.yaml @@ -36,7 +36,7 @@ spec: service.istio.io/canonical-revision: latest {{- end }} annotations: - {{- if .Values.global.meshConfig.enablePrometheusMerge }} + {{- if .Values.meshConfig.enablePrometheusMerge }} prometheus.io/port: "15090" prometheus.io/scrape: "true" prometheus.io/path: "/stats/prometheus" @@ -72,7 +72,7 @@ spec: - /bin/sh args: - -c - - sysctl -w kernel.core_pattern=/var/lib/istio/core.proxy && ulimit -c unlimited + - sysctl -w kernel.core_pattern=/var/lib/istio/data/core.proxy && ulimit -c unlimited securityContext: runAsUser: 0 runAsGroup: 0 @@ -93,8 +93,10 @@ spec: {{- range $key, $val := $gateway.ports }} - containerPort: {{ $val.targetPort | default $val.port }} {{- end }} + {{- if $.Values.global.meshExpansion.enabled }} {{- range $key, $val := $gateway.meshExpansionPorts }} - - containerPort: {{ $val.port }} + - containerPort: {{ $val.targetPort | default $val.port }} + {{- end }} {{- end }} - containerPort: 15090 protocol: TCP @@ -212,7 +214,7 @@ spec: - name: {{ $key }} value: {{ $val }} {{- end }} - {{- range $key, $value := .Values.global.meshConfig.defaultConfig.proxyMetadata }} + {{- range $key, $value := .Values.meshConfig.defaultConfig.proxyMetadata }} - name: {{ $key }} value: "{{ $value }}" {{- end }} @@ -242,7 +244,7 @@ spec: mountPath: /var/run/secrets/tokens readOnly: true {{- end }} - - name: ingressgatewaysdsudspath + - name: gatewaysdsudspath mountPath: /var/run/ingress_gateway {{- if .Values.global.mountMtlsCerts }} # Use the key and cert mounted to /etc/certs/ for the in-cluster mTLS communications. @@ -250,6 +252,8 @@ spec: mountPath: /etc/certs readOnly: true {{- end }} + - mountPath: /var/lib/istio/data + name: istio-data - name: podinfo mountPath: /etc/istio/pod {{- range $gateway.secretVolumes }} @@ -257,6 +261,13 @@ spec: mountPath: {{ .mountPath | quote }} readOnly: true {{- end }} + {{- range $gateway.configVolumes }} + {{- if .mountPath }} + - name: {{ .name }} + mountPath: {{ .mountPath | quote }} + readOnly: true + {{- end }} + {{- end }} {{- if $gateway.additionalContainers }} {{ toYaml $gateway.additionalContainers | indent 8 }} {{- end }} @@ -277,7 +288,9 @@ spec: fieldPath: metadata.annotations - name: istio-envoy emptyDir: {} - - name: ingressgatewaysdsudspath + - name: gatewaysdsudspath + emptyDir: {} + - name: istio-data emptyDir: {} {{- if eq .Values.global.jwtPolicy "third-party-jwt" }} - name: istio-token diff --git a/kubernetes/cray-istio/charts/istio/templates/meshexpansion.yaml b/kubernetes/cray-istio/charts/istio/templates/meshexpansion.yaml index c4b382a..4160aa5 100644 --- a/kubernetes/cray-istio/charts/istio/templates/meshexpansion.yaml +++ b/kubernetes/cray-istio/charts/istio/templates/meshexpansion.yaml @@ -74,4 +74,3 @@ spec: mode: DISABLE {{- end }} - diff --git a/kubernetes/cray-istio/charts/istio/templates/poddisruptionbudget.yaml b/kubernetes/cray-istio/charts/istio/templates/poddisruptionbudget.yaml index 5aabc51..d4966ac 100644 --- a/kubernetes/cray-istio/charts/istio/templates/poddisruptionbudget.yaml +++ b/kubernetes/cray-istio/charts/istio/templates/poddisruptionbudget.yaml @@ -13,5 +13,4 @@ spec: selector: matchLabels: {{ $gateway.labels | toYaml | trim | indent 6 }} - release: {{ .Release.Name }} {{- end }} diff --git a/kubernetes/cray-istio/charts/istio/templates/preconfigured.yaml b/kubernetes/cray-istio/charts/istio/templates/preconfigured.yaml index cd6c522..68cf284 100644 --- a/kubernetes/cray-istio/charts/istio/templates/preconfigured.yaml +++ b/kubernetes/cray-istio/charts/istio/templates/preconfigured.yaml @@ -13,7 +13,7 @@ spec: istio: ingressgateway servers: - hosts: - - "*.global" + - "*.{{ .Values.global.multiCluster.globalDomainSuffix | trim }}" port: name: tls number: 15443 @@ -21,6 +21,7 @@ spec: tls: mode: AUTO_PASSTHROUGH --- +{{- if .Values.global.multiCluster.includeEnvoyFilter }} apiVersion: networking.istio.io/v1alpha3 kind: EnvoyFilter metadata: @@ -46,11 +47,13 @@ spec: operation: INSERT_AFTER value: name: "envoy.filters.network.tcp_cluster_rewrite" - config: - cluster_pattern: "\\.global$" + typed_config: + "@type": "type.googleapis.com/istio.envoy.config.filter.network.tcp_cluster_rewrite.v2alpha1.TcpClusterRewrite" + cluster_pattern: "\\.{{ .Values.global.multiCluster.globalDomainSuffix | trim }}$" cluster_replacement: ".svc.{{ .Values.global.proxy.clusterDomain }}" --- -## To ensure all traffic to *.global is using mTLS +{{- end }} +## To ensure all traffic to globalDomainSuffix is using mTLS apiVersion: networking.istio.io/v1alpha3 kind: DestinationRule metadata: @@ -60,7 +63,7 @@ metadata: {{ $gateway.labels | toYaml | indent 4 }} release: {{ .Release.Name }} spec: - host: "*.global" + host: "*.{{ .Values.global.multiCluster.globalDomainSuffix | trim }}" {{- if .Values.global.defaultConfigVisibilitySettings }} exportTo: - '*' diff --git a/kubernetes/cray-istio/charts/istio/values.yaml b/kubernetes/cray-istio/charts/istio/values.yaml index 64fd4ce..16d3e8a 100644 --- a/kubernetes/cray-istio/charts/istio/values.yaml +++ b/kubernetes/cray-istio/charts/istio/values.yaml @@ -134,8 +134,206 @@ gateways: podAntiAffinityTermLabelSelector: [] # whether to run the gateway in a privileged container - runAsRoot: true + runAsRoot: false # Revision is set as 'version' label and part of the resource names when installing multiple control planes. revision: "" +global: + # set the default set of namespaces to which services, service entries, virtual services, destination + # rules should be exported to. Currently only one value can be provided in this list. This value + # should be one of the following two options: + # * implies these objects are visible to all namespaces, enabling any sidecar to talk to any other sidecar. + # . implies these objects are visible to only to sidecars in the same namespace, or if imported as a Sidecar.egress.host + defaultConfigVisibilitySettings: [] + + # enable pod disruption budget for the control plane, which is used to + # ensure Istio control plane components are gradually upgraded or recovered. + defaultPodDisruptionBudget: + enabled: true + + # A minimal set of requested resources to applied to all deployments so that + # Horizontal Pod Autoscaler will be able to function (if set). + # Each component can overwrite these default values by adding its own resources + # block in the relevant section below and setting the desired resources values. + defaultResources: + requests: + cpu: 10m + # memory: 128Mi + # limits: + # cpu: 100m + # memory: 128Mi + + # Default node tolerations to be applied to all deployments so that all pods can be + # scheduled to a particular nodes with matching taints. Each component can overwrite + # these default values by adding its tolerations block in the relevant section below + # and setting the desired values. + # Configure this field in case that all pods of Istio control plane are expected to + # be scheduled to particular nodes with specified taints. + defaultTolerations: [] + + # Default hub for Istio images. + # Releases are published to docker hub under 'istio' project. + # Dev builds from prow are on gcr.io + hub: gcr.io/istio-testing + + # Default tag for Istio images. + tag: latest + + # Specify image pull policy if default behavior isn't desired. + # Default behavior: latest images will be Always else IfNotPresent. + imagePullPolicy: "" + + # ImagePullSecrets for all ServiceAccount, list of secrets in the same namespace + # to use for pulling any images in pods that reference this ServiceAccount. + # For components that don't use ServiceAccounts (i.e. grafana, servicegraph, tracing) + # ImagePullSecrets will be added to the corresponding Deployment(StatefulSet) objects. + # Must be set for any cluster configured with private docker registry. + imagePullSecrets: [] + # - private-registry-key + + # To output all istio components logs in json format by adding --log_as_json argument to each container argument + logAsJson: false + + # Comma-separated minimum per-scope logging level of messages to output, in the form of :,: + # The control plane has different scopes depending on component, but can configure default log level across all components + # If empty, default scope and level will be used as configured in code + logging: + level: "default:info" + + # If set to true, the pilot and citadel mtls will be exposed on the + # ingress gateway + meshExpansion: + enabled: false + # If set to true, the pilot and citadel mtls and the plain text pilot ports + # will be exposed on an internal gateway + useILB: false + + # Kubernetes >=v1.11.0 will create two PriorityClass, including system-cluster-critical and + # system-node-critical, it is better to configure this in order to make sure your Istio pods + # will not be killed because of low priority class. + # Refer to https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass + # for more detail. + priorityClassName: "" + + proxy: + image: proxyv2 + + # CAUTION: It is important to ensure that all Istio helm charts specify the same clusterDomain value + # cluster domain. Default value is "cluster.local". + clusterDomain: "cluster.local" + + # Per Component log level for proxy, applies to gateways and sidecars. If a component level is + # not set, then the global "logLevel" will be used. + componentLogLevel: "misc:error" + + # If set, newly injected sidecars will have core dumps enabled. + enableCoreDump: false + + # Log level for proxy, applies to gateways and sidecars. + # Expected values are: trace|debug|info|warning|error|critical|off + logLevel: warning + + ############################################################################################## + # The following values are found in other charts. To effectively modify these values, make # + # make sure they are consistent across your Istio helm charts # + ############################################################################################## + + # The customized CA address to retrieve certificates for the pods in the cluster. + # CSR clients such as the Istio Agent and ingress gateways can use this to specify the CA endpoint. + caAddress: "" + + # Used to locate istio-pilot. + # Default is to install pilot in a dedicated namespace, istio-pilot11. You can use multiple namespaces, but + # for each 'profile' you need to match the control plane namespace and the value of istioNamespace + # It is assumed that istio-system is running either 1.0 or an upgraded version of 1.1, but only security components are + # used (citadel generating the secrets). + istioNamespace: istio-system + + # Configure the policy for validating JWT. + # Currently, two options are supported: "third-party-jwt" and "first-party-jwt". + jwtPolicy: "third-party-jwt" + + # Mesh ID means Mesh Identifier. It should be unique within the scope where + # meshes will interact with each other, but it is not required to be + # globally/universally unique. For example, if any of the following are true, + # then two meshes must have different Mesh IDs: + # - Meshes will have their telemetry aggregated in one place + # - Meshes will be federated together + # - Policy will be written referencing one mesh from the other + # + # If an administrator expects that any of these conditions may become true in + # the future, they should ensure their meshes have different Mesh IDs + # assigned. + # + # Within a multicluster mesh, each cluster must be (manually or auto) + # configured to have the same Mesh ID value. If an existing cluster 'joins' a + # multicluster mesh, it will need to be migrated to the new mesh ID. Details + # of migration TBD, and it may be a disruptive operation to change the Mesh + # ID post-install. + # + # If the mesh admin does not specify a value, Istio will use the value of the + # mesh's Trust Domain. The best practice is to select a proper Trust Domain + # value. + meshID: "" + + # Use the user-specified, secret volume mounted key and certs for Pilot and workloads. + mountMtlsCerts: false + + multiCluster: + # Set to true to connect two kubernetes clusters via their respective + # ingressgateway services when pods in each cluster cannot directly + # talk to one another. All clusters should be using Istio mTLS and must + # have a shared root CA for this model to work. + enabled: false + # Should be set to the name of the cluster this installation will run in. This is required for sidecar injection + # to properly label proxies + clusterName: "" + # The suffix for global service names + globalDomainSuffix: "global" + # Enable envoy filter to translate `globalDomainSuffix` to cluster local suffix for cross cluster communication + includeEnvoyFilter: true + + # Network defines the network this cluster belong to. This name + # corresponds to the networks in the map of mesh networks. + network: "" + + # Configure the certificate provider for control plane communication. + # Currently, two providers are supported: "kubernetes" and "istiod". + # As some platforms may not have kubernetes signing APIs, + # Istiod is the default + pilotCertProvider: istiod + + sds: + # The JWT token for SDS and the aud field of such JWT. See RFC 7519, section 4.1.3. + # When a CSR is sent from Citadel Agent to the CA (e.g. Citadel), this aud is to make sure the + # JWT is intended for the CA. + token: + aud: istio-ca + + sts: + # The service port used by Security Token Service (STS) server to handle token exchange requests. + # Setting this port to a non-zero value enables STS server. + servicePort: 0 + + # The trust domain corresponds to the trust root of a system + # Refer to https://github.com/spiffe/spiffe/blob/master/standards/SPIFFE-ID.md#21-trust-domain + # Indicate the domain used in SPIFFE identity URL + # The default depends on the environment. + # kubernetes: cluster.local + # else: default dns domain + trustDomain: "cluster.local" + +meshConfig: + enablePrometheusMerge: true + defaultConfig: + proxyMetadata: {} + tracing: + # tlsSettings: + # mode: DISABLE # DISABLE, SIMPLE, MUTUAL, ISTIO_MUTUAL + # clientCertificate: # example: /etc/istio/tracer/cert-chain.pem + # privateKey: # example: /etc/istio/tracer/key.pem + # caCertificates: # example: /etc/istio/tracer/root-cert.pem + # sni: # example: tracer.somedomain + # subjectAltNames: [] + # - tracer.somedomain diff --git a/kubernetes/cray-istio/files/modify_runtime.sh b/kubernetes/cray-istio/files/modify_runtime.sh deleted file mode 100644 index 74ffbaf..0000000 --- a/kubernetes/cray-istio/files/modify_runtime.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh -# -# Copyright 2021 Hewlett Packard Enterprise Development LP -# -# See comment in README.md that describes why we need to alter -# this runtime envoy setting in istio 1.5.4 and 1.6.13 (at -# a minimum). -# -lines=$(kubectl get pods -A -o=jsonpath='{range .items[*]}{"\n"}{.metadata.name}{"."}{.metadata.namespace}{"."}{.status.phase}{"."}{range .spec.containers[*]}{.name}{"."}{end}{end}' | grep Running | grep istio-proxy) -echo "Ensuring istio-proxy container in the following pods have the setting 'reject_unsupported_transfer_encodings' set to 'false':" -echo "" -for line in $lines -do - pod=$(echo $line | awk 'BEGIN { FS = "." } ; {print $1}') - ns=$(echo $line | awk 'BEGIN { FS = "." } ; {print $2}') - output=$(kubectl -n $ns exec -i $pod -c istio-proxy -- /bin/sh -c 'curl -s -X POST http://localhost:15000/runtime_modify?envoy.reloadable_features.reject_unsupported_transfer_encodings=false' 2>&1) - echo " $pod ($ns): $output" -done diff --git a/kubernetes/cray-istio/templates/proxy-mem-leak-workaround.yaml b/kubernetes/cray-istio/templates/proxy-mem-leak-workaround.yaml deleted file mode 100644 index ff12ae6..0000000 --- a/kubernetes/cray-istio/templates/proxy-mem-leak-workaround.yaml +++ /dev/null @@ -1,55 +0,0 @@ -{{- /* -Copyright 2021 Hewlett Packard Enterprise Development LP -*/ -}} ---- -kind: ServiceAccount -apiVersion: v1 -metadata: - name: proxy-mem-leak-workaround - namespace: {{ .Release.Namespace }} ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: proxy-mem-leak-workaround -rules: - - apiGroups: ["networking.istio.io"] - resources: [envoyfilters] - verbs: [delete] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: proxy-mem-leak-workaround -subjects: - - kind: ServiceAccount - name: proxy-mem-leak-workaround - namespace: {{ .Release.Namespace }} -roleRef: - kind: ClusterRole - name: proxy-mem-leak-workaround - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: batch/v1beta1 -kind: CronJob -metadata: - name: proxy-mem-leak-workaround - namespace: {{ .Release.Namespace }} -spec: - successfulJobsHistoryLimit: 1 - failedJobsHistoryLimit: 1 - schedule: "0/10 * * * *" - jobTemplate: - spec: - backoffLimit: 1 - template: - spec: - restartPolicy: Never - serviceAccountName: "proxy-mem-leak-workaround" - containers: - - name: proxy-mem-leak-workaround - # - # Using common image that can do kubectl stuff - # - image: {{ include "cray-istio.image-prefix" . }}loftsman/docker-kubectl:0.2.0 - command: [ '/bin/sh', '-c', 'kubectl delete envoyfilter -n istio-system tcp-stats-filter-1.6 || true' ] diff --git a/kubernetes/cray-istio/templates/te-bug-workaround.yaml b/kubernetes/cray-istio/templates/te-bug-workaround.yaml deleted file mode 100644 index cb9adf4..0000000 --- a/kubernetes/cray-istio/templates/te-bug-workaround.yaml +++ /dev/null @@ -1,78 +0,0 @@ -{{- /* -Copyright 2021 Hewlett Packard Enterprise Development LP -*/ -}} ---- -kind: ServiceAccount -apiVersion: v1 -metadata: - name: istio-te-bug-workaround - namespace: {{ .Release.Namespace }} ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: istio-te-bug-workaround -rules: - - apiGroups: [""] - resources: [pods] - verbs: [list, get] - - apiGroups: [""] - resources: [pods/exec] - verbs: [create] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: istio-te-bug-workaround -subjects: - - kind: ServiceAccount - name: istio-te-bug-workaround - namespace: {{ .Release.Namespace }} -roleRef: - kind: ClusterRole - name: istio-te-bug-workaround - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: istio-te-bug-workaround -data: - modify_runtime.sh: |- - {{- .Files.Get "files/modify_runtime.sh" | nindent 4 }} ---- -apiVersion: batch/v1beta1 -kind: CronJob -metadata: - name: istio-te-bug-workaround - namespace: {{ .Release.Namespace }} -spec: - successfulJobsHistoryLimit: 1 - failedJobsHistoryLimit: 1 - schedule: "0/10 * * * *" - jobTemplate: - spec: - backoffLimit: 1 - template: - spec: - restartPolicy: Never - serviceAccountName: "istio-te-bug-workaround" - containers: - - name: istio-te-bug-workaround - # - # Using common image that can do kubectl stuff - # - image: {{ include "cray-istio.image-prefix" . }}loftsman/docker-kubectl:0.2.0 - command: - - /bin/sh - - -c - - "/scripts/modify_runtime.sh" - volumeMounts: - - name: istio-te-bug-workaround - mountPath: /scripts - volumes: - - name: istio-te-bug-workaround - configMap: - name: istio-te-bug-workaround - defaultMode: 0744 - diff --git a/kubernetes/cray-istio/values.yaml b/kubernetes/cray-istio/values.yaml index 1923228..928bf9e 100644 --- a/kubernetes/cray-istio/values.yaml +++ b/kubernetes/cray-istio/values.yaml @@ -105,7 +105,7 @@ global: hub: dtr.dev.cray.com/cray # Default tag for Istio images. - tag: 1.6.13-cray1 + tag: 1.7.8-cray1 # Comma-separated minimum per-scope logging level of messages to output, in the form of :,: # The control plane has different scopes depending on component, but can configure default log level across all components @@ -671,6 +671,7 @@ istio: gateways: istio-ingressgateway: + runAsRoot: true # This is necessary because the ports below are 80 and 443. enabled: true type: LoadBalancer autoscaleEnabled: true