From 19461fa86cdc1a91d8b95c819f5e0bfd92bb4e22 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Tue, 30 Aug 2022 13:01:03 -0400 Subject: [PATCH 01/25] Split web and task containers into separate deployments --- roles/common/templates/labels/common.yaml.j2 | 1 - .../tasks/resources_configuration.yml | 9 +- .../{deployment.yaml.j2 => task.yaml.j2} | 5 +- .../templates/deployments/web.yaml.j2 | 288 ++++++++++++++++++ .../templates/networking/service.yaml.j2 | 4 +- 5 files changed, 298 insertions(+), 9 deletions(-) rename roles/installer/templates/deployments/{deployment.yaml.j2 => task.yaml.j2} (99%) create mode 100644 roles/installer/templates/deployments/web.yaml.j2 diff --git a/roles/common/templates/labels/common.yaml.j2 b/roles/common/templates/labels/common.yaml.j2 index 4e133e876..7287ec9d9 100644 --- a/roles/common/templates/labels/common.yaml.j2 +++ b/roles/common/templates/labels/common.yaml.j2 @@ -1,5 +1,4 @@ # https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/ -app.kubernetes.io/name: '{{ ansible_operator_meta.name }}' app.kubernetes.io/part-of: '{{ ansible_operator_meta.name }}' app.kubernetes.io/managed-by: '{{ deployment_type }}-operator' app.kubernetes.io/component: '{{ deployment_type }}' diff --git a/roles/installer/tasks/resources_configuration.yml b/roles/installer/tasks/resources_configuration.yml index f42864435..33df18f5f 100644 --- a/roles/installer/tasks/resources_configuration.yml +++ b/roles/installer/tasks/resources_configuration.yml @@ -249,9 +249,12 @@ - name: Apply deployment resources k8s: apply: yes - definition: "{{ lookup('template', 'deployments/deployment.yaml.j2') }}" + definition: "{{ lookup('template', 'deployments/{{ item }}.yaml.j2') }}" wait: yes - wait_timeout: "{{ 120 * replicas or 120 }}" + wait_timeout: "{{ (120 * replicas) or 120 }}" + loop: + - web + - task register: this_deployment_result - block: @@ -260,7 +263,7 @@ kind: Pod namespace: '{{ ansible_operator_meta.namespace }}' label_selectors: - - "app.kubernetes.io/name={{ ansible_operator_meta.name }}" + - "app.kubernetes.io/name={{ ansible_operator_meta.name }}-task" - "app.kubernetes.io/managed-by={{ deployment_type }}-operator" - "app.kubernetes.io/component={{ deployment_type }}" field_selectors: diff --git a/roles/installer/templates/deployments/deployment.yaml.j2 b/roles/installer/templates/deployments/task.yaml.j2 similarity index 99% rename from roles/installer/templates/deployments/deployment.yaml.j2 rename to roles/installer/templates/deployments/task.yaml.j2 index fe02e7f13..5d28a3e59 100644 --- a/roles/installer/templates/deployments/deployment.yaml.j2 +++ b/roles/installer/templates/deployments/task.yaml.j2 @@ -1,9 +1,8 @@ -# AWX Deployment. --- apiVersion: apps/v1 kind: Deployment metadata: - name: '{{ ansible_operator_meta.name }}' + name: '{{ ansible_operator_meta.name }}-task' namespace: '{{ ansible_operator_meta.namespace }}' labels: {{ lookup("template", "../common/templates/labels/common.yaml.j2") | indent(width=4) | trim }} @@ -12,7 +11,7 @@ spec: replicas: {{ replicas }} selector: matchLabels: - app.kubernetes.io/name: '{{ ansible_operator_meta.name }}' + app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-task' app.kubernetes.io/managed-by: '{{ deployment_type }}-operator' app.kubernetes.io/component: '{{ deployment_type }}' template: diff --git a/roles/installer/templates/deployments/web.yaml.j2 b/roles/installer/templates/deployments/web.yaml.j2 new file mode 100644 index 000000000..a868e27ee --- /dev/null +++ b/roles/installer/templates/deployments/web.yaml.j2 @@ -0,0 +1,288 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: '{{ ansible_operator_meta.name }}-web' + namespace: '{{ ansible_operator_meta.namespace }}' + labels: + app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-web' + {{ lookup("template", "labels/common.yaml.j2") | indent(width=4) | trim }} + {{ lookup("template", "labels/version.yaml.j2") | indent(width=4) | trim }} +spec: + replicas: {{ replicas }} + selector: + matchLabels: + app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-web' + app.kubernetes.io/managed-by: '{{ deployment_type }}-operator' + app.kubernetes.io/component: '{{ deployment_type }}' + template: + metadata: + labels: + app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-web' + {{ lookup("template", "labels/common.yaml.j2") | indent(width=8) | trim }} + {{ lookup("template", "labels/version.yaml.j2") | indent(width=8) | trim }} +{% if annotations %} + annotations: + {{ annotations | indent(width=8) }} +{% endif %} + spec: + serviceAccountName: '{{ ansible_operator_meta.name }}' +{% if image_pull_secret is defined %} + imagePullSecrets: + - name: {{ image_pull_secret }} +{% elif image_pull_secrets | length > 0 %} + imagePullSecrets: +{% for secret in image_pull_secrets %} + - name: {{ secret }} +{% endfor %} +{% endif %} +{% if control_plane_priority_class is defined %} + priorityClassName: '{{ control_plane_priority_class }}' +{% endif %} + initContainers: +{% if bundle_ca_crt or projects_persistence|bool or init_container_extra_commands %} + - name: init + image: '{{ _init_container_image }}' + imagePullPolicy: '{{ image_pull_policy }}' + command: + - /bin/sh + - -c + - | +{% if bundle_ca_crt %} + mkdir -p /etc/pki/ca-trust/extracted/{java,pem,openssl,edk2} + update-ca-trust +{% endif %} +{% if projects_persistence|bool %} + chmod 775 /var/lib/awx/projects + chgrp 1000 /var/lib/awx/projects +{% endif %} +{% if init_container_extra_commands %} + {{ init_container_extra_commands | indent(width=14) }} +{% endif %} + volumeMounts: +{% if bundle_ca_crt %} + - name: "ca-trust-extracted" + mountPath: "/etc/pki/ca-trust/extracted" + - name: "{{ ansible_operator_meta.name }}-bundle-cacert" + mountPath: /etc/pki/ca-trust/source/anchors/bundle-ca.crt + subPath: bundle-ca.crt + readOnly: true +{% endif %} +{% if projects_persistence|bool %} + - name: "{{ ansible_operator_meta.name }}-projects" + mountPath: "/var/lib/awx/projects" +{% endif %} +{% if init_container_extra_volume_mounts -%} + {{ init_container_extra_volume_mounts | indent(width=12, first=True) }} +{% endif %} +{% endif %} + containers: + - image: '{{ _redis_image }}' + imagePullPolicy: '{{ image_pull_policy }}' + name: redis +{% if redis_capabilities is defined and redis_capabilities %} + securityContext: + capabilities: + add: {{ redis_capabilities }} +{% endif %} + args: ["redis-server", "/etc/redis.conf"] + volumeMounts: + - name: {{ ansible_operator_meta.name }}-redis-config + mountPath: "/etc/redis.conf" + subPath: redis.conf + readOnly: true + - name: {{ ansible_operator_meta.name }}-redis-socket + mountPath: "/var/run/redis" + - name: "{{ ansible_operator_meta.name }}-redis-data" + mountPath: "/data" + resources: {{ redis_resource_requirements }} + - image: '{{ _image }}' + name: '{{ ansible_operator_meta.name }}-web' +{% if web_command %} + command: {{ web_command }} +{% endif %} +{% if web_args %} + args: {{ web_args }} +{% endif %} + imagePullPolicy: '{{ image_pull_policy }}' + ports: + - containerPort: 8052 +{% if ingress_type | lower == 'route' and route_tls_termination_mechanism | lower == 'passthrough' %} + - containerPort: 8053 +{% endif %} + volumeMounts: +{% if bundle_ca_crt %} + - name: "ca-trust-extracted" + mountPath: "/etc/pki/ca-trust/extracted" + - name: "{{ ansible_operator_meta.name }}-bundle-cacert" + mountPath: /etc/pki/ca-trust/source/anchors/bundle-ca.crt + subPath: bundle-ca.crt + readOnly: true +{% endif %} + - name: "{{ ansible_operator_meta.name }}-application-credentials" + mountPath: "/etc/tower/conf.d/execution_environments.py" + subPath: execution_environments.py + readOnly: true + - name: "{{ ansible_operator_meta.name }}-application-credentials" + mountPath: "/etc/tower/conf.d/credentials.py" + subPath: credentials.py + readOnly: true + - name: "{{ ansible_operator_meta.name }}-application-credentials" + mountPath: "/etc/tower/conf.d/ldap.py" + subPath: ldap.py + readOnly: true +{% if ingress_type | lower == 'route' and route_tls_termination_mechanism | lower == 'passthrough' %} + - name: "{{ ansible_operator_meta.name }}-nginx-certs" + mountPath: "/etc/nginx/pki" + readOnly: true +{% endif %} +{% if ldap_cacert_ca_crt %} + - name: "{{ ansible_operator_meta.name }}-ldap-cacert" + mountPath: /etc/openldap/certs/ldap-ca.crt + subPath: ldap-ca.crt + readOnly: true +{% endif %} + - name: "{{ secret_key_secret_name }}" + mountPath: /etc/tower/SECRET_KEY + subPath: SECRET_KEY + readOnly: true + - name: {{ ansible_operator_meta.name }}-settings + mountPath: /etc/tower/settings.py + subPath: settings.py + readOnly: true + - name: {{ ansible_operator_meta.name }}-nginx-conf + mountPath: /etc/nginx/nginx.conf + subPath: nginx.conf + readOnly: true + - name: {{ ansible_operator_meta.name }}-redis-socket + mountPath: "/var/run/redis" + - name: supervisor-socket + mountPath: "/var/run/supervisor" + - name: rsyslog-socket + mountPath: "/var/run/awx-rsyslog" + - name: rsyslog-dir + mountPath: "/var/lib/awx/rsyslog" + - name: "{{ ansible_operator_meta.name }}-projects" + mountPath: "/var/lib/awx/projects" +{% if development_mode | bool %} + - name: awx-devel + mountPath: "/awx_devel" +{% endif %} +{% if web_extra_volume_mounts -%} + {{ web_extra_volume_mounts | indent(width=12, first=True) }} +{% endif %} + env: + - name: MY_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: UWSGI_MOUNT_PATH + value: "{{ ingress_path }}" +{% if development_mode | bool %} + - name: AWX_KUBE_DEVEL + value: "1" +{% endif %} +{% if web_extra_env -%} + {{ web_extra_env | indent(width=12, first=True) }} +{% endif %} + resources: {{ web_resource_requirements }} + volumes: +{% if bundle_ca_crt %} + - name: "ca-trust-extracted" + emptyDir: {} + - name: "{{ ansible_operator_meta.name }}-bundle-cacert" + secret: + secretName: "{{ bundle_cacert_secret }}" + items: + - key: bundle-ca.crt + path: 'bundle-ca.crt' +{% endif %} +{% if ingress_type | lower == 'route' and route_tls_termination_mechanism | lower == 'passthrough' %} + - name: "{{ ansible_operator_meta.name }}-nginx-certs" + secret: + secretName: "{{ route_tls_secret }}" + items: + - key: tls.key + path: 'web.key' + - key: tls.crt + path: 'web.crt' +{% endif %} +{% if ldap_cacert_ca_crt %} + - name: "{{ ansible_operator_meta.name }}-ldap-cacert" + secret: + secretName: "{{ ldap_cacert_secret }}" + items: + - key: ldap-ca.crt + path: 'ldap-ca.crt' +{% endif %} + - name: "{{ ansible_operator_meta.name }}-application-credentials" + secret: + secretName: "{{ ansible_operator_meta.name }}-app-credentials" + items: + - key: credentials.py + path: 'credentials.py' + - key: ldap.py + path: 'ldap.py' + - key: execution_environments.py + path: 'execution_environments.py' + - name: "{{ secret_key_secret_name }}" + secret: + secretName: '{{ secret_key_secret_name }}' + items: + - key: secret_key + path: SECRET_KEY + - name: {{ ansible_operator_meta.name }}-settings + configMap: + name: '{{ ansible_operator_meta.name }}-{{ deployment_type }}-configmap' + items: + - key: settings + path: settings.py + - name: {{ ansible_operator_meta.name }}-nginx-conf + configMap: + name: '{{ ansible_operator_meta.name }}-{{ deployment_type }}-configmap' + items: + - key: nginx_conf + path: nginx.conf + - name: {{ ansible_operator_meta.name }}-redis-config + configMap: + name: {{ ansible_operator_meta.name }}-{{ deployment_type }}-configmap + items: + - key: redis_conf + path: redis.conf + - name: {{ ansible_operator_meta.name }}-redis-socket + emptyDir: {} + - name: {{ ansible_operator_meta.name }}-redis-data + emptyDir: {} + - name: supervisor-socket + emptyDir: {} + - name: rsyslog-socket + emptyDir: {} + - name: receptor-socket + emptyDir: {} + - name: rsyslog-dir + emptyDir: {} + - name: {{ ansible_operator_meta.name }}-receptor-config + configMap: + name: '{{ ansible_operator_meta.name }}-{{ deployment_type }}-configmap' + items: + - key: receptor_conf + path: receptor.conf + - name: "{{ ansible_operator_meta.name }}-projects" +{% if projects_persistence|bool %} + persistentVolumeClaim: +{% if projects_existing_claim %} + claimName: {{ projects_existing_claim }} +{% else %} + claimName: '{{ ansible_operator_meta.name }}-projects-claim' +{% endif %} +{% else %} + emptyDir: {} +{% endif %} +{% if development_mode | bool %} + - name: awx-devel + hostPath: + path: /awx_devel +{% endif %} +{% if extra_volumes -%} + {{ extra_volumes | indent(width=8, first=True) }} +{% endif %} diff --git a/roles/installer/templates/networking/service.yaml.j2 b/roles/installer/templates/networking/service.yaml.j2 index a006d01ee..ab09bca9d 100644 --- a/roles/installer/templates/networking/service.yaml.j2 +++ b/roles/installer/templates/networking/service.yaml.j2 @@ -13,7 +13,7 @@ metadata: {% endif %} spec: ports: - + {% if service_type | lower == "nodeport" %} - port: 80 protocol: TCP @@ -46,7 +46,7 @@ spec: name: http {% endif %} selector: - app.kubernetes.io/name: '{{ ansible_operator_meta.name }}' + app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-web' app.kubernetes.io/managed-by: '{{ deployment_type }}-operator' app.kubernetes.io/component: '{{ deployment_type }}' {% if service_type | lower == "nodeport" %} From 64774687296ff539ac34158b0d875ae6bd80263e Mon Sep 17 00:00:00 2001 From: Rick Elrod Date: Wed, 30 Nov 2022 00:35:17 -0600 Subject: [PATCH 02/25] drop awx-web container from awx-task deployment Signed-off-by: Rick Elrod --- .../templates/deployments/task.yaml.j2 | 102 ------------------ 1 file changed, 102 deletions(-) diff --git a/roles/installer/templates/deployments/task.yaml.j2 b/roles/installer/templates/deployments/task.yaml.j2 index 5d28a3e59..6af21d775 100644 --- a/roles/installer/templates/deployments/task.yaml.j2 +++ b/roles/installer/templates/deployments/task.yaml.j2 @@ -157,108 +157,6 @@ spec: /var/lib/pre-stop/scripts/termination-waiter {% endif %} resources: {{ redis_resource_requirements }} - - image: '{{ _image }}' - name: '{{ ansible_operator_meta.name }}-web' -{% if web_command %} - command: {{ web_command }} -{% endif %} -{% if web_args %} - args: {{ web_args }} -{% endif %} - imagePullPolicy: '{{ image_pull_policy }}' - ports: - - containerPort: 8052 -{% if ingress_type | lower == 'route' and route_tls_termination_mechanism | lower == 'passthrough' %} - - containerPort: 8053 -{% endif %} - volumeMounts: -{% if bundle_ca_crt %} - - name: "ca-trust-extracted" - mountPath: "/etc/pki/ca-trust/extracted" - - name: "{{ ansible_operator_meta.name }}-bundle-cacert" - mountPath: /etc/pki/ca-trust/source/anchors/bundle-ca.crt - subPath: bundle-ca.crt - readOnly: true -{% endif %} - - name: "{{ ansible_operator_meta.name }}-application-credentials" - mountPath: "/etc/tower/conf.d/execution_environments.py" - subPath: execution_environments.py - readOnly: true - - name: "{{ ansible_operator_meta.name }}-application-credentials" - mountPath: "/etc/tower/conf.d/credentials.py" - subPath: credentials.py - readOnly: true - - name: "{{ ansible_operator_meta.name }}-application-credentials" - mountPath: "/etc/tower/conf.d/ldap.py" - subPath: ldap.py - readOnly: true -{% if ingress_type | lower == 'route' and route_tls_termination_mechanism | lower == 'passthrough' %} - - name: "{{ ansible_operator_meta.name }}-nginx-certs" - mountPath: "/etc/nginx/pki" - readOnly: true -{% endif %} -{% if ldap_cacert_ca_crt %} - - name: "{{ ansible_operator_meta.name }}-ldap-cacert" - mountPath: /etc/openldap/certs/ldap-ca.crt - subPath: ldap-ca.crt - readOnly: true -{% endif %} - - name: "{{ secret_key_secret_name }}" - mountPath: /etc/tower/SECRET_KEY - subPath: SECRET_KEY - readOnly: true - - name: {{ ansible_operator_meta.name }}-settings - mountPath: /etc/tower/settings.py - subPath: settings.py - readOnly: true - - name: {{ ansible_operator_meta.name }}-nginx-conf - mountPath: /etc/nginx/nginx.conf - subPath: nginx.conf - readOnly: true - - name: {{ ansible_operator_meta.name }}-redis-socket - mountPath: "/var/run/redis" - - name: supervisor-socket - mountPath: "/var/run/supervisor" - - name: rsyslog-socket - mountPath: "/var/run/awx-rsyslog" - - name: rsyslog-dir - mountPath: "/var/lib/awx/rsyslog" - - name: "{{ ansible_operator_meta.name }}-projects" - mountPath: "/var/lib/awx/projects" - - name: "{{ ansible_operator_meta.name }}-receptor-work-signing" - mountPath: "/etc/receptor/signing/work-public-key.pem" - subPath: "work-public-key.pem" - readOnly: true - - name: "{{ ansible_operator_meta.name }}-receptor-ca" - mountPath: "/etc/receptor/tls/ca/receptor-ca.crt" - subPath: "tls.crt" - readOnly: true - - name: "{{ ansible_operator_meta.name }}-receptor-ca" - mountPath: "/etc/receptor/tls/ca/receptor-ca.key" - subPath: "tls.key" - readOnly: true -{% if development_mode | bool %} - - name: awx-devel - mountPath: "/awx_devel" -{% endif %} -{% if web_extra_volume_mounts -%} - {{ web_extra_volume_mounts | indent(width=12, first=True) }} -{% endif %} - env: - - name: MY_POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: UWSGI_MOUNT_PATH - value: "{{ ingress_path }}" -{% if development_mode | bool %} - - name: AWX_KUBE_DEVEL - value: "1" -{% endif %} -{% if web_extra_env -%} - {{ web_extra_env | indent(width=12, first=True) }} -{% endif %} - resources: {{ web_resource_requirements }} - image: '{{ _image }}' name: '{{ ansible_operator_meta.name }}-task' imagePullPolicy: '{{ image_pull_policy }}' From 12e51ad9ade3c66f2831c3a6fa18c552ece5aa56 Mon Sep 17 00:00:00 2001 From: Rick Elrod Date: Wed, 30 Nov 2022 03:37:13 -0600 Subject: [PATCH 03/25] add $MY_POD_IP back to web deployment Signed-off-by: Rick Elrod --- roles/installer/templates/deployments/web.yaml.j2 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/roles/installer/templates/deployments/web.yaml.j2 b/roles/installer/templates/deployments/web.yaml.j2 index a868e27ee..48da18fcf 100644 --- a/roles/installer/templates/deployments/web.yaml.j2 +++ b/roles/installer/templates/deployments/web.yaml.j2 @@ -176,6 +176,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP - name: UWSGI_MOUNT_PATH value: "{{ ingress_path }}" {% if development_mode | bool %} From ebc040fe75c314732cb868676d4b4f3e2b7e020d Mon Sep 17 00:00:00 2001 From: Rick Elrod Date: Sat, 10 Dec 2022 22:06:25 -0600 Subject: [PATCH 04/25] scale web/task replicas separately This is a breaking change since 'replicas' is no longer used. Signed-off-by: Rick Elrod --- config/crd/bases/awx.ansible.com_awxs.yaml | 9 +++++++-- roles/installer/defaults/main.yml | 3 ++- roles/installer/templates/deployments/task.yaml.j2 | 2 +- roles/installer/templates/deployments/web.yaml.j2 | 2 +- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/config/crd/bases/awx.ansible.com_awxs.yaml b/config/crd/bases/awx.ansible.com_awxs.yaml index 9e30b9cb6..1105eebb1 100644 --- a/config/crd/bases/awx.ansible.com_awxs.yaml +++ b/config/crd/bases/awx.ansible.com_awxs.yaml @@ -699,8 +699,13 @@ spec: service_account_annotations: description: ServiceAccount annotations type: string - replicas: - description: Number of instance replicas + web_replicas: + description: Number of web instance replicas + type: integer + default: 1 + format: int32 + task_replicas: + description: Number of task instance replicas type: integer default: 1 format: int32 diff --git a/roles/installer/defaults/main.yml b/roles/installer/defaults/main.yml index 85676ee9f..f72fa7f5d 100644 --- a/roles/installer/defaults/main.yml +++ b/roles/installer/defaults/main.yml @@ -172,7 +172,8 @@ _init_projects_container_image: quay.io/centos/centos:stream9 create_preload_data: true -replicas: "1" +web_replicas: "1" +task_replicas: "1" task_args: - /usr/bin/launch_awx_task.sh diff --git a/roles/installer/templates/deployments/task.yaml.j2 b/roles/installer/templates/deployments/task.yaml.j2 index 6af21d775..44cc0d59a 100644 --- a/roles/installer/templates/deployments/task.yaml.j2 +++ b/roles/installer/templates/deployments/task.yaml.j2 @@ -8,7 +8,7 @@ metadata: {{ lookup("template", "../common/templates/labels/common.yaml.j2") | indent(width=4) | trim }} {{ lookup("template", "../common/templates/labels/version.yaml.j2") | indent(width=4) | trim }} spec: - replicas: {{ replicas }} + replicas: {{ task_replicas }} selector: matchLabels: app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-task' diff --git a/roles/installer/templates/deployments/web.yaml.j2 b/roles/installer/templates/deployments/web.yaml.j2 index 48da18fcf..4eb829227 100644 --- a/roles/installer/templates/deployments/web.yaml.j2 +++ b/roles/installer/templates/deployments/web.yaml.j2 @@ -9,7 +9,7 @@ metadata: {{ lookup("template", "labels/common.yaml.j2") | indent(width=4) | trim }} {{ lookup("template", "labels/version.yaml.j2") | indent(width=4) | trim }} spec: - replicas: {{ replicas }} + replicas: {{ web_replicas }} selector: matchLabels: app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-web' From d9f3a428d416759f8186112ae52e53dc2cfe48d8 Mon Sep 17 00:00:00 2001 From: Jake Jackson Date: Mon, 6 Feb 2023 16:08:35 -0500 Subject: [PATCH 05/25] [web/task split] split web and task deployment + a few supporting bits (#1189) * first pass, still WIP, need tolerations etc * add tolerations that don't work bc idk * bug hunting * local push, still a WIP * affinity still needs testfor to_nice_yaml, tolerations logic is working * fixed task deployment and affinity for both --- config/crd/bases/awx.ansible.com_awxs.yaml | 1104 +++++++++++++++++ roles/installer/defaults/main.yml | 61 + .../tasks/resources_configuration.yml | 2 +- .../templates/deployments/task.yaml.j2 | 61 +- .../templates/deployments/web.yaml.j2 | 59 +- 5 files changed, 1265 insertions(+), 22 deletions(-) diff --git a/config/crd/bases/awx.ansible.com_awxs.yaml b/config/crd/bases/awx.ansible.com_awxs.yaml index 1105eebb1..53e3a5eb0 100644 --- a/config/crd/bases/awx.ansible.com_awxs.yaml +++ b/config/crd/bases/awx.ansible.com_awxs.yaml @@ -542,6 +542,1110 @@ spec: tolerations: description: node tolerations for the pods type: string + task_tolerations: + description: node tolerations for the task pods + type: string + web_tolerations: + description: node tolerations for the web pods + type: string + affinity: + description: If specified, the pod's scheduling constraints + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + type: array + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + web_affinity: + description: If specified, the pod's scheduling constraints + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + type: array + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + task_affinity: + description: If specified, the pod's scheduling constraints + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + type: array + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object image: description: Registry path to the application container to use type: string diff --git a/roles/installer/defaults/main.yml b/roles/installer/defaults/main.yml index f72fa7f5d..12107f25d 100644 --- a/roles/installer/defaults/main.yml +++ b/roles/installer/defaults/main.yml @@ -93,6 +93,58 @@ affinity: {} # effect: "NoSchedule" tolerations: '' +# Add node tolerations for the task pods. Specify as literal block. E.g.: +# task_tolerations: | +# - key: "dedicated" +# operator: "Equal" +# value: "AWXtask" +# effect: "NoSchedule" +task_tolerations: '' + +# Add node tolerations for the web pods. Specify as literal block. E.g.: +# web_tolerations: | +# - key: "dedicated" +# operator: "Equal" +# value: "AWXweb" +# effect: "NoSchedule" +web_tolerations: '' + +# Add affinities for all pods +# affinity: +# nodeAffinity: +# requiredDuringSchedulingIgnoredDuringExecution: +# nodeSelectorTerms: +# - matchExpressions: +# - key: app.kubernetes.io/component +# operator: In +# values: +# - awx +affinity: {} + +# Add affinities for all task pods +# affinity: +# nodeAffinity: +# requiredDuringSchedulingIgnoredDuringExecution: +# nodeSelectorTerms: +# - matchExpressions: +# - key: app.kubernetes.io/name +# operator: In +# values: +# - awx-task +task_affinity: {} + +# Add affinities for all web pods +# affinity: +# nodeAffinity: +# requiredDuringSchedulingIgnoredDuringExecution: +# nodeSelectorTerms: +# - matchExpressions: +# - key: app.kubernetes.io/name +# operator: In +# values: +# - awx-web +web_affinity: {} + # Add annotations to awx pods. Specify as literal block. E.g.: # annotations: | # my.annotation/1: value @@ -181,6 +233,9 @@ task_command: [] web_args: - /usr/bin/launch_awx.sh web_command: [] +ryslog_args: + - /usr/bin/launch_awx_rsyslog.sh +rsyslog_command: [] task_resource_requirements: requests: @@ -197,6 +252,12 @@ ee_resource_requirements: cpu: 100m memory: 64Mi +# TODO: validate default resource requirements +rsyslog_resource_requirements: + requests: + cpu: 100m + memory: 128Mi + # Customize CSRF options csrf_cookie_secure: False session_cookie_secure: False diff --git a/roles/installer/tasks/resources_configuration.yml b/roles/installer/tasks/resources_configuration.yml index 33df18f5f..00349dc28 100644 --- a/roles/installer/tasks/resources_configuration.yml +++ b/roles/installer/tasks/resources_configuration.yml @@ -6,7 +6,7 @@ kind: Pod namespace: '{{ ansible_operator_meta.namespace }}' label_selectors: - - "app.kubernetes.io/name={{ ansible_operator_meta.name }}" + - "app.kubernetes.io/name={{ ansible_operator_meta.name }}-task" - "app.kubernetes.io/managed-by={{ deployment_type }}-operator" - "app.kubernetes.io/component={{ deployment_type }}" field_selectors: diff --git a/roles/installer/templates/deployments/task.yaml.j2 b/roles/installer/templates/deployments/task.yaml.j2 index 44cc0d59a..a17c74804 100644 --- a/roles/installer/templates/deployments/task.yaml.j2 +++ b/roles/installer/templates/deployments/task.yaml.j2 @@ -201,12 +201,8 @@ spec: readOnly: true - name: {{ ansible_operator_meta.name }}-redis-socket mountPath: "/var/run/redis" - - name: supervisor-socket - mountPath: "/var/run/supervisor" - name: rsyslog-socket mountPath: "/var/run/awx-rsyslog" - - name: rsyslog-dir - mountPath: "/var/lib/awx/rsyslog" - name: "{{ ansible_operator_meta.name }}-receptor-config" mountPath: "/etc/receptor/" - name: "{{ ansible_operator_meta.name }}-receptor-work-signing" @@ -326,6 +322,43 @@ spec: {% endif %} {% if ee_extra_env -%} {{ ee_extra_env | indent(width=12, first=True) }} +{% endif %} + - image: '{{ _image }}' + name: '{{ ansible_operator_meta.name }}-rsyslog' +{% if rsyslog_command %} + command: {{ rsyslog_command }} +{% endif %} +{% if ryslog_args %} + args: {{ ryslog_args }} +{% endif %} + imagePullPolicy: '{{ image_pull_policy }}' + volumeMounts: + - name: "{{ ansible_operator_meta.name }}-application-credentials" + mountPath: "/etc/tower/conf.d/credentials.py" + subPath: credentials.py + readOnly: true + - name: "{{ secret_key_secret_name }}" + mountPath: /etc/tower/SECRET_KEY + subPath: SECRET_KEY + readOnly: true + - name: {{ ansible_operator_meta.name }}-settings + mountPath: "/etc/tower/settings.py" + subPath: settings.py + readOnly: true + - name: {{ ansible_operator_meta.name }}-redis-socket + mountPath: "/var/run/redis" + - name: rsyslog-socket + mountPath: "/var/run/awx-rsyslog" +{% if development_mode | bool %} + - name: awx-devel + mountPath: "/awx_devel" +{% endif %} + env: + - name: SUPERVISOR_WEB_CONFIG_PATH + value: "/etc/supervisor_rsyslog.conf" +{% if development_mode | bool %} + - name: AWX_KUBE_DEVEL + value: "1" {% endif %} {% if node_selector %} nodeSelector: @@ -335,14 +368,20 @@ spec: topologySpreadConstraints: {{ topology_spread_constraints | indent(width=8) }} {% endif %} -{% if affinity | length %} +{% if task_tolerations %} + tolerations: + {{ task_tolerations | to_nice_yaml | indent(width=8) }} +{% elif tolerations %} + tolerations: + {{ tolerations | to_nice_yaml | indent(width=8) }} +{% endif %} +{% if task_affinity %} + affinity: + {{ task_affinity | to_nice_yaml | indent(width=8) }} +{% elif affinity %} affinity: {{ affinity | to_nice_yaml | indent(width=8) }} {% endif %} -{% if tolerations %} - tolerations: - {{ tolerations | indent(width=8) }} -{% endif %} {% if (projects_persistence|bool and is_k8s|bool) or (security_context_settings|length) %} securityContext: {% if projects_persistence|bool and is_k8s|bool %} @@ -438,14 +477,10 @@ spec: emptyDir: {} - name: {{ ansible_operator_meta.name }}-redis-data emptyDir: {} - - name: supervisor-socket - emptyDir: {} - name: rsyslog-socket emptyDir: {} - name: receptor-socket emptyDir: {} - - name: rsyslog-dir - emptyDir: {} - name: {{ ansible_operator_meta.name }}-receptor-config emptyDir: {} - name: {{ ansible_operator_meta.name }}-default-receptor-config diff --git a/roles/installer/templates/deployments/web.yaml.j2 b/roles/installer/templates/deployments/web.yaml.j2 index 4eb829227..9ba84c891 100644 --- a/roles/installer/templates/deployments/web.yaml.j2 +++ b/roles/installer/templates/deployments/web.yaml.j2 @@ -156,12 +156,8 @@ spec: readOnly: true - name: {{ ansible_operator_meta.name }}-redis-socket mountPath: "/var/run/redis" - - name: supervisor-socket - mountPath: "/var/run/supervisor" - name: rsyslog-socket mountPath: "/var/run/awx-rsyslog" - - name: rsyslog-dir - mountPath: "/var/lib/awx/rsyslog" - name: "{{ ansible_operator_meta.name }}-projects" mountPath: "/var/lib/awx/projects" {% if development_mode | bool %} @@ -190,6 +186,57 @@ spec: {{ web_extra_env | indent(width=12, first=True) }} {% endif %} resources: {{ web_resource_requirements }} + - image: '{{ _image }}' + name: '{{ ansible_operator_meta.name }}-rsyslog' +{% if rsyslog_command %} + command: {{ rsyslog_command }} +{% endif %} +{% if ryslog_args %} + args: {{ ryslog_args }} +{% endif %} + imagePullPolicy: '{{ image_pull_policy }}' + volumeMounts: + - name: "{{ ansible_operator_meta.name }}-application-credentials" + mountPath: "/etc/tower/conf.d/credentials.py" + subPath: credentials.py + readOnly: true + - name: "{{ secret_key_secret_name }}" + mountPath: /etc/tower/SECRET_KEY + subPath: SECRET_KEY + readOnly: true + - name: {{ ansible_operator_meta.name }}-settings + mountPath: "/etc/tower/settings.py" + subPath: settings.py + readOnly: true + - name: {{ ansible_operator_meta.name }}-redis-socket + mountPath: "/var/run/redis" + - name: rsyslog-socket + mountPath: "/var/run/awx-rsyslog" +{% if development_mode | bool %} + - name: awx-devel + mountPath: "/awx_devel" +{% endif %} + env: + - name: SUPERVISOR_WEB_CONFIG_PATH + value: "/etc/supervisor_rsyslog.conf" +{% if development_mode | bool %} + - name: AWX_KUBE_DEVEL + value: "1" +{% endif %} +{% if web_tolerations %} + tolerations: + {{ web_tolerations| indent(width=8) }} +{% elif tolerations %} + tolerations: + {{ tolerations| indent(width=8) }} +{% endif %} +{% if web_affinity %} + affinity: + {{ web_affinity | to_nice_yaml | indent(width=8) }} +{% elif affinity %} + affinity: + {{ affinity | to_nice_yaml | indent(width=8) }} +{% endif %} volumes: {% if bundle_ca_crt %} - name: "ca-trust-extracted" @@ -257,14 +304,10 @@ spec: emptyDir: {} - name: {{ ansible_operator_meta.name }}-redis-data emptyDir: {} - - name: supervisor-socket - emptyDir: {} - name: rsyslog-socket emptyDir: {} - name: receptor-socket emptyDir: {} - - name: rsyslog-dir - emptyDir: {} - name: {{ ansible_operator_meta.name }}-receptor-config configMap: name: '{{ ansible_operator_meta.name }}-{{ deployment_type }}-configmap' From c1bbd85ea948f07c30f2207f5a7ca932b131bf54 Mon Sep 17 00:00:00 2001 From: Jake Jackson Date: Mon, 6 Feb 2023 16:44:17 -0500 Subject: [PATCH 06/25] add nodeSelector work (#1219) --- config/crd/bases/awx.ansible.com_awxs.yaml | 6 ++++++ roles/installer/defaults/main.yml | 16 ++++++++++++++++ .../installer/templates/deployments/task.yaml.j2 | 5 ++++- .../installer/templates/deployments/web.yaml.j2 | 7 +++++++ 4 files changed, 33 insertions(+), 1 deletion(-) diff --git a/config/crd/bases/awx.ansible.com_awxs.yaml b/config/crd/bases/awx.ansible.com_awxs.yaml index 53e3a5eb0..c44143e24 100644 --- a/config/crd/bases/awx.ansible.com_awxs.yaml +++ b/config/crd/bases/awx.ansible.com_awxs.yaml @@ -160,6 +160,12 @@ spec: node_selector: description: nodeSelector for the pods type: string + web_node_selector: + description: nodeSelector for the web pods + type: string + task_node_selector: + description: nodeSelector for the task pods + type: string topology_spread_constraints: description: topology rule(s) for the pods type: string diff --git a/roles/installer/defaults/main.yml b/roles/installer/defaults/main.yml index 12107f25d..f8620d71f 100644 --- a/roles/installer/defaults/main.yml +++ b/roles/installer/defaults/main.yml @@ -72,6 +72,22 @@ hostname: '' # kubernetes.io/os: linux node_selector: '' +# Add a nodeSelector for the AWX pods. It must match a node's labels for the pod +# to be scheduled on that node. Specify as literal block. E.g.: +# node_selector: | +# disktype: ssd +# kubernetes.io/arch: amd64 +# kubernetes.io/os: linux +web_node_selector: '' + +# Add a nodeSelector for the AWX pods. It must match a node's labels for the pod +# to be scheduled on that node. Specify as literal block. E.g.: +# node_selector: | +# disktype: ssd +# kubernetes.io/arch: amd64 +# kubernetes.io/os: linux +task_node_selector: '' + # Add a topologySpreadConstraints for the AWX pods. # Specify as literal block. E.g.: # topology_spread_constraints: | diff --git a/roles/installer/templates/deployments/task.yaml.j2 b/roles/installer/templates/deployments/task.yaml.j2 index a17c74804..fa5f96bcb 100644 --- a/roles/installer/templates/deployments/task.yaml.j2 +++ b/roles/installer/templates/deployments/task.yaml.j2 @@ -360,7 +360,10 @@ spec: - name: AWX_KUBE_DEVEL value: "1" {% endif %} -{% if node_selector %} +{% if task_node_selector %} + nodeSelector: + {{ task_node_selector | indent(width=8) }} +{% elif node_selector %} nodeSelector: {{ node_selector | indent(width=8) }} {% endif %} diff --git a/roles/installer/templates/deployments/web.yaml.j2 b/roles/installer/templates/deployments/web.yaml.j2 index 9ba84c891..2d05552b3 100644 --- a/roles/installer/templates/deployments/web.yaml.j2 +++ b/roles/installer/templates/deployments/web.yaml.j2 @@ -223,6 +223,13 @@ spec: - name: AWX_KUBE_DEVEL value: "1" {% endif %} +{% if web_node_selector %} + nodeSelector: + {{ web_node_selector | indent(width=8) }} +{% elif node_selector %} + nodeSelector: + {{ node_selector | indent(width=8) }} +{% endif %} {% if web_tolerations %} tolerations: {{ web_tolerations| indent(width=8) }} From 6192cab6535a84d4910e2d6ea06d92ea77d4e9c5 Mon Sep 17 00:00:00 2001 From: Jake Jackson Date: Tue, 7 Feb 2023 15:52:38 -0500 Subject: [PATCH 07/25] fix affinity duplicate (#1225) --- config/crd/bases/awx.ansible.com_awxs.yaml | 373 --------------------- roles/installer/defaults/main.yml | 2 - 2 files changed, 375 deletions(-) diff --git a/config/crd/bases/awx.ansible.com_awxs.yaml b/config/crd/bases/awx.ansible.com_awxs.yaml index c44143e24..8a30f9b02 100644 --- a/config/crd/bases/awx.ansible.com_awxs.yaml +++ b/config/crd/bases/awx.ansible.com_awxs.yaml @@ -169,379 +169,6 @@ spec: topology_spread_constraints: description: topology rule(s) for the pods type: string - affinity: - description: If specified, the pod's scheduling constraints - properties: - nodeAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - preference: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - x-kubernetes-map-type: atomic - weight: - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - properties: - nodeSelectorTerms: - items: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - x-kubernetes-map-type: atomic - type: array - required: - - nodeSelectorTerms - type: object - x-kubernetes-map-type: atomic - type: object - podAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - type: object - podAntiAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - type: object - type: object - termination_grace_period_seconds: - description: Optional duration in seconds pods needs to terminate gracefully - type: integer - format: int32 - service_labels: - description: Additional labels to apply to the service - type: string annotations: description: annotations for the pods type: string diff --git a/roles/installer/defaults/main.yml b/roles/installer/defaults/main.yml index f8620d71f..63f2765bb 100644 --- a/roles/installer/defaults/main.yml +++ b/roles/installer/defaults/main.yml @@ -99,8 +99,6 @@ task_node_selector: '' # app.kubernetes.io/name: "" topology_spread_constraints: '' -affinity: {} - # Add node tolerations for the AWX pods. Specify as literal block. E.g.: # tolerations: | # - key: "dedicated" From 918e4874221380074cb0c0e066804f1d271e3297 Mon Sep 17 00:00:00 2001 From: Hao Liu Date: Wed, 8 Feb 2023 12:56:41 -0500 Subject: [PATCH 08/25] Add web and task replicas to the CRD (#1227) add web_replicas and task_replicas Co-Authored-By: Jake Jackson <24478650+thedoubl3j@users.noreply.github.com> --- config/crd/bases/awx.ansible.com_awxs.yaml | 7 +++++-- roles/installer/defaults/main.yml | 5 +++-- roles/installer/templates/deployments/task.yaml.j2 | 5 +++++ roles/installer/templates/deployments/web.yaml.j2 | 12 ++++++++---- 4 files changed, 21 insertions(+), 8 deletions(-) diff --git a/config/crd/bases/awx.ansible.com_awxs.yaml b/config/crd/bases/awx.ansible.com_awxs.yaml index 8a30f9b02..97ca31966 100644 --- a/config/crd/bases/awx.ansible.com_awxs.yaml +++ b/config/crd/bases/awx.ansible.com_awxs.yaml @@ -1436,15 +1436,18 @@ spec: service_account_annotations: description: ServiceAccount annotations type: string + replicas: + description: Number of instance replicas + type: integer + default: 1 + format: int32 web_replicas: description: Number of web instance replicas type: integer - default: 1 format: int32 task_replicas: description: Number of task instance replicas type: integer - default: 1 format: int32 garbage_collect_secrets: description: Whether or not to remove secrets upon instance removal diff --git a/roles/installer/defaults/main.yml b/roles/installer/defaults/main.yml index 63f2765bb..e5a3ff12f 100644 --- a/roles/installer/defaults/main.yml +++ b/roles/installer/defaults/main.yml @@ -238,8 +238,9 @@ _init_projects_container_image: quay.io/centos/centos:stream9 create_preload_data: true -web_replicas: "1" -task_replicas: "1" +replicas: "1" +web_replicas: '' +task_replicas: '' task_args: - /usr/bin/launch_awx_task.sh diff --git a/roles/installer/templates/deployments/task.yaml.j2 b/roles/installer/templates/deployments/task.yaml.j2 index fa5f96bcb..d0be1739b 100644 --- a/roles/installer/templates/deployments/task.yaml.j2 +++ b/roles/installer/templates/deployments/task.yaml.j2 @@ -8,7 +8,11 @@ metadata: {{ lookup("template", "../common/templates/labels/common.yaml.j2") | indent(width=4) | trim }} {{ lookup("template", "../common/templates/labels/version.yaml.j2") | indent(width=4) | trim }} spec: +{% if task_replicas %} replicas: {{ task_replicas }} +{% elif replicas %} + replicas: {{ replicas }} +{% endif %} selector: matchLabels: app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-task' @@ -17,6 +21,7 @@ spec: template: metadata: labels: + app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-task' {{ lookup("template", "../common/templates/labels/common.yaml.j2") | indent(width=8) | trim }} {{ lookup("template", "../common/templates/labels/version.yaml.j2") | indent(width=8) | trim }} annotations: diff --git a/roles/installer/templates/deployments/web.yaml.j2 b/roles/installer/templates/deployments/web.yaml.j2 index 2d05552b3..855861cd0 100644 --- a/roles/installer/templates/deployments/web.yaml.j2 +++ b/roles/installer/templates/deployments/web.yaml.j2 @@ -6,10 +6,14 @@ metadata: namespace: '{{ ansible_operator_meta.namespace }}' labels: app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-web' - {{ lookup("template", "labels/common.yaml.j2") | indent(width=4) | trim }} - {{ lookup("template", "labels/version.yaml.j2") | indent(width=4) | trim }} + {{ lookup("template", "../common/templates/labels/common.yaml.j2") | indent(width=4) | trim }} + {{ lookup("template", "../common/templates/labels//version.yaml.j2") | indent(width=4) | trim }} spec: +{% if web_replicas %} replicas: {{ web_replicas }} +{% elif replicas %} + replicas: {{ replicas }} +{% endif %} selector: matchLabels: app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-web' @@ -19,8 +23,8 @@ spec: metadata: labels: app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-web' - {{ lookup("template", "labels/common.yaml.j2") | indent(width=8) | trim }} - {{ lookup("template", "labels/version.yaml.j2") | indent(width=8) | trim }} + {{ lookup("template", "../common/templates/labels/common.yaml.j2") | indent(width=8) | trim }} + {{ lookup("template", "../common/templates/labels//version.yaml.j2") | indent(width=8) | trim }} {% if annotations %} annotations: {{ annotations | indent(width=8) }} From d40683cbb9b2f76518d40de80ff46f7f0d18d0c6 Mon Sep 17 00:00:00 2001 From: Jake Jackson Date: Mon, 13 Feb 2023 11:32:53 -0500 Subject: [PATCH 09/25] add topology constraint for each deployment (#1234) --- config/crd/bases/awx.ansible.com_awxs.yaml | 6 +++++ roles/installer/defaults/main.yml | 22 +++++++++++++++++++ .../templates/deployments/task.yaml.j2 | 5 ++++- .../templates/deployments/web.yaml.j2 | 7 ++++++ 4 files changed, 39 insertions(+), 1 deletion(-) diff --git a/config/crd/bases/awx.ansible.com_awxs.yaml b/config/crd/bases/awx.ansible.com_awxs.yaml index 97ca31966..927259ffd 100644 --- a/config/crd/bases/awx.ansible.com_awxs.yaml +++ b/config/crd/bases/awx.ansible.com_awxs.yaml @@ -169,6 +169,12 @@ spec: topology_spread_constraints: description: topology rule(s) for the pods type: string + task_topology_spread_constraints: + description: topology rule(s) for the task pods + type: string + web_topology_spread_constraints: + description: topology rule(s) for the web pods + type: string annotations: description: annotations for the pods type: string diff --git a/roles/installer/defaults/main.yml b/roles/installer/defaults/main.yml index e5a3ff12f..a6d5656d0 100644 --- a/roles/installer/defaults/main.yml +++ b/roles/installer/defaults/main.yml @@ -99,6 +99,28 @@ task_node_selector: '' # app.kubernetes.io/name: "" topology_spread_constraints: '' +# Add a topologySpreadConstraints for the task pods. +# Specify as literal block. E.g.: +# task_topology_spread_constraints: | +# - maxSkew: 100 +# topologyKey: "topology.kubernetes.io/zone" +# whenUnsatisfiable: "ScheduleAnyway" +# labelSelector: +# matchLabels: +# app.kubernetes.io/name: """ +task_topology_spread_constraints: '' + +# Add a topologySpreadConstraints for the web pods. +# Specify as literal block. E.g.: +# web_topology_spread_constraints: | +# - maxSkew: 100 +# topologyKey: "topology.kubernetes.io/zone" +# whenUnsatisfiable: "ScheduleAnyway" +# labelSelector: +# matchLabels: +# app.kubernetes.io/name: "" +web_topology_spread_constraints: '' + # Add node tolerations for the AWX pods. Specify as literal block. E.g.: # tolerations: | # - key: "dedicated" diff --git a/roles/installer/templates/deployments/task.yaml.j2 b/roles/installer/templates/deployments/task.yaml.j2 index d0be1739b..830463dde 100644 --- a/roles/installer/templates/deployments/task.yaml.j2 +++ b/roles/installer/templates/deployments/task.yaml.j2 @@ -372,7 +372,10 @@ spec: nodeSelector: {{ node_selector | indent(width=8) }} {% endif %} -{% if topology_spread_constraints %} +{% if task_topology_spread_constraints %} + topologySpreadConstraints: + {{ task_topology_spread_constraints | indent(width=8) }} +{% elif topology_spread_constraints %} topologySpreadConstraints: {{ topology_spread_constraints | indent(width=8) }} {% endif %} diff --git a/roles/installer/templates/deployments/web.yaml.j2 b/roles/installer/templates/deployments/web.yaml.j2 index 855861cd0..eaa19082a 100644 --- a/roles/installer/templates/deployments/web.yaml.j2 +++ b/roles/installer/templates/deployments/web.yaml.j2 @@ -234,6 +234,13 @@ spec: nodeSelector: {{ node_selector | indent(width=8) }} {% endif %} +{% if web_topology_spread_constraints %} + topologySpreadConstraints: + {{ web_topology_spread_constraints | indent(width=8) }} +{% elif topology_spread_constraints %} + topologySpreadConstraints: + {{ topology_spread_constraints | indent(width=8) }} +{% endif %} {% if web_tolerations %} tolerations: {{ web_tolerations| indent(width=8) }} From 41e84be0d3bf6781df04a055f6364d43119b5da7 Mon Sep 17 00:00:00 2001 From: jessicamack Date: Mon, 13 Feb 2023 16:03:15 -0500 Subject: [PATCH 10/25] added descriptors for new fields (#1238) Signed-off-by: jessicamack --- .../awx-operator.clusterserviceversion.yaml | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/config/manifests/bases/awx-operator.clusterserviceversion.yaml b/config/manifests/bases/awx-operator.clusterserviceversion.yaml index 13cf8d1d8..656a07be7 100644 --- a/config/manifests/bases/awx-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/awx-operator.clusterserviceversion.yaml @@ -360,6 +360,16 @@ spec: x-descriptors: - urn:alm:descriptor:com.tectonic.ui:advanced - urn:alm:descriptor:com.tectonic.ui:number + - displayName: Web Replicas + path: web_replicas + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced + - urn:alm:descriptor:com.tectonic.ui:number + - displayName: Task Replicas + path: task_replicas + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced + - urn:alm:descriptor:com.tectonic.ui:number - displayName: Remove used secrets on instance removal? path: garbage_collect_secrets x-descriptors: @@ -608,16 +618,46 @@ spec: x-descriptors: - urn:alm:descriptor:com.tectonic.ui:advanced - urn:alm:descriptor:com.tectonic.ui:hidden + - displayName: Web Node Selector + path: web_node_selector + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced + - urn:alm:descriptor:com.tectonic.ui:hidden + - displayName: Task Node Selector + path: task_node_selector + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced + - urn:alm:descriptor:com.tectonic.ui:hidden - displayName: Topology Spread Constraints path: topology_spread_constraints x-descriptors: - urn:alm:descriptor:com.tectonic.ui:advanced - urn:alm:descriptor:com.tectonic.ui:hidden + - displayName: Web Topology Spread Constraints + path: web_topology_spread_constraints + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced + - urn:alm:descriptor:com.tectonic.ui:hidden + - displayName: Task Topology Spread Constraints + path: task_topology_spread_constraints + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced + - urn:alm:descriptor:com.tectonic.ui:hidden - displayName: Affinity path: affinity x-descriptors: - urn:alm:descriptor:com.tectonic.ui:advanced - urn:alm:descriptor:com.tectonic.ui:hidden + - displayName: Web Affinity + path: web_affinity + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced + - urn:alm:descriptor:com.tectonic.ui:hidden + - displayName: Task Affinity + path: task_affinity + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced + - urn:alm:descriptor:com.tectonic.ui:hidden - displayName: Control Plane Priority Class path: control_plane_priority_class x-descriptors: @@ -649,6 +689,16 @@ spec: x-descriptors: - urn:alm:descriptor:com.tectonic.ui:advanced - urn:alm:descriptor:com.tectonic.ui:hidden + - displayName: Web Tolerations + path: web_tolerations + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced + - urn:alm:descriptor:com.tectonic.ui:hidden + - displayName: Task Tolerations + path: task_tolerations + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced + - urn:alm:descriptor:com.tectonic.ui:hidden - displayName: CSRF Cookie Secure Setting path: csrf_cookie_secure x-descriptors: From 84b766ac408f87306799a4e4ca96235a40d50ed0 Mon Sep 17 00:00:00 2001 From: Hao Liu <44379968+TheRealHaoLiu@users.noreply.github.com> Date: Tue, 14 Feb 2023 17:30:17 -0500 Subject: [PATCH 11/25] update auto_upgrade logic (#1241) update logic for determining if install.yml task should be run to respect the auto_upgrade field in awx resource conditions and expected behavior ``` auto_upgrade awx awx-web awx-task run install.yml -------------- ----- --------- ---------- ----------------- T - - - T F T - - F F - T T F F - T F T F - F T T F - F F T ``` --- roles/installer/tasks/main.yml | 33 ++++++++++++++++--- .../tasks/resources_configuration.yml | 5 ++- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/roles/installer/tasks/main.yml b/roles/installer/tasks/main.yml index bbad1e360..234caa3f4 100644 --- a/roles/installer/tasks/main.yml +++ b/roles/installer/tasks/main.yml @@ -1,13 +1,36 @@ --- -- name: Check for presence of Deployment +- name: Check for presence of old awx Deployment k8s_info: api_version: apps/v1 kind: Deployment name: "{{ ansible_operator_meta.name }}" namespace: "{{ ansible_operator_meta.namespace }}" - register: tower_deployment + register: awx_deployment -# Just execute deployment steps when auto_upgrade is true or when no deployment exists -- name: Start installation +- name: Check for presence of awx-task Deployment + k8s_info: + api_version: v1 + kind: Deployment + name: "{{ ansible_operator_meta.name }}-task" + namespace: "{{ ansible_operator_meta.namespace }}" + register: awx_task_deployment + +- name: Check for presence of awx-web Deployment + k8s_info: + api_version: v1 + kind: Deployment + name: "{{ ansible_operator_meta.name }}-web" + namespace: "{{ ansible_operator_meta.namespace }}" + register: awx_web_deployment + +- name: Start installation if auto_upgrade is true + include_tasks: install.yml + when: + - auto_upgrade | bool + +- name: Start installation if auto_upgrade is false and deployment is missing include_tasks: install.yml - when: (tower_deployment['resources'] | length > 0 and auto_upgrade | bool ) or (tower_deployment['resources'] | length == 0) + when: + - not (auto_upgrade | bool) + - not (awx_deployment['resources'] | length > 0) + - not (awx_web_deployment['resources'] | length > 0 and awx_task_deployment['resources'] | length > 0) diff --git a/roles/installer/tasks/resources_configuration.yml b/roles/installer/tasks/resources_configuration.yml index 00349dc28..8981b5d05 100644 --- a/roles/installer/tasks/resources_configuration.yml +++ b/roles/installer/tasks/resources_configuration.yml @@ -1,6 +1,5 @@ --- - -- name: Get the current resource pod information. +- name: Get the current resource task pod information. k8s_info: api_version: v1 kind: Pod @@ -11,7 +10,7 @@ - "app.kubernetes.io/component={{ deployment_type }}" field_selectors: - status.phase=Running - register: tower_pod + register: awx_task_pod - name: Set the resource pod as a variable. set_fact: From 3c0ea07a5505cfbae0d07cb9e9f3eecb60333337 Mon Sep 17 00:00:00 2001 From: thedoubl3j Date: Tue, 14 Feb 2023 15:55:55 -0500 Subject: [PATCH 12/25] update documentation for the new fields added --- README.md | 46 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index aaa252592..cbcb0cfde 100644 --- a/README.md +++ b/README.md @@ -716,7 +716,14 @@ spec: control_plane_priority_class: awx-demo-high-priority postgres_priority_class: awx-demo-medium-priority ``` +#### Scaling the Web and Task Pods independently +You can scale replicas up or down for each deployment by using the `web_replicas` or `task_replicas` respectively. You can scale all pods across both deployments by using `replicas` as well. The logic behind these CRD keys acts as such: + +- If you specify the `replicas` field, the key passed will scale both the `web` and `task` replicas to the same number. +- If `web_replicas` or `task_replicas` is ever passed, it will override the existing `replicas` field on the specific deployment with the new key value. + +These new replicas can be constrained in a similar manner to previous single deployments by appending the particular deployment name in front of the constraint used. More about those new constraints can be found below in the [Assigning AWX pods to specific nodes](#assigning-awx-pods-to-specific-nodes) section. #### Assigning AWX pods to specific nodes You can constrain the AWX pods created by the operator to run on a certain subset of nodes. `node_selector` and `postgres_selector` constrains @@ -725,18 +732,28 @@ pods to be scheduled onto nodes with matching taints. The ability to specify topologySpreadConstraints is also allowed through `topology_spread_constraints` If you want to use affinity rules for your AWX pod you can use the `affinity` option. - -| Name | Description | Default | -| --------------------------- | ----------------------------------- | ------- | -| postgres_image | Path of the image to pull | postgres | -| postgres_image_version | Image version to pull | 13 | -| node_selector | AWX pods' nodeSelector | '' | -| topology_spread_constraints | AWX pods' topologySpreadConstraints | '' | -| affinity | AWX pods' affinity rules | '' | -| tolerations | AWX pods' tolerations | '' | -| annotations | AWX pods' annotations | '' | -| postgres_selector | Postgres pods' nodeSelector | '' | -| postgres_tolerations | Postgres pods' tolerations | '' | +If you want to constrain the web and task pods individually, you can do so by specificying the deployment type before the specific setting. For +example, specifying `task_tolerations` will allow the AWX task pod to be scheduled onto nodes with matching taints. + +| Name | Description | Default | +| -------------------------------- | ---------------------------------------- | ------- | +| postgres_image | Path of the image to pull | postgres | +| postgres_image_version | Image version to pull | 13 | +| node_selector | AWX pods' nodeSelector | '' | +| web_node_selector | AWX web pods' nodeSelector | '' | +| task_node_selector | AWX task pods' nodeSelector | '' | +| topology_spread_constraints | AWX pods' topologySpreadConstraints | '' | +| web_topology_spread_constraints | AWX web pods' topologySpreadConstraints | '' | +| task_topology_spread_constraints | AWX task pods' topologySpreadConstraints | '' | +| affinity | AWX pods' affinity rules | '' | +| web_affinity | AWX web pods' affinity rules | '' | +| task_affinity | AWX task pods' affinity rules | '' | +| tolerations | AWX pods' tolerations | '' | +| web_tolerations | AWX web pods' tolerations | '' | +| task_tolerations | AWX task pods' tolerations | '' | +| annotations | AWX pods' annotations | '' | +| postgres_selector | Postgres pods' nodeSelector | '' | +| postgres_tolerations | Postgres pods' tolerations | '' | Example of customization could be: @@ -760,6 +777,11 @@ spec: operator: "Equal" value: "AWX" effect: "NoSchedule" + task_tolerations: | + - key: "dedicated" + operator: "Equal" + value: "AWX_task" + effect: "NoSchedule" postgres_selector: | disktype: ssd kubernetes.io/arch: amd64 From 5894a4ad255120954702e9e84de6d32fea06a32a Mon Sep 17 00:00:00 2001 From: thedoubl3j Date: Thu, 23 Feb 2023 16:58:42 -0500 Subject: [PATCH 13/25] remove old deployment during upgrade --- roles/installer/tasks/install.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/roles/installer/tasks/install.yml b/roles/installer/tasks/install.yml index faca317ef..91aed4c0d 100644 --- a/roles/installer/tasks/install.yml +++ b/roles/installer/tasks/install.yml @@ -1,4 +1,12 @@ --- +- name: Delete old deployment for before installing during upgrade + k8s: + kind: Deployment + api_version: v1 + namespace: "{{ ansible_operator_meta.namespace }}" + name: "{{ ansible_operator_meta.name }}" + state: absent + - name: Patching labels to AWX kind k8s: state: present From a0fdf653fb02d634bb1ca19aa8e66181b2250cab Mon Sep 17 00:00:00 2001 From: Hao Liu Date: Thu, 2 Mar 2023 14:24:38 -0500 Subject: [PATCH 14/25] enable web container to be restarted when configmap/secret change --- .../templates/deployments/web.yaml.j2 | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/roles/installer/templates/deployments/web.yaml.j2 b/roles/installer/templates/deployments/web.yaml.j2 index eaa19082a..814b22508 100644 --- a/roles/installer/templates/deployments/web.yaml.j2 +++ b/roles/installer/templates/deployments/web.yaml.j2 @@ -25,8 +25,26 @@ spec: app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-web' {{ lookup("template", "../common/templates/labels/common.yaml.j2") | indent(width=8) | trim }} {{ lookup("template", "../common/templates/labels//version.yaml.j2") | indent(width=8) | trim }} -{% if annotations %} annotations: +{% for template in [ + "configmaps/config", + "configmaps/pre_stop_scripts", + "secrets/app_credentials", + "storage/persistent", + ] %} + checksum-{{ template | replace('/', '-') }}: "{{ lookup('template', template + '.yaml.j2') | md5 }}" +{% endfor %} +{% for secret in [ + "bundle_cacert", + "route_tls", + "ldap_cacert", + "secret_key", + "receptor_ca", + "receptor_work_signing", + ] %} + checksum-secret-{{ secret }}: "{{ lookup('ansible.builtin.vars', secret, default='')["resources"][0]["data"] | default('') | md5 }}" +{% endfor %} +{% if annotations %} {{ annotations | indent(width=8) }} {% endif %} spec: @@ -102,13 +120,13 @@ spec: resources: {{ redis_resource_requirements }} - image: '{{ _image }}' name: '{{ ansible_operator_meta.name }}-web' + imagePullPolicy: '{{ image_pull_policy }}' {% if web_command %} command: {{ web_command }} {% endif %} {% if web_args %} args: {{ web_args }} {% endif %} - imagePullPolicy: '{{ image_pull_policy }}' ports: - containerPort: 8052 {% if ingress_type | lower == 'route' and route_tls_termination_mechanism | lower == 'passthrough' %} From c2f0c214ebd16cb5b8eb8b283096e192719364c8 Mon Sep 17 00:00:00 2001 From: Hao Liu Date: Thu, 2 Mar 2023 15:11:09 -0500 Subject: [PATCH 15/25] rename tower_pod to awx_task_pod --- roles/installer/tasks/initialize_django.yml | 16 ++++++++-------- roles/installer/tasks/install.yml | 4 ++-- .../installer/tasks/resources_configuration.yml | 14 +++++++------- roles/installer/tasks/update_status.yml | 2 +- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/roles/installer/tasks/initialize_django.yml b/roles/installer/tasks/initialize_django.yml index bc75a06ab..158a77573 100644 --- a/roles/installer/tasks/initialize_django.yml +++ b/roles/installer/tasks/initialize_django.yml @@ -2,7 +2,7 @@ - name: Check if there are any super users defined. k8s_exec: namespace: "{{ ansible_operator_meta.namespace }}" - pod: "{{ tower_pod_name }}" + pod: "{{ awx_task_pod_name }}" container: "{{ ansible_operator_meta.name }}-task" command: >- bash -c "echo 'from django.contrib.auth.models import User; @@ -16,7 +16,7 @@ - name: Create super user via Django if it doesn't exist. k8s_exec: namespace: "{{ ansible_operator_meta.namespace }}" - pod: "{{ tower_pod_name }}" + pod: "{{ awx_task_pod_name }}" container: "{{ ansible_operator_meta.name }}-task" command: awx-manage createsuperuser --username={{ admin_user | quote }} --email={{ admin_email | quote }} --noinput register: result @@ -28,7 +28,7 @@ - name: Update Django super user password k8s_exec: namespace: "{{ ansible_operator_meta.namespace }}" - pod: "{{ tower_pod_name }}" + pod: "{{ awx_task_pod_name }}" container: "{{ ansible_operator_meta.name }}-task" command: awx-manage update_password --username='{{ admin_user }}' --password='{{ admin_password }}' register: result @@ -39,7 +39,7 @@ - name: Check if legacy queue is present k8s_exec: namespace: "{{ ansible_operator_meta.namespace }}" - pod: "{{ tower_pod_name }}" + pod: "{{ awx_task_pod_name }}" container: "{{ ansible_operator_meta.name }}-task" command: >- bash -c "awx-manage list_instances | grep '^\[tower capacity=[0-9]*\]'" @@ -50,7 +50,7 @@ - name: Unregister legacy queue k8s_exec: namespace: "{{ ansible_operator_meta.namespace }}" - pod: "{{ tower_pod_name }}" + pod: "{{ awx_task_pod_name }}" container: "{{ ansible_operator_meta.name }}-task" command: >- bash -c "awx-manage unregister_queue --queuename=tower" @@ -74,7 +74,7 @@ - name: Register default execution environments (without authentication) k8s_exec: namespace: "{{ ansible_operator_meta.namespace }}" - pod: "{{ tower_pod_name }}" + pod: "{{ awx_task_pod_name }}" container: "{{ ansible_operator_meta.name }}-task" command: >- bash -c "awx-manage register_default_execution_environments" @@ -95,7 +95,7 @@ - name: Register default execution environments (with authentication) k8s_exec: namespace: "{{ ansible_operator_meta.namespace }}" - pod: "{{ tower_pod_name }}" + pod: "{{ awx_task_pod_name }}" container: "{{ ansible_operator_meta.name }}-task" command: >- bash -c "awx-manage register_default_execution_environments @@ -111,7 +111,7 @@ - name: Create preload data if necessary. # noqa 305 k8s_exec: namespace: "{{ ansible_operator_meta.namespace }}" - pod: "{{ tower_pod_name }}" + pod: "{{ awx_task_pod_name }}" container: "{{ ansible_operator_meta.name }}-task" command: >- bash -c "awx-manage create_preload_data" diff --git a/roles/installer/tasks/install.yml b/roles/installer/tasks/install.yml index 91aed4c0d..ecf945d57 100644 --- a/roles/installer/tasks/install.yml +++ b/roles/installer/tasks/install.yml @@ -83,7 +83,7 @@ - name: Check for pending migrations k8s_exec: namespace: "{{ ansible_operator_meta.namespace }}" - pod: "{{ tower_pod_name }}" + pod: "{{ awx_task_pod_name }}" container: "{{ ansible_operator_meta.name }}-task" command: >- bash -c "awx-manage showmigrations | grep -v '[X]' | grep '[ ]' | wc -l" @@ -93,7 +93,7 @@ - name: Migrate the database if the K8s resources were updated. # noqa 305 k8s_exec: namespace: "{{ ansible_operator_meta.namespace }}" - pod: "{{ tower_pod_name }}" + pod: "{{ awx_task_pod_name }}" container: "{{ ansible_operator_meta.name }}-task" command: >- bash -c "awx-manage migrate --noinput" diff --git a/roles/installer/tasks/resources_configuration.yml b/roles/installer/tasks/resources_configuration.yml index 8981b5d05..d385891f0 100644 --- a/roles/installer/tasks/resources_configuration.yml +++ b/roles/installer/tasks/resources_configuration.yml @@ -14,15 +14,15 @@ - name: Set the resource pod as a variable. set_fact: - tower_pod: >- - {{ tower_pod['resources'] + awx_task_pod: >- + {{ awx_task_pod['resources'] | rejectattr('metadata.deletionTimestamp', 'defined') | sort(attribute='metadata.creationTimestamp') | first | default({}) }} - name: Set the resource pod name as a variable. set_fact: - tower_pod_name: "{{ tower_pod['metadata']['name'] | default('') }}" + awx_task_pod_name: "{{ awx_task_pod['metadata']['name'] | default('') }}" - name: Set user provided control plane ee image set_fact: @@ -252,8 +252,8 @@ wait: yes wait_timeout: "{{ (120 * replicas) or 120 }}" loop: - - web - task + - web register: this_deployment_result - block: @@ -271,7 +271,7 @@ - name: Update new resource pod as a variable. set_fact: - tower_pod: >- + awx_task_pod: >- {{ _new_pod['resources'] | rejectattr('metadata.deletionTimestamp', 'defined') | sort(attribute='metadata.creationTimestamp') @@ -279,11 +279,11 @@ - name: Update new resource pod name as a variable. set_fact: - tower_pod_name: '{{ tower_pod["metadata"]["name"] | default("")}}' + awx_task_pod_name: '{{ awx_task_pod["metadata"]["name"] | default("")}}' when: - this_deployment_result.changed - name: Verify the resource pod name is populated. assert: - that: tower_pod_name != '' + that: awx_task_pod_name != '' fail_msg: "Could not find the tower pod's name." diff --git a/roles/installer/tasks/update_status.yml b/roles/installer/tasks/update_status.yml index f557f1f47..e6641e773 100644 --- a/roles/installer/tasks/update_status.yml +++ b/roles/installer/tasks/update_status.yml @@ -47,7 +47,7 @@ - name: Retrieve instance version k8s_exec: namespace: "{{ ansible_operator_meta.namespace }}" - pod: "{{ tower_pod_name }}" + pod: "{{ awx_task_pod_name }}" container: "{{ ansible_operator_meta.name }}-task" command: >- bash -c "awx-manage --version" From 942cce05e7100ef8177913dcc8cb9c11b429b064 Mon Sep 17 00:00:00 2001 From: Hao Liu Date: Thu, 2 Mar 2023 16:19:38 -0500 Subject: [PATCH 16/25] resolve merge error --- config/crd/bases/awx.ansible.com_awxs.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/config/crd/bases/awx.ansible.com_awxs.yaml b/config/crd/bases/awx.ansible.com_awxs.yaml index 927259ffd..3e0fef0f9 100644 --- a/config/crd/bases/awx.ansible.com_awxs.yaml +++ b/config/crd/bases/awx.ansible.com_awxs.yaml @@ -175,6 +175,13 @@ spec: web_topology_spread_constraints: description: topology rule(s) for the web pods type: string + termination_grace_period_seconds: + description: Optional duration in seconds pods needs to terminate gracefully + type: integer + format: int32 + service_labels: + description: Additional labels to apply to the service + type: string annotations: description: annotations for the pods type: string From 6fd5ca918b9fa30c160051d75b3843b5e70f9cac Mon Sep 17 00:00:00 2001 From: thedoubl3j Date: Thu, 9 Mar 2023 15:41:51 -0500 Subject: [PATCH 17/25] remove any reference to project persistance in web deployment --- .../templates/deployments/web.yaml.j2 | 23 +------------------ 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/roles/installer/templates/deployments/web.yaml.j2 b/roles/installer/templates/deployments/web.yaml.j2 index 814b22508..900a82e8c 100644 --- a/roles/installer/templates/deployments/web.yaml.j2 +++ b/roles/installer/templates/deployments/web.yaml.j2 @@ -62,7 +62,7 @@ spec: priorityClassName: '{{ control_plane_priority_class }}' {% endif %} initContainers: -{% if bundle_ca_crt or projects_persistence|bool or init_container_extra_commands %} +{% if bundle_ca_crt or init_container_extra_commands %} - name: init image: '{{ _init_container_image }}' imagePullPolicy: '{{ image_pull_policy }}' @@ -74,10 +74,6 @@ spec: mkdir -p /etc/pki/ca-trust/extracted/{java,pem,openssl,edk2} update-ca-trust {% endif %} -{% if projects_persistence|bool %} - chmod 775 /var/lib/awx/projects - chgrp 1000 /var/lib/awx/projects -{% endif %} {% if init_container_extra_commands %} {{ init_container_extra_commands | indent(width=14) }} {% endif %} @@ -90,10 +86,6 @@ spec: subPath: bundle-ca.crt readOnly: true {% endif %} -{% if projects_persistence|bool %} - - name: "{{ ansible_operator_meta.name }}-projects" - mountPath: "/var/lib/awx/projects" -{% endif %} {% if init_container_extra_volume_mounts -%} {{ init_container_extra_volume_mounts | indent(width=12, first=True) }} {% endif %} @@ -180,8 +172,6 @@ spec: mountPath: "/var/run/redis" - name: rsyslog-socket mountPath: "/var/run/awx-rsyslog" - - name: "{{ ansible_operator_meta.name }}-projects" - mountPath: "/var/lib/awx/projects" {% if development_mode | bool %} - name: awx-devel mountPath: "/awx_devel" @@ -350,17 +340,6 @@ spec: items: - key: receptor_conf path: receptor.conf - - name: "{{ ansible_operator_meta.name }}-projects" -{% if projects_persistence|bool %} - persistentVolumeClaim: -{% if projects_existing_claim %} - claimName: {{ projects_existing_claim }} -{% else %} - claimName: '{{ ansible_operator_meta.name }}-projects-claim' -{% endif %} -{% else %} - emptyDir: {} -{% endif %} {% if development_mode | bool %} - name: awx-devel hostPath: From 1106ef54ee034568cccfd0277facd74c9fa6561a Mon Sep 17 00:00:00 2001 From: Hao Liu Date: Wed, 22 Mar 2023 15:34:54 -0400 Subject: [PATCH 18/25] add AWX_COMPONENT env var for awx to be able to identify which component it is running in code --- roles/installer/templates/deployments/task.yaml.j2 | 2 ++ roles/installer/templates/deployments/web.yaml.j2 | 2 ++ 2 files changed, 4 insertions(+) diff --git a/roles/installer/templates/deployments/task.yaml.j2 b/roles/installer/templates/deployments/task.yaml.j2 index 830463dde..c9ac5bea0 100644 --- a/roles/installer/templates/deployments/task.yaml.j2 +++ b/roles/installer/templates/deployments/task.yaml.j2 @@ -238,6 +238,8 @@ spec: - /var/lib/pre-stop/scripts/termination-master {% endif %} env: + - name: AWX_COMPONENT + value: "task" - name: SUPERVISOR_WEB_CONFIG_PATH value: "/etc/supervisord.conf" - name: AWX_SKIP_MIGRATIONS diff --git a/roles/installer/templates/deployments/web.yaml.j2 b/roles/installer/templates/deployments/web.yaml.j2 index 900a82e8c..7fd205cc3 100644 --- a/roles/installer/templates/deployments/web.yaml.j2 +++ b/roles/installer/templates/deployments/web.yaml.j2 @@ -180,6 +180,8 @@ spec: {{ web_extra_volume_mounts | indent(width=12, first=True) }} {% endif %} env: + - name: AWX_COMPONENT + value: "web" - name: MY_POD_NAMESPACE valueFrom: fieldRef: From 25dfc970479666f9b2f3914dfa59377bbe8c4501 Mon Sep 17 00:00:00 2001 From: Lila Date: Wed, 22 Mar 2023 11:42:02 -0400 Subject: [PATCH 19/25] Updated task deployment to fix receptor certs issue. Added the following volume mounts to the web container: - receptor-work-signing - receptor-ca - work-public-key.pem Also added these corresponding volumes to the web deployments: - receptor-ca - receptor-work-signing --- .../templates/deployments/web.yaml.j2 | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/roles/installer/templates/deployments/web.yaml.j2 b/roles/installer/templates/deployments/web.yaml.j2 index 7fd205cc3..682f63220 100644 --- a/roles/installer/templates/deployments/web.yaml.j2 +++ b/roles/installer/templates/deployments/web.yaml.j2 @@ -172,6 +172,18 @@ spec: mountPath: "/var/run/redis" - name: rsyslog-socket mountPath: "/var/run/awx-rsyslog" + - name: "{{ ansible_operator_meta.name }}-receptor-ca" + mountPath: "/etc/receptor/tls/ca/receptor-ca.crt" + subPath: "tls.crt" + readOnly: true + - name: "{{ ansible_operator_meta.name }}-receptor-ca" + mountPath: "/etc/receptor/tls/ca/receptor-ca.key" + subPath: "tls.key" + readOnly: true + - name: "{{ ansible_operator_meta.name }}-receptor-work-signing" + mountPath: "/etc/receptor/signing/work-public-key.pem" + subPath: "work-public-key.pem" + readOnly: true {% if development_mode | bool %} - name: awx-devel mountPath: "/awx_devel" @@ -266,6 +278,12 @@ spec: {{ affinity | to_nice_yaml | indent(width=8) }} {% endif %} volumes: + - name: "{{ ansible_operator_meta.name }}-receptor-ca" + secret: + secretName: "{{ ansible_operator_meta.name }}-receptor-ca" + - name: "{{ ansible_operator_meta.name }}-receptor-work-signing" + secret: + secretName: "{{ ansible_operator_meta.name }}-receptor-work-signing" {% if bundle_ca_crt %} - name: "ca-trust-extracted" emptyDir: {} From 7218e427717b0a75e087d8fe95e69dca04127ac2 Mon Sep 17 00:00:00 2001 From: Jake Jackson Date: Thu, 23 Mar 2023 14:53:14 -0400 Subject: [PATCH 20/25] [web/task split] fix scale down bug (#1295) - rename scale_down vars to the new deployments since the old one no longer exists - rename postgres.yml scale down vars as it references the old ones as well --- roles/installer/tasks/scale_down_deployment.yml | 5 ++++- roles/restore/tasks/postgres.yml | 7 +++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/roles/installer/tasks/scale_down_deployment.yml b/roles/installer/tasks/scale_down_deployment.yml index be731fa0b..6f0a63ef7 100644 --- a/roles/installer/tasks/scale_down_deployment.yml +++ b/roles/installer/tasks/scale_down_deployment.yml @@ -12,8 +12,11 @@ kubernetes.core.k8s_scale: api_version: apps/v1 kind: Deployment - name: "{{ ansible_operator_meta.name }}" + name: "{{ item }}" namespace: "{{ ansible_operator_meta.namespace }}" replicas: 0 wait: yes + loop: + - "{{ ansible_operator_meta.name }}-task" + - "{{ ansible_operator_meta.name }}-web" when: this_deployment['resources'] | length diff --git a/roles/restore/tasks/postgres.yml b/roles/restore/tasks/postgres.yml index 417513b48..8c10c1b7c 100644 --- a/roles/restore/tasks/postgres.yml +++ b/roles/restore/tasks/postgres.yml @@ -50,7 +50,7 @@ k8s_info: api_version: apps/v1 kind: Deployment - name: "{{ deployment_name }}" + name: "{{ ansible_operator_meta.namespace }}-task" namespace: "{{ ansible_operator_meta.namespace }}" register: this_deployment @@ -58,10 +58,13 @@ k8s_scale: api_version: apps/v1 kind: Deployment - name: "{{ deployment_name }}" + name: "{{ item }}" namespace: "{{ ansible_operator_meta.namespace }}" replicas: 0 wait: yes + loop: + - "{{ ansible_operator_meta.name }}-task" + - "{{ ansible_operator_meta.name }}-web" when: this_deployment['resources'] | length - name: Set full resolvable host name for postgres pod From b7e6987aec28a2c82ead97ef8804bcf176c3fe8e Mon Sep 17 00:00:00 2001 From: Lila Date: Wed, 29 Mar 2023 15:11:10 -0400 Subject: [PATCH 21/25] Removed errant quotes in main.yml. --- roles/installer/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/installer/defaults/main.yml b/roles/installer/defaults/main.yml index a6d5656d0..80c544a1c 100644 --- a/roles/installer/defaults/main.yml +++ b/roles/installer/defaults/main.yml @@ -107,7 +107,7 @@ topology_spread_constraints: '' # whenUnsatisfiable: "ScheduleAnyway" # labelSelector: # matchLabels: -# app.kubernetes.io/name: """ +# app.kubernetes.io/name: "" task_topology_spread_constraints: '' # Add a topologySpreadConstraints for the web pods. From 4bec3f709807b577489aea721675116829e4cc2c Mon Sep 17 00:00:00 2001 From: Lila Date: Wed, 29 Mar 2023 15:14:25 -0400 Subject: [PATCH 22/25] Removed extra space. --- roles/installer/templates/networking/service.yaml.j2 | 1 - 1 file changed, 1 deletion(-) diff --git a/roles/installer/templates/networking/service.yaml.j2 b/roles/installer/templates/networking/service.yaml.j2 index ab09bca9d..d178a0291 100644 --- a/roles/installer/templates/networking/service.yaml.j2 +++ b/roles/installer/templates/networking/service.yaml.j2 @@ -13,7 +13,6 @@ metadata: {% endif %} spec: ports: - {% if service_type | lower == "nodeport" %} - port: 80 protocol: TCP From 02328076e57c243f16c7d60759b1f50ab42e09d0 Mon Sep 17 00:00:00 2001 From: Hao Liu Date: Wed, 29 Mar 2023 19:52:22 -0400 Subject: [PATCH 23/25] Update arg for launching web container related to https://github.com/ansible/awx/pull/13777 make sure that the launch script name reflect the component that its launching --- roles/installer/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/installer/defaults/main.yml b/roles/installer/defaults/main.yml index 80c544a1c..69cc5a4a7 100644 --- a/roles/installer/defaults/main.yml +++ b/roles/installer/defaults/main.yml @@ -268,7 +268,7 @@ task_args: - /usr/bin/launch_awx_task.sh task_command: [] web_args: - - /usr/bin/launch_awx.sh + - /usr/bin/launch_awx_web.sh web_command: [] ryslog_args: - /usr/bin/launch_awx_rsyslog.sh From cd3bfbef96838cad33a835f77806678ecfdcad72 Mon Sep 17 00:00:00 2001 From: Hao Liu Date: Wed, 29 Mar 2023 20:20:13 -0400 Subject: [PATCH 24/25] Rename SUPERVISOR_WEB_CONFIG_PATH SUPERVISOR_WEB_CONFIG_PATH is used in the old deployment for task container to reach into the web container and restart services this is no longer possible/needed after splitting the deployment renaming SUPERVISOR_WEB_CONFIG_PATH to SUPERVISOR_CONFIG_PATH and setting it to the supervisor file for the container this can still be useful to help run `supervisorctl -c $SUPERVISOR_CONFIG_PATH` --- roles/installer/templates/deployments/task.yaml.j2 | 6 +++--- roles/installer/templates/deployments/web.yaml.j2 | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/roles/installer/templates/deployments/task.yaml.j2 b/roles/installer/templates/deployments/task.yaml.j2 index c9ac5bea0..db3c59a9d 100644 --- a/roles/installer/templates/deployments/task.yaml.j2 +++ b/roles/installer/templates/deployments/task.yaml.j2 @@ -240,8 +240,8 @@ spec: env: - name: AWX_COMPONENT value: "task" - - name: SUPERVISOR_WEB_CONFIG_PATH - value: "/etc/supervisord.conf" + - name: SUPERVISOR_CONFIG_PATH + value: "/etc/supervisord_task.conf" - name: AWX_SKIP_MIGRATIONS value: "1" - name: MY_POD_UID @@ -361,7 +361,7 @@ spec: mountPath: "/awx_devel" {% endif %} env: - - name: SUPERVISOR_WEB_CONFIG_PATH + - name: SUPERVISOR_CONFIG_PATH value: "/etc/supervisor_rsyslog.conf" {% if development_mode | bool %} - name: AWX_KUBE_DEVEL diff --git a/roles/installer/templates/deployments/web.yaml.j2 b/roles/installer/templates/deployments/web.yaml.j2 index 682f63220..b0ee702a4 100644 --- a/roles/installer/templates/deployments/web.yaml.j2 +++ b/roles/installer/templates/deployments/web.yaml.j2 @@ -194,6 +194,8 @@ spec: env: - name: AWX_COMPONENT value: "web" + - name: SUPERVISOR_CONFIG_PATH + value: "/etc/supervisor_web.conf" - name: MY_POD_NAMESPACE valueFrom: fieldRef: @@ -243,7 +245,7 @@ spec: mountPath: "/awx_devel" {% endif %} env: - - name: SUPERVISOR_WEB_CONFIG_PATH + - name: SUPERVISOR_CONFIG_PATH value: "/etc/supervisor_rsyslog.conf" {% if development_mode | bool %} - name: AWX_KUBE_DEVEL From 4a6d1855dcc9cccfe76e7128c9e28f9ea74f6ec0 Mon Sep 17 00:00:00 2001 From: thedoubl3j Date: Wed, 29 Mar 2023 21:49:51 -0400 Subject: [PATCH 25/25] remove disktype key from example --- roles/installer/defaults/main.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/roles/installer/defaults/main.yml b/roles/installer/defaults/main.yml index 69cc5a4a7..e97647c51 100644 --- a/roles/installer/defaults/main.yml +++ b/roles/installer/defaults/main.yml @@ -67,7 +67,6 @@ hostname: '' # Add a nodeSelector for the AWX pods. It must match a node's labels for the pod # to be scheduled on that node. Specify as literal block. E.g.: # node_selector: | -# disktype: ssd # kubernetes.io/arch: amd64 # kubernetes.io/os: linux node_selector: '' @@ -75,7 +74,6 @@ node_selector: '' # Add a nodeSelector for the AWX pods. It must match a node's labels for the pod # to be scheduled on that node. Specify as literal block. E.g.: # node_selector: | -# disktype: ssd # kubernetes.io/arch: amd64 # kubernetes.io/os: linux web_node_selector: '' @@ -83,7 +81,6 @@ web_node_selector: '' # Add a nodeSelector for the AWX pods. It must match a node's labels for the pod # to be scheduled on that node. Specify as literal block. E.g.: # node_selector: | -# disktype: ssd # kubernetes.io/arch: amd64 # kubernetes.io/os: linux task_node_selector: ''