From 5372771bac00f39eb672071568334887b5568684 Mon Sep 17 00:00:00 2001 From: Seth Foster Date: Thu, 7 Apr 2022 14:47:08 -0400 Subject: [PATCH 1/4] Add priority class options to high priority pods - Add postgres_priority_class - Add control_plane_priority_class - Add default requests for postgres pod to ensure at a "Burstable" QoS --- config/crd/bases/awx.ansible.com_awxs.yaml | 6 ++++++ molecule/default/templates/awx_cr_molecule.yml.j2 | 1 + roles/installer/defaults/main.yml | 9 ++++++++- roles/installer/templates/deployment.yaml.j2 | 3 +++ roles/installer/templates/postgres.yaml.j2 | 3 +++ 5 files changed, 21 insertions(+), 1 deletion(-) diff --git a/config/crd/bases/awx.ansible.com_awxs.yaml b/config/crd/bases/awx.ansible.com_awxs.yaml index afff46b49..6dbad17af 100644 --- a/config/crd/bases/awx.ansible.com_awxs.yaml +++ b/config/crd/bases/awx.ansible.com_awxs.yaml @@ -165,6 +165,9 @@ spec: control_plane_ee_image: description: Registry path to the Execution Environment container image to use on control plane pods type: string + control_plane_priority_class: + description: Assign a preexisting priority class to the control plane pods + type: string ee_pull_credentials_secret: description: Secret where pull credentials for registered ees can be found type: string @@ -392,6 +395,9 @@ spec: postgres_storage_class: description: Storage class to use for the PostgreSQL PVC type: string + postgres_priority_class: + description: Assign a preexisting priority class to the postgres pod + type: string postgres_data_path: description: Path where the PostgreSQL data are located type: string diff --git a/molecule/default/templates/awx_cr_molecule.yml.j2 b/molecule/default/templates/awx_cr_molecule.yml.j2 index e7c411d7e..2c19db8af 100644 --- a/molecule/default/templates/awx_cr_molecule.yml.j2 +++ b/molecule/default/templates/awx_cr_molecule.yml.j2 @@ -25,3 +25,4 @@ spec: requests: cpu: 200m memory: 64M + postgres_resource_requirements: {} diff --git a/roles/installer/defaults/main.yml b/roles/installer/defaults/main.yml index 6e1537b3d..b7f35f8ab 100644 --- a/roles/installer/defaults/main.yml +++ b/roles/installer/defaults/main.yml @@ -186,6 +186,8 @@ ee_resource_requirements: csrf_cookie_secure: False session_cookie_secure: False +# Assign a preexisting priority class to the control plane pods +control_plane_priority_class: '' # Add extra environment variables to the AWX task/web containers. Specify as # literal block. E.g.: # task_extra_env: | @@ -227,7 +229,12 @@ postgres_storage_requirements: requests: storage: 8Gi postgres_init_container_resource_requirements: {} -postgres_resource_requirements: {} +postgres_resource_requirements: + requests: + cpu: 500m + memory: 1Gi +# Assign a preexisting priority class to the postgres pod +postgres_priority_class: '' postgres_data_path: '/var/lib/postgresql/data/pgdata' # Persistence to the AWX project data folder diff --git a/roles/installer/templates/deployment.yaml.j2 b/roles/installer/templates/deployment.yaml.j2 index 3aff85649..ff150a239 100644 --- a/roles/installer/templates/deployment.yaml.j2 +++ b/roles/installer/templates/deployment.yaml.j2 @@ -41,6 +41,9 @@ spec: {% for secret in image_pull_secrets %} - name: {{ secret }} {% endfor %} +{% endif %} +{% if control_plane_priorityclass is defined %} + priorityClassName: '{{ control_plane_priority_class }}' {% endif %} initContainers: {% if bundle_ca_crt or projects_persistence|bool or init_container_extra_commands %} diff --git a/roles/installer/templates/postgres.yaml.j2 b/roles/installer/templates/postgres.yaml.j2 index 67d12f4cf..6caef41a5 100644 --- a/roles/installer/templates/postgres.yaml.j2 +++ b/roles/installer/templates/postgres.yaml.j2 @@ -41,6 +41,9 @@ spec: {% for secret in image_pull_secrets %} - name: {{ secret }} {% endfor %} +{% endif %} +{% if postgres_priority_class is defined %} + priorityClassName: '{{ postgres_priority_class }}' {% endif %} initContainers: - name: database-check From 21062f070864d7abb8b64a8d1fe50d8387e966ef Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Fri, 15 Apr 2022 15:44:31 -0400 Subject: [PATCH 2/4] Add default resource requests for postgres containers --- README.md | 40 +++++++++++++++---------------- roles/installer/defaults/main.yml | 21 +++++++++------- 2 files changed, 32 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index ce3d0a1e2..c88a011fe 100644 --- a/README.md +++ b/README.md @@ -434,14 +434,14 @@ If you don't have access to an external PostgreSQL service, the AWX operator can The following variables are customizable for the managed PostgreSQL service -| Name | Description | Default | -| --------------------------------------------- | --------------------------------------------- | --------------------------------- | -| postgres_image | Path of the image to pull | postgres:12 | -| postgres_init_container_resource_requirements | Database init container resource requirements | requests: {} | -| postgres_resource_requirements | PostgreSQL container resource requirements | requests: {} | -| postgres_storage_requirements | PostgreSQL container storage requirements | requests: {storage: 8Gi} | -| postgres_storage_class | PostgreSQL PV storage class | Empty string | -| postgres_data_path | PostgreSQL data path | `/var/lib/postgresql/data/pgdata` | +| Name | Description | Default | +| --------------------------------------------- | --------------------------------------------- | ---------------------------------- | +| postgres_image | Path of the image to pull | postgres:12 | +| postgres_init_container_resource_requirements | Database init container resource requirements | requests: {cpu: 10m, memory: 64Mi} | +| postgres_resource_requirements | PostgreSQL container resource requirements | requests: {cpu: 10m, memory: 64Mi} | +| postgres_storage_requirements | PostgreSQL container storage requirements | requests: {storage: 8Gi} | +| postgres_storage_class | PostgreSQL PV storage class | Empty string | +| postgres_data_path | PostgreSQL data path | `/var/lib/postgresql/data/pgdata` | Example of customization could be: @@ -541,11 +541,11 @@ Again, this is the most relaxed SCC that is provided by OpenShift, so be sure to The resource requirements for both, the task and the web containers are configurable - both the lower end (requests) and the upper end (limits). -| Name | Description | Default | -| -------------------------- | ------------------------------------------------ | ----------------------------------- | -| web_resource_requirements | Web container resource requirements | requests: {cpu: 1000m, memory: 2Gi} | -| task_resource_requirements | Task container resource requirements | requests: {cpu: 500m, memory: 1Gi} | -| ee_resource_requirements | EE control plane container resource requirements | requests: {cpu: 500m, memory: 1Gi} | +| Name | Description | Default | +| -------------------------- | ------------------------------------------------ | ------------------------------------ | +| web_resource_requirements | Web container resource requirements | requests: {cpu: 100m, memory: 128Mi} | +| task_resource_requirements | Task container resource requirements | requests: {cpu: 100m, memory: 128Mi} | +| ee_resource_requirements | EE control plane container resource requirements | requests: {cpu: 100m, memory: 128Mi} | Example of customization could be: @@ -555,24 +555,24 @@ spec: ... web_resource_requirements: requests: - cpu: 1000m + cpu: 250m memory: 2Gi limits: - cpu: 2000m + cpu: 1000m memory: 4Gi task_resource_requirements: requests: - cpu: 500m + cpu: 250m memory: 1Gi limits: - cpu: 1000m + cpu: 2000m memory: 2Gi ee_resource_requirements: requests: - cpu: 500m - memory: 1Gi + cpu: 250m + memory: 100Mi limits: - cpu: 1000m + cpu: 500m memory: 2Gi ``` diff --git a/roles/installer/defaults/main.yml b/roles/installer/defaults/main.yml index b7f35f8ab..4197f554f 100644 --- a/roles/installer/defaults/main.yml +++ b/roles/installer/defaults/main.yml @@ -169,18 +169,18 @@ web_command: [] task_resource_requirements: requests: - cpu: 500m - memory: 1Gi + cpu: 100m + memory: 128Mi web_resource_requirements: requests: - cpu: 1000m - memory: 2Gi + cpu: 100m + memory: 128Mi ee_resource_requirements: requests: - cpu: 500m - memory: 1Gi + cpu: 100m + memory: 64Mi # Customize CSRF options csrf_cookie_secure: False @@ -228,11 +228,14 @@ postgres_tolerations: '' postgres_storage_requirements: requests: storage: 8Gi -postgres_init_container_resource_requirements: {} postgres_resource_requirements: requests: - cpu: 500m - memory: 1Gi + cpu: 10m + memory: 64Mi +postgres_init_container_resource_requirements: + requests: + cpu: 10m + memory: 64Mi # Assign a preexisting priority class to the postgres pod postgres_priority_class: '' postgres_data_path: '/var/lib/postgresql/data/pgdata' From dfa0f6d45e0ed08a92a1484115eae62b4ec5b2ef Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Wed, 20 Apr 2022 20:23:33 -0400 Subject: [PATCH 3/4] Add docs for priority classes & fix typo --- README.md | 19 +++++++++++++++++++ roles/installer/templates/deployment.yaml.j2 | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c88a011fe..fee0431b6 100644 --- a/README.md +++ b/README.md @@ -442,6 +442,7 @@ The following variables are customizable for the managed PostgreSQL service | postgres_storage_requirements | PostgreSQL container storage requirements | requests: {storage: 8Gi} | | postgres_storage_class | PostgreSQL PV storage class | Empty string | | postgres_data_path | PostgreSQL data path | `/var/lib/postgresql/data/pgdata` | +| postgres_priority_class | Priority class used for PostgreSQL pod | Empty string | Example of customization could be: @@ -576,6 +577,24 @@ spec: memory: 2Gi ``` +#### Priority Classes + +The AWX and Postgres pods can be assigned a custom PriorityClass to rank their importance compared to other pods in your cluster, which determines which pods get evicted first if resources are running low. +First, [create your PriorityClass](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#priorityclass) if needed. +Then set the name of your priority class to the control plane and postgres pods as shown below. + +```yaml +--- +apiVersion: awx.ansible.com/v1beta1 +kind: AWX +metadata: + name: awx-demo +spec: + ... + control_plane_priority_class: awx-demo-high-priority + postgres_priority_class: awx-demo-medium-priority +``` + #### Assigning AWX pods to specific nodes You can constrain the AWX pods created by the operator to run on a certain subset of nodes. `node_selector` and `postgres_selector` constrains diff --git a/roles/installer/templates/deployment.yaml.j2 b/roles/installer/templates/deployment.yaml.j2 index ff150a239..955e3ae38 100644 --- a/roles/installer/templates/deployment.yaml.j2 +++ b/roles/installer/templates/deployment.yaml.j2 @@ -42,7 +42,7 @@ spec: - name: {{ secret }} {% endfor %} {% endif %} -{% if control_plane_priorityclass is defined %} +{% if control_plane_priority_class is defined %} priorityClassName: '{{ control_plane_priority_class }}' {% endif %} initContainers: From daf15a93bf591132aa78dff9d2dede099cc2461c Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Thu, 21 Apr 2022 15:04:22 -0400 Subject: [PATCH 4/4] Reduce the resources requests for CI runs * GitHub Workflows run in a resource constrained environment, we were asking too much of it, so pods never got scheduled. --- config/crd/bases/awx.ansible.com_awxs.yaml | 2 +- molecule/default/templates/awx_cr_molecule.yml.j2 | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/config/crd/bases/awx.ansible.com_awxs.yaml b/config/crd/bases/awx.ansible.com_awxs.yaml index 6dbad17af..b4fe30929 100644 --- a/config/crd/bases/awx.ansible.com_awxs.yaml +++ b/config/crd/bases/awx.ansible.com_awxs.yaml @@ -187,7 +187,7 @@ spec: type: array items: type: string - image_pull_secret: # deprecated + image_pull_secret: # deprecated description: (Deprecated) Image pull secret for app and database containers type: string task_resource_requirements: diff --git a/molecule/default/templates/awx_cr_molecule.yml.j2 b/molecule/default/templates/awx_cr_molecule.yml.j2 index 2c19db8af..07df757a3 100644 --- a/molecule/default/templates/awx_cr_molecule.yml.j2 +++ b/molecule/default/templates/awx_cr_molecule.yml.j2 @@ -15,14 +15,15 @@ spec: kubernetes.io/ingress.class: nginx web_resource_requirements: requests: - cpu: 250m - memory: 128M + cpu: 100m + memory: 32M task_resource_requirements: requests: - cpu: 250m - memory: 128M + cpu: 100m + memory: 32M ee_resource_requirements: requests: cpu: 200m - memory: 64M + memory: 32M postgres_resource_requirements: {} + postgres_init_container_resource_requirements: {}