From 75755d7f65fb06c6e2e74f805b877774bfa7fcda Mon Sep 17 00:00:00 2001 From: Edith Puclla <58795858+edithturn@users.noreply.github.com> Date: Wed, 19 Jan 2022 06:51:51 -0500 Subject: [PATCH] Verify enough resources for breeze (#20763) Verify resources, memory, cpus and disk for Docker in Python. --- dev/breeze/doc/BREEZE2.md | 4 +- .../ci/libraries/_docker_engine_resources.sh | 2 +- scripts/in_container/run_resource_check.py | 109 ++++++++++++++++++ scripts/in_container/run_resource_check.sh | 69 ----------- 4 files changed, 112 insertions(+), 72 deletions(-) create mode 100755 scripts/in_container/run_resource_check.py delete mode 100755 scripts/in_container/run_resource_check.sh diff --git a/dev/breeze/doc/BREEZE2.md b/dev/breeze/doc/BREEZE2.md index 98e5c65564b3e..19ae7b9735dd1 100644 --- a/dev/breeze/doc/BREEZE2.md +++ b/dev/breeze/doc/BREEZE2.md @@ -26,7 +26,7 @@ But first: Created a TOC for easy reference about BREEZE commands - [Updated BREEZE](#updated-breeze) - [BREEZE setting up autocomplete](#breeze-setting-up-autocomplete) - [Breeze on Linux](#breeze-on-linux) -- [Breeze on Windows](#breeze-on-windows) +- [Breeze2 on Windows](#breeze2-on-windows) @@ -106,7 +106,7 @@ You should set up the autocomplete option automatically by running: You get the auto-completion working when you re-enter the shell. -# Breeze on Windows +# Breeze2 on Windows In Windows environment, you will need to use pipx to install Breeze. diff --git a/scripts/ci/libraries/_docker_engine_resources.sh b/scripts/ci/libraries/_docker_engine_resources.sh index 75daf175e29f9..af836815f98cf 100644 --- a/scripts/ci/libraries/_docker_engine_resources.sh +++ b/scripts/ci/libraries/_docker_engine_resources.sh @@ -46,5 +46,5 @@ function docker_engine_resources::check_all_resources() { docker_v run -t "${EXTRA_DOCKER_FLAGS[@]}" \ --entrypoint "/bin/bash" \ "${AIRFLOW_CI_IMAGE_WITH_TAG}" \ - -c "/opt/airflow/scripts/in_container/run_resource_check.sh" + -c "python /opt/airflow/scripts/in_container/run_resource_check.py" } diff --git a/scripts/in_container/run_resource_check.py b/scripts/in_container/run_resource_check.py new file mode 100755 index 0000000000000..a8f59580b160f --- /dev/null +++ b/scripts/in_container/run_resource_check.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python + + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +from dataclasses import dataclass +from typing import Dict + +import psutil +from rich.console import Console + + +@dataclass +class Resource: + current: int + minimumAllowed: int + + +console = Console(force_terminal=True, color_system="standard", width=180) + + +def get_size(bytes): + """ + Convert Bytes into Gigabytes + 1 Gigabytes = 1024*1024*1024 = 1073741824 bytes + """ + factor = 1024 * 1024 * 1024 + value_gb = bytes // factor + return value_gb + + +def resoure_check(): + """ + Use gsutil to get resources in bytes for memory and disk + """ + MINIMUM_ALLOWED_MEMORY = 4 + MINIMUM_ALLOWED_CPUS = 2 + MINIMUM_ALLOWED_DISK = 20 + print("\nChecking resources.\n") + + # Memory current available + svmem = psutil.virtual_memory() + mem_available = get_size(svmem.available) + + # Cpus current available + cpus_available = psutil.cpu_count(logical=True) + + # Disk current available + partitions = psutil.disk_partitions() + partition_usage = psutil.disk_usage(partitions[0].mountpoint) + disk_available = get_size(partition_usage.free) + + resources: Dict[str, Resource] = { + 'Memory': Resource(current=mem_available, minimumAllowed=MINIMUM_ALLOWED_MEMORY), + 'Cpus': Resource(current=cpus_available, minimumAllowed=MINIMUM_ALLOWED_CPUS), + 'Disk': Resource(current=disk_available, minimumAllowed=MINIMUM_ALLOWED_DISK), + } + return resources + + +def resoure_validate(): + + resources = resoure_check() + warning_resources = False + check = "OK" + + for resource, capacity in resources.items(): + + check = '' if resource == "Cpus" else 'GB' + + if capacity.current < capacity.minimumAllowed: + console.print(f"[yellow]WARNING!!!: Not enough {resource} available for Docker.") + print( + f"At least {capacity.minimumAllowed}{check} of {resource} required. " + f" You have {capacity.current}{check}\n" + ) + warning_resources = True + else: + console.print(f" * {resource} available {capacity.current}{check}. [green]OK.") + + if warning_resources: + console.print("[yellow]WARNING!!!: You have not enough resources to run Airflow (see above)!") + print("Please follow the instructions to increase amount of resources available:") + console.print( + " Please check https://github.com/apache/airflow/blob/main/BREEZE.rst#resources-required" + " for details" + ) + else: + console.print("\n[green]Resource check successful.\n") + + +if __name__ == "__main__": + resoure_validate() diff --git a/scripts/in_container/run_resource_check.sh b/scripts/in_container/run_resource_check.sh deleted file mode 100755 index 584af350d324b..0000000000000 --- a/scripts/in_container/run_resource_check.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# shellcheck source=scripts/in_container/_in_container_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_in_container_script_init.sh" - -function resource_check() { - local one_meg=1048576 - local mem_available - local cpus_available - local disk_available - local warning_resources="false" - echo - echo "Checking resources." - echo - mem_available=$(($(getconf _PHYS_PAGES) * $(getconf PAGE_SIZE) / one_meg)) - cpus_available=$(grep -cE 'cpu[0-9]+' /proc/stat) - disk_available=$(df / | tail -1 | awk '{print $4}') - human_readable_memory=$(numfmt --to iec $((mem_available * one_meg))) - human_readable_disk=$(numfmt --to iec $((disk_available * 1024 ))) - if (( mem_available < 4000 )) ; then - echo "${COLOR_YELLOW}WARNING!!!: Not enough memory available for Docker.${COLOR_RESET}" - echo "At least 4GB of memory required. You have ${human_readable_memory}" - warning_resources="true" - else - echo "* Memory available ${human_readable_memory}. ${COLOR_GREEN}OK.${COLOR_RESET}" - fi - if (( cpus_available < 2 )); then - echo "${COLOR_YELLOW}WARNING!!!: Not enough CPUS available for Docker.${COLOR_RESET}" - echo "At least 2 CPUs recommended. You have ${cpus_available}" - warning_resources="true" - else - echo "* CPUs available ${cpus_available}. ${COLOR_GREEN}OK.${COLOR_RESET}" - fi - if (( disk_available < one_meg*20 )); then - echo "${COLOR_YELLOW}WARNING!!!: Not enough Disk space available for Docker.${COLOR_RESET}" - echo "At least 20 GBs recommended. You have ${human_readable_disk}" - warning_resources="true" - else - echo "* Disk available ${human_readable_disk}. ${COLOR_GREEN}OK.${COLOR_RESET}" - fi - if [[ ${warning_resources} == "true" ]]; then - echo - echo "${COLOR_YELLOW}WARNING!!!: You have not enough resources to run Airflow (see above)!${COLOR_RESET}" - echo "Please follow the instructions to increase amount of resources available:" - echo " Please check https://github.com/apache/airflow/blob/main/BREEZE.rst#resources-required for details" - echo - else - echo - echo "${COLOR_GREEN}Resource check successful.${COLOR_RESET}" - echo - fi -} - -resource_check