Skip to content

Commit

Permalink
Verify enough resources for breeze (#20763)
Browse files Browse the repository at this point in the history
Verify resources, memory, cpus and disk for Docker in Python.
  • Loading branch information
edithturn authored Jan 19, 2022
1 parent b8526ab commit 75755d7
Show file tree
Hide file tree
Showing 4 changed files with 112 additions and 72 deletions.
4 changes: 2 additions & 2 deletions dev/breeze/doc/BREEZE2.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ But first: Created a TOC for easy reference about BREEZE commands
- [Updated BREEZE](#updated-breeze)
- [BREEZE setting up autocomplete](#breeze-setting-up-autocomplete)
- [Breeze on Linux](#breeze-on-linux)
- [Breeze on Windows](#breeze-on-windows)
- [Breeze2 on Windows](#breeze2-on-windows)

<!-- END doctoc generated TOC please keep comment here to allow auto update -->

Expand Down Expand Up @@ -106,7 +106,7 @@ You should set up the autocomplete option automatically by running:

You get the auto-completion working when you re-enter the shell.

# Breeze on Windows
# Breeze2 on Windows

In Windows environment, you will need to use pipx to install Breeze.

Expand Down
2 changes: 1 addition & 1 deletion scripts/ci/libraries/_docker_engine_resources.sh
Original file line number Diff line number Diff line change
Expand Up @@ -46,5 +46,5 @@ function docker_engine_resources::check_all_resources() {
docker_v run -t "${EXTRA_DOCKER_FLAGS[@]}" \
--entrypoint "/bin/bash" \
"${AIRFLOW_CI_IMAGE_WITH_TAG}" \
-c "/opt/airflow/scripts/in_container/run_resource_check.sh"
-c "python /opt/airflow/scripts/in_container/run_resource_check.py"
}
109 changes: 109 additions & 0 deletions scripts/in_container/run_resource_check.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
#!/usr/bin/env python


# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.


from dataclasses import dataclass
from typing import Dict

import psutil
from rich.console import Console


@dataclass
class Resource:
current: int
minimumAllowed: int


console = Console(force_terminal=True, color_system="standard", width=180)


def get_size(bytes):
"""
Convert Bytes into Gigabytes
1 Gigabytes = 1024*1024*1024 = 1073741824 bytes
"""
factor = 1024 * 1024 * 1024
value_gb = bytes // factor
return value_gb


def resoure_check():
"""
Use gsutil to get resources in bytes for memory and disk
"""
MINIMUM_ALLOWED_MEMORY = 4
MINIMUM_ALLOWED_CPUS = 2
MINIMUM_ALLOWED_DISK = 20
print("\nChecking resources.\n")

# Memory current available
svmem = psutil.virtual_memory()
mem_available = get_size(svmem.available)

# Cpus current available
cpus_available = psutil.cpu_count(logical=True)

# Disk current available
partitions = psutil.disk_partitions()
partition_usage = psutil.disk_usage(partitions[0].mountpoint)
disk_available = get_size(partition_usage.free)

resources: Dict[str, Resource] = {
'Memory': Resource(current=mem_available, minimumAllowed=MINIMUM_ALLOWED_MEMORY),
'Cpus': Resource(current=cpus_available, minimumAllowed=MINIMUM_ALLOWED_CPUS),
'Disk': Resource(current=disk_available, minimumAllowed=MINIMUM_ALLOWED_DISK),
}
return resources


def resoure_validate():

resources = resoure_check()
warning_resources = False
check = "OK"

for resource, capacity in resources.items():

check = '' if resource == "Cpus" else 'GB'

if capacity.current < capacity.minimumAllowed:
console.print(f"[yellow]WARNING!!!: Not enough {resource} available for Docker.")
print(
f"At least {capacity.minimumAllowed}{check} of {resource} required. "
f" You have {capacity.current}{check}\n"
)
warning_resources = True
else:
console.print(f" * {resource} available {capacity.current}{check}. [green]OK.")

if warning_resources:
console.print("[yellow]WARNING!!!: You have not enough resources to run Airflow (see above)!")
print("Please follow the instructions to increase amount of resources available:")
console.print(
" Please check https://github.com/apache/airflow/blob/main/BREEZE.rst#resources-required"
" for details"
)
else:
console.print("\n[green]Resource check successful.\n")


if __name__ == "__main__":
resoure_validate()
69 changes: 0 additions & 69 deletions scripts/in_container/run_resource_check.sh

This file was deleted.

0 comments on commit 75755d7

Please sign in to comment.