From 33cf4b3b53715980a71b7375e182a0b50e90739f Mon Sep 17 00:00:00 2001 From: caetano melone Date: Tue, 23 Jan 2024 22:09:26 -0800 Subject: [PATCH 01/37] first prediction functionality --- gantry/prediction.py | 86 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 gantry/prediction.py diff --git a/gantry/prediction.py b/gantry/prediction.py new file mode 100644 index 0000000..36993e3 --- /dev/null +++ b/gantry/prediction.py @@ -0,0 +1,86 @@ +# don't include retried builds in prediction bc of bias? + +# - [ ] only allow upper bounds for now +# - [ ] weigh the most current build higher because of the retried jobs recursion thing +# - [ ] is there a need to set up some sort of cache? +# - [ ] like cache the calculation...hash the combo of parameters and store the prediction in a db table rather than doing calculation on the fly +# - store the current bounds static and make sure you don't go below? how would that work with the build size param...will it still exist? + + +import aiosqlite + +MIN_TRAIN_SAMPLE = 4 + + +async def select_sample(db: aiosqlite.Connection, build: dict) -> list: + """ + Selects a sample of builds to use for prediction + + args: + build: dict that must contain pkg name, pkg version, compiler, compiler version + returns: + list of lists with cpu_mean, cpu_max, mem_mean, mem_max + """ + + param_combos = [ + ("pkg_name", "pkg_version", "compiler_name", "compiler_version"), + ("pkg_name", "compiler_name", "compiler_version"), + ("pkg_name", "pkg_version", "compiler_name"), + ("pkg_name", "compiler_name"), + ("pkg_name", "pkg_version"), + ("pkg_name"), + ] + + for combo in param_combos: + condition_values = [build[param] for param in combo] + query = f"SELECT cpu_mean, cpu_max, mem_mean, mem_max FROM builds WHERE ref='develop' AND {' AND '.join(f'{param}=?' for param in combo)} ORDER BY end DESC LIMIT 5" + async with db.execute(query, condition_values) as cursor: + builds = await cursor.fetchall() + if len(builds) >= MIN_TRAIN_SAMPLE: + return builds + + return [] + + +async def predict_resource_usage(db: aiosqlite.Connection, build: dict) -> dict: + """ + Predict the resource usage of a build + + args: + build: dict that must contain pkg name, pkg version, compiler, compiler version + returns: + dict of predicted resource usage: cpu_request, cpu_limit, mem_request, mem_limit. CPU in cores, mem in MB + """ + + sample = await sample(db, build) + if not sample: + return { + "cpu_request": 1.0, + "cpu_limit": None, + "mem_request": 2000, + "mem_limit": None, + } + + # mapping of sample: [0] cpu_mean, [1] cpu_max, [2] mem_mean, [3] mem_max + return { + # averages the respective metric in the sample + "cpu_request": sum([build[0] for build in sample]) / len(sample), + "cpu_limit": None, + # "cpu_limit": sum([build[1] for build in sample]) / len(sample), + "mem_request": sum([build[2] for build in sample]) / len(sample), + "mem_limit": None, + # "mem_limit": sum([build[3] for build in sample]) / len(sample), + } + + +async def handle_bulk(db: aiosqlite.Connection, builds: list) -> list: + """ + Handles a bulk request of builds + + args: + builds: list of dicts that must contain pkg name, pkg version, compiler, compiler version + returns: + list of dicts with predicted resource usage: cpu_request, cpu_limit, mem_request, mem_limit. CPU in cores, mem in MB + """ + + return [await predict_resource_usage(db, build) for build in builds] \ No newline at end of file From 24cfb6901737ba5375e009530840897c5302a36f Mon Sep 17 00:00:00 2001 From: caetano melone Date: Mon, 29 Jan 2024 21:14:42 -0800 Subject: [PATCH 02/37] 1 --- gantry/prediction.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/gantry/prediction.py b/gantry/prediction.py index 36993e3..cc343d2 100644 --- a/gantry/prediction.py +++ b/gantry/prediction.py @@ -1,10 +1,9 @@ -# don't include retried builds in prediction bc of bias? - # - [ ] only allow upper bounds for now -# - [ ] weigh the most current build higher because of the retried jobs recursion thing -# - [ ] is there a need to set up some sort of cache? -# - [ ] like cache the calculation...hash the combo of parameters and store the prediction in a db table rather than doing calculation on the fly # - store the current bounds static and make sure you don't go below? how would that work with the build size param...will it still exist? +# TODO take in the hash and make that the unique identifier +# TODO make them generic...what happens when they're missing +# TODO make this compatible with the API... +# batch vs single import aiosqlite @@ -33,7 +32,7 @@ async def select_sample(db: aiosqlite.Connection, build: dict) -> list: for combo in param_combos: condition_values = [build[param] for param in combo] - query = f"SELECT cpu_mean, cpu_max, mem_mean, mem_max FROM builds WHERE ref='develop' AND {' AND '.join(f'{param}=?' for param in combo)} ORDER BY end DESC LIMIT 5" + query = f"SELECT cpu_mean, cpu_max, mem_mean, mem_max FROM builds WHERE ref='develop' AND {' AND '.join(f'{param}=?' for param in combo)} ORDER BY end DESC LIMIT {MIN_TRAIN_SAMPLE + 1}" async with db.execute(query, condition_values) as cursor: builds = await cursor.fetchall() if len(builds) >= MIN_TRAIN_SAMPLE: From 8421602599a9c52cd543beb148c096044567d921 Mon Sep 17 00:00:00 2001 From: caetano melone Date: Mon, 29 Jan 2024 22:01:19 -0800 Subject: [PATCH 03/37] add basic routing --- gantry/{ => routes}/prediction.py | 40 ++++++++-------- gantry/views.py | 77 +++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 21 deletions(-) rename gantry/{ => routes}/prediction.py (65%) diff --git a/gantry/prediction.py b/gantry/routes/prediction.py similarity index 65% rename from gantry/prediction.py rename to gantry/routes/prediction.py index cc343d2..4cbc81f 100644 --- a/gantry/prediction.py +++ b/gantry/routes/prediction.py @@ -1,14 +1,13 @@ # - [ ] only allow upper bounds for now # - store the current bounds static and make sure you don't go below? how would that work with the build size param...will it still exist? -# TODO take in the hash and make that the unique identifier -# TODO make them generic...what happens when they're missing -# TODO make this compatible with the API... -# batch vs single +# should we match kubernetes units? do we want to make it generic? in the prediction api MR import aiosqlite MIN_TRAIN_SAMPLE = 4 +DEFAULT_CPU_REQUEST = 1.0 +DEFAULT_MEM_REQUEST = 2000 async def select_sample(db: aiosqlite.Connection, build: dict) -> list: @@ -41,7 +40,7 @@ async def select_sample(db: aiosqlite.Connection, build: dict) -> list: return [] -async def predict_resource_usage(db: aiosqlite.Connection, build: dict) -> dict: +async def predict_single(db: aiosqlite.Connection, build: dict) -> dict: """ Predict the resource usage of a build @@ -53,26 +52,25 @@ async def predict_resource_usage(db: aiosqlite.Connection, build: dict) -> dict: sample = await sample(db, build) if not sample: - return { - "cpu_request": 1.0, - "cpu_limit": None, - "mem_request": 2000, - "mem_limit": None, + vars = { + "cpu_request": DEFAULT_CPU_REQUEST, + "mem_request": DEFAULT_MEM_REQUEST, } - - # mapping of sample: [0] cpu_mean, [1] cpu_max, [2] mem_mean, [3] mem_max + else: + # mapping of sample: [0] cpu_mean, [1] cpu_max, [2] mem_mean, [3] mem_max + vars = { + # averages the respective metric in the sample + "cpu_request": sum([build[0] for build in sample]) / len(sample), + "mem_request": sum([build[2] for build in sample]) / len(sample), + } + return { - # averages the respective metric in the sample - "cpu_request": sum([build[0] for build in sample]) / len(sample), - "cpu_limit": None, - # "cpu_limit": sum([build[1] for build in sample]) / len(sample), - "mem_request": sum([build[2] for build in sample]) / len(sample), - "mem_limit": None, - # "mem_limit": sum([build[3] for build in sample]) / len(sample), + "hash": build["hash"], + "variables": vars, } -async def handle_bulk(db: aiosqlite.Connection, builds: list) -> list: +async def predict_bulk(db: aiosqlite.Connection, builds: list) -> list: """ Handles a bulk request of builds @@ -82,4 +80,4 @@ async def handle_bulk(db: aiosqlite.Connection, builds: list) -> list: list of dicts with predicted resource usage: cpu_request, cpu_limit, mem_request, mem_limit. CPU in cores, mem in MB """ - return [await predict_resource_usage(db, build) for build in builds] \ No newline at end of file + return [await predict_single(db, build) for build in builds] \ No newline at end of file diff --git a/gantry/views.py b/gantry/views.py index 1981e3b..d436c22 100644 --- a/gantry/views.py +++ b/gantry/views.py @@ -6,6 +6,7 @@ from aiohttp import web from gantry.routes.collection import fetch_job +from gantry.routes.prediction import predict_single, predict_bulk logger = logging.getLogger(__name__) routes = web.RouteTableDef() @@ -35,3 +36,79 @@ async def collect_job(request: web.Request) -> web.Response: ) return web.Response(status=200) + + +@routes.get("/v1/resource-mapping") +async def map_resources(request: web.Request) -> web.Response: + """ + acceptable payload: + + { + "hash": "string", + "package": { + "name": "string", + "version": "string" + }, + "compiler": { + "name": "string", + "version": "string" + } + } + + also accepts a list of the above objects + + returns: + + { + "hash": "string", + "variables": { + "cpu_request": "float", + "mem_request": "float", + } + } + + or a list of the above objects + + """ + + try: + payload = await request.json() + except json.decoder.JSONDecodeError: + return web.Response(status=400, text="invalid json") + + def validate_payload(payload): + """ensures that the payload is valid""" + if not isinstance(payload, list): + payload = [payload] + + for item in payload: + required_fields = ["hash", "package", "compiler"] + for field in required_fields: + if not isinstance(item.get(field), dict): + return False + + if not all(isinstance(item.get("package", {}).get(key), str) for key in ["name", "version"]): + return False + + if not all(isinstance(item.get("compiler", {}).get(key), str) for key in ["name", "version"]): + return False + + return True + + if not validate_payload(payload): + return web.Response(status=400, text="invalid payload") + + if isinstance(payload, dict): + response = await predict_single(request.app["db"], payload) + else: + response = await predict_bulk(request.app["db"], payload) + + return web.json_response(response) + + + + + + + + From e7bcc9f4f70c9f73eed483b257ab1f2b4b42c9b0 Mon Sep 17 00:00:00 2001 From: caetano melone Date: Tue, 30 Jan 2024 16:36:24 -0800 Subject: [PATCH 04/37] check whether predictions are lower than current allocation --- gantry/routes/prediction/__init__.py | 0 gantry/routes/prediction/current_mapping.py | 2 + gantry/routes/{ => prediction}/prediction.py | 65 +++++++++++++++----- gantry/views.py | 28 ++++----- 4 files changed, 63 insertions(+), 32 deletions(-) create mode 100644 gantry/routes/prediction/__init__.py create mode 100644 gantry/routes/prediction/current_mapping.py rename gantry/routes/{ => prediction}/prediction.py (53%) diff --git a/gantry/routes/prediction/__init__.py b/gantry/routes/prediction/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gantry/routes/prediction/current_mapping.py b/gantry/routes/prediction/current_mapping.py new file mode 100644 index 0000000..b7a4cf3 --- /dev/null +++ b/gantry/routes/prediction/current_mapping.py @@ -0,0 +1,2 @@ +# fmt: off +pkg_mappings = {'rust': {'build_jobs': 12, 'cpu_request': 12000.0, 'mem_request': 35000.0}, 'py-tensorflow': {'build_jobs': 12, 'cpu_request': 12000.0, 'mem_request': 32000.0}, 'py-jaxlib': {'build_jobs': 12, 'cpu_request': 12000.0, 'mem_request': 29000.0}, 'nvhpc': {'build_jobs': 12, 'cpu_request': 12000.0, 'mem_request': 24000.0}, 'paraview': {'build_jobs': 12, 'cpu_request': 12000.0, 'mem_request': 24000.0}, 'llvm': {'build_jobs': 12, 'cpu_request': 12000.0, 'mem_request': 21000.0}, 'py-torch': {'build_jobs': 12, 'cpu_request': 12000.0, 'mem_request': 21000.0}, 'dealii': {'build_jobs': 12, 'cpu_request': 12000.0, 'mem_request': 19000.0}, 'mxnet': {'build_jobs': 12, 'cpu_request': 12000.0, 'mem_request': 19000.0}, 'py-torchaudio': {'build_jobs': 12, 'cpu_request': 12000.0, 'mem_request': 19000.0}, 'rocblas': {'build_jobs': 12, 'cpu_request': 12000.0, 'mem_request': 19000.0}, 'ecp-data-vis-sdk': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'intel-tbb': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 15000.0}, 'llvm-amdgpu': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 15000.0}, 'salmon': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 15000.0}, 'trilinos': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'ascent': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'atk': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'axom': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'cistem': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'cmake': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'ctffind': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'cuda': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'dray': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'dyninst': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'gcc': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'ginkgo': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'hdf5': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'hipblas': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'hpx': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'kokkos-kernels': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'kokkos-nvcc-wrapper': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'lbann': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'magma': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'mesa': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'mfem': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'mpich': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'netlib-lapack': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'oce': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'openblas': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'openfoam': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'openturns': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'parallelio': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'plumed': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'precice': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'qt': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'raja': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'relion': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'rocfft': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'rocsolver': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'rocsparse': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'slate': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'strumpack': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'sundials': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'umpire': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'visit': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'vtk': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'vtk-h': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'vtk-m': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'warpx': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'wrf': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'wxwidgets': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'adios2': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'amrex': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'archer': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'autoconf-archive': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'binutils': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'blaspp': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'blt': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'boost': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'butterflypack': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'cabana': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'caliper': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'camp': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'chai': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'conduit': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'curl': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'datatransferkit': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'double-conversion': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'eigen': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'faodel': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'ffmpeg': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'fftw': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'fortrilinos': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'gettext': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'gperftools': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'gptune': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'heffte': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'hpctoolkit': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'hwloc': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'hydrogen': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'hypre': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'kokkos': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'lammps': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'lapackpp': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'legion': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'libtool': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'libxml2': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'libzmq': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'llvm-openmp-ompt': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'mbedtls': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'mvapich2': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'nasm': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'netlib-scalapack': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'omega-h': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'openjpeg': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'openmpi': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'openpmd-api': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'pagmo2': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'papyrus': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'parsec': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'pdt': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'pegtl': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'petsc': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'pumi': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'py-beniget': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'py-cinemasci': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'pygmo': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'py-ipython-genutils': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'py-packaging': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'py-petsc4py': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'py-scipy': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'py-statsmodels': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'py-warlock': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'py-warpx': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'samrai': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'slepc': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'slurm': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'sqlite': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'superlu-dist': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'tasmanian': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'tau': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'upcxx': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'zfp': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'alsa-lib': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'ant': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'antlr': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'argobots': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'automake': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'berkeley-db': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'bison': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'bzip2': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'czmq': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'darshan-util': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'diffutils': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'docbook-xml': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'exmcutils': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'expat': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'findutils': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'flit': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'freetype': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'gawk': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'gdbm': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'glib': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'gmake': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'gotcha': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'hpcviewer': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'jansson': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'json-c': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'libbsd': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'libedit': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'libevent': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'libfabric': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'libffi': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'libgcrypt': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'libiconv': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'libidn2': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'libjpeg-turbo': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'libmd': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'libnrm': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'libpciaccess': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'libpng': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'libsigsegv': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'libsodium': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'libunistring': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'libunwind': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'libyaml': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'lua': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'lua-luaposix': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'lz4': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'm4': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'meson': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'metis': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'mpfr': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'ncurses': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'ninja': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'numactl': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'openjdk': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'openssh': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'openssl': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'papi': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'parallel-netcdf': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'pcre': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'pcre2': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'pdsh': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'perl': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'perl-data-dumper': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'pkgconf': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'py-alembic': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'py-cffi': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'py-cycler': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'py-decorator': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'py-idna': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'py-jsonschema': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'py-kiwisolver': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'py-mistune': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'py-pycparser': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'py-setuptools': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'py-setuptools-scm': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'py-six': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'py-testpath': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'py-wheel': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'qhull': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'readline': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'sed': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'snappy': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'superlu': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'swig': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'tar': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'tcl': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'texinfo': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'tut': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'unzip': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'util-linux-uuid': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'util-macros': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'xz': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'yaml-cpp': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'zlib': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'zstd': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}} diff --git a/gantry/routes/prediction.py b/gantry/routes/prediction/prediction.py similarity index 53% rename from gantry/routes/prediction.py rename to gantry/routes/prediction/prediction.py index 4cbc81f..2d1b44e 100644 --- a/gantry/routes/prediction.py +++ b/gantry/routes/prediction/prediction.py @@ -1,15 +1,42 @@ -# - [ ] only allow upper bounds for now -# - store the current bounds static and make sure you don't go below? how would that work with the build size param...will it still exist? -# should we match kubernetes units? do we want to make it generic? in the prediction api MR - - import aiosqlite +from gantry.routes.prediction.current_mapping import pkg_mappings + MIN_TRAIN_SAMPLE = 4 -DEFAULT_CPU_REQUEST = 1.0 +DEFAULT_CPU_REQUEST = 1000 DEFAULT_MEM_REQUEST = 2000 +def preprocess_pred(prediction, pkg_name): + """ + Main goal of this function is to ensure that our + prediction is not lower than the current allocation + for that package. This restriction will likely be + removed in the future as we understand the effectiveness + of the prediction model. + """ + + cur_alloc = pkg_mappings.get(pkg_name) + + if cur_alloc: + prediction["variables"]["cpu_request"] = max( + prediction["variables"]["cpu_request"], cur_alloc["cpu_request"] + ) + prediction["variables"]["mem_request"] = max( + prediction["variables"]["mem_request"], cur_alloc["mem_request"] + ) + + # put into k8s units and should not be a float + prediction["variables"][ + "cpu_request" + ] = f"{int(prediction['variables']['cpu_request'])}m" + prediction["variables"][ + "mem_request" + ] = f"{int(prediction['variables']['mem_request'])}M" + + return prediction + + async def select_sample(db: aiosqlite.Connection, build: dict) -> list: """ Selects a sample of builds to use for prediction @@ -47,37 +74,41 @@ async def predict_single(db: aiosqlite.Connection, build: dict) -> dict: args: build: dict that must contain pkg name, pkg version, compiler, compiler version returns: - dict of predicted resource usage: cpu_request, cpu_limit, mem_request, mem_limit. CPU in cores, mem in MB + dict of predicted resource usage: cpu_request, mem_request + CPU in millicore, mem in MB """ - sample = await sample(db, build) + sample = await select_sample(db, build) if not sample: vars = { - "cpu_request": DEFAULT_CPU_REQUEST, - "mem_request": DEFAULT_MEM_REQUEST, + "KUBERNETES_CPU_REQUEST": DEFAULT_CPU_REQUEST, + "KUBERNETES_MEMORY_REQUEST": DEFAULT_MEM_REQUEST, } else: # mapping of sample: [0] cpu_mean, [1] cpu_max, [2] mem_mean, [3] mem_max vars = { # averages the respective metric in the sample - "cpu_request": sum([build[0] for build in sample]) / len(sample), - "mem_request": sum([build[2] for build in sample]) / len(sample), + "KUBERNETES_CPU_REQUEST": sum([build[0] for build in sample]) / len(sample), + "KUBERNETES_MEMORY_REQUEST": ( + sum([build[2] for build in sample]) / len(sample) + ), } - return { + pred = { "hash": build["hash"], "variables": vars, } + return preprocess_pred(pred, build["pkg_name"]) + async def predict_bulk(db: aiosqlite.Connection, builds: list) -> list: """ Handles a bulk request of builds args: - builds: list of dicts that must contain pkg name, pkg version, compiler, compiler version - returns: - list of dicts with predicted resource usage: cpu_request, cpu_limit, mem_request, mem_limit. CPU in cores, mem in MB + builds: list of dicts (see predict_single) + returns: see predict_single) """ - return [await predict_single(db, build) for build in builds] \ No newline at end of file + return [await predict_single(db, build) for build in builds] diff --git a/gantry/views.py b/gantry/views.py index d436c22..918b012 100644 --- a/gantry/views.py +++ b/gantry/views.py @@ -6,7 +6,7 @@ from aiohttp import web from gantry.routes.collection import fetch_job -from gantry.routes.prediction import predict_single, predict_bulk +from gantry.routes.prediction.prediction import predict_bulk, predict_single logger = logging.getLogger(__name__) routes = web.RouteTableDef() @@ -68,14 +68,14 @@ async def map_resources(request: web.Request) -> web.Response: } or a list of the above objects - + """ try: payload = await request.json() except json.decoder.JSONDecodeError: return web.Response(status=400, text="invalid json") - + def validate_payload(payload): """ensures that the payload is valid""" if not isinstance(payload, list): @@ -87,28 +87,26 @@ def validate_payload(payload): if not isinstance(item.get(field), dict): return False - if not all(isinstance(item.get("package", {}).get(key), str) for key in ["name", "version"]): + if not all( + isinstance(item.get("package", {}).get(key), str) + for key in ["name", "version"] + ): return False - if not all(isinstance(item.get("compiler", {}).get(key), str) for key in ["name", "version"]): + if not all( + isinstance(item.get("compiler", {}).get(key), str) + for key in ["name", "version"] + ): return False return True - + if not validate_payload(payload): return web.Response(status=400, text="invalid payload") - + if isinstance(payload, dict): response = await predict_single(request.app["db"], payload) else: response = await predict_bulk(request.app["db"], payload) return web.json_response(response) - - - - - - - - From c2da0e06d7e69f15aefbad0d63c59c7822c5f7df Mon Sep 17 00:00:00 2001 From: caetano melone Date: Tue, 30 Jan 2024 22:27:36 -0800 Subject: [PATCH 05/37] adjust conversions --- gantry/routes/prediction/prediction.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/gantry/routes/prediction/prediction.py b/gantry/routes/prediction/prediction.py index 2d1b44e..2101ec7 100644 --- a/gantry/routes/prediction/prediction.py +++ b/gantry/routes/prediction/prediction.py @@ -6,6 +6,9 @@ DEFAULT_CPU_REQUEST = 1000 DEFAULT_MEM_REQUEST = 2000 +CORES_TO_MILLICORES = 1000 +BYTES_TO_MEGABYTES = 1/1_000_000 + def preprocess_pred(prediction, pkg_name): """ @@ -29,10 +32,10 @@ def preprocess_pred(prediction, pkg_name): # put into k8s units and should not be a float prediction["variables"][ "cpu_request" - ] = f"{int(prediction['variables']['cpu_request'])}m" + ] = f"{int(prediction['variables']['cpu_request'] * CORES_TO_MILLICORES)}m" prediction["variables"][ "mem_request" - ] = f"{int(prediction['variables']['mem_request'])}M" + ] = f"{int(prediction['variables']['mem_request'] * BYTES_TO_MEGABYTES)}M" return prediction From 169fab5ab3dca52d6b2482ae2a843004e6a1e9dd Mon Sep 17 00:00:00 2001 From: caetano melone Date: Tue, 30 Jan 2024 23:27:10 -0800 Subject: [PATCH 06/37] fix some bugs --- gantry/routes/prediction/current_mapping.py | 301 +++++++++++++++++++- gantry/routes/prediction/prediction.py | 74 +++-- gantry/views.py | 7 +- 3 files changed, 355 insertions(+), 27 deletions(-) diff --git a/gantry/routes/prediction/current_mapping.py b/gantry/routes/prediction/current_mapping.py index b7a4cf3..54f1544 100644 --- a/gantry/routes/prediction/current_mapping.py +++ b/gantry/routes/prediction/current_mapping.py @@ -1,2 +1,299 @@ -# fmt: off -pkg_mappings = {'rust': {'build_jobs': 12, 'cpu_request': 12000.0, 'mem_request': 35000.0}, 'py-tensorflow': {'build_jobs': 12, 'cpu_request': 12000.0, 'mem_request': 32000.0}, 'py-jaxlib': {'build_jobs': 12, 'cpu_request': 12000.0, 'mem_request': 29000.0}, 'nvhpc': {'build_jobs': 12, 'cpu_request': 12000.0, 'mem_request': 24000.0}, 'paraview': {'build_jobs': 12, 'cpu_request': 12000.0, 'mem_request': 24000.0}, 'llvm': {'build_jobs': 12, 'cpu_request': 12000.0, 'mem_request': 21000.0}, 'py-torch': {'build_jobs': 12, 'cpu_request': 12000.0, 'mem_request': 21000.0}, 'dealii': {'build_jobs': 12, 'cpu_request': 12000.0, 'mem_request': 19000.0}, 'mxnet': {'build_jobs': 12, 'cpu_request': 12000.0, 'mem_request': 19000.0}, 'py-torchaudio': {'build_jobs': 12, 'cpu_request': 12000.0, 'mem_request': 19000.0}, 'rocblas': {'build_jobs': 12, 'cpu_request': 12000.0, 'mem_request': 19000.0}, 'ecp-data-vis-sdk': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'intel-tbb': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 15000.0}, 'llvm-amdgpu': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 15000.0}, 'salmon': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 15000.0}, 'trilinos': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'ascent': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'atk': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'axom': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'cistem': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'cmake': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'ctffind': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'cuda': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'dray': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'dyninst': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'gcc': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'ginkgo': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'hdf5': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'hipblas': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'hpx': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'kokkos-kernels': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'kokkos-nvcc-wrapper': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'lbann': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'magma': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'mesa': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'mfem': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'mpich': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'netlib-lapack': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'oce': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'openblas': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'openfoam': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'openturns': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'parallelio': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'plumed': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'precice': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'qt': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'raja': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'relion': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'rocfft': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'rocsolver': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'rocsparse': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'slate': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'strumpack': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'sundials': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'umpire': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'visit': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'vtk': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'vtk-h': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'vtk-m': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'warpx': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'wrf': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'wxwidgets': {'build_jobs': 8, 'cpu_request': 8000.0, 'mem_request': 12000.0}, 'adios2': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'amrex': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'archer': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'autoconf-archive': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'binutils': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'blaspp': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'blt': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'boost': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'butterflypack': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'cabana': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'caliper': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'camp': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'chai': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'conduit': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'curl': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'datatransferkit': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'double-conversion': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'eigen': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'faodel': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'ffmpeg': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'fftw': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'fortrilinos': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'gettext': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'gperftools': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'gptune': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'heffte': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'hpctoolkit': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'hwloc': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'hydrogen': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'hypre': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'kokkos': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'lammps': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'lapackpp': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'legion': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'libtool': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'libxml2': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'libzmq': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'llvm-openmp-ompt': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'mbedtls': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'mvapich2': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'nasm': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'netlib-scalapack': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'omega-h': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'openjpeg': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'openmpi': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'openpmd-api': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'pagmo2': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'papyrus': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'parsec': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'pdt': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'pegtl': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'petsc': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'pumi': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'py-beniget': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'py-cinemasci': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'pygmo': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'py-ipython-genutils': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'py-packaging': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'py-petsc4py': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'py-scipy': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'py-statsmodels': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'py-warlock': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'py-warpx': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'samrai': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'slepc': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'slurm': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'sqlite': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'superlu-dist': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'tasmanian': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'tau': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'upcxx': {'build_jobs': 2, 'cpu_request': 2000.0, 'mem_request': 4000.0}, 'zfp': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'alsa-lib': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'ant': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'antlr': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'argobots': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'automake': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'berkeley-db': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'bison': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'bzip2': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'czmq': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'darshan-util': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'diffutils': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'docbook-xml': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'exmcutils': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'expat': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'findutils': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'flit': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'freetype': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'gawk': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'gdbm': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'glib': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'gmake': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'gotcha': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'hpcviewer': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'jansson': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'json-c': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'libbsd': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'libedit': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'libevent': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'libfabric': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'libffi': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'libgcrypt': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'libiconv': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'libidn2': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'libjpeg-turbo': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'libmd': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'libnrm': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'libpciaccess': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'libpng': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'libsigsegv': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'libsodium': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'libunistring': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'libunwind': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'libyaml': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'lua': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'lua-luaposix': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'lz4': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'm4': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'meson': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'metis': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'mpfr': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'ncurses': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'ninja': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'numactl': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'openjdk': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'openssh': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'openssl': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'papi': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'parallel-netcdf': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'pcre': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'pcre2': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'pdsh': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'perl': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'perl-data-dumper': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'pkgconf': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'py-alembic': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'py-cffi': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'py-cycler': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'py-decorator': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'py-idna': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'py-jsonschema': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'py-kiwisolver': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'py-mistune': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'py-pycparser': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'py-setuptools': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'py-setuptools-scm': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'py-six': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'py-testpath': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'py-wheel': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'qhull': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'readline': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'sed': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'snappy': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'superlu': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'swig': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'tar': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'tcl': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'texinfo': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'tut': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'unzip': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'util-linux-uuid': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'util-macros': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'xz': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'yaml-cpp': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'zlib': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}, 'zstd': {'build_jobs': 1, 'cpu_request': 500.0, 'mem_request': 500.0}} +pkg_mappings = { + "rust": {"build_jobs": 12, "cpu_request": 12.0, "mem_request": 35000000000.0}, + "py-tensorflow": { + "build_jobs": 12, + "cpu_request": 12.0, + "mem_request": 32000000000.0, + }, + "py-jaxlib": {"build_jobs": 12, "cpu_request": 12.0, "mem_request": 29000000000.0}, + "nvhpc": {"build_jobs": 12, "cpu_request": 12.0, "mem_request": 24000000000.0}, + "paraview": {"build_jobs": 12, "cpu_request": 12.0, "mem_request": 24000000000.0}, + "llvm": {"build_jobs": 12, "cpu_request": 12.0, "mem_request": 21000000000.0}, + "py-torch": {"build_jobs": 12, "cpu_request": 12.0, "mem_request": 21000000000.0}, + "dealii": {"build_jobs": 12, "cpu_request": 12.0, "mem_request": 19000000000.0}, + "mxnet": {"build_jobs": 12, "cpu_request": 12.0, "mem_request": 19000000000.0}, + "py-torchaudio": { + "build_jobs": 12, + "cpu_request": 12.0, + "mem_request": 19000000000.0, + }, + "rocblas": {"build_jobs": 12, "cpu_request": 12.0, "mem_request": 19000000000.0}, + "ecp-data-vis-sdk": { + "build_jobs": 8, + "cpu_request": 8.0, + "mem_request": 12000000000.0, + }, + "intel-tbb": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 15000000000.0}, + "llvm-amdgpu": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 15000000000.0}, + "salmon": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 15000000000.0}, + "trilinos": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, + "ascent": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "atk": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, + "axom": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "cistem": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, + "cmake": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, + "ctffind": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, + "cuda": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, + "dray": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "dyninst": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, + "gcc": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, + "ginkgo": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, + "hdf5": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "hipblas": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, + "hpx": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, + "kokkos-kernels": { + "build_jobs": 8, + "cpu_request": 8.0, + "mem_request": 12000000000.0, + }, + "kokkos-nvcc-wrapper": { + "build_jobs": 8, + "cpu_request": 8.0, + "mem_request": 12000000000.0, + }, + "lbann": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, + "magma": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, + "mesa": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, + "mfem": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "mpich": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "netlib-lapack": { + "build_jobs": 8, + "cpu_request": 8.0, + "mem_request": 12000000000.0, + }, + "oce": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, + "openblas": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "openfoam": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, + "openturns": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, + "parallelio": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, + "plumed": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, + "precice": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, + "qt": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, + "raja": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "relion": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, + "rocfft": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, + "rocsolver": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, + "rocsparse": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, + "slate": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, + "strumpack": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "sundials": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "umpire": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, + "visit": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, + "vtk": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "vtk-h": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "vtk-m": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "warpx": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, + "wrf": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, + "wxwidgets": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, + "adios2": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "amrex": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "archer": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "autoconf-archive": { + "build_jobs": 1, + "cpu_request": 0.5, + "mem_request": 500000000.0, + }, + "binutils": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "blaspp": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "blt": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "boost": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "butterflypack": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "cabana": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "caliper": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "camp": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "chai": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "conduit": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "curl": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "datatransferkit": { + "build_jobs": 2, + "cpu_request": 2.0, + "mem_request": 4000000000.0, + }, + "double-conversion": { + "build_jobs": 2, + "cpu_request": 2.0, + "mem_request": 4000000000.0, + }, + "eigen": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "faodel": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "ffmpeg": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "fftw": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "fortrilinos": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "gettext": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "gperftools": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "gptune": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "heffte": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "hpctoolkit": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "hwloc": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "hydrogen": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "hypre": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "kokkos": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "lammps": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "lapackpp": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "legion": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "libtool": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "libxml2": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "libzmq": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "llvm-openmp-ompt": { + "build_jobs": 2, + "cpu_request": 2.0, + "mem_request": 4000000000.0, + }, + "mbedtls": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "mvapich2": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "nasm": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "netlib-scalapack": { + "build_jobs": 2, + "cpu_request": 2.0, + "mem_request": 4000000000.0, + }, + "omega-h": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "openjpeg": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "openmpi": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "openpmd-api": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "pagmo2": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "papyrus": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "parsec": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "pdt": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "pegtl": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "petsc": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "pumi": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "py-beniget": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "py-cinemasci": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "pygmo": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "py-ipython-genutils": { + "build_jobs": 2, + "cpu_request": 2.0, + "mem_request": 4000000000.0, + }, + "py-packaging": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "py-petsc4py": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "py-scipy": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "py-statsmodels": { + "build_jobs": 2, + "cpu_request": 2.0, + "mem_request": 4000000000.0, + }, + "py-warlock": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "py-warpx": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "samrai": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "slepc": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "slurm": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "sqlite": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "superlu-dist": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "tasmanian": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "tau": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "upcxx": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, + "zfp": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "alsa-lib": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "ant": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "antlr": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "argobots": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "automake": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "berkeley-db": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "bison": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "bzip2": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "czmq": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "darshan-util": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "diffutils": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "docbook-xml": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "exmcutils": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "expat": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "findutils": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "flit": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "freetype": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "gawk": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "gdbm": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "glib": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "gmake": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "gotcha": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "hpcviewer": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "jansson": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "json-c": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "libbsd": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "libedit": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "libevent": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "libfabric": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "libffi": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "libgcrypt": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "libiconv": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "libidn2": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "libjpeg-turbo": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "libmd": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "libnrm": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "libpciaccess": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "libpng": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "libsigsegv": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "libsodium": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "libunistring": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "libunwind": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "libyaml": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "lua": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "lua-luaposix": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "lz4": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "m4": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "meson": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "metis": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "mpfr": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "ncurses": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "ninja": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "numactl": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "openjdk": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "openssh": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "openssl": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "papi": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "parallel-netcdf": { + "build_jobs": 1, + "cpu_request": 0.5, + "mem_request": 500000000.0, + }, + "pcre": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "pcre2": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "pdsh": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "perl": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "perl-data-dumper": { + "build_jobs": 1, + "cpu_request": 0.5, + "mem_request": 500000000.0, + }, + "pkgconf": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "py-alembic": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "py-cffi": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "py-cycler": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "py-decorator": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "py-idna": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "py-jsonschema": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "py-kiwisolver": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "py-mistune": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "py-pycparser": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "py-setuptools": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "py-setuptools-scm": { + "build_jobs": 1, + "cpu_request": 0.5, + "mem_request": 500000000.0, + }, + "py-six": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "py-testpath": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "py-wheel": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "qhull": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "readline": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "sed": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "snappy": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "superlu": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "swig": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "tar": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "tcl": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "texinfo": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "tut": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "unzip": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "util-linux-uuid": { + "build_jobs": 1, + "cpu_request": 0.5, + "mem_request": 500000000.0, + }, + "util-macros": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "xz": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "yaml-cpp": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "zlib": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, + "zstd": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, +} diff --git a/gantry/routes/prediction/prediction.py b/gantry/routes/prediction/prediction.py index 2101ec7..b8f11d3 100644 --- a/gantry/routes/prediction/prediction.py +++ b/gantry/routes/prediction/prediction.py @@ -1,13 +1,15 @@ +import logging + import aiosqlite from gantry.routes.prediction.current_mapping import pkg_mappings MIN_TRAIN_SAMPLE = 4 -DEFAULT_CPU_REQUEST = 1000 -DEFAULT_MEM_REQUEST = 2000 +DEFAULT_CPU_REQUEST = 1.0 +DEFAULT_MEM_REQUEST = 2000 * 1_000_000 # 2GB in bytes CORES_TO_MILLICORES = 1000 -BYTES_TO_MEGABYTES = 1/1_000_000 +BYTES_TO_MEGABYTES = 1 / 1_000_000 def preprocess_pred(prediction, pkg_name): @@ -22,25 +24,35 @@ def preprocess_pred(prediction, pkg_name): cur_alloc = pkg_mappings.get(pkg_name) if cur_alloc: - prediction["variables"]["cpu_request"] = max( - prediction["variables"]["cpu_request"], cur_alloc["cpu_request"] + prediction["variables"]["KUBERNETES_CPU_REQUEST"] = max( + prediction["variables"]["KUBERNETES_CPU_REQUEST"], cur_alloc["cpu_request"] ) - prediction["variables"]["mem_request"] = max( - prediction["variables"]["mem_request"], cur_alloc["mem_request"] + prediction["variables"]["KUBERNETES_MEMORY_REQUEST"] = max( + prediction["variables"]["KUBERNETES_MEMORY_REQUEST"], + cur_alloc["mem_request"], ) - # put into k8s units and should not be a float - prediction["variables"][ - "cpu_request" - ] = f"{int(prediction['variables']['cpu_request'] * CORES_TO_MILLICORES)}m" - prediction["variables"][ - "mem_request" - ] = f"{int(prediction['variables']['mem_request'] * BYTES_TO_MEGABYTES)}M" + # put into k8s friendly format + prediction["variables"]["KUBERNETES_CPU_REQUEST"] = ( + str( + int(prediction["variables"]["KUBERNETES_CPU_REQUEST"] * CORES_TO_MILLICORES) + ) + + "m" + ) + prediction["variables"]["KUBERNETES_MEMORY_REQUEST"] = ( + str( + int( + prediction["variables"]["KUBERNETES_MEMORY_REQUEST"] + * BYTES_TO_MEGABYTES + ) + ) + + "M" + ) return prediction -async def select_sample(db: aiosqlite.Connection, build: dict) -> list: +async def get_sample(db: aiosqlite.Connection, build: dict) -> list: """ Selects a sample of builds to use for prediction @@ -50,22 +62,33 @@ async def select_sample(db: aiosqlite.Connection, build: dict) -> list: list of lists with cpu_mean, cpu_max, mem_mean, mem_max """ + flat_build = { + "pkg_name": build["package"]["name"], + "pkg_version": build["package"]["version"], + "compiler_name": build["compiler"]["name"], + "compiler_version": build["compiler"]["version"], + } + param_combos = [ ("pkg_name", "pkg_version", "compiler_name", "compiler_version"), ("pkg_name", "compiler_name", "compiler_version"), ("pkg_name", "pkg_version", "compiler_name"), ("pkg_name", "compiler_name"), ("pkg_name", "pkg_version"), - ("pkg_name"), + ("pkg_name",), ] for combo in param_combos: - condition_values = [build[param] for param in combo] - query = f"SELECT cpu_mean, cpu_max, mem_mean, mem_max FROM builds WHERE ref='develop' AND {' AND '.join(f'{param}=?' for param in combo)} ORDER BY end DESC LIMIT {MIN_TRAIN_SAMPLE + 1}" + condition_values = [flat_build[param] for param in combo] + query = f""" + SELECT cpu_mean, cpu_max, mem_mean, mem_max FROM jobs WHERE ref='develop' + AND {' AND '.join(f'{param}=?' for param in combo)} + ORDER BY end DESC LIMIT {MIN_TRAIN_SAMPLE + 1} + """ async with db.execute(query, condition_values) as cursor: - builds = await cursor.fetchall() - if len(builds) >= MIN_TRAIN_SAMPLE: - return builds + sample = await cursor.fetchall() + if len(sample) >= MIN_TRAIN_SAMPLE: + return sample return [] @@ -81,7 +104,7 @@ async def predict_single(db: aiosqlite.Connection, build: dict) -> dict: CPU in millicore, mem in MB """ - sample = await select_sample(db, build) + sample = await get_sample(db, build) if not sample: vars = { "KUBERNETES_CPU_REQUEST": DEFAULT_CPU_REQUEST, @@ -102,7 +125,12 @@ async def predict_single(db: aiosqlite.Connection, build: dict) -> dict: "variables": vars, } - return preprocess_pred(pred, build["pkg_name"]) + if pred["variables"]["KUBERNETES_CPU_REQUEST"] < 0.25: + logging.warning(f"Warning: CPU request for {build['hash']} is below 0.25 cores") + if pred["variables"]["KUBERNETES_MEMORY_REQUEST"] < 10000000: + logging.warning(f"Warning: Memory request for {build['hash']} is below 10MB") + + return preprocess_pred(pred, build["package"]["name"]) async def predict_bulk(db: aiosqlite.Connection, builds: list) -> list: diff --git a/gantry/views.py b/gantry/views.py index 918b012..9afb4c9 100644 --- a/gantry/views.py +++ b/gantry/views.py @@ -38,7 +38,7 @@ async def collect_job(request: web.Request) -> web.Response: return web.Response(status=200) -@routes.get("/v1/resource-mapping") +@routes.post("/v1/allocate") async def map_resources(request: web.Request) -> web.Response: """ acceptable payload: @@ -82,7 +82,10 @@ def validate_payload(payload): payload = [payload] for item in payload: - required_fields = ["hash", "package", "compiler"] + if not isinstance(item.get("hash"), str): + return False + + required_fields = ["package", "compiler"] for field in required_fields: if not isinstance(item.get(field), dict): return False From 3464898b8d69e96f03be4d6741047ee807d191b5 Mon Sep 17 00:00:00 2001 From: caetano melone Date: Tue, 30 Jan 2024 23:28:07 -0800 Subject: [PATCH 07/37] no black current_mapping --- gantry/routes/prediction/current_mapping.py | 300 +------------------- 1 file changed, 1 insertion(+), 299 deletions(-) diff --git a/gantry/routes/prediction/current_mapping.py b/gantry/routes/prediction/current_mapping.py index 54f1544..bd4fea7 100644 --- a/gantry/routes/prediction/current_mapping.py +++ b/gantry/routes/prediction/current_mapping.py @@ -1,299 +1 @@ -pkg_mappings = { - "rust": {"build_jobs": 12, "cpu_request": 12.0, "mem_request": 35000000000.0}, - "py-tensorflow": { - "build_jobs": 12, - "cpu_request": 12.0, - "mem_request": 32000000000.0, - }, - "py-jaxlib": {"build_jobs": 12, "cpu_request": 12.0, "mem_request": 29000000000.0}, - "nvhpc": {"build_jobs": 12, "cpu_request": 12.0, "mem_request": 24000000000.0}, - "paraview": {"build_jobs": 12, "cpu_request": 12.0, "mem_request": 24000000000.0}, - "llvm": {"build_jobs": 12, "cpu_request": 12.0, "mem_request": 21000000000.0}, - "py-torch": {"build_jobs": 12, "cpu_request": 12.0, "mem_request": 21000000000.0}, - "dealii": {"build_jobs": 12, "cpu_request": 12.0, "mem_request": 19000000000.0}, - "mxnet": {"build_jobs": 12, "cpu_request": 12.0, "mem_request": 19000000000.0}, - "py-torchaudio": { - "build_jobs": 12, - "cpu_request": 12.0, - "mem_request": 19000000000.0, - }, - "rocblas": {"build_jobs": 12, "cpu_request": 12.0, "mem_request": 19000000000.0}, - "ecp-data-vis-sdk": { - "build_jobs": 8, - "cpu_request": 8.0, - "mem_request": 12000000000.0, - }, - "intel-tbb": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 15000000000.0}, - "llvm-amdgpu": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 15000000000.0}, - "salmon": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 15000000000.0}, - "trilinos": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, - "ascent": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "atk": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, - "axom": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "cistem": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, - "cmake": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, - "ctffind": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, - "cuda": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, - "dray": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "dyninst": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, - "gcc": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, - "ginkgo": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, - "hdf5": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "hipblas": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, - "hpx": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, - "kokkos-kernels": { - "build_jobs": 8, - "cpu_request": 8.0, - "mem_request": 12000000000.0, - }, - "kokkos-nvcc-wrapper": { - "build_jobs": 8, - "cpu_request": 8.0, - "mem_request": 12000000000.0, - }, - "lbann": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, - "magma": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, - "mesa": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, - "mfem": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "mpich": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "netlib-lapack": { - "build_jobs": 8, - "cpu_request": 8.0, - "mem_request": 12000000000.0, - }, - "oce": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, - "openblas": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "openfoam": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, - "openturns": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, - "parallelio": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, - "plumed": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, - "precice": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, - "qt": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, - "raja": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "relion": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, - "rocfft": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, - "rocsolver": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, - "rocsparse": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, - "slate": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, - "strumpack": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "sundials": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "umpire": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, - "visit": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, - "vtk": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "vtk-h": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "vtk-m": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "warpx": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, - "wrf": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, - "wxwidgets": {"build_jobs": 8, "cpu_request": 8.0, "mem_request": 12000000000.0}, - "adios2": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "amrex": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "archer": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "autoconf-archive": { - "build_jobs": 1, - "cpu_request": 0.5, - "mem_request": 500000000.0, - }, - "binutils": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "blaspp": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "blt": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "boost": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "butterflypack": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "cabana": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "caliper": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "camp": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "chai": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "conduit": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "curl": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "datatransferkit": { - "build_jobs": 2, - "cpu_request": 2.0, - "mem_request": 4000000000.0, - }, - "double-conversion": { - "build_jobs": 2, - "cpu_request": 2.0, - "mem_request": 4000000000.0, - }, - "eigen": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "faodel": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "ffmpeg": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "fftw": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "fortrilinos": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "gettext": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "gperftools": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "gptune": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "heffte": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "hpctoolkit": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "hwloc": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "hydrogen": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "hypre": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "kokkos": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "lammps": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "lapackpp": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "legion": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "libtool": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "libxml2": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "libzmq": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "llvm-openmp-ompt": { - "build_jobs": 2, - "cpu_request": 2.0, - "mem_request": 4000000000.0, - }, - "mbedtls": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "mvapich2": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "nasm": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "netlib-scalapack": { - "build_jobs": 2, - "cpu_request": 2.0, - "mem_request": 4000000000.0, - }, - "omega-h": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "openjpeg": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "openmpi": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "openpmd-api": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "pagmo2": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "papyrus": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "parsec": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "pdt": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "pegtl": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "petsc": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "pumi": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "py-beniget": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "py-cinemasci": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "pygmo": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "py-ipython-genutils": { - "build_jobs": 2, - "cpu_request": 2.0, - "mem_request": 4000000000.0, - }, - "py-packaging": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "py-petsc4py": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "py-scipy": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "py-statsmodels": { - "build_jobs": 2, - "cpu_request": 2.0, - "mem_request": 4000000000.0, - }, - "py-warlock": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "py-warpx": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "samrai": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "slepc": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "slurm": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "sqlite": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "superlu-dist": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "tasmanian": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "tau": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "upcxx": {"build_jobs": 2, "cpu_request": 2.0, "mem_request": 4000000000.0}, - "zfp": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "alsa-lib": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "ant": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "antlr": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "argobots": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "automake": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "berkeley-db": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "bison": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "bzip2": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "czmq": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "darshan-util": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "diffutils": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "docbook-xml": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "exmcutils": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "expat": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "findutils": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "flit": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "freetype": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "gawk": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "gdbm": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "glib": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "gmake": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "gotcha": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "hpcviewer": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "jansson": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "json-c": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "libbsd": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "libedit": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "libevent": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "libfabric": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "libffi": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "libgcrypt": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "libiconv": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "libidn2": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "libjpeg-turbo": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "libmd": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "libnrm": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "libpciaccess": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "libpng": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "libsigsegv": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "libsodium": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "libunistring": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "libunwind": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "libyaml": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "lua": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "lua-luaposix": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "lz4": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "m4": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "meson": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "metis": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "mpfr": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "ncurses": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "ninja": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "numactl": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "openjdk": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "openssh": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "openssl": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "papi": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "parallel-netcdf": { - "build_jobs": 1, - "cpu_request": 0.5, - "mem_request": 500000000.0, - }, - "pcre": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "pcre2": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "pdsh": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "perl": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "perl-data-dumper": { - "build_jobs": 1, - "cpu_request": 0.5, - "mem_request": 500000000.0, - }, - "pkgconf": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "py-alembic": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "py-cffi": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "py-cycler": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "py-decorator": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "py-idna": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "py-jsonschema": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "py-kiwisolver": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "py-mistune": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "py-pycparser": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "py-setuptools": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "py-setuptools-scm": { - "build_jobs": 1, - "cpu_request": 0.5, - "mem_request": 500000000.0, - }, - "py-six": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "py-testpath": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "py-wheel": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "qhull": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "readline": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "sed": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "snappy": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "superlu": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "swig": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "tar": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "tcl": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "texinfo": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "tut": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "unzip": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "util-linux-uuid": { - "build_jobs": 1, - "cpu_request": 0.5, - "mem_request": 500000000.0, - }, - "util-macros": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "xz": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "yaml-cpp": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "zlib": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, - "zstd": {"build_jobs": 1, "cpu_request": 0.5, "mem_request": 500000000.0}, -} +pkg_mappings = {'rust': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 35000000000.0}, 'py-tensorflow': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 32000000000.0}, 'py-jaxlib': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 29000000000.0}, 'nvhpc': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 24000000000.0}, 'paraview': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 24000000000.0}, 'llvm': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 21000000000.0}, 'py-torch': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 21000000000.0}, 'dealii': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 19000000000.0}, 'mxnet': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 19000000000.0}, 'py-torchaudio': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 19000000000.0}, 'rocblas': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 19000000000.0}, 'ecp-data-vis-sdk': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'intel-tbb': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 15000000000.0}, 'llvm-amdgpu': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 15000000000.0}, 'salmon': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 15000000000.0}, 'trilinos': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'ascent': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'atk': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'axom': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'cistem': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'cmake': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'ctffind': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'cuda': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'dray': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'dyninst': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'gcc': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'ginkgo': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'hdf5': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'hipblas': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'hpx': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'kokkos-kernels': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'kokkos-nvcc-wrapper': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'lbann': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'magma': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'mesa': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'mfem': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'mpich': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'netlib-lapack': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'oce': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'openblas': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'openfoam': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'openturns': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'parallelio': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'plumed': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'precice': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'qt': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'raja': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'relion': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'rocfft': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'rocsolver': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'rocsparse': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'slate': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'strumpack': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'sundials': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'umpire': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'visit': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'vtk': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'vtk-h': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'vtk-m': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'warpx': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'wrf': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'wxwidgets': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'adios2': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'amrex': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'archer': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'autoconf-archive': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'binutils': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'blaspp': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'blt': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'boost': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'butterflypack': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'cabana': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'caliper': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'camp': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'chai': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'conduit': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'curl': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'datatransferkit': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'double-conversion': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'eigen': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'faodel': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'ffmpeg': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'fftw': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'fortrilinos': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'gettext': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'gperftools': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'gptune': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'heffte': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'hpctoolkit': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'hwloc': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'hydrogen': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'hypre': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'kokkos': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'lammps': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'lapackpp': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'legion': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'libtool': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'libxml2': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libzmq': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'llvm-openmp-ompt': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'mbedtls': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'mvapich2': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'nasm': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'netlib-scalapack': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'omega-h': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'openjpeg': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'openmpi': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'openpmd-api': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'pagmo2': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'papyrus': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'parsec': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'pdt': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'pegtl': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'petsc': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'pumi': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'py-beniget': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'py-cinemasci': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'pygmo': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'py-ipython-genutils': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'py-packaging': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'py-petsc4py': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'py-scipy': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'py-statsmodels': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'py-warlock': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'py-warpx': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'samrai': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'slepc': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'slurm': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'sqlite': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'superlu-dist': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'tasmanian': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'tau': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'upcxx': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'zfp': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'alsa-lib': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'ant': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'antlr': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'argobots': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'automake': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'berkeley-db': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'bison': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'bzip2': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'czmq': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'darshan-util': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'diffutils': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'docbook-xml': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'exmcutils': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'expat': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'findutils': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'flit': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'freetype': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'gawk': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'gdbm': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'glib': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'gmake': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'gotcha': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'hpcviewer': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'jansson': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'json-c': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libbsd': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libedit': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libevent': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libfabric': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libffi': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libgcrypt': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libiconv': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libidn2': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libjpeg-turbo': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libmd': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libnrm': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libpciaccess': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libpng': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libsigsegv': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libsodium': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libunistring': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libunwind': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libyaml': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'lua': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'lua-luaposix': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'lz4': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'm4': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'meson': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'metis': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'mpfr': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'ncurses': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'ninja': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'numactl': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'openjdk': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'openssh': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'openssl': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'papi': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'parallel-netcdf': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'pcre': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'pcre2': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'pdsh': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'perl': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'perl-data-dumper': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'pkgconf': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-alembic': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-cffi': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-cycler': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-decorator': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-idna': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-jsonschema': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-kiwisolver': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-mistune': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-pycparser': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-setuptools': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-setuptools-scm': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-six': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-testpath': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-wheel': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'qhull': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'readline': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'sed': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'snappy': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'superlu': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'swig': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'tar': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'tcl': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'texinfo': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'tut': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'unzip': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'util-linux-uuid': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'util-macros': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'xz': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'yaml-cpp': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'zlib': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'zstd': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}} From 2bd4d98276a4d116533d540490cc727183ce2bb5 Mon Sep 17 00:00:00 2001 From: caetano melone Date: Wed, 31 Jan 2024 17:30:12 -0800 Subject: [PATCH 08/37] readability --- gantry/routes/prediction/prediction.py | 165 +++++++++++++------------ gantry/util/k8s.py | 12 ++ 2 files changed, 96 insertions(+), 81 deletions(-) create mode 100644 gantry/util/k8s.py diff --git a/gantry/routes/prediction/prediction.py b/gantry/routes/prediction/prediction.py index b8f11d3..b62665b 100644 --- a/gantry/routes/prediction/prediction.py +++ b/gantry/routes/prediction/prediction.py @@ -1,55 +1,82 @@ import logging +# TODO clean all this up check obsidian notes to make sure everything's implemented + import aiosqlite from gantry.routes.prediction.current_mapping import pkg_mappings +from gantry.util.k8s import convert_bytes, convert_cores -MIN_TRAIN_SAMPLE = 4 +IDEAL_SAMPLE = 4 DEFAULT_CPU_REQUEST = 1.0 DEFAULT_MEM_REQUEST = 2000 * 1_000_000 # 2GB in bytes -CORES_TO_MILLICORES = 1000 -BYTES_TO_MEGABYTES = 1 / 1_000_000 - -def preprocess_pred(prediction, pkg_name): +async def predict_single(db: aiosqlite.Connection, build: dict) -> dict: """ - Main goal of this function is to ensure that our - prediction is not lower than the current allocation - for that package. This restriction will likely be - removed in the future as we understand the effectiveness - of the prediction model. + Predict the resource usage of a build + + args: + build: dict that must contain pkg name, pkg version, compiler, compiler version + returns: + dict of predicted resource usage: cpu_request, mem_request + CPU in millicore, mem in MB """ - cur_alloc = pkg_mappings.get(pkg_name) + sample = await get_sample(db, build) + predictions = {} + if not sample: + predictions = { + "cpu_request": DEFAULT_CPU_REQUEST, + "mem_request": DEFAULT_MEM_REQUEST, + } + else: + # mapping of sample: [0] cpu_mean, [1] cpu_max, [2] mem_mean, [3] mem_max + predictions = { + # averages the respective metric in the sample + "cpu_request": sum([build[0] for build in sample]) / len(sample), + "mem_request": (sum([build[2] for build in sample]) / len(sample)), + } - if cur_alloc: - prediction["variables"]["KUBERNETES_CPU_REQUEST"] = max( - prediction["variables"]["KUBERNETES_CPU_REQUEST"], cur_alloc["cpu_request"] - ) - prediction["variables"]["KUBERNETES_MEMORY_REQUEST"] = max( - prediction["variables"]["KUBERNETES_MEMORY_REQUEST"], - cur_alloc["mem_request"], - ) + ensure_higher_pred(predictions, build["package"]["name"]) - # put into k8s friendly format - prediction["variables"]["KUBERNETES_CPU_REQUEST"] = ( - str( - int(prediction["variables"]["KUBERNETES_CPU_REQUEST"] * CORES_TO_MILLICORES) - ) - + "m" - ) - prediction["variables"]["KUBERNETES_MEMORY_REQUEST"] = ( - str( - int( - prediction["variables"]["KUBERNETES_MEMORY_REQUEST"] - * BYTES_TO_MEGABYTES - ) - ) - + "M" - ) + # warn if the prediction is below some thresholds + if predictions["cpu_request"] < 0.25: + logging.warning(f"Warning: CPU request for {build['hash']} is below 0.25 cores") + predictions["cpu_request"] = DEFAULT_CPU_REQUEST + if predictions["mem_request"] < 10_000_000: + logging.warning(f"Warning: Memory request for {build['hash']} is below 10MB") + predictions["mem_request"] = DEFAULT_MEM_REQUEST + + # convert predictions to k8s friendly format + for k, v in predictions.items(): + if k.startswith("cpu"): + predictions[k] = convert_cores(v) + elif k.startswith("mem"): + predictions[k] = convert_bytes(v) + + return { + "hash": build["hash"], + "variables": { + # spack uses these env vars to set the resource requests + # set them here at the last minute to avoid using these vars + # and clogging up the code + "KUBERNETES_CPU_REQUEST": predictions["cpu_request"], + "KUBERNETES_MEMORY_REQUEST": predictions["mem_request"], + }, + } + + +async def predict_bulk(db: aiosqlite.Connection, builds: list) -> list: + """ + Handles a bulk request of builds + + args: + builds: list of dicts (see predict_single) + returns: see predict_single) + """ - return prediction + return [await predict_single(db, build) for build in builds] async def get_sample(db: aiosqlite.Connection, build: dict) -> list: @@ -69,6 +96,7 @@ async def get_sample(db: aiosqlite.Connection, build: dict) -> list: "compiler_version": build["compiler"]["version"], } + # ranked in order of priority, the params we would like to match on param_combos = [ ("pkg_name", "pkg_version", "compiler_name", "compiler_version"), ("pkg_name", "compiler_name", "compiler_version"), @@ -79,67 +107,42 @@ async def get_sample(db: aiosqlite.Connection, build: dict) -> list: ] for combo in param_combos: + # filter by the params we want condition_values = [flat_build[param] for param in combo] + # we want at least MIN_TRAIN_SAMPLE +1/= rows query = f""" SELECT cpu_mean, cpu_max, mem_mean, mem_max FROM jobs WHERE ref='develop' AND {' AND '.join(f'{param}=?' for param in combo)} - ORDER BY end DESC LIMIT {MIN_TRAIN_SAMPLE + 1} + ORDER BY end DESC LIMIT {IDEAL_SAMPLE} """ async with db.execute(query, condition_values) as cursor: sample = await cursor.fetchall() - if len(sample) >= MIN_TRAIN_SAMPLE: + # we can accept the sample if it's 1 shorter + if len(sample) >= IDEAL_SAMPLE - 1: return sample + # continue if we didn't find enough rows return [] -async def predict_single(db: aiosqlite.Connection, build: dict) -> dict: +def ensure_higher_pred(prediction: dict, pkg_name: str): """ - Predict the resource usage of a build + Ensure that the prediction is higher than the current allocation + for the package. This will be removed in the future as we analyze + the effectiveness of the prediction model. args: - build: dict that must contain pkg name, pkg version, compiler, compiler version - returns: - dict of predicted resource usage: cpu_request, mem_request - CPU in millicore, mem in MB + prediction: dict of predicted resource usage: cpu_request, mem_request + pkg_name: str """ - sample = await get_sample(db, build) - if not sample: - vars = { - "KUBERNETES_CPU_REQUEST": DEFAULT_CPU_REQUEST, - "KUBERNETES_MEMORY_REQUEST": DEFAULT_MEM_REQUEST, - } - else: - # mapping of sample: [0] cpu_mean, [1] cpu_max, [2] mem_mean, [3] mem_max - vars = { - # averages the respective metric in the sample - "KUBERNETES_CPU_REQUEST": sum([build[0] for build in sample]) / len(sample), - "KUBERNETES_MEMORY_REQUEST": ( - sum([build[2] for build in sample]) / len(sample) - ), - } - - pred = { - "hash": build["hash"], - "variables": vars, - } - - if pred["variables"]["KUBERNETES_CPU_REQUEST"] < 0.25: - logging.warning(f"Warning: CPU request for {build['hash']} is below 0.25 cores") - if pred["variables"]["KUBERNETES_MEMORY_REQUEST"] < 10000000: - logging.warning(f"Warning: Memory request for {build['hash']} is below 10MB") - - return preprocess_pred(pred, build["package"]["name"]) - - -async def predict_bulk(db: aiosqlite.Connection, builds: list) -> list: - """ - Handles a bulk request of builds + cur_alloc = pkg_mappings.get(pkg_name) - args: - builds: list of dicts (see predict_single) - returns: see predict_single) - """ + if cur_alloc: + prediction["cpu_request"] = max( + prediction["cpu_request"], cur_alloc["cpu_request"] + ) - return [await predict_single(db, build) for build in builds] + prediction["mem_request"] = max( + prediction["mem_request"], cur_alloc["mem_request"] + ) diff --git a/gantry/util/k8s.py b/gantry/util/k8s.py new file mode 100644 index 0000000..f0218c5 --- /dev/null +++ b/gantry/util/k8s.py @@ -0,0 +1,12 @@ +CORES_TO_MILLICORES = 1000 +BYTES_TO_MEGABYTES = 1 / 1_000_000 + +# these functions convert the predictions to k8s friendly format + +def convert_cores(cores: float) -> str: + """cores to millicores""" + return str(int(cores * CORES_TO_MILLICORES)) + "m" + +def convert_bytes(bytes: float) -> str: + """bytes to megabytes""" + return str(int(bytes * BYTES_TO_MEGABYTES)) + "M" From 08130274effe5a86451b11e68fe3ac72906f3a4c Mon Sep 17 00:00:00 2001 From: Caetano Melone Date: Mon, 5 Feb 2024 16:49:11 -0800 Subject: [PATCH 09/37] allocate -> allocation Co-authored-by: Alec Scott --- gantry/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gantry/views.py b/gantry/views.py index 9afb4c9..4be82d8 100644 --- a/gantry/views.py +++ b/gantry/views.py @@ -38,8 +38,8 @@ async def collect_job(request: web.Request) -> web.Response: return web.Response(status=200) -@routes.post("/v1/allocate") -async def map_resources(request: web.Request) -> web.Response: +@routes.get("/v1/allocation") +async def allocate(request: web.Request) -> web.Response: """ acceptable payload: From 63a1404f3b0518c846d159ee66dd26191733295d Mon Sep 17 00:00:00 2001 From: caetano melone Date: Mon, 5 Feb 2024 16:44:45 -0800 Subject: [PATCH 10/37] use GET endpoint rather than POST for allocation --- gantry/__main__.py | 30 +++++++++++++++++++++++++++--- gantry/views.py | 6 +++++- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/gantry/__main__.py b/gantry/__main__.py index a420f94..4d2c531 100644 --- a/gantry/__main__.py +++ b/gantry/__main__.py @@ -1,3 +1,4 @@ +import asyncio import logging import os @@ -33,13 +34,36 @@ async def init_clients(app: web.Application): ) -def main(): +async def main(): app = web.Application() app.add_routes(routes) app.cleanup_ctx.append(init_db) app.on_startup.append(init_clients) - web.run_app(app) + runner = web.AppRunner( + app, max_line_size=int(os.environ.get("MAX_GET_SIZE", 100_000)) + ) + await runner.setup() + site = web.TCPSite( + runner, + os.environ.get("GANTRY_HOST", "localhost"), + os.environ.get("GANTRY_PORT", 8080), + ) + await site.start() + + print( + f"""Gantry running on {os.environ.get('GANTRY_HOST', 'localhost')}: + {os.environ.get('GANTRY_PORT', 8080)}""" + ) + print("-------------------") + + try: + # wait for finish signal + await asyncio.Future() + except asyncio.CancelledError: + pass + finally: + await runner.cleanup() if __name__ == "__main__": - main() + asyncio.run(main()) diff --git a/gantry/views.py b/gantry/views.py index 4be82d8..54518d0 100644 --- a/gantry/views.py +++ b/gantry/views.py @@ -70,9 +70,13 @@ async def allocate(request: web.Request) -> web.Response: or a list of the above objects """ + payload = request.query.get("query") + + if not payload: + return web.Response(status=400, text="missing query parameter") try: - payload = await request.json() + payload = json.loads(payload) except json.decoder.JSONDecodeError: return web.Response(status=400, text="invalid json") From 960f89bbffdae061729f1c9595d6830c8b4971a0 Mon Sep 17 00:00:00 2001 From: caetano melone Date: Mon, 5 Feb 2024 20:16:41 -0800 Subject: [PATCH 11/37] simplify validation logic --- gantry/views.py | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/gantry/views.py b/gantry/views.py index 54518d0..7e66ceb 100644 --- a/gantry/views.py +++ b/gantry/views.py @@ -80,29 +80,29 @@ async def allocate(request: web.Request) -> web.Response: except json.decoder.JSONDecodeError: return web.Response(status=400, text="invalid json") - def validate_payload(payload): - """ensures that the payload is valid""" - if not isinstance(payload, list): + def validate_payload(payload: dict) -> bool: + """Ensures that the payload from the client is valid.""" + # must be a dict or a list + if not isinstance(payload, (dict, list)): + return False + + if isinstance(payload, dict): + # put dict in list for iteration payload = [payload] for item in payload: - if not isinstance(item.get("hash"), str): - return False - - required_fields = ["package", "compiler"] - for field in required_fields: - if not isinstance(item.get(field), dict): - return False - - if not all( - isinstance(item.get("package", {}).get(key), str) - for key in ["name", "version"] - ): - return False - - if not all( - isinstance(item.get("compiler", {}).get(key), str) - for key in ["name", "version"] + if not ( + # item must be dict + isinstance(item, dict) + # must contain hash field + and isinstance(item.get("hash"), str) + # must contain name and version + # for both package and compiler + and all( + isinstance(item.get(field, {}).get(key), str) + for field in ["package", "compiler"] + for key in ["name", "version"] + ) ): return False @@ -113,7 +113,7 @@ def validate_payload(payload): if isinstance(payload, dict): response = await predict_single(request.app["db"], payload) - else: + elif isinstance(payload, list): response = await predict_bulk(request.app["db"], payload) return web.json_response(response) From 2f65298712e46ab73487de110b947d7853f09465 Mon Sep 17 00:00:00 2001 From: caetano melone Date: Mon, 5 Feb 2024 20:34:12 -0800 Subject: [PATCH 12/37] bulk predictions run in parallel --- gantry/routes/prediction/prediction.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gantry/routes/prediction/prediction.py b/gantry/routes/prediction/prediction.py index b62665b..f9b9045 100644 --- a/gantry/routes/prediction/prediction.py +++ b/gantry/routes/prediction/prediction.py @@ -1,3 +1,4 @@ +import asyncio import logging # TODO clean all this up check obsidian notes to make sure everything's implemented @@ -76,7 +77,7 @@ async def predict_bulk(db: aiosqlite.Connection, builds: list) -> list: returns: see predict_single) """ - return [await predict_single(db, build) for build in builds] + return await asyncio.gather(*(predict_single(db, build) for build in builds)) async def get_sample(db: aiosqlite.Connection, build: dict) -> list: From 5714aae612c2839617c0b832681af756e853a7f4 Mon Sep 17 00:00:00 2001 From: caetano melone Date: Fri, 9 Feb 2024 08:58:17 -0600 Subject: [PATCH 13/37] fmt changes --- gantry/__main__.py | 11 +++++------ gantry/views.py | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/gantry/__main__.py b/gantry/__main__.py index 4d2c531..15bc850 100644 --- a/gantry/__main__.py +++ b/gantry/__main__.py @@ -43,17 +43,16 @@ async def main(): app, max_line_size=int(os.environ.get("MAX_GET_SIZE", 100_000)) ) await runner.setup() + port = os.environ.get("GANTRY_PORT", 8080) + host = os.environ.get("GANTRY_HOST", "localhost") site = web.TCPSite( runner, - os.environ.get("GANTRY_HOST", "localhost"), - os.environ.get("GANTRY_PORT", 8080), + host, + port, ) await site.start() - print( - f"""Gantry running on {os.environ.get('GANTRY_HOST', 'localhost')}: - {os.environ.get('GANTRY_PORT', 8080)}""" - ) + print(f"Gantry running on {host}:{port}") print("-------------------") try: diff --git a/gantry/views.py b/gantry/views.py index 7e66ceb..c1c92e4 100644 --- a/gantry/views.py +++ b/gantry/views.py @@ -39,7 +39,7 @@ async def collect_job(request: web.Request) -> web.Response: @routes.get("/v1/allocation") -async def allocate(request: web.Request) -> web.Response: +async def allocation(request: web.Request) -> web.Response: """ acceptable payload: From 22d87e5f4f2dbb828dfa999990d2e9eaa98170af Mon Sep 17 00:00:00 2001 From: caetano melone Date: Fri, 9 Feb 2024 09:00:58 -0600 Subject: [PATCH 14/37] cpu units now in cores rather than millicores --- gantry/routes/prediction/prediction.py | 10 +++++----- gantry/util/k8s.py | 5 ----- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/gantry/routes/prediction/prediction.py b/gantry/routes/prediction/prediction.py index f9b9045..f0a38e2 100644 --- a/gantry/routes/prediction/prediction.py +++ b/gantry/routes/prediction/prediction.py @@ -6,7 +6,7 @@ import aiosqlite from gantry.routes.prediction.current_mapping import pkg_mappings -from gantry.util.k8s import convert_bytes, convert_cores +from gantry.util import k8s IDEAL_SAMPLE = 4 DEFAULT_CPU_REQUEST = 1.0 @@ -35,8 +35,8 @@ async def predict_single(db: aiosqlite.Connection, build: dict) -> dict: # mapping of sample: [0] cpu_mean, [1] cpu_max, [2] mem_mean, [3] mem_max predictions = { # averages the respective metric in the sample - "cpu_request": sum([build[0] for build in sample]) / len(sample), - "mem_request": (sum([build[2] for build in sample]) / len(sample)), + "cpu_request": round(sum([build[0] for build in sample]) / len(sample)), + "mem_request": sum([build[2] for build in sample]) / len(sample), } ensure_higher_pred(predictions, build["package"]["name"]) @@ -52,9 +52,9 @@ async def predict_single(db: aiosqlite.Connection, build: dict) -> dict: # convert predictions to k8s friendly format for k, v in predictions.items(): if k.startswith("cpu"): - predictions[k] = convert_cores(v) + predictions[k] = str(int(v)) elif k.startswith("mem"): - predictions[k] = convert_bytes(v) + predictions[k] = k8s.convert_bytes(v) return { "hash": build["hash"], diff --git a/gantry/util/k8s.py b/gantry/util/k8s.py index f0218c5..75b70b9 100644 --- a/gantry/util/k8s.py +++ b/gantry/util/k8s.py @@ -1,12 +1,7 @@ -CORES_TO_MILLICORES = 1000 BYTES_TO_MEGABYTES = 1 / 1_000_000 # these functions convert the predictions to k8s friendly format -def convert_cores(cores: float) -> str: - """cores to millicores""" - return str(int(cores * CORES_TO_MILLICORES)) + "m" - def convert_bytes(bytes: float) -> str: """bytes to megabytes""" return str(int(bytes * BYTES_TO_MEGABYTES)) + "M" From 27b07f5f4cf91c99963000240997990943057772 Mon Sep 17 00:00:00 2001 From: caetano melone Date: Fri, 9 Feb 2024 09:10:41 -0600 Subject: [PATCH 15/37] request limit 8MB --- gantry/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gantry/__main__.py b/gantry/__main__.py index 15bc850..0a9854a 100644 --- a/gantry/__main__.py +++ b/gantry/__main__.py @@ -40,7 +40,7 @@ async def main(): app.cleanup_ctx.append(init_db) app.on_startup.append(init_clients) runner = web.AppRunner( - app, max_line_size=int(os.environ.get("MAX_GET_SIZE", 100_000)) + app, max_line_size=int(os.environ.get("MAX_GET_SIZE", 800_000)) ) await runner.setup() port = os.environ.get("GANTRY_PORT", 8080) From 3f2b7ffa32c3cb63181994e84415837480af1333 Mon Sep 17 00:00:00 2001 From: caetano melone Date: Fri, 9 Feb 2024 15:52:02 -0600 Subject: [PATCH 16/37] add variant filtering --- gantry/routes/prediction/prediction.py | 50 ++++++++++++++++++++++---- gantry/views.py | 2 ++ 2 files changed, 46 insertions(+), 6 deletions(-) diff --git a/gantry/routes/prediction/prediction.py b/gantry/routes/prediction/prediction.py index f0a38e2..d02bf79 100644 --- a/gantry/routes/prediction/prediction.py +++ b/gantry/routes/prediction/prediction.py @@ -6,7 +6,7 @@ import aiosqlite from gantry.routes.prediction.current_mapping import pkg_mappings -from gantry.util import k8s +from gantry.util import k8s, spec IDEAL_SAMPLE = 4 DEFAULT_CPU_REQUEST = 1.0 @@ -32,11 +32,11 @@ async def predict_single(db: aiosqlite.Connection, build: dict) -> dict: "mem_request": DEFAULT_MEM_REQUEST, } else: - # mapping of sample: [0] cpu_mean, [1] cpu_max, [2] mem_mean, [3] mem_max + # mapping of sample: [1] cpu_mean, [2] cpu_max, [3] mem_mean, [4] mem_max predictions = { # averages the respective metric in the sample - "cpu_request": round(sum([build[0] for build in sample]) / len(sample)), - "mem_request": sum([build[2] for build in sample]) / len(sample), + "cpu_request": round(sum([build[1] for build in sample]) / len(sample)), + "mem_request": sum([build[3] for build in sample]) / len(sample), } ensure_higher_pred(predictions, build["package"]["name"]) @@ -93,6 +93,7 @@ async def get_sample(db: aiosqlite.Connection, build: dict) -> list: flat_build = { "pkg_name": build["package"]["name"], "pkg_version": build["package"]["version"], + "pkg_variants": build["package"]["variants"], "compiler_name": build["compiler"]["name"], "compiler_version": build["compiler"]["version"], } @@ -112,12 +113,14 @@ async def get_sample(db: aiosqlite.Connection, build: dict) -> list: condition_values = [flat_build[param] for param in combo] # we want at least MIN_TRAIN_SAMPLE +1/= rows query = f""" - SELECT cpu_mean, cpu_max, mem_mean, mem_max FROM jobs WHERE ref='develop' - AND {' AND '.join(f'{param}=?' for param in combo)} + SELECT pkg_variants, cpu_mean, cpu_max, mem_mean, mem_max FROM jobs + WHERE ref='develop' AND {' AND '.join(f'{param}=?' for param in combo)} ORDER BY end DESC LIMIT {IDEAL_SAMPLE} """ + async with db.execute(query, condition_values) as cursor: sample = await cursor.fetchall() + sample = filter_variants(sample, flat_build) # we can accept the sample if it's 1 shorter if len(sample) >= IDEAL_SAMPLE - 1: return sample @@ -126,6 +129,41 @@ async def get_sample(db: aiosqlite.Connection, build: dict) -> list: return [] +def filter_variants(sample: list, build: dict) -> list: + """ + Filter the sample to match the build's variants. + """ + + EXPENSIVE_VARIANTS = { + "sycl", + "mpi", + "rocm", + "cuda", + "python", + "fortran", + "openmp", + "hdf5", + } + + matched_rows = [] + build_variants = spec.spec_variants(build["pkg_variants"]) + + # we prefer exact matches but + # we can accept the row if all the values of + # EXPENSIVE_VARIANTS match + for row in sample: + row_variants = spec.spec_variants(row[0]) + if build_variants == row_variants: + matched_rows.append(row) + elif all( + row_variants.get(variant) == build_variants.get(variant) + for variant in EXPENSIVE_VARIANTS + ): + matched_rows.append(row) + + return matched_rows + + def ensure_higher_pred(prediction: dict, pkg_name: str): """ Ensure that the prediction is higher than the current allocation diff --git a/gantry/views.py b/gantry/views.py index c1c92e4..1d4dffd 100644 --- a/gantry/views.py +++ b/gantry/views.py @@ -103,6 +103,8 @@ def validate_payload(payload: dict) -> bool: for field in ["package", "compiler"] for key in ["name", "version"] ) + # look for variants inside package + and isinstance(item.get("package", {}).get("variants"), str) ): return False From a87f0f1404fc51d60e8508f73df75f4fc7b29644 Mon Sep 17 00:00:00 2001 From: caetano melone Date: Wed, 31 Jan 2024 17:49:09 -0800 Subject: [PATCH 17/37] small fixes --- gantry/routes/prediction/prediction.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/gantry/routes/prediction/prediction.py b/gantry/routes/prediction/prediction.py index d02bf79..ecb5ccf 100644 --- a/gantry/routes/prediction/prediction.py +++ b/gantry/routes/prediction/prediction.py @@ -1,14 +1,12 @@ import asyncio import logging -# TODO clean all this up check obsidian notes to make sure everything's implemented - import aiosqlite from gantry.routes.prediction.current_mapping import pkg_mappings from gantry.util import k8s, spec -IDEAL_SAMPLE = 4 +IDEAL_SAMPLE = 5 DEFAULT_CPU_REQUEST = 1.0 DEFAULT_MEM_REQUEST = 2000 * 1_000_000 # 2GB in bytes @@ -74,7 +72,7 @@ async def predict_bulk(db: aiosqlite.Connection, builds: list) -> list: args: builds: list of dicts (see predict_single) - returns: see predict_single) + returns: see predict_single """ return await asyncio.gather(*(predict_single(db, build) for build in builds)) From 031706e91af7cac175f94339fcbaed84a8c1540b Mon Sep 17 00:00:00 2001 From: caetano melone Date: Wed, 14 Feb 2024 23:38:52 -0800 Subject: [PATCH 18/37] updated standard allocations --- gantry/routes/prediction/current_mapping.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gantry/routes/prediction/current_mapping.py b/gantry/routes/prediction/current_mapping.py index bd4fea7..a1727af 100644 --- a/gantry/routes/prediction/current_mapping.py +++ b/gantry/routes/prediction/current_mapping.py @@ -1 +1 @@ -pkg_mappings = {'rust': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 35000000000.0}, 'py-tensorflow': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 32000000000.0}, 'py-jaxlib': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 29000000000.0}, 'nvhpc': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 24000000000.0}, 'paraview': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 24000000000.0}, 'llvm': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 21000000000.0}, 'py-torch': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 21000000000.0}, 'dealii': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 19000000000.0}, 'mxnet': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 19000000000.0}, 'py-torchaudio': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 19000000000.0}, 'rocblas': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 19000000000.0}, 'ecp-data-vis-sdk': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'intel-tbb': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 15000000000.0}, 'llvm-amdgpu': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 15000000000.0}, 'salmon': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 15000000000.0}, 'trilinos': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'ascent': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'atk': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'axom': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'cistem': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'cmake': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'ctffind': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'cuda': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'dray': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'dyninst': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'gcc': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'ginkgo': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'hdf5': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'hipblas': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'hpx': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'kokkos-kernels': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'kokkos-nvcc-wrapper': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'lbann': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'magma': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'mesa': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'mfem': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'mpich': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'netlib-lapack': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'oce': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'openblas': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'openfoam': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'openturns': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'parallelio': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'plumed': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'precice': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'qt': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'raja': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'relion': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'rocfft': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'rocsolver': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'rocsparse': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'slate': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'strumpack': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'sundials': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'umpire': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'visit': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'vtk': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'vtk-h': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'vtk-m': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'warpx': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'wrf': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'wxwidgets': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'adios2': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'amrex': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'archer': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'autoconf-archive': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'binutils': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'blaspp': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'blt': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'boost': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'butterflypack': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'cabana': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'caliper': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'camp': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'chai': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'conduit': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'curl': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'datatransferkit': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'double-conversion': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'eigen': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'faodel': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'ffmpeg': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'fftw': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'fortrilinos': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'gettext': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'gperftools': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'gptune': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'heffte': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'hpctoolkit': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'hwloc': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'hydrogen': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'hypre': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'kokkos': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'lammps': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'lapackpp': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'legion': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'libtool': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'libxml2': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libzmq': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'llvm-openmp-ompt': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'mbedtls': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'mvapich2': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'nasm': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'netlib-scalapack': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'omega-h': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'openjpeg': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'openmpi': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'openpmd-api': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'pagmo2': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'papyrus': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'parsec': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'pdt': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'pegtl': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'petsc': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'pumi': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'py-beniget': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'py-cinemasci': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'pygmo': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'py-ipython-genutils': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'py-packaging': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'py-petsc4py': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'py-scipy': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'py-statsmodels': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'py-warlock': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'py-warpx': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'samrai': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'slepc': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'slurm': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'sqlite': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'superlu-dist': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'tasmanian': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'tau': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'upcxx': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'zfp': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'alsa-lib': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'ant': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'antlr': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'argobots': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'automake': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'berkeley-db': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'bison': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'bzip2': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'czmq': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'darshan-util': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'diffutils': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'docbook-xml': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'exmcutils': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'expat': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'findutils': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'flit': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'freetype': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'gawk': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'gdbm': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'glib': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'gmake': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'gotcha': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'hpcviewer': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'jansson': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'json-c': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libbsd': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libedit': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libevent': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libfabric': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libffi': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libgcrypt': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libiconv': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libidn2': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libjpeg-turbo': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libmd': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libnrm': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libpciaccess': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libpng': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libsigsegv': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libsodium': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libunistring': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libunwind': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libyaml': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'lua': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'lua-luaposix': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'lz4': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'm4': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'meson': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'metis': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'mpfr': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'ncurses': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'ninja': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'numactl': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'openjdk': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'openssh': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'openssl': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'papi': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'parallel-netcdf': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'pcre': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'pcre2': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'pdsh': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'perl': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'perl-data-dumper': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'pkgconf': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-alembic': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-cffi': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-cycler': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-decorator': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-idna': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-jsonschema': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-kiwisolver': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-mistune': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-pycparser': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-setuptools': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-setuptools-scm': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-six': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-testpath': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-wheel': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'qhull': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'readline': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'sed': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'snappy': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'superlu': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'swig': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'tar': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'tcl': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'texinfo': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'tut': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'unzip': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'util-linux-uuid': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'util-macros': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'xz': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'yaml-cpp': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'zlib': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'zstd': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}} +pkg_mappings = {'py-torch': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 48000000000.0}, 'rust': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 35000000000.0}, 'py-tensorflow': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 32000000000.0}, 'py-torchaudio': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 32000000000.0}, 'py-jaxlib': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 29000000000.0}, 'nvhpc': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 24000000000.0}, 'paraview': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 24000000000.0}, 'llvm': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 21000000000.0}, 'dealii': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 19000000000.0}, 'mxnet': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 19000000000.0}, 'rocblas': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 19000000000.0}, 'ecp-data-vis-sdk': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'intel-tbb': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 15000000000.0}, 'llvm-amdgpu': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 15000000000.0}, 'salmon': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 15000000000.0}, 'trilinos': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'ascent': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'axom': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'cistem': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'cuda': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'dray': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'gcc': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'ginkgo': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'hdf5': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'kokkos-kernels': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'mfem': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'mpich': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'netlib-lapack': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'openblas': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'openfoam': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'openturns': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'raja': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'relion': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'rocsolver': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'rocsparse': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'strumpack': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'sundials': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'visit': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'vtk': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'vtk-h': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'vtk-m': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'hpx': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 9000000000.0}, 'slate': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 9000000000.0}, 'warpx': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 9000000000.0}, 'hipblas': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 8000000000.0}, 'rocfft': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 8000000000.0}, 'umpire': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 8000000000.0}, 'lbann': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 7000000000.0}, 'magma': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 7000000000.0}, 'mesa': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 7000000000.0}, 'qt': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 7000000000.0}, 'dyninst': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 6000000000.0}, 'precice': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 6000000000.0}, 'cmake': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 5000000000.0}, 'plumed': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 5000000000.0}, 'wrf': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 5000000000.0}, 'parallelio': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 3000000000.0}, 'adios2': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'amrex': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'archer': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'autoconf-archive': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'blt': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'boost': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'butterflypack': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'cabana': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'caliper': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'camp': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'chai': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'conduit': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'curl': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'datatransferkit': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'faodel': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'fortrilinos': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'gettext': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'gptune': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'heffte': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'hpctoolkit': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'hwloc': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'hydrogen': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'hypre': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'kokkos': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'lammps': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'lapackpp': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'legion': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'libxml2': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libzmq': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'llvm-openmp-ompt': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'mbedtls': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'mvapich2': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'netlib-scalapack': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'omega-h': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'openjpeg': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'openmpi': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'openpmd-api': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'pagmo2': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'papyrus': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'parsec': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'petsc': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'pumi': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'py-beniget': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'py-cinemasci': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'pygmo': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'py-ipython-genutils': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'py-packaging': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'py-petsc4py': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'py-scipy': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'py-statsmodels': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'py-warlock': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'py-warpx': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'slepc': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'slurm': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'sqlite': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'superlu-dist': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'tasmanian': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'tau': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'upcxx': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'zfp': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'oce': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 3000000000.0}, 'binutils': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 2000000000.0}, 'blaspp': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 2000000000.0}, 'double-conversion': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 2000000000.0}, 'eigen': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 2000000000.0}, 'fftw': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 2000000000.0}, 'libtool': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 2000000000.0}, 'nasm': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 2000000000.0}, 'pegtl': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 2000000000.0}, 'pdt': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 2000000000.0}, 'kokkos-nvcc-wrapper': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 1000000000.0}, 'ffmpeg': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 1000000000.0}, 'gperftools': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 1000000000.0}, 'samrai': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 1000000000.0}, 'alsa-lib': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'ant': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'antlr': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'argobots': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'automake': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'berkeley-db': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'bison': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'bzip2': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'czmq': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'darshan-util': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'diffutils': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'docbook-xml': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'exmcutils': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'expat': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'findutils': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'flit': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'freetype': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'gawk': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'gdbm': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'glib': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'gmake': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'gotcha': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'hpcviewer': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'jansson': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'json-c': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libbsd': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libedit': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libevent': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libfabric': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libffi': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libgcrypt': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libiconv': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libidn2': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libjpeg-turbo': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libmd': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libnrm': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libpciaccess': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libpng': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libsigsegv': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libsodium': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libunistring': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libunwind': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libyaml': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'lua': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'lua-luaposix': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'lz4': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'm4': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'meson': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'metis': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'mpfr': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'ncurses': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'ninja': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'numactl': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'openjdk': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'openssh': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'openssl': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'papi': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'parallel-netcdf': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'pcre': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'pcre2': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'pdsh': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'perl': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'perl-data-dumper': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'pkgconf': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-alembic': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-cffi': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-cycler': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-decorator': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-idna': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-jsonschema': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-kiwisolver': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-mistune': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-pycparser': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-setuptools': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-setuptools-scm': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-six': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-testpath': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-wheel': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'qhull': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'readline': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'sed': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'snappy': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'superlu': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'swig': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'tar': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'tcl': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'texinfo': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'unzip': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'util-linux-uuid': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'util-macros': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'xz': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'yaml-cpp': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'zlib': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'zstd': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}} From 1125971e3a8eaa1a22f06a83d6d6608a3b2cb164 Mon Sep 17 00:00:00 2001 From: caetano melone Date: Thu, 15 Feb 2024 16:10:31 -0800 Subject: [PATCH 19/37] Re-write the variant filtering logic. Rather than grabbing the 4-5 rows that match based on other params (name, version for both pkg and compiler) and then filtering by variant, these queries are now split up into two. The program will first try to do an exact match on variants. If that doesn't work, it'll try to match based on expensive variants. This increases the amount of queries being done, but is a more robust system and will result in more matches. --- gantry/routes/prediction/prediction.py | 141 +++++++++++++++---------- 1 file changed, 84 insertions(+), 57 deletions(-) diff --git a/gantry/routes/prediction/prediction.py b/gantry/routes/prediction/prediction.py index ecb5ccf..4f5e0b0 100644 --- a/gantry/routes/prediction/prediction.py +++ b/gantry/routes/prediction/prediction.py @@ -8,7 +8,17 @@ IDEAL_SAMPLE = 5 DEFAULT_CPU_REQUEST = 1.0 -DEFAULT_MEM_REQUEST = 2000 * 1_000_000 # 2GB in bytes +DEFAULT_MEM_REQUEST = 2 * 1_000_000_000 # 2GB in bytes +EXPENSIVE_VARIANTS = { + "sycl", + "mpi", + "rocm", + "cuda", + "python", + "fortran", + "openmp", + "hdf5", +} async def predict_single(db: aiosqlite.Connection, build: dict) -> dict: @@ -30,11 +40,12 @@ async def predict_single(db: aiosqlite.Connection, build: dict) -> dict: "mem_request": DEFAULT_MEM_REQUEST, } else: - # mapping of sample: [1] cpu_mean, [2] cpu_max, [3] mem_mean, [4] mem_max + # mapping of sample: [0] cpu_mean, [1] cpu_max, [2] mem_mean, [3] mem_max predictions = { # averages the respective metric in the sample - "cpu_request": round(sum([build[1] for build in sample]) / len(sample)), - "mem_request": sum([build[3] for build in sample]) / len(sample), + # cpu should always be whole number + "cpu_request": round(sum([build[0] for build in sample]) / len(sample)), + "mem_request": sum([build[2] for build in sample]) / len(sample), } ensure_higher_pred(predictions, build["package"]["name"]) @@ -97,69 +108,85 @@ async def get_sample(db: aiosqlite.Connection, build: dict) -> list: } # ranked in order of priority, the params we would like to match on - param_combos = [ - ("pkg_name", "pkg_version", "compiler_name", "compiler_version"), - ("pkg_name", "compiler_name", "compiler_version"), - ("pkg_name", "pkg_version", "compiler_name"), - ("pkg_name", "compiler_name"), - ("pkg_name", "pkg_version"), - ("pkg_name",), - ] + param_combos = ( + ( + "pkg_name", + "pkg_variants", + "pkg_version", + "compiler_name", + "compiler_version", + ), + ("pkg_name", "pkg_variants", "compiler_name", "compiler_version"), + ("pkg_name", "pkg_variants", "pkg_version", "compiler_name"), + ("pkg_name", "pkg_variants", "compiler_name"), + ("pkg_name", "pkg_variants", "pkg_version"), + ("pkg_name", "pkg_variants"), + ) + + async def select_sample(query: str, filters: dict, extra_params: list = []) -> list: + async with db.execute(query, list(filters.values()) + extra_params) as cursor: + sample = await cursor.fetchall() + # we can accept the sample if it's 1 shorter + if len(sample) >= IDEAL_SAMPLE - 1: + return sample + return [] for combo in param_combos: - # filter by the params we want - condition_values = [flat_build[param] for param in combo] - # we want at least MIN_TRAIN_SAMPLE +1/= rows + filters = {param: flat_build[param] for param in combo} + + # the first attempt at getting a sample is to match on all the params + # within this combo, variants included query = f""" - SELECT pkg_variants, cpu_mean, cpu_max, mem_mean, mem_max FROM jobs - WHERE ref='develop' AND {' AND '.join(f'{param}=?' for param in combo)} + SELECT cpu_mean, cpu_max, mem_mean, mem_max FROM jobs + WHERE ref='develop' AND {' AND '.join(f'{param}=?' for param in filters.keys())} ORDER BY end DESC LIMIT {IDEAL_SAMPLE} """ - async with db.execute(query, condition_values) as cursor: - sample = await cursor.fetchall() - sample = filter_variants(sample, flat_build) - # we can accept the sample if it's 1 shorter - if len(sample) >= IDEAL_SAMPLE - 1: - return sample - # continue if we didn't find enough rows - - return [] + if sample := await select_sample(query, filters): + return sample + + # if we are not able to get a sufficient sample, we'll try to filter + # by expensive variants, rather than an exact variant match + + filters.pop("pkg_variants") + variants = spec.spec_variants(flat_build["pkg_variants"]) + + exp_variant_conditions = [] + exp_variant_values = [] + + # iterate through all the expensive variants and create a set of conditions + # for the select query + for var in EXPENSIVE_VARIANTS: + if var in variants: + # if the client has queried for an expensive variant, we want to ensure + # that the sample has the same exact value + exp_variant_conditions.append( + f"json_extract(pkg_variants, '$.{var}')=?" + ) + exp_variant_values.append(int(variants.get(var, 0))) + else: + # if an expensive variant was not queried for, + # we want to make sure that the variant was not set within the sample + # as we want to ensure that the sample is not biased towards + # the presence of expensive variants (or lack thereof) + exp_variant_conditions.append( + f"json_extract(pkg_variants, '$.{var}') IS NULL" + ) + query = f""" + SELECT cpu_mean, cpu_max, mem_mean, mem_max FROM jobs + WHERE ref='develop' AND {' AND '.join(f'{param}=?' for param in filters.keys())} + AND {' AND '.join(exp_variant_conditions)} + ORDER BY end DESC LIMIT {IDEAL_SAMPLE} + """ -def filter_variants(sample: list, build: dict) -> list: - """ - Filter the sample to match the build's variants. - """ + if sample := await select_sample(query, filters, exp_variant_values): + return sample - EXPENSIVE_VARIANTS = { - "sycl", - "mpi", - "rocm", - "cuda", - "python", - "fortran", - "openmp", - "hdf5", - } + # go to the next combo, we want to ensure that expensive variants + # are taken into account - matched_rows = [] - build_variants = spec.spec_variants(build["pkg_variants"]) - - # we prefer exact matches but - # we can accept the row if all the values of - # EXPENSIVE_VARIANTS match - for row in sample: - row_variants = spec.spec_variants(row[0]) - if build_variants == row_variants: - matched_rows.append(row) - elif all( - row_variants.get(variant) == build_variants.get(variant) - for variant in EXPENSIVE_VARIANTS - ): - matched_rows.append(row) - - return matched_rows + return [] def ensure_higher_pred(prediction: dict, pkg_name: str): From ce9bab3c8517466a9c8a11847edf4d969af7c571 Mon Sep 17 00:00:00 2001 From: caetano melone Date: Thu, 15 Feb 2024 16:16:17 -0800 Subject: [PATCH 20/37] formatting --- gantry/routes/prediction/current_mapping.py | 1 + gantry/util/k8s.py | 1 + 2 files changed, 2 insertions(+) diff --git a/gantry/routes/prediction/current_mapping.py b/gantry/routes/prediction/current_mapping.py index a1727af..1eb6418 100644 --- a/gantry/routes/prediction/current_mapping.py +++ b/gantry/routes/prediction/current_mapping.py @@ -1 +1,2 @@ +# fmt: off pkg_mappings = {'py-torch': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 48000000000.0}, 'rust': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 35000000000.0}, 'py-tensorflow': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 32000000000.0}, 'py-torchaudio': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 32000000000.0}, 'py-jaxlib': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 29000000000.0}, 'nvhpc': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 24000000000.0}, 'paraview': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 24000000000.0}, 'llvm': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 21000000000.0}, 'dealii': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 19000000000.0}, 'mxnet': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 19000000000.0}, 'rocblas': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 19000000000.0}, 'ecp-data-vis-sdk': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'intel-tbb': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 15000000000.0}, 'llvm-amdgpu': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 15000000000.0}, 'salmon': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 15000000000.0}, 'trilinos': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'ascent': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'axom': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'cistem': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'cuda': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'dray': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'gcc': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'ginkgo': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'hdf5': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'kokkos-kernels': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'mfem': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'mpich': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'netlib-lapack': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'openblas': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'openfoam': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'openturns': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'raja': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'relion': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'rocsolver': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'rocsparse': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'strumpack': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'sundials': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'visit': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'vtk': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'vtk-h': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'vtk-m': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'hpx': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 9000000000.0}, 'slate': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 9000000000.0}, 'warpx': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 9000000000.0}, 'hipblas': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 8000000000.0}, 'rocfft': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 8000000000.0}, 'umpire': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 8000000000.0}, 'lbann': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 7000000000.0}, 'magma': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 7000000000.0}, 'mesa': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 7000000000.0}, 'qt': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 7000000000.0}, 'dyninst': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 6000000000.0}, 'precice': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 6000000000.0}, 'cmake': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 5000000000.0}, 'plumed': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 5000000000.0}, 'wrf': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 5000000000.0}, 'parallelio': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 3000000000.0}, 'adios2': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'amrex': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'archer': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'autoconf-archive': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'blt': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'boost': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'butterflypack': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'cabana': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'caliper': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'camp': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'chai': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'conduit': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'curl': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'datatransferkit': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'faodel': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'fortrilinos': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'gettext': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'gptune': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'heffte': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'hpctoolkit': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'hwloc': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'hydrogen': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'hypre': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'kokkos': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'lammps': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'lapackpp': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'legion': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'libxml2': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libzmq': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'llvm-openmp-ompt': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'mbedtls': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'mvapich2': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'netlib-scalapack': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'omega-h': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'openjpeg': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'openmpi': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'openpmd-api': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'pagmo2': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'papyrus': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'parsec': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'petsc': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'pumi': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'py-beniget': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'py-cinemasci': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'pygmo': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'py-ipython-genutils': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'py-packaging': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'py-petsc4py': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'py-scipy': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'py-statsmodels': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'py-warlock': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'py-warpx': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'slepc': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'slurm': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'sqlite': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'superlu-dist': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'tasmanian': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'tau': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'upcxx': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'zfp': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'oce': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 3000000000.0}, 'binutils': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 2000000000.0}, 'blaspp': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 2000000000.0}, 'double-conversion': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 2000000000.0}, 'eigen': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 2000000000.0}, 'fftw': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 2000000000.0}, 'libtool': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 2000000000.0}, 'nasm': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 2000000000.0}, 'pegtl': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 2000000000.0}, 'pdt': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 2000000000.0}, 'kokkos-nvcc-wrapper': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 1000000000.0}, 'ffmpeg': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 1000000000.0}, 'gperftools': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 1000000000.0}, 'samrai': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 1000000000.0}, 'alsa-lib': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'ant': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'antlr': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'argobots': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'automake': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'berkeley-db': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'bison': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'bzip2': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'czmq': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'darshan-util': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'diffutils': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'docbook-xml': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'exmcutils': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'expat': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'findutils': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'flit': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'freetype': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'gawk': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'gdbm': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'glib': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'gmake': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'gotcha': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'hpcviewer': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'jansson': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'json-c': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libbsd': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libedit': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libevent': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libfabric': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libffi': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libgcrypt': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libiconv': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libidn2': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libjpeg-turbo': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libmd': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libnrm': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libpciaccess': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libpng': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libsigsegv': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libsodium': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libunistring': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libunwind': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libyaml': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'lua': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'lua-luaposix': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'lz4': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'm4': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'meson': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'metis': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'mpfr': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'ncurses': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'ninja': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'numactl': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'openjdk': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'openssh': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'openssl': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'papi': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'parallel-netcdf': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'pcre': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'pcre2': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'pdsh': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'perl': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'perl-data-dumper': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'pkgconf': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-alembic': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-cffi': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-cycler': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-decorator': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-idna': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-jsonschema': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-kiwisolver': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-mistune': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-pycparser': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-setuptools': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-setuptools-scm': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-six': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-testpath': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-wheel': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'qhull': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'readline': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'sed': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'snappy': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'superlu': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'swig': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'tar': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'tcl': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'texinfo': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'unzip': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'util-linux-uuid': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'util-macros': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'xz': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'yaml-cpp': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'zlib': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'zstd': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}} diff --git a/gantry/util/k8s.py b/gantry/util/k8s.py index 75b70b9..5e2eb46 100644 --- a/gantry/util/k8s.py +++ b/gantry/util/k8s.py @@ -2,6 +2,7 @@ # these functions convert the predictions to k8s friendly format + def convert_bytes(bytes: float) -> str: """bytes to megabytes""" return str(int(bytes * BYTES_TO_MEGABYTES)) + "M" From 00795b2d525779203e22b4938f533f37f5e19440 Mon Sep 17 00:00:00 2001 From: caetano melone Date: Thu, 15 Feb 2024 16:19:05 -0800 Subject: [PATCH 21/37] =?UTF-8?q?formatting=20=F0=9F=98=B5=E2=80=8D?= =?UTF-8?q?=F0=9F=92=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- gantry/routes/prediction/current_mapping.py | 1 + 1 file changed, 1 insertion(+) diff --git a/gantry/routes/prediction/current_mapping.py b/gantry/routes/prediction/current_mapping.py index 1eb6418..823e831 100644 --- a/gantry/routes/prediction/current_mapping.py +++ b/gantry/routes/prediction/current_mapping.py @@ -1,2 +1,3 @@ # fmt: off +# flake8: noqa pkg_mappings = {'py-torch': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 48000000000.0}, 'rust': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 35000000000.0}, 'py-tensorflow': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 32000000000.0}, 'py-torchaudio': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 32000000000.0}, 'py-jaxlib': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 29000000000.0}, 'nvhpc': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 24000000000.0}, 'paraview': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 24000000000.0}, 'llvm': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 21000000000.0}, 'dealii': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 19000000000.0}, 'mxnet': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 19000000000.0}, 'rocblas': {'build_jobs': 12, 'cpu_request': 12.0, 'mem_request': 19000000000.0}, 'ecp-data-vis-sdk': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'intel-tbb': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 15000000000.0}, 'llvm-amdgpu': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 15000000000.0}, 'salmon': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 15000000000.0}, 'trilinos': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'ascent': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'axom': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'cistem': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'cuda': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'dray': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'gcc': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'ginkgo': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'hdf5': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'kokkos-kernels': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'mfem': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'mpich': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'netlib-lapack': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'openblas': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'openfoam': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'openturns': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'raja': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'relion': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'rocsolver': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'rocsparse': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'strumpack': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'sundials': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'visit': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 12000000000.0}, 'vtk': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'vtk-h': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'vtk-m': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'hpx': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 9000000000.0}, 'slate': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 9000000000.0}, 'warpx': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 9000000000.0}, 'hipblas': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 8000000000.0}, 'rocfft': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 8000000000.0}, 'umpire': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 8000000000.0}, 'lbann': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 7000000000.0}, 'magma': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 7000000000.0}, 'mesa': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 7000000000.0}, 'qt': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 7000000000.0}, 'dyninst': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 6000000000.0}, 'precice': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 6000000000.0}, 'cmake': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 5000000000.0}, 'plumed': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 5000000000.0}, 'wrf': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 5000000000.0}, 'parallelio': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 3000000000.0}, 'adios2': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'amrex': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'archer': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'autoconf-archive': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'blt': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'boost': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'butterflypack': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'cabana': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'caliper': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'camp': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'chai': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'conduit': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'curl': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'datatransferkit': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'faodel': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'fortrilinos': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'gettext': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'gptune': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'heffte': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'hpctoolkit': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'hwloc': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'hydrogen': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'hypre': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'kokkos': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'lammps': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'lapackpp': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'legion': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'libxml2': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libzmq': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'llvm-openmp-ompt': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'mbedtls': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'mvapich2': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'netlib-scalapack': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'omega-h': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'openjpeg': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'openmpi': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'openpmd-api': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'pagmo2': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'papyrus': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'parsec': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'petsc': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'pumi': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'py-beniget': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'py-cinemasci': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'pygmo': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'py-ipython-genutils': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'py-packaging': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'py-petsc4py': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'py-scipy': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'py-statsmodels': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'py-warlock': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'py-warpx': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'slepc': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'slurm': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'sqlite': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'superlu-dist': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'tasmanian': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'tau': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'upcxx': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 4000000000.0}, 'zfp': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'oce': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 3000000000.0}, 'binutils': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 2000000000.0}, 'blaspp': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 2000000000.0}, 'double-conversion': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 2000000000.0}, 'eigen': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 2000000000.0}, 'fftw': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 2000000000.0}, 'libtool': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 2000000000.0}, 'nasm': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 2000000000.0}, 'pegtl': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 2000000000.0}, 'pdt': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 2000000000.0}, 'kokkos-nvcc-wrapper': {'build_jobs': 8, 'cpu_request': 8.0, 'mem_request': 1000000000.0}, 'ffmpeg': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 1000000000.0}, 'gperftools': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 1000000000.0}, 'samrai': {'build_jobs': 2, 'cpu_request': 2.0, 'mem_request': 1000000000.0}, 'alsa-lib': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'ant': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'antlr': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'argobots': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'automake': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'berkeley-db': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'bison': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'bzip2': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'czmq': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'darshan-util': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'diffutils': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'docbook-xml': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'exmcutils': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'expat': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'findutils': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'flit': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'freetype': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'gawk': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'gdbm': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'glib': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'gmake': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'gotcha': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'hpcviewer': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'jansson': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'json-c': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libbsd': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libedit': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libevent': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libfabric': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libffi': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libgcrypt': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libiconv': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libidn2': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libjpeg-turbo': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libmd': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libnrm': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libpciaccess': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libpng': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libsigsegv': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libsodium': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libunistring': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libunwind': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'libyaml': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'lua': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'lua-luaposix': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'lz4': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'm4': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'meson': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'metis': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'mpfr': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'ncurses': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'ninja': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'numactl': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'openjdk': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'openssh': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'openssl': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'papi': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'parallel-netcdf': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'pcre': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'pcre2': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'pdsh': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'perl': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'perl-data-dumper': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'pkgconf': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-alembic': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-cffi': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-cycler': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-decorator': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-idna': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-jsonschema': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-kiwisolver': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-mistune': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-pycparser': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-setuptools': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-setuptools-scm': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-six': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-testpath': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'py-wheel': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'qhull': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'readline': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'sed': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'snappy': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'superlu': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'swig': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'tar': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'tcl': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'texinfo': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'unzip': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'util-linux-uuid': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'util-macros': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'xz': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'yaml-cpp': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'zlib': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}, 'zstd': {'build_jobs': 1, 'cpu_request': 0.5, 'mem_request': 500000000.0}} From 9affc34dac610971221330118f6f58b9b48208f9 Mon Sep 17 00:00:00 2001 From: caetano melone Date: Thu, 15 Feb 2024 22:27:18 -0800 Subject: [PATCH 22/37] logging -> logger usage --- gantry/routes/prediction/prediction.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gantry/routes/prediction/prediction.py b/gantry/routes/prediction/prediction.py index 4f5e0b0..1277881 100644 --- a/gantry/routes/prediction/prediction.py +++ b/gantry/routes/prediction/prediction.py @@ -6,6 +6,8 @@ from gantry.routes.prediction.current_mapping import pkg_mappings from gantry.util import k8s, spec +logger = logging.getLogger(__name__) + IDEAL_SAMPLE = 5 DEFAULT_CPU_REQUEST = 1.0 DEFAULT_MEM_REQUEST = 2 * 1_000_000_000 # 2GB in bytes @@ -52,10 +54,10 @@ async def predict_single(db: aiosqlite.Connection, build: dict) -> dict: # warn if the prediction is below some thresholds if predictions["cpu_request"] < 0.25: - logging.warning(f"Warning: CPU request for {build['hash']} is below 0.25 cores") + logger.warning(f"Warning: CPU request for {build['hash']} is below 0.25 cores") predictions["cpu_request"] = DEFAULT_CPU_REQUEST if predictions["mem_request"] < 10_000_000: - logging.warning(f"Warning: Memory request for {build['hash']} is below 10MB") + logger.warning(f"Warning: Memory request for {build['hash']} is below 10MB") predictions["mem_request"] = DEFAULT_MEM_REQUEST # convert predictions to k8s friendly format From 777c175ca268aa7e097df1f62799503b9a9e5b55 Mon Sep 17 00:00:00 2001 From: caetano melone Date: Thu, 15 Feb 2024 22:35:57 -0800 Subject: [PATCH 23/37] remove unnecessary comment [ci skip] --- gantry/routes/prediction/prediction.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/gantry/routes/prediction/prediction.py b/gantry/routes/prediction/prediction.py index 1277881..62fc8b8 100644 --- a/gantry/routes/prediction/prediction.py +++ b/gantry/routes/prediction/prediction.py @@ -185,9 +185,6 @@ async def select_sample(query: str, filters: dict, extra_params: list = []) -> l if sample := await select_sample(query, filters, exp_variant_values): return sample - # go to the next combo, we want to ensure that expensive variants - # are taken into account - return [] From 7aeac6836e04af23a2b6aadce976c23ea3b44b4d Mon Sep 17 00:00:00 2001 From: caetano melone Date: Tue, 27 Feb 2024 00:36:23 -0800 Subject: [PATCH 24/37] move validate payload for prediction into util for easier testing --- gantry/util/prediction.py | 33 +++++++++++++++++++++++++++++++++ gantry/views.py | 31 +------------------------------ 2 files changed, 34 insertions(+), 30 deletions(-) create mode 100644 gantry/util/prediction.py diff --git a/gantry/util/prediction.py b/gantry/util/prediction.py new file mode 100644 index 0000000..951901e --- /dev/null +++ b/gantry/util/prediction.py @@ -0,0 +1,33 @@ +from gantry.util.spec import spec_variants + + +def validate_payload(payload: dict) -> bool: + """Ensures that the payload from the client is valid.""" + # must be a dict or a list + if not isinstance(payload, (dict, list)): + return False + + if isinstance(payload, dict): + # put dict in list for iteration + payload = [payload] + + for item in payload: + if not ( + # item must be dict + isinstance(item, dict) + # must contain hash field + and isinstance(item.get("hash"), str) + # must contain name and version + # for both package and compiler + and all( + isinstance(item.get(field, {}).get(key), str) + for field in ["package", "compiler"] + for key in ["name", "version"] + ) + # look for variants inside package + and isinstance(item.get("package", {}).get("variants"), str) + and spec_variants(item["package"]["variants"]) != {} + ): + return False + + return True diff --git a/gantry/views.py b/gantry/views.py index 1d4dffd..ab62f51 100644 --- a/gantry/views.py +++ b/gantry/views.py @@ -7,6 +7,7 @@ from gantry.routes.collection import fetch_job from gantry.routes.prediction.prediction import predict_bulk, predict_single +from gantry.util.prediction import validate_payload logger = logging.getLogger(__name__) routes = web.RouteTableDef() @@ -80,36 +81,6 @@ async def allocation(request: web.Request) -> web.Response: except json.decoder.JSONDecodeError: return web.Response(status=400, text="invalid json") - def validate_payload(payload: dict) -> bool: - """Ensures that the payload from the client is valid.""" - # must be a dict or a list - if not isinstance(payload, (dict, list)): - return False - - if isinstance(payload, dict): - # put dict in list for iteration - payload = [payload] - - for item in payload: - if not ( - # item must be dict - isinstance(item, dict) - # must contain hash field - and isinstance(item.get("hash"), str) - # must contain name and version - # for both package and compiler - and all( - isinstance(item.get(field, {}).get(key), str) - for field in ["package", "compiler"] - for key in ["name", "version"] - ) - # look for variants inside package - and isinstance(item.get("package", {}).get("variants"), str) - ): - return False - - return True - if not validate_payload(payload): return web.Response(status=400, text="invalid payload") From feda9c654b865d82411b5b05f96914583830e888 Mon Sep 17 00:00:00 2001 From: caetano melone Date: Tue, 27 Feb 2024 00:39:35 -0800 Subject: [PATCH 25/37] make a couple changes to prediction to make testing easier --- gantry/routes/prediction/prediction.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/gantry/routes/prediction/prediction.py b/gantry/routes/prediction/prediction.py index 62fc8b8..4a3e1e3 100644 --- a/gantry/routes/prediction/prediction.py +++ b/gantry/routes/prediction/prediction.py @@ -1,5 +1,7 @@ import asyncio +import json import logging +import os import aiosqlite @@ -50,7 +52,8 @@ async def predict_single(db: aiosqlite.Connection, build: dict) -> dict: "mem_request": sum([build[2] for build in sample]) / len(sample), } - ensure_higher_pred(predictions, build["package"]["name"]) + if os.environ.get("PREDICT_STRATEGY") == "ensure_higher": + ensure_higher_pred(predictions, build["package"]["name"]) # warn if the prediction is below some thresholds if predictions["cpu_request"] < 0.25: @@ -101,10 +104,12 @@ async def get_sample(db: aiosqlite.Connection, build: dict) -> list: list of lists with cpu_mean, cpu_max, mem_mean, mem_max """ + pkg_variants = spec.spec_variants(build["package"]["variants"]) flat_build = { "pkg_name": build["package"]["name"], "pkg_version": build["package"]["version"], - "pkg_variants": build["package"]["variants"], + # variants are represented as JSON in the database + "pkg_variants": json.dumps(pkg_variants), "compiler_name": build["compiler"]["name"], "compiler_version": build["compiler"]["version"], } @@ -151,7 +156,6 @@ async def select_sample(query: str, filters: dict, extra_params: list = []) -> l # by expensive variants, rather than an exact variant match filters.pop("pkg_variants") - variants = spec.spec_variants(flat_build["pkg_variants"]) exp_variant_conditions = [] exp_variant_values = [] @@ -159,13 +163,13 @@ async def select_sample(query: str, filters: dict, extra_params: list = []) -> l # iterate through all the expensive variants and create a set of conditions # for the select query for var in EXPENSIVE_VARIANTS: - if var in variants: + if var in pkg_variants: # if the client has queried for an expensive variant, we want to ensure # that the sample has the same exact value exp_variant_conditions.append( f"json_extract(pkg_variants, '$.{var}')=?" ) - exp_variant_values.append(int(variants.get(var, 0))) + exp_variant_values.append(int(pkg_variants.get(var, 0))) else: # if an expensive variant was not queried for, # we want to make sure that the variant was not set within the sample From 29b99ad05aa07207c97fa9f56c37d4a60cc0b783 Mon Sep 17 00:00:00 2001 From: caetano melone Date: Tue, 27 Feb 2024 01:00:31 -0800 Subject: [PATCH 26/37] remove bulk prediction functionality we have two options for accepting multiple predictions: 1) instructing the client to send a request for each prediction 2) allowing the client to pass a list of specs to predict for the second option presents a problem when we get into the weeds of GET request length limits. since specs can vary widely in length when running json.dumps() on them, it would be difficult to either instruct the client to limit their length, or to somehow validate it. when we were allowing bulk prediction in one GET request, it was handled through an asyncio.gather() call. this approach is approximately 2x faster than individual HTTP requests when testing 5000 consecutive calls to the API (4s vs 8s). In this case, 8 seconds is not very long so it's not a problem, but we can revisit this in the future if we run into performance issues. --- gantry/routes/prediction/prediction.py | 13 -------- gantry/util/prediction.py | 41 +++++++++++--------------- gantry/views.py | 15 ++-------- 3 files changed, 21 insertions(+), 48 deletions(-) diff --git a/gantry/routes/prediction/prediction.py b/gantry/routes/prediction/prediction.py index 4a3e1e3..146dca5 100644 --- a/gantry/routes/prediction/prediction.py +++ b/gantry/routes/prediction/prediction.py @@ -1,4 +1,3 @@ -import asyncio import json import logging import os @@ -82,18 +81,6 @@ async def predict_single(db: aiosqlite.Connection, build: dict) -> dict: } -async def predict_bulk(db: aiosqlite.Connection, builds: list) -> list: - """ - Handles a bulk request of builds - - args: - builds: list of dicts (see predict_single) - returns: see predict_single - """ - - return await asyncio.gather(*(predict_single(db, build) for build in builds)) - - async def get_sample(db: aiosqlite.Connection, build: dict) -> list: """ Selects a sample of builds to use for prediction diff --git a/gantry/util/prediction.py b/gantry/util/prediction.py index 951901e..e6b336e 100644 --- a/gantry/util/prediction.py +++ b/gantry/util/prediction.py @@ -4,30 +4,25 @@ def validate_payload(payload: dict) -> bool: """Ensures that the payload from the client is valid.""" # must be a dict or a list - if not isinstance(payload, (dict, list)): + if not isinstance(payload, dict): return False - if isinstance(payload, dict): - # put dict in list for iteration - payload = [payload] - - for item in payload: - if not ( - # item must be dict - isinstance(item, dict) - # must contain hash field - and isinstance(item.get("hash"), str) - # must contain name and version - # for both package and compiler - and all( - isinstance(item.get(field, {}).get(key), str) - for field in ["package", "compiler"] - for key in ["name", "version"] - ) - # look for variants inside package - and isinstance(item.get("package", {}).get("variants"), str) - and spec_variants(item["package"]["variants"]) != {} - ): - return False + if not ( + # item must be dict + isinstance(payload, dict) + # must contain hash field + and isinstance(payload.get("hash"), str) + # must contain name and version + # for both package and compiler + and all( + isinstance(payload.get(field, {}).get(key), str) + for field in ["package", "compiler"] + for key in ["name", "version"] + ) + # look for variants inside package + and isinstance(payload.get("package", {}).get("variants"), str) + and spec_variants(payload["package"]["variants"]) != {} + ): + return False return True diff --git a/gantry/views.py b/gantry/views.py index ab62f51..292d376 100644 --- a/gantry/views.py +++ b/gantry/views.py @@ -6,7 +6,7 @@ from aiohttp import web from gantry.routes.collection import fetch_job -from gantry.routes.prediction.prediction import predict_bulk, predict_single +from gantry.routes.prediction.prediction import predict_single from gantry.util.prediction import validate_payload logger = logging.getLogger(__name__) @@ -49,6 +49,7 @@ async def allocation(request: web.Request) -> web.Response: "package": { "name": "string", "version": "string" + "variants": "string" }, "compiler": { "name": "string", @@ -56,8 +57,6 @@ async def allocation(request: web.Request) -> web.Response: } } - also accepts a list of the above objects - returns: { @@ -67,9 +66,6 @@ async def allocation(request: web.Request) -> web.Response: "mem_request": "float", } } - - or a list of the above objects - """ payload = request.query.get("query") @@ -84,9 +80,4 @@ async def allocation(request: web.Request) -> web.Response: if not validate_payload(payload): return web.Response(status=400, text="invalid payload") - if isinstance(payload, dict): - response = await predict_single(request.app["db"], payload) - elif isinstance(payload, list): - response = await predict_bulk(request.app["db"], payload) - - return web.json_response(response) + return web.json_response(await predict_single(request.app["db"], payload)) From d8b0598030939c4f484f14ebaf4ca54d4db870f9 Mon Sep 17 00:00:00 2001 From: caetano melone Date: Tue, 27 Feb 2024 14:31:52 -0800 Subject: [PATCH 27/37] fix: remove unnecessary condition in validate_payload --- gantry/util/prediction.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/gantry/util/prediction.py b/gantry/util/prediction.py index e6b336e..e24dfc6 100644 --- a/gantry/util/prediction.py +++ b/gantry/util/prediction.py @@ -3,9 +3,6 @@ def validate_payload(payload: dict) -> bool: """Ensures that the payload from the client is valid.""" - # must be a dict or a list - if not isinstance(payload, dict): - return False if not ( # item must be dict From 50aead89335ff2406c6b53a9ed327b16b899e570 Mon Sep 17 00:00:00 2001 From: caetano melone Date: Mon, 4 Mar 2024 10:20:58 -0800 Subject: [PATCH 28/37] fix: remove custom GET length, 8K is sufficient for one url-encoded json spec --- docs/deploy.md | 3 --- gantry/__main__.py | 29 +++-------------------------- 2 files changed, 3 insertions(+), 29 deletions(-) diff --git a/docs/deploy.md b/docs/deploy.md index 7c2d95a..829fc95 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -28,9 +28,6 @@ The following variables should be exposed to the container. Those **bolded** are - **`GITLAB_API_TOKEN`** - this token should have API read access - **`GITLAB_WEBHOOK_TOKEN`** - coordinate this value with the collection webhook - **`DB_FILE`** - path where the application can access the SQLite file -- `MAX_GET_SIZE` - the maximum `GET` request (in bytes), default is 8MB -- `GANTRY_HOST` - web app hostname, default is `localhost` -- `GANTRY_PORT` - web app port, default is `8080` - `PREDICT_STRATEGY` - optional mode for the prediction algorithm - options: - `ensure_higher`: if the predicted resource usage is below current levels, it will disregard the prediction and keep what would be allocated without Gantry's intervention diff --git a/gantry/__main__.py b/gantry/__main__.py index 0a9854a..a420f94 100644 --- a/gantry/__main__.py +++ b/gantry/__main__.py @@ -1,4 +1,3 @@ -import asyncio import logging import os @@ -34,35 +33,13 @@ async def init_clients(app: web.Application): ) -async def main(): +def main(): app = web.Application() app.add_routes(routes) app.cleanup_ctx.append(init_db) app.on_startup.append(init_clients) - runner = web.AppRunner( - app, max_line_size=int(os.environ.get("MAX_GET_SIZE", 800_000)) - ) - await runner.setup() - port = os.environ.get("GANTRY_PORT", 8080) - host = os.environ.get("GANTRY_HOST", "localhost") - site = web.TCPSite( - runner, - host, - port, - ) - await site.start() - - print(f"Gantry running on {host}:{port}") - print("-------------------") - - try: - # wait for finish signal - await asyncio.Future() - except asyncio.CancelledError: - pass - finally: - await runner.cleanup() + web.run_app(app) if __name__ == "__main__": - asyncio.run(main()) + main() From f5f23638e61226e63056a65ed56ede5f24cd7df4 Mon Sep 17 00:00:00 2001 From: caetano melone Date: Mon, 4 Mar 2024 11:49:49 -0800 Subject: [PATCH 29/37] fix: remove hash from GET payload --- gantry/routes/prediction/prediction.py | 5 ++--- gantry/util/prediction.py | 2 -- gantry/views.py | 2 -- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/gantry/routes/prediction/prediction.py b/gantry/routes/prediction/prediction.py index 146dca5..fd3688d 100644 --- a/gantry/routes/prediction/prediction.py +++ b/gantry/routes/prediction/prediction.py @@ -56,10 +56,10 @@ async def predict_single(db: aiosqlite.Connection, build: dict) -> dict: # warn if the prediction is below some thresholds if predictions["cpu_request"] < 0.25: - logger.warning(f"Warning: CPU request for {build['hash']} is below 0.25 cores") + logger.warning(f"Warning: CPU request for {build} is below 0.25 cores") predictions["cpu_request"] = DEFAULT_CPU_REQUEST if predictions["mem_request"] < 10_000_000: - logger.warning(f"Warning: Memory request for {build['hash']} is below 10MB") + logger.warning(f"Warning: Memory request for {build} is below 10MB") predictions["mem_request"] = DEFAULT_MEM_REQUEST # convert predictions to k8s friendly format @@ -70,7 +70,6 @@ async def predict_single(db: aiosqlite.Connection, build: dict) -> dict: predictions[k] = k8s.convert_bytes(v) return { - "hash": build["hash"], "variables": { # spack uses these env vars to set the resource requests # set them here at the last minute to avoid using these vars diff --git a/gantry/util/prediction.py b/gantry/util/prediction.py index e24dfc6..ea94c8a 100644 --- a/gantry/util/prediction.py +++ b/gantry/util/prediction.py @@ -7,8 +7,6 @@ def validate_payload(payload: dict) -> bool: if not ( # item must be dict isinstance(payload, dict) - # must contain hash field - and isinstance(payload.get("hash"), str) # must contain name and version # for both package and compiler and all( diff --git a/gantry/views.py b/gantry/views.py index 292d376..abf706b 100644 --- a/gantry/views.py +++ b/gantry/views.py @@ -45,7 +45,6 @@ async def allocation(request: web.Request) -> web.Response: acceptable payload: { - "hash": "string", "package": { "name": "string", "version": "string" @@ -60,7 +59,6 @@ async def allocation(request: web.Request) -> web.Response: returns: { - "hash": "string", "variables": { "cpu_request": "float", "mem_request": "float", From 341c5a36a019ce562c562e263033819960663c1e Mon Sep 17 00:00:00 2001 From: caetano melone Date: Wed, 6 Mar 2024 09:54:59 -0800 Subject: [PATCH 30/37] fix: update docstring for allocation view --- gantry/views.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/gantry/views.py b/gantry/views.py index abf706b..7c18011 100644 --- a/gantry/views.py +++ b/gantry/views.py @@ -42,6 +42,9 @@ async def collect_job(request: web.Request) -> web.Response: @routes.get("/v1/allocation") async def allocation(request: web.Request) -> web.Response: """ + Given a spec in JSON format, return environment variables + that set resource allocations based on historical data. + acceptable payload: { @@ -59,11 +62,12 @@ async def allocation(request: web.Request) -> web.Response: returns: { - "variables": { - "cpu_request": "float", - "mem_request": "float", - } + "variables": {} } + + the variables key contains the environment variables + that should be set within the build environment + example: KUBERNETES_CPU_REQUEST, KUBERNETES_CPU_LIMIT, etc. """ payload = request.query.get("query") From 558d57571a8323de1e2abc06a50c4c003d7ca58d Mon Sep 17 00:00:00 2001 From: caetano melone Date: Thu, 7 Mar 2024 12:27:44 -0800 Subject: [PATCH 31/37] allocation API now accepts spec string rather than JSON payload --- gantry/routes/prediction/prediction.py | 39 ++++++++++------------ gantry/util/prediction.py | 23 ------------- gantry/util/spec.py | 46 ++++++++++++++++++++++++++ gantry/views.py | 38 +++++++-------------- 4 files changed, 76 insertions(+), 70 deletions(-) delete mode 100644 gantry/util/prediction.py diff --git a/gantry/routes/prediction/prediction.py b/gantry/routes/prediction/prediction.py index fd3688d..642572f 100644 --- a/gantry/routes/prediction/prediction.py +++ b/gantry/routes/prediction/prediction.py @@ -5,7 +5,7 @@ import aiosqlite from gantry.routes.prediction.current_mapping import pkg_mappings -from gantry.util import k8s, spec +from gantry.util import k8s logger = logging.getLogger(__name__) @@ -24,18 +24,19 @@ } -async def predict_single(db: aiosqlite.Connection, build: dict) -> dict: +async def predict_single(db: aiosqlite.Connection, spec: dict) -> dict: """ - Predict the resource usage of a build + Predict the resource usage of a spec args: - build: dict that must contain pkg name, pkg version, compiler, compiler version + spec: dict that contains pkg_name, pkg_version, pkg_variants, + compiler_name, compiler_version returns: dict of predicted resource usage: cpu_request, mem_request CPU in millicore, mem in MB """ - sample = await get_sample(db, build) + sample = await get_sample(db, spec) predictions = {} if not sample: predictions = { @@ -52,14 +53,14 @@ async def predict_single(db: aiosqlite.Connection, build: dict) -> dict: } if os.environ.get("PREDICT_STRATEGY") == "ensure_higher": - ensure_higher_pred(predictions, build["package"]["name"]) + ensure_higher_pred(predictions, spec["pkg_name"]) # warn if the prediction is below some thresholds if predictions["cpu_request"] < 0.25: - logger.warning(f"Warning: CPU request for {build} is below 0.25 cores") + logger.warning(f"Warning: CPU request for {spec} is below 0.25 cores") predictions["cpu_request"] = DEFAULT_CPU_REQUEST if predictions["mem_request"] < 10_000_000: - logger.warning(f"Warning: Memory request for {build} is below 10MB") + logger.warning(f"Warning: Memory request for {spec} is below 10MB") predictions["mem_request"] = DEFAULT_MEM_REQUEST # convert predictions to k8s friendly format @@ -80,25 +81,21 @@ async def predict_single(db: aiosqlite.Connection, build: dict) -> dict: } -async def get_sample(db: aiosqlite.Connection, build: dict) -> list: +async def get_sample(db: aiosqlite.Connection, spec: dict) -> list: """ Selects a sample of builds to use for prediction args: - build: dict that must contain pkg name, pkg version, compiler, compiler version + spec: see predict_single returns: list of lists with cpu_mean, cpu_max, mem_mean, mem_max """ - pkg_variants = spec.spec_variants(build["package"]["variants"]) - flat_build = { - "pkg_name": build["package"]["name"], - "pkg_version": build["package"]["version"], - # variants are represented as JSON in the database - "pkg_variants": json.dumps(pkg_variants), - "compiler_name": build["compiler"]["name"], - "compiler_version": build["compiler"]["version"], - } + # store the pkg_variants as a dict which is used in some of the queries + pkg_variants = spec["pkg_variants"] + # variants are represented as JSON in the database + # so we convert the dict to a string for comparison + spec["pkg_variants"] = json.dumps(spec["pkg_variants"]) # ranked in order of priority, the params we would like to match on param_combos = ( @@ -125,7 +122,7 @@ async def select_sample(query: str, filters: dict, extra_params: list = []) -> l return [] for combo in param_combos: - filters = {param: flat_build[param] for param in combo} + filters = {param: spec[param] for param in combo} # the first attempt at getting a sample is to match on all the params # within this combo, variants included @@ -180,7 +177,7 @@ async def select_sample(query: str, filters: dict, extra_params: list = []) -> l def ensure_higher_pred(prediction: dict, pkg_name: str): """ - Ensure that the prediction is higher than the current allocation + Ensure that the prediction is higher than the current allospecion for the package. This will be removed in the future as we analyze the effectiveness of the prediction model. diff --git a/gantry/util/prediction.py b/gantry/util/prediction.py deleted file mode 100644 index ea94c8a..0000000 --- a/gantry/util/prediction.py +++ /dev/null @@ -1,23 +0,0 @@ -from gantry.util.spec import spec_variants - - -def validate_payload(payload: dict) -> bool: - """Ensures that the payload from the client is valid.""" - - if not ( - # item must be dict - isinstance(payload, dict) - # must contain name and version - # for both package and compiler - and all( - isinstance(payload.get(field, {}).get(key), str) - for field in ["package", "compiler"] - for key in ["name", "version"] - ) - # look for variants inside package - and isinstance(payload.get("package", {}).get("variants"), str) - and spec_variants(payload["package"]["variants"]) != {} - ): - return False - - return True diff --git a/gantry/util/spec.py b/gantry/util/spec.py index eb1b33d..d3eb552 100644 --- a/gantry/util/spec.py +++ b/gantry/util/spec.py @@ -1,3 +1,6 @@ +import re + + def spec_variants(spec: str) -> dict: """Given a spec's concrete variants, return a dict in name: value format.""" # example: +adios2~advanced_debug patches=02253c7,acb3805,b724e6a use_vtkm=on @@ -27,3 +30,46 @@ def spec_variants(spec: str) -> dict: variants[part[1:]] = False return variants + + +def parse_alloc_spec(spec: str) -> dict: + """ + Parses a spec in the format emacs@29.2 +json+native+treesitter%gcc@12.3.0 + and returns a dictionary with the following keys: + - pkg_name: str + - pkg_version: str + - pkg_variants: dict + - compiler: str + - compiler_version: str + + Returns an empty dict if the spec is invalid. + + This format is specifically used for the allocation API and is documented + for the client. + """ + + # example: emacs@29.2 +json+native+treesitter%gcc@12.3.0 + spec_pattern = re.compile(r"(.+?)@([\d.]+)\s+(.+?)%([\w-]+)@([\d.]+)") + + match = spec_pattern.match(spec) + if not match: + return {} + + # groups in order + # create a dictionary with the keys and values + pkg_name, pkg_version, pkg_variants, compiler_name, compiler_version = ( + match.groups() + ) + + spec_dict = { + "pkg_name": pkg_name, + "pkg_version": pkg_version, + "pkg_variants": spec_variants(pkg_variants), + "compiler_name": compiler_name, + "compiler_version": compiler_version, + } + + if not spec_dict["pkg_variants"]: + return {} + + return spec_dict diff --git a/gantry/views.py b/gantry/views.py index 7c18011..c94d92e 100644 --- a/gantry/views.py +++ b/gantry/views.py @@ -7,7 +7,7 @@ from gantry.routes.collection import fetch_job from gantry.routes.prediction.prediction import predict_single -from gantry.util.prediction import validate_payload +from gantry.util.spec import parse_alloc_spec logger = logging.getLogger(__name__) routes = web.RouteTableDef() @@ -42,22 +42,12 @@ async def collect_job(request: web.Request) -> web.Response: @routes.get("/v1/allocation") async def allocation(request: web.Request) -> web.Response: """ - Given a spec in JSON format, return environment variables + Given a spec return environment variables that set resource allocations based on historical data. - acceptable payload: - - { - "package": { - "name": "string", - "version": "string" - "variants": "string" - }, - "compiler": { - "name": "string", - "version": "string" - } - } + acceptable spec format: + pkg_name@pkg_version +variant1+variant2%compiler@compiler_version + NOTE: there must be a space between the package version and the variants returns: @@ -69,17 +59,13 @@ async def allocation(request: web.Request) -> web.Response: that should be set within the build environment example: KUBERNETES_CPU_REQUEST, KUBERNETES_CPU_LIMIT, etc. """ - payload = request.query.get("query") - - if not payload: - return web.Response(status=400, text="missing query parameter") + spec = request.query.get("spec") - try: - payload = json.loads(payload) - except json.decoder.JSONDecodeError: - return web.Response(status=400, text="invalid json") + if not spec: + return web.Response(status=400, text="missing spec parameter") - if not validate_payload(payload): - return web.Response(status=400, text="invalid payload") + parsed_spec = parse_alloc_spec(spec) + if not parsed_spec: + return web.Response(status=400, text="invalid spec") - return web.json_response(await predict_single(request.app["db"], payload)) + return web.json_response(await predict_single(request.app["db"], parsed_spec)) From 54658ce225c901ef16a6b1b6e91f0afcb4c20aab Mon Sep 17 00:00:00 2001 From: caetano melone Date: Thu, 7 Mar 2024 13:04:38 -0800 Subject: [PATCH 32/37] formatting fixes --- gantry/routes/prediction/prediction.py | 2 +- gantry/util/spec.py | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/gantry/routes/prediction/prediction.py b/gantry/routes/prediction/prediction.py index 642572f..7b22996 100644 --- a/gantry/routes/prediction/prediction.py +++ b/gantry/routes/prediction/prediction.py @@ -177,7 +177,7 @@ async def select_sample(query: str, filters: dict, extra_params: list = []) -> l def ensure_higher_pred(prediction: dict, pkg_name: str): """ - Ensure that the prediction is higher than the current allospecion + Ensure that the prediction is higher than the current allocation for the package. This will be removed in the future as we analyze the effectiveness of the prediction model. diff --git a/gantry/util/spec.py b/gantry/util/spec.py index d3eb552..422face 100644 --- a/gantry/util/spec.py +++ b/gantry/util/spec.py @@ -57,7 +57,13 @@ def parse_alloc_spec(spec: str) -> dict: # groups in order # create a dictionary with the keys and values - pkg_name, pkg_version, pkg_variants, compiler_name, compiler_version = ( + ( + pkg_name, + pkg_version, + pkg_variants, + compiler_name, + compiler_version + ) = ( match.groups() ) From 1d4278c2209bbd0d56f9b37b4af9c2fc16a93633 Mon Sep 17 00:00:00 2001 From: caetano melone Date: Thu, 7 Mar 2024 13:06:36 -0800 Subject: [PATCH 33/37] formatting fixes --- gantry/util/spec.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/gantry/util/spec.py b/gantry/util/spec.py index 422face..f1c14e0 100644 --- a/gantry/util/spec.py +++ b/gantry/util/spec.py @@ -63,9 +63,7 @@ def parse_alloc_spec(spec: str) -> dict: pkg_variants, compiler_name, compiler_version - ) = ( - match.groups() - ) + ) = match.groups() spec_dict = { "pkg_name": pkg_name, From b552cfc7df5be19d02874c42b66c264120c2fafc Mon Sep 17 00:00:00 2001 From: caetano melone Date: Thu, 7 Mar 2024 13:09:51 -0800 Subject: [PATCH 34/37] black is not being consistent :( --- gantry/util/spec.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/gantry/util/spec.py b/gantry/util/spec.py index f1c14e0..36aed05 100644 --- a/gantry/util/spec.py +++ b/gantry/util/spec.py @@ -58,11 +58,11 @@ def parse_alloc_spec(spec: str) -> dict: # groups in order # create a dictionary with the keys and values ( - pkg_name, - pkg_version, - pkg_variants, - compiler_name, - compiler_version + pkg_name, + pkg_version, + pkg_variants, + compiler_name, + compiler_version, ) = match.groups() spec_dict = { From 67e043843bd6ba1db7b4a69994407fd07b0e5d57 Mon Sep 17 00:00:00 2001 From: caetano melone Date: Thu, 7 Mar 2024 20:51:36 -0800 Subject: [PATCH 35/37] fix: race condition where spec passed by reference was modified by `get_sample` --- gantry/routes/prediction/prediction.py | 12 +++--------- gantry/util/spec.py | 18 +++++++++++++----- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/gantry/routes/prediction/prediction.py b/gantry/routes/prediction/prediction.py index 7b22996..9566ae9 100644 --- a/gantry/routes/prediction/prediction.py +++ b/gantry/routes/prediction/prediction.py @@ -1,4 +1,3 @@ -import json import logging import os @@ -91,12 +90,6 @@ async def get_sample(db: aiosqlite.Connection, spec: dict) -> list: list of lists with cpu_mean, cpu_max, mem_mean, mem_max """ - # store the pkg_variants as a dict which is used in some of the queries - pkg_variants = spec["pkg_variants"] - # variants are represented as JSON in the database - # so we convert the dict to a string for comparison - spec["pkg_variants"] = json.dumps(spec["pkg_variants"]) - # ranked in order of priority, the params we would like to match on param_combos = ( ( @@ -146,13 +139,14 @@ async def select_sample(query: str, filters: dict, extra_params: list = []) -> l # iterate through all the expensive variants and create a set of conditions # for the select query for var in EXPENSIVE_VARIANTS: - if var in pkg_variants: + if var in spec["pkg_variants_dict"]: # if the client has queried for an expensive variant, we want to ensure # that the sample has the same exact value exp_variant_conditions.append( f"json_extract(pkg_variants, '$.{var}')=?" ) - exp_variant_values.append(int(pkg_variants.get(var, 0))) + + exp_variant_values.append(int(spec["pkg_variants_dict"].get(var, 0))) else: # if an expensive variant was not queried for, # we want to make sure that the variant was not set within the sample diff --git a/gantry/util/spec.py b/gantry/util/spec.py index 36aed05..3eaa547 100644 --- a/gantry/util/spec.py +++ b/gantry/util/spec.py @@ -1,3 +1,4 @@ +import json import re @@ -38,7 +39,8 @@ def parse_alloc_spec(spec: str) -> dict: and returns a dictionary with the following keys: - pkg_name: str - pkg_version: str - - pkg_variants: dict + - pkg_variants: str + - pkg_variants_dict: dict - compiler: str - compiler_version: str @@ -65,15 +67,21 @@ def parse_alloc_spec(spec: str) -> dict: compiler_version, ) = match.groups() + pkg_variants_dict = spec_variants(pkg_variants) + if not pkg_variants_dict: + return {} + spec_dict = { "pkg_name": pkg_name, "pkg_version": pkg_version, - "pkg_variants": spec_variants(pkg_variants), + # two representations of the variants are returned here + # to cut down on repeated conversions in later functions + # variants are represented as JSON in the database + "pkg_variants": json.dumps(pkg_variants_dict), + # variants dict is also returned for the client + "pkg_variants_dict": pkg_variants_dict, "compiler_name": compiler_name, "compiler_version": compiler_version, } - if not spec_dict["pkg_variants"]: - return {} - return spec_dict From e3b260212f13f0c3af8a2a945d8d06fa9fdd3211 Mon Sep 17 00:00:00 2001 From: Caetano Melone Date: Thu, 25 Apr 2024 14:51:02 -0700 Subject: [PATCH 36/37] `predict_single` -> `predict` Co-authored-by: Alec Scott --- gantry/routes/prediction/prediction.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gantry/routes/prediction/prediction.py b/gantry/routes/prediction/prediction.py index 9566ae9..035cbe1 100644 --- a/gantry/routes/prediction/prediction.py +++ b/gantry/routes/prediction/prediction.py @@ -23,7 +23,7 @@ } -async def predict_single(db: aiosqlite.Connection, spec: dict) -> dict: +async def predict(db: aiosqlite.Connection, spec: dict) -> dict: """ Predict the resource usage of a spec From 00821d4e8da8e517a1fcb17ce285ca8bb5105de4 Mon Sep 17 00:00:00 2001 From: caetano melone Date: Thu, 25 Apr 2024 16:56:38 -0700 Subject: [PATCH 37/37] decouple prediction strategy from env variable --- docs/deploy.md | 3 --- gantry/routes/prediction/prediction.py | 11 +++++++---- gantry/views.py | 7 +++++-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/docs/deploy.md b/docs/deploy.md index 829fc95..fd17d0b 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -28,6 +28,3 @@ The following variables should be exposed to the container. Those **bolded** are - **`GITLAB_API_TOKEN`** - this token should have API read access - **`GITLAB_WEBHOOK_TOKEN`** - coordinate this value with the collection webhook - **`DB_FILE`** - path where the application can access the SQLite file -- `PREDICT_STRATEGY` - optional mode for the prediction algorithm - - options: - - `ensure_higher`: if the predicted resource usage is below current levels, it will disregard the prediction and keep what would be allocated without Gantry's intervention diff --git a/gantry/routes/prediction/prediction.py b/gantry/routes/prediction/prediction.py index 035cbe1..725c9f9 100644 --- a/gantry/routes/prediction/prediction.py +++ b/gantry/routes/prediction/prediction.py @@ -1,5 +1,4 @@ import logging -import os import aiosqlite @@ -23,13 +22,17 @@ } -async def predict(db: aiosqlite.Connection, spec: dict) -> dict: +async def predict(db: aiosqlite.Connection, spec: dict, strategy: str = None) -> dict: """ Predict the resource usage of a spec args: spec: dict that contains pkg_name, pkg_version, pkg_variants, compiler_name, compiler_version + strategy (optional): custom prediction behavior + "ensure_higher": if the predicted resource usage is + below current levels, it will disregard the prediction and + keep what would be allocated without Gantry's intervention returns: dict of predicted resource usage: cpu_request, mem_request CPU in millicore, mem in MB @@ -51,7 +54,7 @@ async def predict(db: aiosqlite.Connection, spec: dict) -> dict: "mem_request": sum([build[2] for build in sample]) / len(sample), } - if os.environ.get("PREDICT_STRATEGY") == "ensure_higher": + if strategy == "ensure_higher": ensure_higher_pred(predictions, spec["pkg_name"]) # warn if the prediction is below some thresholds @@ -85,7 +88,7 @@ async def get_sample(db: aiosqlite.Connection, spec: dict) -> list: Selects a sample of builds to use for prediction args: - spec: see predict_single + spec: see predict returns: list of lists with cpu_mean, cpu_max, mem_mean, mem_max """ diff --git a/gantry/views.py b/gantry/views.py index c94d92e..47b693d 100644 --- a/gantry/views.py +++ b/gantry/views.py @@ -6,7 +6,7 @@ from aiohttp import web from gantry.routes.collection import fetch_job -from gantry.routes.prediction.prediction import predict_single +from gantry.routes.prediction.prediction import predict from gantry.util.spec import parse_alloc_spec logger = logging.getLogger(__name__) @@ -68,4 +68,7 @@ async def allocation(request: web.Request) -> web.Response: if not parsed_spec: return web.Response(status=400, text="invalid spec") - return web.json_response(await predict_single(request.app["db"], parsed_spec)) + # we want to keep predictions >= current levels (with ensure_higher strategy) + return web.json_response( + await predict(request.app["db"], parsed_spec, strategy="ensure_higher") + )