From b09b010fa37fbccc665b8ebda51611710bc5f8f4 Mon Sep 17 00:00:00 2001 From: Allan Carter Date: Mon, 4 Nov 2024 17:48:04 +0000 Subject: [PATCH] Fix Lambda python version for previous versions of ParallelCluster. PC 3.11.1 updated the Lambda layer to use Python 3.12. Previous version required Python 3.9. Set the Lambda Python runtime based on the ParallelCluster version so that the application continues to work on older versions of PC. This bug was introduced in #270. Related to #270 Resolves #280 --- source/cdk/cdk_slurm_stack.py | 24 ++++++++++++------------ source/cdk/config_schema.py | 11 +++++++++++ 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/source/cdk/cdk_slurm_stack.py b/source/cdk/cdk_slurm_stack.py index 17f21ec..870c61d 100644 --- a/source/cdk/cdk_slurm_stack.py +++ b/source/cdk/cdk_slurm_stack.py @@ -53,7 +53,7 @@ import boto3 from botocore.exceptions import ClientError import config_schema -from config_schema import get_PARALLEL_CLUSTER_MUNGE_VERSION, get_PARALLEL_CLUSTER_PYTHON_VERSION, get_PC_SLURM_VERSION, get_SLURM_VERSION +from config_schema import get_PARALLEL_CLUSTER_LAMBDA_RUNTIME, get_PARALLEL_CLUSTER_MUNGE_VERSION, get_PARALLEL_CLUSTER_PYTHON_VERSION, get_PC_SLURM_VERSION, get_SLURM_VERSION from constructs import Construct from copy import copy, deepcopy from hashlib import sha512 @@ -1427,7 +1427,7 @@ def create_parallel_cluster_lambdas(self): aws_lambda.Architecture.X86_64, ], compatible_runtimes = [ - aws_lambda.Runtime.PYTHON_3_12, + get_PARALLEL_CLUSTER_LAMBDA_RUNTIME(parse_version(self.config['slurm']['ParallelClusterConfig']['Version'])) ], ) @@ -1437,7 +1437,7 @@ def create_parallel_cluster_lambdas(self): function_name=f"{self.stack_name}-CreateBuildFiles", description="Create ParallelCluster build configuration files", memory_size=2048, - runtime=aws_lambda.Runtime.PYTHON_3_12, + runtime=get_PARALLEL_CLUSTER_LAMBDA_RUNTIME(parse_version(self.config['slurm']['ParallelClusterConfig']['Version'])), architecture=aws_lambda.Architecture.X86_64, timeout=Duration.minutes(2), log_retention=logs.RetentionDays.INFINITE, @@ -1499,7 +1499,7 @@ def create_parallel_cluster_lambdas(self): function_name=f"{self.stack_name}-CreateParallelClusterConfig", description="Create ParallelCluster config", memory_size=2048, - runtime=aws_lambda.Runtime.PYTHON_3_12, + runtime=get_PARALLEL_CLUSTER_LAMBDA_RUNTIME(parse_version(self.config['slurm']['ParallelClusterConfig']['Version'])), architecture=aws_lambda.Architecture.X86_64, timeout=Duration.minutes(15), log_retention=logs.RetentionDays.INFINITE, @@ -1547,7 +1547,7 @@ def create_parallel_cluster_lambdas(self): function_name=f"{self.stack_name}-CreateParallelCluster", description="Create ParallelCluster", memory_size=2048, - runtime=aws_lambda.Runtime.PYTHON_3_12, + runtime=get_PARALLEL_CLUSTER_LAMBDA_RUNTIME(parse_version(self.config['slurm']['ParallelClusterConfig']['Version'])), architecture=aws_lambda.Architecture.X86_64, timeout=Duration.minutes(15), log_retention=logs.RetentionDays.INFINITE, @@ -1846,7 +1846,7 @@ def create_parallel_cluster_lambdas(self): function_name=f"{self.stack_name}-CreateHeadNodeARecord", description="Create head node A record", memory_size=2048, - runtime=aws_lambda.Runtime.PYTHON_3_12, + runtime=get_PARALLEL_CLUSTER_LAMBDA_RUNTIME(parse_version(self.config['slurm']['ParallelClusterConfig']['Version'])), architecture=aws_lambda.Architecture.X86_64, timeout=Duration.minutes(15), log_retention=logs.RetentionDays.INFINITE, @@ -1893,7 +1893,7 @@ def create_parallel_cluster_lambdas(self): function_name=f"{self.stack_name}-UpdateHeadNode", description="Update head node", memory_size=2048, - runtime=aws_lambda.Runtime.PYTHON_3_12, + runtime=get_PARALLEL_CLUSTER_LAMBDA_RUNTIME(parse_version(self.config['slurm']['ParallelClusterConfig']['Version'])), architecture=aws_lambda.Architecture.X86_64, timeout=Duration.minutes(15), log_retention=logs.RetentionDays.INFINITE, @@ -1935,7 +1935,7 @@ def create_parallel_cluster_lambdas(self): function_name=f"{self.stack_name}-ConfigUsersGroupsJson", description="Configure users and groups json file", memory_size=2048, - runtime=aws_lambda.Runtime.PYTHON_3_12, + runtime=get_PARALLEL_CLUSTER_LAMBDA_RUNTIME(parse_version(self.config['slurm']['ParallelClusterConfig']['Version'])), architecture=aws_lambda.Architecture.X86_64, timeout=Duration.minutes(15), log_retention=logs.RetentionDays.INFINITE, @@ -1983,7 +1983,7 @@ def create_parallel_cluster_lambdas(self): function_name=f"{self.stack_name}-ConfigExternalLoginNodes", description="Configure external login nodes", memory_size=2048, - runtime=aws_lambda.Runtime.PYTHON_3_12, + runtime=get_PARALLEL_CLUSTER_LAMBDA_RUNTIME(parse_version(self.config['slurm']['ParallelClusterConfig']['Version'])), architecture=aws_lambda.Architecture.X86_64, timeout=Duration.minutes(15), log_retention=logs.RetentionDays.INFINITE, @@ -2030,7 +2030,7 @@ def create_parallel_cluster_lambdas(self): function_name=f"{self.stack_name}-DeconfigUsersGroupsJson", description="Deconfigure RES users and groups json file", memory_size=2048, - runtime=aws_lambda.Runtime.PYTHON_3_12, + runtime=get_PARALLEL_CLUSTER_LAMBDA_RUNTIME(parse_version(self.config['slurm']['ParallelClusterConfig']['Version'])), architecture=aws_lambda.Architecture.X86_64, timeout=Duration.minutes(15), log_retention=logs.RetentionDays.INFINITE, @@ -2072,7 +2072,7 @@ def create_parallel_cluster_lambdas(self): function_name=f"{self.stack_name}-DeconfigExternalLoginNodes", description="Deconfigure external login nodes", memory_size=2048, - runtime=aws_lambda.Runtime.PYTHON_3_12, + runtime=get_PARALLEL_CLUSTER_LAMBDA_RUNTIME(parse_version(self.config['slurm']['ParallelClusterConfig']['Version'])), architecture=aws_lambda.Architecture.X86_64, timeout=Duration.minutes(15), log_retention=logs.RetentionDays.INFINITE, @@ -2114,7 +2114,7 @@ def create_callSlurmRestApiLambda(self): function_name=f"{self.stack_name}-CallSlurmRestApiLambda", description="Example showing how to call Slurm REST API", memory_size=128, - runtime=aws_lambda.Runtime.PYTHON_3_12, + runtime=get_PARALLEL_CLUSTER_LAMBDA_RUNTIME(parse_version(self.config['slurm']['ParallelClusterConfig']['Version'])), architecture=aws_lambda.Architecture.ARM_64, timeout=Duration.minutes(1), log_retention=logs.RetentionDays.INFINITE, diff --git a/source/cdk/config_schema.py b/source/cdk/config_schema.py index 61c91bc..5f77ef7 100644 --- a/source/cdk/config_schema.py +++ b/source/cdk/config_schema.py @@ -16,6 +16,9 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ +from aws_cdk import ( + aws_lambda as aws_lambda +) import boto3 from botocore.client import ClientError import json @@ -318,6 +321,14 @@ def PARALLEL_CLUSTER_SUPPORTS_CUSTOM_MUNGE_KEY(parallel_cluster_version): def PARALLEL_CLUSTER_SUPPORTS_HOME_MOUNT(parallel_cluster_version): return parallel_cluster_version >= PARALLEL_CLUSTER_SUPPORTS_HOME_MOUNT_VERSION +# Version 3.11.0 + +def get_PARALLEL_CLUSTER_LAMBDA_RUNTIME(parallel_cluster_version): + if parallel_cluster_version < parse_version('3.11.1'): + return aws_lambda.Runtime.PYTHON_3_9 + else: + return aws_lambda.Runtime.PYTHON_3_12 + # Determine all AWS regions available on the account. default_region = environ.get("AWS_DEFAULT_REGION", "us-east-1") ec2_client = boto3.client("ec2", region_name=default_region)