From db2f38d08ec31006a88f855c53f518880a80b9d3 Mon Sep 17 00:00:00 2001 From: Allan Carter Date: Mon, 4 Nov 2024 17:48:04 +0000 Subject: [PATCH] Fix Lambda python version for previous versions of ParallelCluster. PC 3.11.1 updated the Lambda layer to use Python 3.12. Previous version required Python 3.9. Set the Lambda Python runtime based on the ParallelCluster version so that the application continues to work on older versions of PC. This bug was introduced in #270. Related to #270 Resolves #280 --- source/cdk/cdk_slurm_stack.py | 24 ++++++++++++------------ source/cdk/config_schema.py | 11 +++++++++++ 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/source/cdk/cdk_slurm_stack.py b/source/cdk/cdk_slurm_stack.py index 17f21ec..99b9f4a 100644 --- a/source/cdk/cdk_slurm_stack.py +++ b/source/cdk/cdk_slurm_stack.py @@ -53,7 +53,7 @@ import boto3 from botocore.exceptions import ClientError import config_schema -from config_schema import get_PARALLEL_CLUSTER_MUNGE_VERSION, get_PARALLEL_CLUSTER_PYTHON_VERSION, get_PC_SLURM_VERSION, get_SLURM_VERSION +from config_schema import get_PARALLEL_CLUSTER_LAMBDA_LAYER_PYTHON_VERSION, get_PARALLEL_CLUSTER_MUNGE_VERSION, get_PARALLEL_CLUSTER_PYTHON_VERSION, get_PC_SLURM_VERSION, get_SLURM_VERSION from constructs import Construct from copy import copy, deepcopy from hashlib import sha512 @@ -1427,7 +1427,7 @@ def create_parallel_cluster_lambdas(self): aws_lambda.Architecture.X86_64, ], compatible_runtimes = [ - aws_lambda.Runtime.PYTHON_3_12, + get_PARALLEL_CLUSTER_LAMBDA_LAYER_PYTHON_VERSION(parse_version(self.config['slurm']['ParallelClusterConfig']['Version'])) ], ) @@ -1437,7 +1437,7 @@ def create_parallel_cluster_lambdas(self): function_name=f"{self.stack_name}-CreateBuildFiles", description="Create ParallelCluster build configuration files", memory_size=2048, - runtime=aws_lambda.Runtime.PYTHON_3_12, + runtime=get_PARALLEL_CLUSTER_LAMBDA_LAYER_PYTHON_VERSION(parse_version(self.config['slurm']['ParallelClusterConfig']['Version'])), architecture=aws_lambda.Architecture.X86_64, timeout=Duration.minutes(2), log_retention=logs.RetentionDays.INFINITE, @@ -1499,7 +1499,7 @@ def create_parallel_cluster_lambdas(self): function_name=f"{self.stack_name}-CreateParallelClusterConfig", description="Create ParallelCluster config", memory_size=2048, - runtime=aws_lambda.Runtime.PYTHON_3_12, + runtime=get_PARALLEL_CLUSTER_LAMBDA_LAYER_PYTHON_VERSION(parse_version(self.config['slurm']['ParallelClusterConfig']['Version'])), architecture=aws_lambda.Architecture.X86_64, timeout=Duration.minutes(15), log_retention=logs.RetentionDays.INFINITE, @@ -1547,7 +1547,7 @@ def create_parallel_cluster_lambdas(self): function_name=f"{self.stack_name}-CreateParallelCluster", description="Create ParallelCluster", memory_size=2048, - runtime=aws_lambda.Runtime.PYTHON_3_12, + runtime=get_PARALLEL_CLUSTER_LAMBDA_LAYER_PYTHON_VERSION(parse_version(self.config['slurm']['ParallelClusterConfig']['Version'])), architecture=aws_lambda.Architecture.X86_64, timeout=Duration.minutes(15), log_retention=logs.RetentionDays.INFINITE, @@ -1846,7 +1846,7 @@ def create_parallel_cluster_lambdas(self): function_name=f"{self.stack_name}-CreateHeadNodeARecord", description="Create head node A record", memory_size=2048, - runtime=aws_lambda.Runtime.PYTHON_3_12, + runtime=get_PARALLEL_CLUSTER_LAMBDA_LAYER_PYTHON_VERSION(parse_version(self.config['slurm']['ParallelClusterConfig']['Version'])), architecture=aws_lambda.Architecture.X86_64, timeout=Duration.minutes(15), log_retention=logs.RetentionDays.INFINITE, @@ -1893,7 +1893,7 @@ def create_parallel_cluster_lambdas(self): function_name=f"{self.stack_name}-UpdateHeadNode", description="Update head node", memory_size=2048, - runtime=aws_lambda.Runtime.PYTHON_3_12, + runtime=get_PARALLEL_CLUSTER_LAMBDA_LAYER_PYTHON_VERSION(parse_version(self.config['slurm']['ParallelClusterConfig']['Version'])), architecture=aws_lambda.Architecture.X86_64, timeout=Duration.minutes(15), log_retention=logs.RetentionDays.INFINITE, @@ -1935,7 +1935,7 @@ def create_parallel_cluster_lambdas(self): function_name=f"{self.stack_name}-ConfigUsersGroupsJson", description="Configure users and groups json file", memory_size=2048, - runtime=aws_lambda.Runtime.PYTHON_3_12, + runtime=get_PARALLEL_CLUSTER_LAMBDA_LAYER_PYTHON_VERSION(parse_version(self.config['slurm']['ParallelClusterConfig']['Version'])), architecture=aws_lambda.Architecture.X86_64, timeout=Duration.minutes(15), log_retention=logs.RetentionDays.INFINITE, @@ -1983,7 +1983,7 @@ def create_parallel_cluster_lambdas(self): function_name=f"{self.stack_name}-ConfigExternalLoginNodes", description="Configure external login nodes", memory_size=2048, - runtime=aws_lambda.Runtime.PYTHON_3_12, + runtime=get_PARALLEL_CLUSTER_LAMBDA_LAYER_PYTHON_VERSION(parse_version(self.config['slurm']['ParallelClusterConfig']['Version'])), architecture=aws_lambda.Architecture.X86_64, timeout=Duration.minutes(15), log_retention=logs.RetentionDays.INFINITE, @@ -2030,7 +2030,7 @@ def create_parallel_cluster_lambdas(self): function_name=f"{self.stack_name}-DeconfigUsersGroupsJson", description="Deconfigure RES users and groups json file", memory_size=2048, - runtime=aws_lambda.Runtime.PYTHON_3_12, + runtime=get_PARALLEL_CLUSTER_LAMBDA_LAYER_PYTHON_VERSION(parse_version(self.config['slurm']['ParallelClusterConfig']['Version'])), architecture=aws_lambda.Architecture.X86_64, timeout=Duration.minutes(15), log_retention=logs.RetentionDays.INFINITE, @@ -2072,7 +2072,7 @@ def create_parallel_cluster_lambdas(self): function_name=f"{self.stack_name}-DeconfigExternalLoginNodes", description="Deconfigure external login nodes", memory_size=2048, - runtime=aws_lambda.Runtime.PYTHON_3_12, + runtime=get_PARALLEL_CLUSTER_LAMBDA_LAYER_PYTHON_VERSION(parse_version(self.config['slurm']['ParallelClusterConfig']['Version'])), architecture=aws_lambda.Architecture.X86_64, timeout=Duration.minutes(15), log_retention=logs.RetentionDays.INFINITE, @@ -2114,7 +2114,7 @@ def create_callSlurmRestApiLambda(self): function_name=f"{self.stack_name}-CallSlurmRestApiLambda", description="Example showing how to call Slurm REST API", memory_size=128, - runtime=aws_lambda.Runtime.PYTHON_3_12, + runtime=get_PARALLEL_CLUSTER_LAMBDA_LAYER_PYTHON_VERSION(parse_version(self.config['slurm']['ParallelClusterConfig']['Version'])), architecture=aws_lambda.Architecture.ARM_64, timeout=Duration.minutes(1), log_retention=logs.RetentionDays.INFINITE, diff --git a/source/cdk/config_schema.py b/source/cdk/config_schema.py index 61c91bc..15ad0b5 100644 --- a/source/cdk/config_schema.py +++ b/source/cdk/config_schema.py @@ -16,6 +16,9 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ +from aws_cdk import ( + aws_lambda as aws_lambda +) import boto3 from botocore.client import ClientError import json @@ -318,6 +321,14 @@ def PARALLEL_CLUSTER_SUPPORTS_CUSTOM_MUNGE_KEY(parallel_cluster_version): def PARALLEL_CLUSTER_SUPPORTS_HOME_MOUNT(parallel_cluster_version): return parallel_cluster_version >= PARALLEL_CLUSTER_SUPPORTS_HOME_MOUNT_VERSION +# Version 3.11.0 + +def get_PARALLEL_CLUSTER_LAMBDA_LAYER_PYTHON_VERSION(parallel_cluster_version): + if parallel_cluster_version < parse_version('3.11.1'): + return aws_lambda.Runtime.PYTHON_3_9 + else: + return aws_lambda.Runtime.PYTHON_3_12 + # Determine all AWS regions available on the account. default_region = environ.get("AWS_DEFAULT_REGION", "us-east-1") ec2_client = boto3.client("ec2", region_name=default_region)