Add Node 10.20.1 to supported runtime versions

api/assistants/task_spawner/task_spawner_assistant.py

@@ -2834,6 +2834,14 @@ def _build_lambda( app_config, aws_client_factory, credentials, lambda_object ):
 				lambda_object.code,
 				lambda_object.libraries
 			)
+		elif lambda_object.language == "nodejs10.20.1":
+			package_zip_data = TaskSpawner._build_nodejs_10201_lambda(
+				app_config,
+				aws_client_factory,
+				credentials,
+				lambda_object.code,
+				lambda_object.libraries
+			)
 		elif lambda_object.language == "go1.12":
 			lambda_object.code = TaskSpawner._get_go_112_base_code(
 				app_config,
@@ -4228,7 +4236,7 @@ def _get_nodejs_10163_lambda_base_zip( aws_client_factory, credentials, librarie
 			libraries_object[ str( library ) ] = "latest"
 
 		final_s3_package_zip_path = TaskSpawner._get_final_zip_package_path(
-			"nodejs10.16.3",
+			"nodejs10.20.1",
 			libraries_object
 		)
 
@@ -4260,7 +4268,7 @@ def _get_nodejs_10163_lambda_base_zip( aws_client_factory, credentials, librarie
 	@run_on_executor
 	@emit_runtime_metrics( "start_node10163_codebuild" )
 	def start_node10163_codebuild( self, credentials, libraries_object ):
-		return TaskSpawner._start_node810_codebuild(
+		return TaskSpawner._start_node10163_codebuild(
 			self.aws_client_factory,
 			credentials,
 			libraries_object
@@ -4365,6 +4373,208 @@ def _start_node10163_codebuild( aws_client_factory, credentials, libraries_objec
 
 		return build_id
 
+	@staticmethod
+	def _get_nodejs_10201_base_code( app_config, code ):
+		code = re.sub(
+			r"function main\([^\)]+\)[^{]\{",
+			"function main( blockInput ) {",
+			code
+		)
+
+		code = re.sub(
+			r"function mainCallback\([^\)]+\)[^{]\{",
+			"function mainCallback( blockInput, callback ) {",
+			code
+		)
+
+		code = code + "\n\n" + app_config.get( "LAMDBA_BASE_CODES" )[ "nodejs10.20.1" ]
+		return code
+
+	@staticmethod
+	def _build_nodejs_10201_lambda( app_config, aws_client_factory, credentials, code, libraries ):
+		code = TaskSpawner._get_nodejs_10201_base_code(
+			app_config,
+			code
+		)
+
+		# Use CodeBuilder to get a base zip of the libraries
+		base_zip_data = copy.deepcopy( EMPTY_ZIP_DATA )
+		if len( libraries ) > 0:
+			base_zip_data = TaskSpawner._get_nodejs_10201_lambda_base_zip(
+				aws_client_factory,
+				credentials,
+				libraries
+			)
+
+		# Create a virtual file handler for the Lambda zip package
+		lambda_package_zip = io.BytesIO( base_zip_data )
+
+		with zipfile.ZipFile( lambda_package_zip, "a", zipfile.ZIP_DEFLATED ) as zip_file_handler:
+			info = zipfile.ZipInfo(
+				"lambda"
+			)
+			info.external_attr = 0777 << 16L
+
+			# Write lambda.py into new .zip
+			zip_file_handler.writestr(
+				info,
+				str( code )
+			)
+
+		lambda_package_zip_data = lambda_package_zip.getvalue()
+		lambda_package_zip.close()
+
+		return lambda_package_zip_data
+
+	@staticmethod
+	def _get_nodejs_10201_lambda_base_zip( aws_client_factory, credentials, libraries ):
+		s3_client = aws_client_factory.get_aws_client(
+			"s3",
+			credentials
+		)
+
+		libraries_object = {}
+		for library in libraries:
+			libraries_object[ str( library ) ] = "latest"
+
+		final_s3_package_zip_path = TaskSpawner._get_final_zip_package_path(
+			"nodejs10.16.3",
+			libraries_object
+		)
+
+		if TaskSpawner._s3_object_exists( aws_client_factory, credentials, credentials[ "lambda_packages_bucket" ], final_s3_package_zip_path ):
+			return TaskSpawner._read_from_s3(
+				aws_client_factory,
+				credentials,
+				credentials[ "lambda_packages_bucket" ],
+				final_s3_package_zip_path
+			)
+
+		# Kick off CodeBuild for the libraries to get a zip artifact of
+		# all of the libraries.
+		build_id = TaskSpawner._start_node10201_codebuild(
+			aws_client_factory,
+			credentials,
+			libraries_object
+		)
+
+		# This continually polls for the CodeBuild build to finish
+		# Once it does it returns the raw artifact zip data.
+		return TaskSpawner._get_codebuild_artifact_zip_data(
+			aws_client_factory,
+			credentials,
+			build_id,
+			final_s3_package_zip_path
+		)
+
+	@run_on_executor
+	@emit_runtime_metrics( "start_node10201_codebuild" )
+	def start_node10201_codebuild( self, credentials, libraries_object ):
+		return TaskSpawner._start_node10201_codebuild(
+			self.aws_client_factory,
+			credentials,
+			libraries_object
+		)
+
+	@staticmethod
+	def _start_node10201_codebuild( aws_client_factory, credentials, libraries_object ):
+		"""
+		Returns a build ID to be polled at a later time
+		"""
+		codebuild_client = aws_client_factory.get_aws_client(
+			"codebuild",
+			credentials
+		)
+
+		s3_client = aws_client_factory.get_aws_client(
+			"s3",
+			credentials
+		)
+
+		package_json_template = {
+			"name": "refinery-lambda",
+			"version": "1.0.0",
+			"description": "Lambda created by Refinery",
+			"main": "main.js",
+			"dependencies": libraries_object,
+			"devDependencies": {},
+			"scripts": {}
+		}
+
+		# Create empty zip file
+		codebuild_zip = io.BytesIO( EMPTY_ZIP_DATA )
+
+		buildspec_template = {
+			"artifacts": {
+				"files": [
+					"**/*"
+				]
+			},
+			"phases": {
+				"build": {
+					"commands": [
+						"npm install"
+					]
+				},
+				"install": {
+					"runtime-versions": {
+						"nodejs": 10
+					}
+				}
+			},
+			"version": 0.2
+		}
+
+		with zipfile.ZipFile( codebuild_zip, "a", zipfile.ZIP_DEFLATED ) as zip_file_handler:
+			# Write buildspec.yml defining the build process
+			buildspec = zipfile.ZipInfo(
+				"buildspec.yml"
+			)
+			buildspec.external_attr = 0777 << 16L
+			zip_file_handler.writestr(
+				buildspec,
+				yaml.dump(
+					buildspec_template
+				)
+			)
+
+			# Write the package.json
+			package_json = zipfile.ZipInfo(
+				"package.json"
+			)
+			package_json.external_attr = 0777 << 16L
+			zip_file_handler.writestr(
+				package_json,
+				json.dumps(
+					package_json_template
+				)
+			)
+
+		codebuild_zip_data = codebuild_zip.getvalue()
+		codebuild_zip.close()
+
+		# S3 object key of the build package, randomly generated.
+		s3_key = "buildspecs/" + str( uuid.uuid4() ) + ".zip"
+
+		# Write the CodeBuild build package to S3
+		s3_response = s3_client.put_object(
+			Bucket=credentials[ "lambda_packages_bucket" ],
+			Body=codebuild_zip_data,
+			Key=s3_key,
+			ACL="public-read", # THIS HAS TO BE PUBLIC READ FOR SOME FUCKED UP REASON I DONT KNOW WHY
+		)
+
+		# Fire-off the build
+		codebuild_response = codebuild_client.start_build(
+			projectName="refinery-builds",
+			sourceTypeOverride="S3",
+			sourceLocationOverride=credentials[ "lambda_packages_bucket" ] + "/" + s3_key,
+		)
+
+		build_id = codebuild_response[ "build" ][ "id" ]
+
+		return build_id
+
 	@staticmethod
 	def _get_nodejs_810_base_code( app_config, code ):
 		code = re.sub(

api/controller/aws/actions.py

@@ -27,7 +27,7 @@ def get_layers_for_lambda( language ):
 	You must do the following:
 	* Extensively test the new custom runtime.
 	* Upload the new layer version to the root AWS account.
-	* Run the following command on the root account to publically allow use of the layer:
+	* Run the following command on the root account to publicly allow use of the layer:
 
 	aws lambda add-layer-version-permission \
 	--layer-name REPLACE_ME_WITH_LAYER_NAME \
@@ -53,6 +53,10 @@ def get_layers_for_lambda( language ):
 		new_layers.append(
 			"arn:aws:lambda:us-west-2:134071937287:layer:refinery-nodejs10-custom-runtime:9"
 		)
+	elif language == "nodejs10.20.1":
+		new_layers.append(
+			"arn:aws:lambda:us-west-2:134071937287:layer:refinery-nodejs1020-custom-runtime:1"
+		)
 	elif language == "php7.3":
 		new_layers.append(
 			"arn:aws:lambda:us-west-2:134071937287:layer:refinery-php73-custom-runtime:28"
@@ -89,7 +93,7 @@ def get_language_specific_environment_variables( language ):
 			"key": "PYTHONUNBUFFERED",
 			"value": "1",
 		})
-	elif language == "nodejs8.10" or language == "nodejs10.16.3":
+	elif language == "nodejs8.10" or language == "nodejs10.16.3" or language == "nodejs10.20.1":
 		environment_variables_list.append({
 			"key": "NODE_PATH",
 			"value": "/var/task/node_modules/",
@@ -240,6 +244,11 @@ def get_base_lambda_code( app_config, language, code ):
 			app_config,
 			code
 		)
+	elif language == "nodejs10.20.1":
+		return TaskSpawner._get_nodejs_10201_base_code(
+			app_config,
+			code
+		)
 	elif language == "php7.3":
 		return TaskSpawner._get_php_73_base_code(
 			app_config,

api/controller/lambdas/controllers.py

@@ -220,6 +220,11 @@ def post( self ):
 				credentials,
 				libraries_dict
 			)
+		elif self.json[ "language" ] == "nodejs10.20.1":
+			build_id = yield self.task_spawner.start_node10201_codebuild(
+				credentials,
+				libraries_dict
+			)
 		elif self.json[ "language" ] == "php7.3":
 			build_id = yield self.task_spawner.start_php73_codebuild(
 				credentials,

api/custom-runtime/build_all.sh

@@ -6,6 +6,9 @@ cd ../
 cd node10.16.3/
 ./build.sh
 cd ../
+cd node10.20.1/
+./build.sh
+cd ../
 cd node8.10/
 ./build.sh
 cd ../

api/custom-runtime/node10.16.3/build.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 echo "Building Node 10.16.3 Refinery custom runtime layer package..."
+mkdir -p ./layer-contents;
 rm -rf ./layer-contents/*
 cp runtime ./layer-contents/
 cp -r ../base-src/* ./layer-contents/

api/custom-runtime/node10.20.1/.gitignore

@@ -0,0 +1,2 @@
+layer-contents/*
+custom-runtime.zip

api/custom-runtime/node10.20.1/build.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+echo "Building Node 10.20.1 Refinery custom runtime layer package..."
+mkdir -p ./layer-contents;
+rm -rf ./layer-contents/*
+cp runtime ./layer-contents/
+cp -r ../base-src/* ./layer-contents/
+cd ./layer-contents/
+zip -qr custom-runtime.zip *
+mv custom-runtime.zip ../
+cd ..
+rm -rf /layer-contents/*
+#aws s3 cp custom-runtime.zip s3://refinery-custom-runtime-layers-packages-testing/node-810-custom-runtime.zip
+#aws lambda publish-layer-version --layer-name refinery-node810-custom-runtime --description "Refinery Node 8.10 custom runtime layer." --content "S3Bucket=refinery-custom-runtime-layers-packages-testing,S3Key=node-810-custom-runtime.zip" --compatible-runtimes "provided" "python2.7"

api/custom-runtime/node8.10/build.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 echo "Building Node 8.10 Refinery custom runtime layer package..."
+mkdir -p ./layer-contents;
 rm -rf ./layer-contents/*
 cp runtime ./layer-contents/
 cp -r ../base-src/* ./layer-contents/