diff --git a/api/assistants/task_spawner/task_spawner_assistant.py b/api/assistants/task_spawner/task_spawner_assistant.py index 46466c510..9bee45f42 100644 --- a/api/assistants/task_spawner/task_spawner_assistant.py +++ b/api/assistants/task_spawner/task_spawner_assistant.py @@ -2834,6 +2834,14 @@ def _build_lambda( app_config, aws_client_factory, credentials, lambda_object ): lambda_object.code, lambda_object.libraries ) + elif lambda_object.language == "nodejs10.20.1": + package_zip_data = TaskSpawner._build_nodejs_10201_lambda( + app_config, + aws_client_factory, + credentials, + lambda_object.code, + lambda_object.libraries + ) elif lambda_object.language == "go1.12": lambda_object.code = TaskSpawner._get_go_112_base_code( app_config, @@ -4228,7 +4236,7 @@ def _get_nodejs_10163_lambda_base_zip( aws_client_factory, credentials, librarie libraries_object[ str( library ) ] = "latest" final_s3_package_zip_path = TaskSpawner._get_final_zip_package_path( - "nodejs10.16.3", + "nodejs10.20.1", libraries_object ) @@ -4260,7 +4268,7 @@ def _get_nodejs_10163_lambda_base_zip( aws_client_factory, credentials, librarie @run_on_executor @emit_runtime_metrics( "start_node10163_codebuild" ) def start_node10163_codebuild( self, credentials, libraries_object ): - return TaskSpawner._start_node810_codebuild( + return TaskSpawner._start_node10163_codebuild( self.aws_client_factory, credentials, libraries_object @@ -4365,6 +4373,208 @@ def _start_node10163_codebuild( aws_client_factory, credentials, libraries_objec return build_id + @staticmethod + def _get_nodejs_10201_base_code( app_config, code ): + code = re.sub( + r"function main\([^\)]+\)[^{]\{", + "function main( blockInput ) {", + code + ) + + code = re.sub( + r"function mainCallback\([^\)]+\)[^{]\{", + "function mainCallback( blockInput, callback ) {", + code + ) + + code = code + "\n\n" + app_config.get( "LAMDBA_BASE_CODES" )[ "nodejs10.20.1" ] + return code + + @staticmethod + def _build_nodejs_10201_lambda( app_config, aws_client_factory, credentials, code, libraries ): + code = TaskSpawner._get_nodejs_10201_base_code( + app_config, + code + ) + + # Use CodeBuilder to get a base zip of the libraries + base_zip_data = copy.deepcopy( EMPTY_ZIP_DATA ) + if len( libraries ) > 0: + base_zip_data = TaskSpawner._get_nodejs_10201_lambda_base_zip( + aws_client_factory, + credentials, + libraries + ) + + # Create a virtual file handler for the Lambda zip package + lambda_package_zip = io.BytesIO( base_zip_data ) + + with zipfile.ZipFile( lambda_package_zip, "a", zipfile.ZIP_DEFLATED ) as zip_file_handler: + info = zipfile.ZipInfo( + "lambda" + ) + info.external_attr = 0777 << 16L + + # Write lambda.py into new .zip + zip_file_handler.writestr( + info, + str( code ) + ) + + lambda_package_zip_data = lambda_package_zip.getvalue() + lambda_package_zip.close() + + return lambda_package_zip_data + + @staticmethod + def _get_nodejs_10201_lambda_base_zip( aws_client_factory, credentials, libraries ): + s3_client = aws_client_factory.get_aws_client( + "s3", + credentials + ) + + libraries_object = {} + for library in libraries: + libraries_object[ str( library ) ] = "latest" + + final_s3_package_zip_path = TaskSpawner._get_final_zip_package_path( + "nodejs10.16.3", + libraries_object + ) + + if TaskSpawner._s3_object_exists( aws_client_factory, credentials, credentials[ "lambda_packages_bucket" ], final_s3_package_zip_path ): + return TaskSpawner._read_from_s3( + aws_client_factory, + credentials, + credentials[ "lambda_packages_bucket" ], + final_s3_package_zip_path + ) + + # Kick off CodeBuild for the libraries to get a zip artifact of + # all of the libraries. + build_id = TaskSpawner._start_node10201_codebuild( + aws_client_factory, + credentials, + libraries_object + ) + + # This continually polls for the CodeBuild build to finish + # Once it does it returns the raw artifact zip data. + return TaskSpawner._get_codebuild_artifact_zip_data( + aws_client_factory, + credentials, + build_id, + final_s3_package_zip_path + ) + + @run_on_executor + @emit_runtime_metrics( "start_node10201_codebuild" ) + def start_node10201_codebuild( self, credentials, libraries_object ): + return TaskSpawner._start_node10201_codebuild( + self.aws_client_factory, + credentials, + libraries_object + ) + + @staticmethod + def _start_node10201_codebuild( aws_client_factory, credentials, libraries_object ): + """ + Returns a build ID to be polled at a later time + """ + codebuild_client = aws_client_factory.get_aws_client( + "codebuild", + credentials + ) + + s3_client = aws_client_factory.get_aws_client( + "s3", + credentials + ) + + package_json_template = { + "name": "refinery-lambda", + "version": "1.0.0", + "description": "Lambda created by Refinery", + "main": "main.js", + "dependencies": libraries_object, + "devDependencies": {}, + "scripts": {} + } + + # Create empty zip file + codebuild_zip = io.BytesIO( EMPTY_ZIP_DATA ) + + buildspec_template = { + "artifacts": { + "files": [ + "**/*" + ] + }, + "phases": { + "build": { + "commands": [ + "npm install" + ] + }, + "install": { + "runtime-versions": { + "nodejs": 10 + } + } + }, + "version": 0.2 + } + + with zipfile.ZipFile( codebuild_zip, "a", zipfile.ZIP_DEFLATED ) as zip_file_handler: + # Write buildspec.yml defining the build process + buildspec = zipfile.ZipInfo( + "buildspec.yml" + ) + buildspec.external_attr = 0777 << 16L + zip_file_handler.writestr( + buildspec, + yaml.dump( + buildspec_template + ) + ) + + # Write the package.json + package_json = zipfile.ZipInfo( + "package.json" + ) + package_json.external_attr = 0777 << 16L + zip_file_handler.writestr( + package_json, + json.dumps( + package_json_template + ) + ) + + codebuild_zip_data = codebuild_zip.getvalue() + codebuild_zip.close() + + # S3 object key of the build package, randomly generated. + s3_key = "buildspecs/" + str( uuid.uuid4() ) + ".zip" + + # Write the CodeBuild build package to S3 + s3_response = s3_client.put_object( + Bucket=credentials[ "lambda_packages_bucket" ], + Body=codebuild_zip_data, + Key=s3_key, + ACL="public-read", # THIS HAS TO BE PUBLIC READ FOR SOME FUCKED UP REASON I DONT KNOW WHY + ) + + # Fire-off the build + codebuild_response = codebuild_client.start_build( + projectName="refinery-builds", + sourceTypeOverride="S3", + sourceLocationOverride=credentials[ "lambda_packages_bucket" ] + "/" + s3_key, + ) + + build_id = codebuild_response[ "build" ][ "id" ] + + return build_id + @staticmethod def _get_nodejs_810_base_code( app_config, code ): code = re.sub( diff --git a/api/controller/aws/actions.py b/api/controller/aws/actions.py index 0a4d03132..a70af3a28 100644 --- a/api/controller/aws/actions.py +++ b/api/controller/aws/actions.py @@ -27,7 +27,7 @@ def get_layers_for_lambda( language ): You must do the following: * Extensively test the new custom runtime. * Upload the new layer version to the root AWS account. - * Run the following command on the root account to publically allow use of the layer: + * Run the following command on the root account to publicly allow use of the layer: aws lambda add-layer-version-permission \ --layer-name REPLACE_ME_WITH_LAYER_NAME \ @@ -53,6 +53,10 @@ def get_layers_for_lambda( language ): new_layers.append( "arn:aws:lambda:us-west-2:134071937287:layer:refinery-nodejs10-custom-runtime:9" ) + elif language == "nodejs10.20.1": + new_layers.append( + "arn:aws:lambda:us-west-2:134071937287:layer:refinery-nodejs1020-custom-runtime:1" + ) elif language == "php7.3": new_layers.append( "arn:aws:lambda:us-west-2:134071937287:layer:refinery-php73-custom-runtime:28" @@ -89,7 +93,7 @@ def get_language_specific_environment_variables( language ): "key": "PYTHONUNBUFFERED", "value": "1", }) - elif language == "nodejs8.10" or language == "nodejs10.16.3": + elif language == "nodejs8.10" or language == "nodejs10.16.3" or language == "nodejs10.20.1": environment_variables_list.append({ "key": "NODE_PATH", "value": "/var/task/node_modules/", @@ -240,6 +244,11 @@ def get_base_lambda_code( app_config, language, code ): app_config, code ) + elif language == "nodejs10.20.1": + return TaskSpawner._get_nodejs_10201_base_code( + app_config, + code + ) elif language == "php7.3": return TaskSpawner._get_php_73_base_code( app_config, diff --git a/api/controller/lambdas/controllers.py b/api/controller/lambdas/controllers.py index fa053d13e..9c8a86779 100644 --- a/api/controller/lambdas/controllers.py +++ b/api/controller/lambdas/controllers.py @@ -220,6 +220,11 @@ def post( self ): credentials, libraries_dict ) + elif self.json[ "language" ] == "nodejs10.20.1": + build_id = yield self.task_spawner.start_node10201_codebuild( + credentials, + libraries_dict + ) elif self.json[ "language" ] == "php7.3": build_id = yield self.task_spawner.start_php73_codebuild( credentials, diff --git a/api/custom-runtime/build_all.sh b/api/custom-runtime/build_all.sh index 58a0d7541..790b8975e 100755 --- a/api/custom-runtime/build_all.sh +++ b/api/custom-runtime/build_all.sh @@ -6,6 +6,9 @@ cd ../ cd node10.16.3/ ./build.sh cd ../ +cd node10.20.1/ +./build.sh +cd ../ cd node8.10/ ./build.sh cd ../ diff --git a/api/custom-runtime/node10.16.3/build.sh b/api/custom-runtime/node10.16.3/build.sh index d04a8bf7c..9d78556ec 100755 --- a/api/custom-runtime/node10.16.3/build.sh +++ b/api/custom-runtime/node10.16.3/build.sh @@ -1,5 +1,6 @@ #!/bin/bash echo "Building Node 10.16.3 Refinery custom runtime layer package..." +mkdir -p ./layer-contents; rm -rf ./layer-contents/* cp runtime ./layer-contents/ cp -r ../base-src/* ./layer-contents/ diff --git a/api/custom-runtime/node10.20.1/.gitignore b/api/custom-runtime/node10.20.1/.gitignore new file mode 100644 index 000000000..d31e89e5f --- /dev/null +++ b/api/custom-runtime/node10.20.1/.gitignore @@ -0,0 +1,2 @@ +layer-contents/* +custom-runtime.zip diff --git a/api/custom-runtime/node10.20.1/build.sh b/api/custom-runtime/node10.20.1/build.sh new file mode 100755 index 000000000..66ba4f5e9 --- /dev/null +++ b/api/custom-runtime/node10.20.1/build.sh @@ -0,0 +1,13 @@ +#!/bin/bash +echo "Building Node 10.20.1 Refinery custom runtime layer package..." +mkdir -p ./layer-contents; +rm -rf ./layer-contents/* +cp runtime ./layer-contents/ +cp -r ../base-src/* ./layer-contents/ +cd ./layer-contents/ +zip -qr custom-runtime.zip * +mv custom-runtime.zip ../ +cd .. +rm -rf /layer-contents/* +#aws s3 cp custom-runtime.zip s3://refinery-custom-runtime-layers-packages-testing/node-810-custom-runtime.zip +#aws lambda publish-layer-version --layer-name refinery-node810-custom-runtime --description "Refinery Node 8.10 custom runtime layer." --content "S3Bucket=refinery-custom-runtime-layers-packages-testing,S3Key=node-810-custom-runtime.zip" --compatible-runtimes "provided" "python2.7" diff --git a/api/custom-runtime/node10.20.1/runtime b/api/custom-runtime/node10.20.1/runtime new file mode 100755 index 000000000..977303b7a Binary files /dev/null and b/api/custom-runtime/node10.20.1/runtime differ diff --git a/api/custom-runtime/node8.10/build.sh b/api/custom-runtime/node8.10/build.sh index 857a8054f..6b7d2ca57 100755 --- a/api/custom-runtime/node8.10/build.sh +++ b/api/custom-runtime/node8.10/build.sh @@ -1,5 +1,6 @@ #!/bin/bash echo "Building Node 8.10 Refinery custom runtime layer package..." +mkdir -p ./layer-contents; rm -rf ./layer-contents/* cp runtime ./layer-contents/ cp -r ../base-src/* ./layer-contents/ diff --git a/api/lambda_bases/nodejs10.20.1 b/api/lambda_bases/nodejs10.20.1 new file mode 100644 index 000000000..5d7f45a63 --- /dev/null +++ b/api/lambda_bases/nodejs10.20.1 @@ -0,0 +1,123 @@ +backpack = {}; +__input_data = ""; + +process.stdin.setEncoding("utf8"); + +process.stdin.on('readable', function() { + var chunk; + while (chunk = process.stdin.read()) { + __input_data += chunk; + } +}); + +process.stdin.on('end', function () { + __rfn_init(); +}); + +function __returnResult(programOutput) { + console.log( + "" + JSON.stringify( + programOutput + ) + "" + ); +} + +/** + * Wraps a function with a promise. Returns a new function that can be called with await. + */ +function __promisify(fn) { + return function promiseWrappedFunction() { + return new Promise((resolve, reject) => { + function wrapCallback(err, result) { + // If error, reject the promise. + if (err) { + return reject(err); + } + // Call resolve to close the promise + resolve(result); + } + + // Concatenate to arguments list the wrapCallback function + const argumentsList = [ + ...(Array.prototype.slice.apply(arguments)), + wrapCallback + ]; + + // Call the wrapped function + fn.apply(this, argumentsList); + }); + } +} + +async function __rfn_init() { + try { + if (process.stdout._handle) { + process.stdout._handle.setBlocking(true); + } + + // Inside of the try catch statement to ensure that any exceptions thrown here are logged. + const inputData = JSON.parse( __input_data ); + const lambdaInput = inputData[ "lambda_input" ]; + backpack = inputData[ "backpack" ]; + + const mainCallbackDefined = typeof mainCallback !== 'undefined'; + const mainDefined = typeof main !== 'undefined'; + + if (!mainCallbackDefined && !mainDefined) { + throw new Error('No main function was defined for block. You must specify a function named either `main` or `mainCallback` for a block to be valid and for it to run.'); + } + + if (mainCallbackDefined && mainDefined) { + throw new Error('Both main and mainCallback were both defined, only one one entry point may be defined per block per block. Please delete or rename one of them.'); + } + + // Identify which function we will call as our entry point based on what was defined in the script. + // If it's a callback, we wrap it with a promise so that we may await it. + const mainEntrypoint = mainDefined ? main : __promisify(mainCallback); + + const output = await mainEntrypoint(lambdaInput); + + __returnResult({ + "output": output, + "backpack": backpack, + }); + + backpack = {}; + flushProcessOutputsAndExit__refinery(0); + } catch ( e ) { + if( e.stack ) { + e = e.stack.toString() + } else { + e = e.toString(); + } + console.log( + "" + JSON.stringify({ + "output": e, + "backpack": backpack + }) + "" + ); + backpack = {}; + flushProcessOutputsAndExit__refinery(-1); + } +} + +function flushProcessOutputsAndExit__refinery(exitCode) { + // Configure the streams to be blocking + makeBlockingStream__refinery(process.stdout); + makeBlockingStream__refinery(process.stderr); + + + // Allow Node to cleanup any internals before the next process tick + setImmediate(function callProcessExitWithCode() { + process.exit(exitCode); + }); +} + +function makeBlockingStream__refinery(stream) { + if (!stream || !stream._handle || !stream._handle.setBlocking) { + // Not able to set blocking so just bail out + return; + } + + stream._handle.setBlocking(true); +} diff --git a/api/pyconstants/project_constants.py b/api/pyconstants/project_constants.py index 23a956af6..dd469c530 100644 --- a/api/pyconstants/project_constants.py +++ b/api/pyconstants/project_constants.py @@ -18,6 +18,7 @@ "python2.7", "nodejs8.10", "nodejs10.16.3", + "nodejs10.20.1", "php7.3", "go1.12", "ruby2.6.4" @@ -28,6 +29,7 @@ CUSTOM_RUNTIME_LANGUAGES = [ "nodejs8.10", "nodejs10.16.3", + "nodejs10.20.1", "php7.3", "go1.12", "python2.7", @@ -41,6 +43,7 @@ "python2.7": [], "nodejs8.10": [], "nodejs10.16.3": [], + "nodejs10.20.1": [], "php7.3": [], "go1.12": [], "ruby2.6.4": [] diff --git a/front-end/src/constants/project-editor-constants.ts b/front-end/src/constants/project-editor-constants.ts index 040228b8f..85e853e43 100644 --- a/front-end/src/constants/project-editor-constants.ts +++ b/front-end/src/constants/project-editor-constants.ts @@ -324,6 +324,11 @@ def main(block_input, backpack): async function main(blockInput, backpack) { return 'Hello World!'; } +`, + [SupportedLanguage.NODEJS_1020]: ` +async function main(blockInput, backpack) { + return 'Hello World!'; +} `, [SupportedLanguage.NODEJS_8]: ` async function main(blockInput, backpack) { diff --git a/front-end/src/types/graph.ts b/front-end/src/types/graph.ts index 1987107ab..ef40752b6 100644 --- a/front-end/src/types/graph.ts +++ b/front-end/src/types/graph.ts @@ -25,6 +25,7 @@ export enum SupportedLanguage { PYTHON_2 = 'python2.7', NODEJS_8 = 'nodejs8.10', NODEJS_10 = 'nodejs10.16.3', + NODEJS_1020 = 'nodejs10.20.1', PHP7 = 'php7.3', GO1_12 = 'go1.12' } diff --git a/front-end/src/types/project-editor-types.ts b/front-end/src/types/project-editor-types.ts index 398c099d7..10ecc8769 100644 --- a/front-end/src/types/project-editor-types.ts +++ b/front-end/src/types/project-editor-types.ts @@ -89,6 +89,7 @@ export interface MonacoLanguageLookup extends SupportedLanguageToAceLang { export const languageToAceLangMap: MonacoLanguageLookup = { [SupportedLanguage.NODEJS_8]: 'javascript', [SupportedLanguage.NODEJS_10]: 'javascript', + [SupportedLanguage.NODEJS_1020]: 'javascript', [SupportedLanguage.PYTHON_2]: 'python', [SupportedLanguage.PYTHON_3]: 'python', [SupportedLanguage.GO1_12]: 'go', @@ -104,6 +105,7 @@ export type LanguageToBaseRepoURL = { [key in SupportedLanguage]: string | null export const LanguageToBaseRepoURLMap: LanguageToBaseRepoURL = { [SupportedLanguage.NODEJS_8]: 'https://www.npmjs.com', [SupportedLanguage.NODEJS_10]: 'https://www.npmjs.com', + [SupportedLanguage.NODEJS_1020]: 'https://www.npmjs.com', [SupportedLanguage.PYTHON_2]: 'https://pypi.org', [SupportedLanguage.PYTHON_3]: 'https://pypi.org', [SupportedLanguage.GO1_12]: null, @@ -115,7 +117,8 @@ export type LanguageToLibraryRepoURL = { [key in SupportedLanguage]: string | nu export const LanguageToLibraryRepoURLMap: LanguageToLibraryRepoURL = { [SupportedLanguage.NODEJS_8]: LanguageToBaseRepoURLMap[SupportedLanguage.NODEJS_8] + '/package/', - [SupportedLanguage.NODEJS_10]: LanguageToBaseRepoURLMap[SupportedLanguage.NODEJS_8] + '/package/', + [SupportedLanguage.NODEJS_10]: LanguageToBaseRepoURLMap[SupportedLanguage.NODEJS_10] + '/package/', + [SupportedLanguage.NODEJS_1020]: LanguageToBaseRepoURLMap[SupportedLanguage.NODEJS_1020] + '/package/', [SupportedLanguage.PYTHON_3]: LanguageToBaseRepoURLMap[SupportedLanguage.PYTHON_3] + '/project/', [SupportedLanguage.PYTHON_2]: LanguageToBaseRepoURLMap[SupportedLanguage.PYTHON_2] + '/project/', [SupportedLanguage.GO1_12]: null, diff --git a/front-end/src/utils/project-debug-utils.ts b/front-end/src/utils/project-debug-utils.ts index 7176dd542..d2d2ce7ea 100644 --- a/front-end/src/utils/project-debug-utils.ts +++ b/front-end/src/utils/project-debug-utils.ts @@ -32,6 +32,7 @@ export type LanguageToStringLookup = { [key in SupportedLanguage]: string }; export const languageToRunScriptLoopup: LanguageToStringLookup = { [SupportedLanguage.NODEJS_8]: 'node block-code.js', [SupportedLanguage.NODEJS_10]: 'node block-code.js', + [SupportedLanguage.NODEJS_1020]: 'node block-code.js', [SupportedLanguage.PYTHON_3]: 'python block-code.py', [SupportedLanguage.PYTHON_2]: 'python block-code.py', [SupportedLanguage.GO1_12]: 'go run block-code.go', @@ -41,6 +42,7 @@ export const languageToRunScriptLoopup: LanguageToStringLookup = { export const languageToFileExtension: LanguageToStringLookup = { [SupportedLanguage.NODEJS_10]: 'js', + [SupportedLanguage.NODEJS_1020]: 'js', [SupportedLanguage.NODEJS_8]: 'js', [SupportedLanguage.PYTHON_3]: 'py', [SupportedLanguage.PYTHON_2]: 'py', @@ -60,6 +62,7 @@ import sys export const languageToPrependedCode: LanguageToStringLookup = { [SupportedLanguage.NODEJS_8]: '// This file was partially generated by Refinery', [SupportedLanguage.NODEJS_10]: '// This file was partially generated by Refinery', + [SupportedLanguage.NODEJS_1020]: '// This file was partially generated by Refinery', [SupportedLanguage.PYTHON_3]: pythonPrependedCode, [SupportedLanguage.PYTHON_2]: pythonPrependedCode, [SupportedLanguage.GO1_12]: '// This file was partially generated by Refinery', @@ -121,6 +124,7 @@ if (typeof main !== undefined) { export const languageToAppendedCode: LanguageToStringLookup = { [SupportedLanguage.NODEJS_8]: nodeAppendedCode, [SupportedLanguage.NODEJS_10]: nodeAppendedCode, + [SupportedLanguage.NODEJS_1020]: nodeAppendedCode, [SupportedLanguage.PYTHON_3]: pythonAppendedCode, [SupportedLanguage.PYTHON_2]: pythonAppendedCode, [SupportedLanguage.GO1_12]: