Improve cleanup of per-Lambda vendor dirs upon startup; fixes #1426

changelog.md

@@ -11,7 +11,8 @@
 
 ### Fixed
 
-- Improve error handling during possible `@tables` port conflicts on startup; partially fixes #1441, thanks @jlipps!
+- Improved cleanup of per-Lambda vendor dirs (`node_modules`, `vendor`) upon startup; fixes #1426
+- Improved error handling during possible `@tables` port conflicts on startup; partially fixes #1441, thanks @jlipps!
 
 ---
 

package.json

@@ -30,7 +30,7 @@
   "dependencies": {
     "@architect/asap": "~6.0.3",
     "@architect/create": "~4.2.3-RC.0",
-    "@architect/hydrate": "~3.3.0-RC.1",
+    "@architect/hydrate": "~3.3.0-RC.2",
     "@architect/inventory": "~3.6.0",
     "@architect/utils": "~3.1.9",
     "@aws-sdk/client-apigatewaymanagementapi": "^3.316.0",

readme.md

@@ -39,6 +39,7 @@ npx arc sandbox
 - `-v`, `--verbose` - Enable verbose logging
 - `-d`, `--debug` - Enable debug logging
 - `-q`, `--quiet` - Disable (most) logging
+- `--disable-delete-vendor` - Disable deleting Lambda vendor dirs upon startup
 - `--disable-symlinks` - Disable symlinking `src/shared` into all functions and use file copying instead
 
 
@@ -79,6 +80,8 @@ Methods may be passed an options object containing the following parameters:
   - Defaults to `http`
   - Can be one of `http` (aliased to `httpv2`), `httpv1`, `rest`
 - `cwd` - **String** - Specify a working directory (handy for aiming Sandbox at test mocks)
+- `deleteVendor` - **Boolean** - Delete Lambda vendor dirs upon startup
+  - Defaults to `true`
 - `env` - **Object** - Environment variables for Lambda invocations in automated testing
   - String values overwrite env vars of the same name set via `.env` or `prefs.arc` files
   - `undefined` values delete any env vars of the same name set via `.env` or `prefs.arc` files

src/cli/_flags.js

@@ -13,14 +13,17 @@ module.exports = function getFlags () {
     quiet: [ 'q' ],
     verbose: [ 'v' ],
   }
-  let boolean = [ 'disable-symlinks' ]
+  let boolean = [ 'disable-delete-vendor', 'disable-symlinks' ]
   let args = minimist(process.argv.slice(2), { alias, boolean })
 
   // Log levels (defaults to `normal` in the updater)
   let logLevel
   if (args.verbose) logLevel = 'verbose'
   if (args.debug) logLevel = 'debug'
 
+  // Disable node_modules / vendor dir deletion upon startup
+  let deleteVendor = !args['disable-delete-vendor']
+
   // Main user-facing @http port
   let port = Number(args.port) || undefined
 
@@ -38,6 +41,7 @@ module.exports = function getFlags () {
   let symlink = !args['disable-symlinks']
 
   let flags = {
+    deleteVendor,
     port,
     host,
     quiet,

src/sandbox/index.js

@@ -61,7 +61,7 @@ function _start (params, callback) {
       http.start(params, callback)
     },
 
-    // Loop through functions and see if any need dependency hydration
+    // Loop through functions, clear vendor dirs, and see if any need dependency hydration
     function (callback) {
       maybeHydrate(params, callback)
     },

src/sandbox/maybe-hydrate.js

@@ -1,48 +1,63 @@
 let chalk = require('chalk')
 let depStatus = require('depstatus')
-let { existsSync: exists } = require('fs')
-let { globSync } = require('glob')
+let { existsSync: exists, readFileSync, rmSync } = require('fs')
 let { join } = require('path')
 let hydrate = require('@architect/hydrate')
 let series = require('run-series')
-let { chars, pathToUnix } = require('@architect/utils')
+let { chars } = require('@architect/utils')
 
 /**
- * Checks for the existence of supported dependency manifests, and auto-hydrates each function's dependencies as necessary
- *
- * Supported manifests:
- * - package.json
- * - requirements.txt
- * - Gemfile
- * - (more to come!)
- *
- * Not responsible for src/shared + views, handled elsewhere to optimize load time
+ * Checks for the existence of supported dependency manifests, and auto-hydrates dependencies as necessary
+ * Supported manifests: `package.json`, `requirements.txt`, `Gemfile`
  */
-module.exports = function maybeHydrate ({ cwd, inventory, quiet }, callback) {
+module.exports = function maybeHydrate ({ cwd, inventory, quiet, deleteVendor }, callback) {
   let { inv } = inventory
   if (!inv.lambdaSrcDirs || !inv.lambdaSrcDirs.length) {
     callback()
   }
   else {
+    // Enable vendor dir deletion by default
+    let del = deleteVendor === undefined ? true : deleteVendor
+    if (inv._project.preferences?.sandbox?.['delete-vendor'] !== undefined) {
+      del = inv._project.preferences.sandbox['delete-vendor']
+    }
+
     let notify = () => {
       if (!quiet) console.log(chalk.grey(chars.done, 'Found new functions to hydrate!'))
     }
+    let destroy = filepath => del && rmSync(filepath, { recursive: true, force: true })
     let notified = false
     // Make a new array, don't inventory
-    let lambdaSrcDirs = [ ...inv.lambdaSrcDirs ]
+    let srcDirs = [ ...inv.lambdaSrcDirs ]
 
     let shared = inv?.shared?.src || join(cwd, 'src', 'shared')
     let views = inv?.views?.src || join(cwd, 'src', 'views')
-    lambdaSrcDirs.push(shared)
-    lambdaSrcDirs.push(views)
+    srcDirs.push(shared)
+    srcDirs.push(views)
 
-    let ops = lambdaSrcDirs.map(path => {
+    let ops = srcDirs.map(path => {
       return function (callback) {
+        let isNode, isPython, isRuby
+        let lambda = inv.lambdasBySrcDir[path]
+        if (lambda) {
+          if (Array.isArray(lambda)) lambda = lambda[0] // Normalize possible multi-tenant Lambdas
+          let { config } = lambda
+          let { runtime, runtimeConfig } = config
+          isNode = runtime.startsWith('nodejs') || runtimeConfig?.baseruntime?.startsWith('nodejs')
+          isPython = runtime.startsWith('python')
+          isRuby = runtime.startsWith('ruby')
+        }
+
+        // Possible manifests
+        let packageJson = join(path, 'package.json')
+        let requirementsTxt = join(path, 'requirements.txt')
+        let gemfile = join(path, 'Gemfile')
+
         /**
          * Check each of our supported dependency manifests
          * - Try to generally minimize file hits
          * - Try not to go any deeper into the filesystem than necessary (dep trees can take a long time to walk!)
-         * - Assumes Architect deps will have their own deps, helping indicate hydration status
+         * - Generally be aggressive with destroying vendor dirs
          */
         function install (callback) {
           if (!notified) notify()
@@ -53,47 +68,64 @@ module.exports = function maybeHydrate ({ cwd, inventory, quiet }, callback) {
           let hydrateShared = path === shared || path === views || false
           hydrate.install({ cwd, inventory, basepath: path, copyShared, hydrateShared, quiet }, callback)
         }
-        series([
-          function _packageJson (callback) {
-            let packageJson = exists(join(path, 'package.json'))
-            if (packageJson) {
-              let result = depStatus(path)
-              let { missing, outdated, warn } = result
-              let installDeps = missing.length || outdated.length || warn.length
-              if (installDeps) {
-                install(callback)
-              }
-              else callback()
-            }
-            else callback()
-          },
-          function _requirementsTxt (callback) {
-            let requirementsTxt = exists(join(path, 'requirements.txt'))
-            if (requirementsTxt) {
-              let pattern = pathToUnix(path + '/vendor/*')
-              let arcDir = join(path, 'vendor', 'architect-functions')
-              let hydrated = globSync(pattern).some(file => !file.includes(arcDir))
-              if (!hydrated) {
-                install(callback)
-              }
-              else callback()
+
+        if (isNode || (!lambda && exists(packageJson))) {
+          let nodeModules = join(path, 'node_modules')
+
+          // Don't start by destroying `node_modules`, as `depStatus` may help us hydrate whatever is missing
+          if (exists(packageJson)) {
+            // Check to see if there was a failed or aborted deploy
+            let pkg = JSON.parse(readFileSync(packageJson))
+            if (pkg?._arc === 'autoinstall') {
+              destroy(packageJson)
+              destroy(nodeModules)
+              return callback()
             }
+
+            let result = depStatus(path)
+            let { missing, outdated, warn } = result
+            let installDeps = missing.length || outdated.length || warn.length
+
+            if (installDeps) install(callback)
+            // Looks like deps are all good here, no need to destroy `node_modules`
             else callback()
-          },
-          function _gemfile (callback) {
-            let gemfile = exists(join(path, 'Gemfile'))
-            if (gemfile) {
-              let pattern = pathToUnix(path + '/vendor/bundle/*')
-              let arcDir = join(path, 'vendor', 'bundle', 'architect-functions')
-              let hydrated = globSync(pattern).some(file => !file.includes(arcDir))
-              if (!hydrated) {
-                install(callback)
-              }
-              else callback()
+          }
+          else {
+            destroy(nodeModules)
+            callback()
+          }
+        }
+
+        else if (isPython || (!lambda && exists(requirementsTxt))) {
+          let vendor = join(path, 'vendor')
+          // Always start fresh
+          destroy(vendor)
+
+          if (exists(requirementsTxt)) {
+            // Check to see if there was a failed or aborted deploy
+            let reqs = readFileSync(requirementsTxt).toString()
+            if (reqs.includes('# _arc: autoinstall')) {
+              destroy(requirementsTxt)
+              return callback()
             }
-            else callback()
-          },
-        ], callback)
+            install(callback)
+          }
+          else callback()
+        }
+
+        else if (isRuby || (!lambda && exists(gemfile))) {
+          let vendor = join(path, 'vendor')
+          // Always start fresh
+          destroy(vendor)
+
+          if (exists(gemfile)) {
+            install(callback)
+          }
+          else callback()
+        }
+
+        // All other runtimes, shared + views
+        else callback()
       }
     })
     series(ops, callback)

test/mock/dep-warn/all-packages/prefs.arc

@@ -0,0 +1,2 @@
+@sandbox
+delete-vendor false

test/mock/dep-warn/basic/prefs.arc

@@ -0,0 +1,2 @@
+@sandbox
+delete-vendor false

test/mock/dep-warn/lambda-packages/prefs.arc

@@ -0,0 +1,2 @@
+@sandbox
+delete-vendor false

test/mock/dep-warn/no-packages/prefs.arc

@@ -0,0 +1,2 @@
+@sandbox
+delete-vendor false

test/mock/dep-warn/shared-packages/prefs.arc

@@ -0,0 +1,2 @@
+@sandbox
+delete-vendor false