feat: add support for n install auto, add N_PREFIX cache, replace…

… node repo with `n`, fixes ddev#6418 (ddev#6420)

containers/ddev-php-base/Dockerfile

@@ -84,8 +84,6 @@ http://nginx.org/packages/debian `lsb_release -cs` nginx" > /etc/apt/sources.lis
 RUN curl -sSLo /tmp/debsuryorg-archive-keyring.deb https://packages.sury.org/debsuryorg-archive-keyring.deb && \
     dpkg -i /tmp/debsuryorg-archive-keyring.deb && rm -f /tmp/debsuryorg-archive-keyring.deb && \
     echo "deb [signed-by=/usr/share/keyrings/deb.sury.org-php.gpg] https://packages.sury.org/php/ $(lsb_release -sc) main" > /etc/apt/sources.list.d/php.list && apt-get update
-RUN curl -sSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /usr/share/keyrings/nodesource.gpg && \
-    echo "deb [signed-by=/usr/share/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_VERSION.x nodistro main" > /etc/apt/sources.list.d/nodesource.list
 
 RUN apt-get -qq update
 RUN apt-get -qq install --no-install-recommends --no-install-suggests -y \
@@ -98,9 +96,14 @@ RUN apt-get -qq install --no-install-recommends --no-install-suggests -y \
     jq \
     msmtp \
     nginx \
-    nodejs \
     sqlite3
 
+# Install n to manage nodejs_version, make a symlink for nodejs
+RUN curl -fsSL https://raw.githubusercontent.com/tj/n/master/bin/n | bash -s "${NODE_VERSION}" && \
+    npm install -g n && \
+    n rm "${NODE_VERSION}" && \
+    ln -sf "$(which node)" "$(which node)js"
+
 RUN npm install --unsafe-perm=true --global gulp-cli yarn
 # Normal user needs to be able to write to php sessions
 RUN set -eu -o pipefail && LATEST=$(curl -L --fail --silent "https://api.github.com/repos/nvm-sh/nvm/releases/latest" | jq -r .tag_name) && curl --fail -sL https://raw.githubusercontent.com/nvm-sh/nvm/${LATEST}/install.sh -o /usr/local/bin/install_nvm.sh && chmod +x /usr/local/bin/install_nvm.sh

containers/ddev-webserver/Dockerfile

@@ -3,7 +3,7 @@
 ### Build ddev-php-base from ddev-webserver-base
 ### ddev-php-base is the basic of ddev-php-prod
 ### and ddev-webserver-* (For DDEV local Usage)
-FROM ddev/ddev-php-base:20240729_rfay_php8.4 as ddev-webserver-base
+FROM ddev/ddev-php-base:20240723_stasadev_n_install_auto as ddev-webserver-base
 
 ENV BACKDROP_DRUSH_VERSION=1.4.0
 ENV DEBIAN_FRONTEND=noninteractive
@@ -165,7 +165,7 @@ RUN chmod -R 777 /var/log
 # we need to create the /var/cache/linux and /var/lib/nginx manually for the arm64 image and chmod them, please don't remove them!
 RUN mkdir -p /mnt/ddev-global-cache/mkcert /run/{php,blackfire} /var/cache/nginx /var/lib/nginx && chmod -R ugo+rw /mnt/ddev-global-cache/
 
-RUN chmod -fR ugo+w /usr/sbin /usr/bin /etc/nginx /var/cache/nginx /var/lib/nginx /run /var/www /etc/php/*/*/conf.d/ /var/lib/php/modules /etc/alternatives /usr/lib/node_modules /etc/php /etc/apache2 /var/log/apache2/ /var/run/apache2 /var/lib/apache2 /mnt/ddev-global-cache/*
+RUN chmod -fR ugo+w /usr/sbin /usr/bin /etc/nginx /var/cache/nginx /var/lib/nginx /run /var/www /etc/php/*/*/conf.d/ /var/lib/php/modules /etc/alternatives /usr/local/lib/node_modules /etc/php /etc/apache2 /var/log/apache2/ /var/run/apache2 /var/lib/apache2 /mnt/ddev-global-cache/*
 
 RUN mkdir -p /var/xhprof && curl --fail  -o /tmp/xhprof.tgz -sSL https://pecl.php.net/get/xhprof && tar -zxf /tmp/xhprof.tgz --strip-components=1 -C /var/xhprof && chmod 777 /var/xhprof/xhprof_html && rm /tmp/xhprof.tgz
 
@@ -279,7 +279,7 @@ RUN chmod -R 777 /var/log
 # we need to create the /var/cache/linux and /var/lib/nginx manually for the arm64 image and chmod them, please don't remove them!
 RUN mkdir -p /mnt/ddev-global-cache/mkcert /run/php /var/cache/nginx /var/lib/nginx && chmod -R ugo+rw /home /mnt/ddev-global-cache/
 
-RUN chmod -fR ugo+w /usr/sbin /usr/bin /etc/nginx /var/cache/nginx /var/lib/nginx /run /var/www /etc/php/*/*/conf.d/ /var/lib/php/modules /etc/alternatives /usr/lib/node_modules /etc/php /etc/apache2 /var/lock/apache2 /var/log/apache2/ /var/run/apache2 /var/lib/apache2 /mnt/ddev-global-cache/*
+RUN chmod -fR ugo+w /usr/sbin /usr/bin /etc/nginx /var/cache/nginx /var/lib/nginx /run /var/www /etc/php/*/*/conf.d/ /var/lib/php/modules /etc/alternatives /usr/local/lib/node_modules /etc/php /etc/apache2 /var/lock/apache2 /var/log/apache2/ /var/run/apache2 /var/lib/apache2 /mnt/ddev-global-cache/*
 
 RUN touch /var/log/nginx/error.log /var/log/nginx/access.log /var/log/php-fpm.log && \
     chmod 666 /var/log/nginx/error.log /var/log/nginx/access.log /var/log/php-fpm.log

containers/ddev-webserver/ddev-webserver-base-files/usr/local/bin/n-install.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+# This script is used to install a matching Node.js version
+# using nodejs_version from .ddev/config.yaml in ddev-webserver
+# It requires N_PREFIX and N_INSTALL_VERSION to be set (normally in a docker build phase)
+# This script is intended to be run in /start.sh without root privileges
+
+set -eu -o pipefail
+
+if [ "${N_PREFIX:-}" = "" ]; then
+  echo "This script requires N_PREFIX to be set" && exit 1
+fi
+
+if [ "${N_INSTALL_VERSION:-}" = "" ]; then
+  echo "This script requires N_INSTALL_VERSION to be set" && exit 2
+fi
+
+if [ "${HOSTNAME:-}" = "" ]; then
+  echo "This script requires HOSTNAME to be set" && exit 3
+fi
+
+if [ ! -d "/mnt/ddev-global-cache/n_prefix/${HOSTNAME}" ]; then
+  echo "This script requires the directory /mnt/ddev-global-cache/n_prefix/${HOSTNAME}" && exit 4
+fi
+
+system_node_dir="$(dirname "$(which node)")"
+
+if [ ! -w "${system_node_dir}" ]; then
+  echo "This script cannot write to the directory ${system_node_dir}" && exit 5
+fi
+
+ln -sf "/mnt/ddev-global-cache/n_prefix/${HOSTNAME}" "${N_PREFIX}"
+
+# try normal install that also uses cache
+n_install_result=true
+timeout 30 n install "${N_INSTALL_VERSION}" 2> >(tee /tmp/n-install-stderr.txt >&2) || n_install_result=false
+
+# try offline install on fail, but only if there is no error for invalid ${N_INSTALL_VERSION}
+if [ "${n_install_result}" = "false" ] && ! grep -q "invalid version" /tmp/n-install-stderr.txt; then
+  timeout 30 n install "${N_INSTALL_VERSION}" --offline 2> >(tee -a /tmp/n-install-stderr.txt >&2) && n_install_result=true
+fi
+
+if [ "${n_install_result}" = "true" ]; then
+  # create symlinks on success
+  for node_binary in "${N_PREFIX}/bin/"*; do
+    if [ -f "${node_binary}" ]; then
+      ln -sf "${node_binary}" "${system_node_dir}"
+    fi
+  done
+  ln -sf "${system_node_dir}/node" "${system_node_dir}/nodejs"
+  # we don't need this file if everything is fine
+  rm -f /tmp/n-install-stderr.txt
+else
+  # remove the symlink on error so that the system Node.js can be used
+  rm -f "${N_PREFIX}"
+fi
+
+hash -r

containers/ddev-webserver/ddev-webserver-base-scripts/start.sh

@@ -28,9 +28,9 @@ DDEV_WEBSERVER_TYPE="${DDEV_WEBSERVER_TYPE:-nginx-fpm}"
 # Update the default PHP and FPM versions a DDEV_PHP_VERSION like '5.6' or '7.0' is provided
 # Otherwise it will use the default version configured in the Dockerfile
 if [ -n "$DDEV_PHP_VERSION" ] ; then
-	update-alternatives --set php /usr/bin/php${DDEV_PHP_VERSION}
-	ln -sf /usr/sbin/php-fpm${DDEV_PHP_VERSION} /usr/sbin/php-fpm
-	export PHP_INI=/etc/php/${DDEV_PHP_VERSION}/fpm/php.ini
+  update-alternatives --set php /usr/bin/php${DDEV_PHP_VERSION}
+  ln -sf /usr/sbin/php-fpm${DDEV_PHP_VERSION} /usr/sbin/php-fpm
+  export PHP_INI=/etc/php/${DDEV_PHP_VERSION}/fpm/php.ini
 fi
 
 # Set PHP timezone to configured $TZ if there is one
@@ -93,8 +93,13 @@ ls /var/www/html >/dev/null || (echo "/var/www/html does not seem to be healthy/
 # Make sure the TERMINUS_CACHE_DIR (/mnt/ddev-global-cache/terminus/cache) exists
 sudo mkdir -p ${TERMINUS_CACHE_DIR}
 
-sudo mkdir -p /mnt/ddev-global-cache/{bashhistory/${HOSTNAME},mysqlhistory/${HOSTNAME},nvm_dir/${HOSTNAME},npm,yarn/classic,yarn/berry,corepack}
+sudo mkdir -p /mnt/ddev-global-cache/{bashhistory/${HOSTNAME},mysqlhistory/${HOSTNAME},n_prefix/${HOSTNAME},nvm_dir/${HOSTNAME},npm,yarn/classic,yarn/berry,corepack}
 sudo chown -R "$(id -u):$(id -g)" /mnt/ddev-global-cache/ /var/lib/php
+
+if [ "${N_PREFIX:-}" != "" ] && [ "${N_INSTALL_VERSION:-}" != "" ]; then
+  n-install.sh || true
+fi
+
 # The following ensures a persistent and shared "global" cache for
 # yarn classic (frozen v1) and yarn berry (active). In the case of berry, the global cache
 # will only be used if the project is configured to use it through it's own

containers/ddev-webserver/ddev-webserver-prod-scripts/start.sh

@@ -28,9 +28,9 @@ DDEV_WEBSERVER_TYPE="${DDEV_WEBSERVER_TYPE:-nginx-fpm}"
 # Update the default PHP and FPM versions a DDEV_PHP_VERSION like '5.6' or '7.0' is provided
 # Otherwise it will use the default version configured in the Dockerfile
 if [ -n "$DDEV_PHP_VERSION" ] ; then
-	update-alternatives --set php /usr/bin/php${DDEV_PHP_VERSION}
-	ln -sf /usr/sbin/php-fpm${DDEV_PHP_VERSION} /usr/sbin/php-fpm
-	export PHP_INI=/etc/php/${DDEV_PHP_VERSION}/fpm/php.ini
+  update-alternatives --set php /usr/bin/php${DDEV_PHP_VERSION}
+  ln -sf /usr/sbin/php-fpm${DDEV_PHP_VERSION} /usr/sbin/php-fpm
+  export PHP_INI=/etc/php/${DDEV_PHP_VERSION}/fpm/php.ini
 fi
 
 # Set PHP timezone to configured $TZ if there is one
@@ -90,7 +90,11 @@ disable_xhprof
 
 ls /var/www/html >/dev/null || (echo "/var/www/html does not seem to be healthy/mounted; docker may not be mounting it., exiting" && exit 101)
 
-mkdir -p /mnt/ddev-global-cache/{bashhistory/${HOSTNAME},mysqlhistory/${HOSTNAME},nvm_dir/${HOSTNAME},npm,yarn/classic,yarn/berry,corepack}
+mkdir -p /mnt/ddev-global-cache/{bashhistory/${HOSTNAME},mysqlhistory/${HOSTNAME},n_prefix/${HOSTNAME},nvm_dir/${HOSTNAME},npm,yarn/classic,yarn/berry,corepack}
+
+if [ "${N_PREFIX:-}" != "" ] && [ "${N_INSTALL_VERSION:-}" != "" ]; then
+  n-install.sh || true
+fi
 
 # The following ensures a persistent and shared "global" cache for
 # yarn classic (frozen v1) and yarn berry (active). In the case of berry, the global cache

docs/content/users/configuration/config.md

@@ -385,13 +385,39 @@ Whether to skip mounting project into web container.
 
 ## `nodejs_version`
 
-Node.js version for the web container’s “system” version.
+Node.js version for the web container’s “system” version. [`n`](https://www.npmjs.com/package/n) tool is under the hood.
+
+There is no need to reconfigure `nodejs_version` unless you want a version other than the version already specified, which will be the default version at the time the project was configured.
 
 | Type | Default | Usage
 | -- | -- | --
 | :octicons-file-directory-16: project | current LTS version | any [node version](https://www.npmjs.com/package/n#specifying-nodejs-versions), like `16`, `18.2`, `18.19.2`, etc.
 
-There is no need to configure `nodejs_version` unless you want a version other than the default version.
+!!!tip "How to install the Node.js version from a file"
+    Your project team may specify the Node.js version in a more general way than in the `.ddev/config.yaml`. For example, you may use a `.nvmrc` file, the `package.json`, or a similar technique. In that case, DDEV can use the external configuration provided by that file.
+
+    There is an `auto` label (see [full documentation](https://www.npmjs.com/package/n#specifying-nodejs-versions)):
+
+    ```bash
+    ddev config --nodejs-version=auto
+    ```
+
+    It reads the target version from a file in the [DDEV_APPROOT](../extend/custom-commands.md#environment-variables-provided) directory, or any parent directory.
+
+    `n` looks for in order:
+
+    * `.n-node-version` : version on single line. Custom to `n`.
+    * `.node-version` : version on single line. Used by [multiple tools](https://github.com/shadowspawn/node-version-usage).
+    * `.nvmrc` : version on single line. Used by `nvm`.
+    * if no version file found, look for `engine` as below.
+
+    The `engine` label looks for a `package.json` file and reads the engines field to determine compatible Node.js.
+
+    If your file is not in the `DDEV_APPROOT` directory, you can create a link to the parent folder, so that `n` can find it. For example, if you have `frontend/.nvmrc`, create a `.ddev/web-build/Dockerfile.nvmrc` file:
+
+    ```dockerfile
+    RUN ln -sf /var/www/html/frontend/.nvmrc /var/www/.nvmrc
+    ```
 
 !!!note "Switching from `nvm` to `nodejs_version`"
     If switching from using `nvm` to using `nodejs_version`, you may find that the container continues to use the previously specified version. If this happens, use `ddev nvm alias default system` or `ddev ssh` into the container (`ddev ssh`) and run `rm -rf /mnt/ddev-global-cache/nvm_dir/${DDEV_PROJECT}-web`, then `ddev restart`.

pkg/ddevapp/config.go

@@ -957,8 +957,10 @@ func (app *DdevApp) RenderComposeYAML() (string, error) {
 	extraWebContent := "\nRUN mkdir -p /home/$username && chown $username /home/$username && chmod 600 /home/$username/.pgpass"
 	extraWebContent = extraWebContent + "\nENV NVM_DIR=/home/$username/.nvm"
 	if app.NodeJSVersion != nodeps.NodeJSDefault {
-		extraWebContent = extraWebContent + "\nRUN npm install -g n"
-		extraWebContent = extraWebContent + fmt.Sprintf("\nRUN n install %s && ln -sf /usr/local/bin/node /usr/local/bin/nodejs", app.NodeJSVersion)
+		extraWebContent = extraWebContent + fmt.Sprintf(`
+ENV N_PREFIX=/home/$username/.n
+ENV N_INSTALL_VERSION="%s"
+`, app.NodeJSVersion)
 	}
 	if app.CorepackEnable {
 		extraWebContent = extraWebContent + "\nRUN corepack enable"

pkg/ddevapp/ddevapp.go

@@ -1473,6 +1473,16 @@ Fix with 'ddev config global --required-docker-compose-version="" --use-docker-c
 		util.Warning("Something is wrong with your Docker provider and /mnt/ddev_config is not mounted from the project .ddev folder. Your project cannot normally function successfully with this situation. Is your project in your home directory?")
 	}
 
+	if app.NodeJSVersion != nodeps.NodeJSDefault {
+		util.Debug(`checking nodejs_version: "%s" install for errors`, app.NodeJSVersion)
+		nInstallStderr, _, _ := app.Exec(&ExecOpts{
+			Cmd: "cat /tmp/n-install-stderr.txt 2>/dev/null || true",
+		})
+		if nInstallStderr != "" {
+			util.Warning("Unable to install nodejs_version: \"%s\".\nError output from `n install %s`:\n%s", app.NodeJSVersion, app.NodeJSVersion, nInstallStderr)
+		}
+	}
+
 	if !IsRouterDisabled(app) {
 		output.UserOut.Printf("Starting ddev-router if necessary...")
 		err = StartDdevRouter()

pkg/ddevapp/nodejs_test.go

@@ -1,12 +1,15 @@
 package ddevapp_test
 
 import (
+	"encoding/json"
 	"os"
+	"path/filepath"
 	"strings"
 	"testing"
 
 	"github.com/ddev/ddev/pkg/ddevapp"
 	"github.com/ddev/ddev/pkg/exec"
+	"github.com/ddev/ddev/pkg/fileutil"
 	"github.com/ddev/ddev/pkg/testcommon"
 	"github.com/ddev/ddev/pkg/util"
 	asrt "github.com/stretchr/testify/assert"
@@ -37,12 +40,35 @@ func TestNodeJSVersions(t *testing.T) {
 		assert.NoError(err)
 	})
 
+	nvmrcFile := filepath.Join(app.AppRoot, ".nvmrc")
+	err = fileutil.CopyFile(filepath.Join(origDir, "testdata", t.Name(), ".nvmrc"), nvmrcFile)
+	require.NoError(t, err)
+	nvmrcFileContents, err := os.ReadFile(nvmrcFile)
+	require.NoError(t, err, "Unable to read %s: %v", nvmrcFile, err)
+	nvmrcVersion := strings.TrimSpace(string(nvmrcFileContents))
+
+	packageJSONFile := filepath.Join(app.AppRoot, "package.json")
+	err = fileutil.CopyFile(filepath.Join(origDir, "testdata", t.Name(), "package.json"), packageJSONFile)
+	require.NoError(t, err)
+	packageJSONFileContents, err := os.ReadFile(packageJSONFile)
+	require.NoError(t, err, "Unable to read %s: %v", packageJSONFile, err)
+	var packageJSON map[string]interface{}
+	err = json.Unmarshal(packageJSONFileContents, &packageJSON)
+	require.NoError(t, err, "Unable to unmarshal %s: %v", packageJSONFile, err)
+	engines := packageJSON["engines"].(map[string]interface{})
+	packageJSONVersion := engines["node"].(string)
+
 	err = app.Start()
 	require.NoError(t, err)
 
-	// Testing some random versions, both complete and incomplete
-	for _, v := range []string{"6", "10", "14.20", "16.0.0", "20"} {
+	// Testing some random versions, complete, incomplete, and labels
+	for _, v := range []string{"6", "auto", "engine", "16.0.0", "20"} {
 		app.NodeJSVersion = v
+		if app.NodeJSVersion == "auto" {
+			v = nvmrcVersion
+		} else if app.NodeJSVersion == "engine" {
+			v = packageJSONVersion
+		}
 		err = app.Restart()
 		assert.NoError(err)
 		out, _, err := app.Exec(&ddevapp.ExecOpts{
@@ -99,7 +125,7 @@ func TestCorepackEnable(t *testing.T) {
 	err = app.Start()
 	require.NoError(t, err)
 	out, _, err := app.Exec(&ddevapp.ExecOpts{
-		Cmd: `ls -l /usr/bin/yarn`,
+		Cmd: `ls -l /usr/local/bin/yarn`,
 	})
 	require.NoError(t, err)
 	require.NotContains(t, out, "corepack")
@@ -113,7 +139,7 @@ func TestCorepackEnable(t *testing.T) {
 	err = app.Start()
 	require.NoError(t, err)
 	out, _, err = app.Exec(&ddevapp.ExecOpts{
-		Cmd: `ls -l /usr/bin/yarn`,
+		Cmd: `ls -l /usr/local/bin/yarn`,
 	})
 	require.NoError(t, err)
 	require.Contains(t, out, "corepack")

pkg/ddevapp/templates.go

@@ -68,12 +68,10 @@ const ConfigInstructions = `
 
 # nodejs_version: "20"
 # change from the default system Node.js version to any other version.
-# Numeric version numbers can be complete (i.e. 18.15.0) or
-# incomplete (18, 17.2, 16). 'lts' and 'latest' can be used as well along with
-# other named releases.
-# see https://www.npmjs.com/package/n#specifying-nodejs-versions
-# Note that you can continue using 'ddev nvm' or nvm inside the web container
-# to change the project's installed node version if you need to.
+# See https://ddev.readthedocs.io/en/stable/users/configuration/config/#nodejs_version for more information
+# and https://www.npmjs.com/package/n#specifying-nodejs-versions for the full documentation,
+# Note that using of 'ddev nvm' is discouraged because "nodejs_version" is much easier to use,
+# can specify any version, and is more robust than using 'nvm'.
 
 # corepack_enable: false
 # Change to 'true' to 'corepack enable' and gain access to latest versions of yarn/pnpm

pkg/ddevapp/testdata/TestNodeJSVersions/.nvmrc

@@ -0,0 +1 @@
+10

pkg/ddevapp/testdata/TestNodeJSVersions/package.json

@@ -0,0 +1,5 @@
+{
+    "engines": {
+        "node": "14.20"
+    }
+}

pkg/versionconstants/versionconstants.go

@@ -11,7 +11,7 @@ var AmplitudeAPIKey = ""
 var WebImg = "ddev/ddev-webserver"
 
 // WebTag defines the default web image tag
-var WebTag = "20240729_rfay_php8.4" // Note that this can be overridden by make
+var WebTag = "20240723_stasadev_n_install_auto" // Note that this can be overridden by make
 
 // DBImg defines the default db image used for applications.
 var DBImg = "ddev/ddev-dbserver"