heroku · edmorley · Oct 9, 2024 · Oct 7, 2024
@@ -2,6 +2,10 @@
 
 ## [Unreleased]
 
+- Improved build log output about the detected Python version. ([#1658](https://github.com/heroku/heroku-buildpack-python/pull/1658))
+- Improved error messages shown when the requested Python version is not a valid version string or is for an unknown/non-existent major Python version. ([#1658](https://github.com/heroku/heroku-buildpack-python/pull/1658))
+- Improved error messages shown when `Pipfile.lock` is not valid JSON. ([#1658](https://github.com/heroku/heroku-buildpack-python/pull/1658))
+- Fixed invalid Python versions being silently ignored when they were specified via the `python_version` field in `Pipfile.lock`. ([#1658](https://github.com/heroku/heroku-buildpack-python/pull/1658))
 - Added support for Python 3.9 on Heroku-24. ([#1656](https://github.com/heroku/heroku-buildpack-python/pull/1656))
 - Added buildpack metrics for use of outdated Python patch versions and occurrences of internal errors. ([#1657](https://github.com/heroku/heroku-buildpack-python/pull/1657))
 - Improved the robustness of buildpack error handling by enabling `inherit_errexit`. ([#1655](https://github.com/heroku/heroku-buildpack-python/pull/1655))

@@ -24,6 +24,7 @@ source "${BUILDPACK_DIR}/lib/output.sh"
 source "${BUILDPACK_DIR}/lib/package_manager.sh"
 source "${BUILDPACK_DIR}/lib/pip.sh"
 source "${BUILDPACK_DIR}/lib/pipenv.sh"
+source "${BUILDPACK_DIR}/lib/python_version.sh"
 source "${BUILDPACK_DIR}/lib/utils.sh"
 
 compile_start_time=$(nowms)
@@ -46,9 +47,6 @@ export BUILD_DIR CACHE_DIR ENV_DIR
 S3_BASE_URL="${BUILDPACK_S3_BASE_URL:-"https://heroku-buildpack-python.s3.us-east-1.amazonaws.com"}"
 # This has to be exported since it's used by the geo-libs step which is run in a subshell.
 
-# Default Python Versions
-source "${BUILDPACK_DIR}/bin/default_pythons"
-
 # Common Problem Warnings:
 # This section creates a temporary file in which to stick the output of `pip install`.
 # The `warnings` subscript then greps through this for common problems and guides
@@ -123,10 +121,14 @@ fi
 # Runs a `bin/pre_compile` script if found in the app source, allowing build customisation.
 source "${BUILDPACK_DIR}/bin/steps/hooks/pre_compile"
 
-# Sticky runtimes. If there was a previous build, and it used a given version of Python,
-# continue to use that version of Python in perpetuity.
+# TODO: Clear the cache if this isn't a valid version, as part of the cache refactor.
+# (Currently the version is instead validated in `read_requested_python_version()`)
 if [[ -f "$CACHE_DIR/.heroku/python-version" ]]; then
-	CACHED_PYTHON_VERSION=$(cat "$CACHE_DIR/.heroku/python-version")
+	cached_python_version="$(cat "${CACHE_DIR}/.heroku/python-version")"
+	# `python-X.Y.Z` -> `X.Y`
+	cached_python_version="${cached_python_version#python-}"
+else
+	cached_python_version=
 fi
 
 # We didn't always record the stack version.
@@ -140,29 +142,37 @@ fi
 package_manager="$(package_manager::determine_package_manager "${BUILD_DIR}")"
 meta_set "package_manager" "${package_manager}"
 
-# Pipenv Python version support.
-# Detect the version of Python requested from a Pipfile (e.g. python_version or python_full_version).
-# Convert it to a runtime.txt file.
-source "${BUILDPACK_DIR}/bin/steps/pipenv-python-version"
-
-if [[ -f runtime.txt ]]; then
-	# PYTHON_VERSION_SOURCE may have already been set by the pipenv-python-version step.
-	# TODO: Refactor this and stop pipenv-python-version using runtime.txt as an API.
-	PYTHON_VERSION_SOURCE=${PYTHON_VERSION_SOURCE:-"runtime.txt"}
-	puts-step "Using Python version specified in ${PYTHON_VERSION_SOURCE}"
-	meta_set "python_version_reason" "specified"
-elif [[ -n "${CACHED_PYTHON_VERSION:-}" ]]; then
-	puts-step "No Python version was specified. Using the same version as the last build: ${CACHED_PYTHON_VERSION}"
-	echo "       To use a different version, see: https://devcenter.heroku.com/articles/python-runtimes"
-	meta_set "python_version_reason" "cached"
-	echo "${CACHED_PYTHON_VERSION}" >runtime.txt
-else
-	puts-step "No Python version was specified. Using the buildpack default: ${DEFAULT_PYTHON_VERSION}"
-	echo "       To use a different version, see: https://devcenter.heroku.com/articles/python-runtimes"
-	meta_set "python_version_reason" "default"
-	echo "${DEFAULT_PYTHON_VERSION}" >runtime.txt
+# TODO: Move this warning to lib/package_manager.sh once `output::warning()` exists
+# (puts-warn outputs to stdout, which would break `determine_package_manager()` as is).
+# TODO: Adjust this warning to mention support for missing Pipfile.lock will be removed soon.
+if [[ "${package_manager}" == "pipenv" && ! -f "${BUILD_DIR}/Pipfile.lock" ]]; then
+	puts-warn "No 'Pipfile.lock' found! We recommend you commit this into your repository."
 fi
 
+# We use the Bash 4.3+ `nameref` feature to pass back multiple values from this function
+# without having to hardcode globals. See: https://stackoverflow.com/a/38997681
+python_version::read_requested_python_version "${BUILD_DIR}" "${package_manager}" "${cached_python_version}" requested_python_version python_version_origin
+meta_set "python_version_reason" "${python_version_origin}"
+
+case "${python_version_origin}" in
+	default)
+		puts-step "No Python version was specified. Using the buildpack default: Python ${requested_python_version}"
+		echo "       To use a different version, see: https://devcenter.heroku.com/articles/python-runtimes"
+		;;
+	cached)
+		puts-step "No Python version was specified. Using the same version as the last build: Python ${requested_python_version}"
+		echo "       To use a different version, see: https://devcenter.heroku.com/articles/python-runtimes"
+		;;
+	*)
+		puts-step "Using Python ${requested_python_version} specified in ${python_version_origin}"
+		;;
+esac
+
+python_full_version="$(python_version::resolve_python_version "${requested_python_version}" "${python_version_origin}")"
+python_major_version="${python_full_version%.*}"
+meta_set "python_version" "${python_full_version}"
+meta_set "python_version_major" "${python_major_version}"
+
 # The directory for the .profile.d scripts.
 mkdir -p "$(dirname "$PROFILE_PATH")"
 # The directory for editable VCS dependencies.

@@ -4,54 +4,21 @@
 
 set -euo pipefail
 
-PYTHON_VERSION=$(cat runtime.txt)
-# Remove leading and trailing whitespace. Note: This implementation relies upon
-# `extglob` being set, which is the case thanks to `bin/utils` being run earlier.
-PYTHON_VERSION="${PYTHON_VERSION##+([[:space:]])}"
-PYTHON_VERSION="${PYTHON_VERSION%%+([[:space:]])}"
-
-function eol_python_version_error() {
-	local major_version="${1}"
-	local eol_date="${2}"
-	display_error <<-EOF
-		Error: Python ${major_version} is no longer supported.
-
-		Python ${major_version} reached upstream end-of-life on ${eol_date}, and is
-		therefore no longer receiving security updates:
-		https://devguide.python.org/versions/#supported-versions
-
-		As such, it is no longer supported by this buildpack.
-
-		Please upgrade to a newer Python version.
-
-		For a list of the supported Python versions, see:
-		https://devcenter.heroku.com/articles/python-support#supported-runtimes
-	EOF
-	meta_set "failure_reason" "python-version-eol"
-	exit 1
-}
-
-# We check for EOL prior to checking if the archive exists on S3, to ensure the more specific EOL error
-# message is still shown for newer stacks where the EOL Python versions might not have been built.
-case "${PYTHON_VERSION}" in
-	python-3.7.+([0-9]))
-		eol_python_version_error "3.7" "June 27th, 2023"
-		;;
-	python-3.6.+([0-9]))
-		eol_python_version_error "3.6" "December 23rd, 2021"
-		;;
-	*) ;;
-esac
-
 # The Python runtime archive filename is of form: 'python-X.Y.Z-ubuntu-22.04-amd64.tar.zst'
 # The Ubuntu version is calculated from `STACK` since it's faster than calling `lsb_release`.
 UBUNTU_VERSION="${STACK/heroku-/}.04"
 ARCH=$(dpkg --print-architecture)
-PYTHON_URL="${S3_BASE_URL}/${PYTHON_VERSION}-ubuntu-${UBUNTU_VERSION}-${ARCH}.tar.zst"
-
+PYTHON_URL="${S3_BASE_URL}/python-${python_full_version}-ubuntu-${UBUNTU_VERSION}-${ARCH}.tar.zst"
+
+# The Python version validation earlier will have filtered out most unsupported versions.
+# However, the version might still not be found if either:
+# 1. It's a Python major version we've deprecated and so is only available on older stacks (i.e: Python 3.8).
+# 2. If an exact Python version was requested and the patch version doesn't exist (e.g. 3.12.999).
+# 3. The user has pinned to an older buildpack version and the S3 bucket location or layout has changed since.
+# TODO: Update this message to be more specific once Python 3.8 support is dropped.
 if ! curl --output /dev/null --silent --head --fail --retry 3 --retry-connrefused --connect-timeout 10 "${PYTHON_URL}"; then
 	display_error <<-EOF
-		Error: Requested runtime '${PYTHON_VERSION}' is not available for this stack (${STACK}).
+		Error: Python ${python_full_version} is not available for this stack (${STACK}).
 
 		For a list of the supported Python versions, see:
 		https://devcenter.heroku.com/articles/python-support#supported-runtimes
@@ -60,20 +27,17 @@ if ! curl --output /dev/null --silent --head --fail --retry 3 --retry-connrefuse
 	exit 1
 fi
 
-# TODO: Refactor Python version usage to use the non-prefixed form everywhere.
-python_version_without_prefix="${PYTHON_VERSION#python-}"
-meta_set "python_version" "${python_version_without_prefix}"
-meta_set "python_version_major" "${python_version_without_prefix%.*}"
-
 function warn_if_patch_update_available() {
-	local requested_version="${1}"
-	local latest_patch_version="${2}"
+	local requested_full_version="${1}"
+	local requested_major_version="${2}"
+	local latest_patch_version
+	latest_patch_version="$(python_version::resolve_python_version "${requested_major_version}" "${python_version_origin}")"
 	# Extract the patch version component of the version strings (ie: the '5' in '3.10.5').
-	local requested_patch_number="${requested_version##*.}"
+	local requested_patch_number="${requested_full_version##*.}"
 	local latest_patch_number="${latest_patch_version##*.}"
 	if ((requested_patch_number < latest_patch_number)); then
 		puts-warn
-		puts-warn "A Python security update is available! Upgrade as soon as possible to: ${latest_patch_version}"
+		puts-warn "A Python security update is available! Upgrade as soon as possible to: Python ${latest_patch_version}"
 		puts-warn "See: https://devcenter.heroku.com/articles/python-runtimes"
 		puts-warn
 		meta_set "python_version_outdated" "true"
@@ -84,45 +48,31 @@ function warn_if_patch_update_available() {
 
 # We wait until now to display outdated Python version warnings, since we only want to show them
 # if there weren't any errors with the version to avoid adding noise to the error messages.
-case "${PYTHON_VERSION}" in
-	python-3.12.*)
-		warn_if_patch_update_available "${PYTHON_VERSION}" "${LATEST_312}"
-		;;
-	python-3.11.*)
-		warn_if_patch_update_available "${PYTHON_VERSION}" "${LATEST_311}"
-		;;
-	python-3.10.*)
-		warn_if_patch_update_available "${PYTHON_VERSION}" "${LATEST_310}"
-		;;
-	python-3.9.*)
-		warn_if_patch_update_available "${PYTHON_VERSION}" "${LATEST_39}"
-		;;
-	python-3.8.*)
-		puts-warn
-		puts-warn "Python 3.8 will reach its upstream end-of-life in October 2024, at which"
-		puts-warn "point it will no longer receive security updates:"
-		puts-warn "https://devguide.python.org/versions/#supported-versions"
-		puts-warn
-		puts-warn "Support for Python 3.8 will be removed from this buildpack on December 4th, 2024."
-		puts-warn
-		puts-warn "Upgrade to a newer Python version as soon as possible to keep your app secure."
-		puts-warn "See: https://devcenter.heroku.com/articles/python-runtimes"
-		puts-warn
-		warn_if_patch_update_available "${PYTHON_VERSION}" "${LATEST_38}"
-		;;
-	# TODO: Make this case an error, since it should be unreachable.
-	*) ;;
-esac
+# TODO: Move this into lib/ as part of the warnings refactor.
+if [[ "${python_major_version}" == "3.8" ]]; then
+	puts-warn
+	puts-warn "Python 3.8 will reach its upstream end-of-life in October 2024, at which"
+	puts-warn "point it will no longer receive security updates:"
+	puts-warn "https://devguide.python.org/versions/#supported-versions"
+	puts-warn
+	puts-warn "Support for Python 3.8 will be removed from this buildpack on December 4th, 2024."
+	puts-warn
+	puts-warn "Upgrade to a newer Python version as soon as possible to keep your app secure."
+	puts-warn "See: https://devcenter.heroku.com/articles/python-runtimes"
+	puts-warn
+fi
+
+warn_if_patch_update_available "${python_full_version}" "${python_major_version}"
 
 if [[ "$STACK" != "$CACHED_PYTHON_STACK" ]]; then
 	puts-step "Stack has changed from $CACHED_PYTHON_STACK to $STACK, clearing cache"
 	rm -rf .heroku/python-stack .heroku/python-version .heroku/python .heroku/vendor .heroku/python .heroku/python-sqlite3-version
 fi
 
+# TODO: Clean this up as part of the cache refactor.
 if [[ -f .heroku/python-version ]]; then
-	# shellcheck disable=SC2312 # TODO: Invoke this command separately to avoid masking its return value.
-	if [[ ! "$(cat .heroku/python-version)" == "$PYTHON_VERSION" ]]; then
-		puts-step "Python version has changed from $(cat .heroku/python-version) to ${PYTHON_VERSION}, clearing cache"
+	if [[ "${cached_python_version}" != "${python_full_version}" ]]; then
+		puts-step "Python version has changed from ${cached_python_version} to ${python_full_version}, clearing cache"
 		rm -rf .heroku/python
 	else
 		SKIP_INSTALL=1
@@ -153,23 +103,23 @@ if [[ -f "${BUILD_DIR}/requirements.txt" ]]; then
 fi
 
 if [[ "${SKIP_INSTALL:-0}" == "1" ]]; then
-	puts-step "Using cached install of ${PYTHON_VERSION}"
+	puts-step "Using cached install of Python ${python_full_version}"
 else
-	puts-step "Installing ${PYTHON_VERSION}"
+	puts-step "Installing Python ${python_full_version}"
 
 	# Prepare destination directory.
 	mkdir -p .heroku/python
 
 	if ! curl --silent --show-error --fail --retry 3 --retry-connrefused --connect-timeout 10 "${PYTHON_URL}" | tar --zstd --extract --directory .heroku/python; then
 		# The Python version was confirmed to exist previously, so any failure here is due to
 		# a networking issue or archive/buildpack bug rather than the runtime not existing.
-		display_error "Error: Failed to download/install ${PYTHON_VERSION}."
+		display_error "Error: Failed to download/install Python ${python_full_version}."
 		meta_set "failure_reason" "python-download"
 		exit 1
 	fi
 
 	# Record for future reference.
-	echo "$PYTHON_VERSION" >.heroku/python-version
+	echo "python-${python_full_version}" >.heroku/python-version
 	echo "$STACK" >.heroku/python-stack
 
 	hash -r