diff --git a/Android/README.md b/Android/README.md index f5f463ca116589..bae9150ef057ac 100644 --- a/Android/README.md +++ b/Android/README.md @@ -25,7 +25,7 @@ you don't already have the SDK, here's how to install it: The `android.py` script also requires the following commands to be on the `PATH`: * `curl` -* `java` +* `java` (or set the `JAVA_HOME` environment variable) * `tar` * `unzip` @@ -80,18 +80,54 @@ call. For example, if you want a pydebug build that also caches the results from ## Testing -To run the Python test suite on Android: - -* Install Android Studio, if you don't already have it. -* Follow the instructions in the previous section to build all supported - architectures. -* Run `./android.py setup-testbed` to download the Gradle wrapper. -* Open the `testbed` directory in Android Studio. -* In the *Device Manager* dock, connect a device or start an emulator. - Then select it from the drop-down list in the toolbar. -* Click the "Run" button in the toolbar. -* The testbed app displays nothing on screen while running. To see its output, - open the [Logcat window](https://developer.android.com/studio/debug/logcat). - -To run specific tests, or pass any other arguments to the test suite, edit the -command line in testbed/app/src/main/python/main.py. +The tests can be run on Linux, macOS, or Windows, although on Windows you'll +have to build the `cross-build/HOST` subdirectory on one of the other platforms +and copy it over. + +The test suite can usually be run on a device with 2 GB of RAM, though for some +configurations or test orders you may need to increase this. As of Android +Studio Koala, 2 GB is the default for all emulators, although the user interface +may indicate otherwise. The effective setting is `hw.ramSize` in +~/.android/avd/*.avd/hardware-qemu.ini, whereas Android Studio displays the +value from config.ini. Changing the value in Android Studio will update both of +these files. + +Before running the test suite, follow the instructions in the previous section +to build the architecture you want to test. Then run the test script in one of +the following modes: + +* In `--connected` mode, it runs on a device or emulator you have already + connected to the build machine. List the available devices with + `$ANDROID_HOME/platform-tools/adb devices -l`, then pass a device ID to the + script like this: + + ```sh + ./android.py test --connected emulator-5554 + ``` + +* In `--managed` mode, it uses a temporary headless emulator defined in the + `managedDevices` section of testbed/app/build.gradle.kts. This mode is slower, + but more reproducible. + + We currently define two devices: `minVersion` and `maxVersion`, corresponding + to our minimum and maximum supported Android versions. For example: + + ```sh + ./android.py test --managed maxVersion + ``` + +By default, the only messages the script will show are Python's own stdout and +stderr. Add the `-v` option to also show Gradle output, and non-Python logcat +messages. + +Any other arguments on the `android.py test` command line will be passed through +to `python -m test` – use `--` to separate them from android.py's own options. +See the [Python Developer's +Guide](https://devguide.python.org/testing/run-write-tests/) for common options +– most of them will work on Android, except for those that involve subprocesses, +such as `-j`. + +Every time you run `android.py test`, changes in pure-Python files in the +repository's `Lib` directory will be picked up immediately. Changes in C files, +and architecture-specific files such as sysconfigdata, will not take effect +until you re-run `android.py make-host` or `build`. diff --git a/Android/android-env.sh b/Android/android-env.sh index 545d559d93ab36..93372e3fe1c7ee 100644 --- a/Android/android-env.sh +++ b/Android/android-env.sh @@ -28,7 +28,7 @@ ndk_version=26.2.11394342 ndk=$ANDROID_HOME/ndk/$ndk_version if ! [ -e $ndk ]; then - log "Installing NDK: this may take several minutes" + log "Installing NDK - this may take several minutes" yes | $ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager "ndk;$ndk_version" fi diff --git a/Android/android.py b/Android/android.py index a78b15c9c4e58c..b5403b5d2a4a5e 100755 --- a/Android/android.py +++ b/Android/android.py @@ -1,21 +1,51 @@ #!/usr/bin/env python3 +import asyncio import argparse from glob import glob import os import re +import shlex import shutil +import signal import subprocess import sys import sysconfig +from asyncio import wait_for +from contextlib import asynccontextmanager from os.path import basename, relpath from pathlib import Path +from subprocess import CalledProcessError from tempfile import TemporaryDirectory + SCRIPT_NAME = Path(__file__).name CHECKOUT = Path(__file__).resolve().parent.parent +ANDROID_DIR = CHECKOUT / "Android" +TESTBED_DIR = ANDROID_DIR / "testbed" CROSS_BUILD_DIR = CHECKOUT / "cross-build" +APP_ID = "org.python.testbed" +DECODE_ARGS = ("UTF-8", "backslashreplace") + + +try: + android_home = Path(os.environ['ANDROID_HOME']) +except KeyError: + sys.exit("The ANDROID_HOME environment variable is required.") + +adb = Path( + f"{android_home}/platform-tools/adb" + + (".exe" if os.name == "nt" else "") +) + +gradlew = Path( + f"{TESTBED_DIR}/gradlew" + + (".bat" if os.name == "nt" else "") +) + +logcat_started = False + def delete_glob(pattern): # Path.glob doesn't accept non-relative patterns. @@ -42,10 +72,14 @@ def subdir(name, *, clean=None): return path -def run(command, *, host=None, **kwargs): - env = os.environ.copy() +def run(command, *, host=None, env=None, log=True, **kwargs): + kwargs.setdefault("check", True) + if env is None: + env = os.environ.copy() + original_env = env.copy() + if host: - env_script = CHECKOUT / "Android/android-env.sh" + env_script = ANDROID_DIR / "android-env.sh" env_output = subprocess.run( f"set -eu; " f"HOST={host}; " @@ -66,15 +100,13 @@ def run(command, *, host=None, **kwargs): print(line) env[key] = value - if env == os.environ: + if env == original_env: raise ValueError(f"Found no variables in {env_script.name} output:\n" + env_output) - print(">", " ".join(map(str, command))) - try: - subprocess.run(command, check=True, env=env, **kwargs) - except subprocess.CalledProcessError as e: - sys.exit(e) + if log: + print(">", " ".join(map(str, command))) + return subprocess.run(command, env=env, **kwargs) def build_python_path(): @@ -180,31 +212,334 @@ def clean_all(context): delete_glob(CROSS_BUILD_DIR) +def setup_sdk(): + sdkmanager = android_home / ( + "cmdline-tools/latest/bin/sdkmanager" + + (".bat" if os.name == "nt" else "") + ) + + # Gradle will fail if it needs to install an SDK package whose license + # hasn't been accepted, so pre-accept all licenses. + if not all((android_home / "licenses" / path).exists() for path in [ + "android-sdk-arm-dbt-license", "android-sdk-license" + ]): + run([sdkmanager, "--licenses"], text=True, input="y\n" * 100) + + # Gradle may install this automatically, but we can't rely on that because + # we need to run adb within the logcat task. + if not adb.exists(): + run([sdkmanager, "platform-tools"]) + + # To avoid distributing compiled artifacts without corresponding source code, # the Gradle wrapper is not included in the CPython repository. Instead, we # extract it from the Gradle release. -def setup_testbed(context): +def setup_testbed(): + if all((TESTBED_DIR / path).exists() for path in [ + "gradlew", "gradlew.bat", "gradle/wrapper/gradle-wrapper.jar", + ]): + return + ver_long = "8.7.0" ver_short = ver_long.removesuffix(".0") - testbed_dir = CHECKOUT / "Android/testbed" for filename in ["gradlew", "gradlew.bat"]: out_path = download( f"https://raw.githubusercontent.com/gradle/gradle/v{ver_long}/{filename}", - testbed_dir) + TESTBED_DIR) os.chmod(out_path, 0o755) with TemporaryDirectory(prefix=SCRIPT_NAME) as temp_dir: - os.chdir(temp_dir) bin_zip = download( - f"https://services.gradle.org/distributions/gradle-{ver_short}-bin.zip") + f"https://services.gradle.org/distributions/gradle-{ver_short}-bin.zip", + temp_dir) outer_jar = f"gradle-{ver_short}/lib/plugins/gradle-wrapper-{ver_short}.jar" - run(["unzip", bin_zip, outer_jar]) - run(["unzip", "-o", "-d", f"{testbed_dir}/gradle/wrapper", outer_jar, - "gradle-wrapper.jar"]) + run(["unzip", "-d", temp_dir, bin_zip, outer_jar]) + run(["unzip", "-o", "-d", f"{TESTBED_DIR}/gradle/wrapper", + f"{temp_dir}/{outer_jar}", "gradle-wrapper.jar"]) -def main(): +# run_testbed will build the app automatically, but it hides the Gradle output +# by default, so it's useful to have this as a separate command for the buildbot. +def build_testbed(context): + setup_sdk() + setup_testbed() + run( + [gradlew, "--console", "plain", "packageDebug", "packageDebugAndroidTest"], + cwd=TESTBED_DIR, + ) + + +# Work around a bug involving sys.exit and TaskGroups +# (https://github.com/python/cpython/issues/101515). +def exit(*args): + raise MySystemExit(*args) + + +class MySystemExit(Exception): + pass + + +# The `test` subcommand runs all subprocesses through this context manager so +# that no matter what happens, they can always be cancelled from another task, +# and they will always be cleaned up on exit. +@asynccontextmanager +async def async_process(*args, **kwargs): + process = await asyncio.create_subprocess_exec(*args, **kwargs) + try: + yield process + finally: + if process.returncode is None: + # Allow a reasonably long time for Gradle to clean itself up, + # because we don't want stale emulators left behind. + timeout = 10 + process.terminate() + try: + await wait_for(process.wait(), timeout) + except TimeoutError: + print( + f"Command {args} did not terminate after {timeout} seconds " + f" - sending SIGKILL" + ) + process.kill() + + # Even after killing the process we must still wait for it, + # otherwise we'll get the warning "Exception ignored in __del__". + await wait_for(process.wait(), timeout=1) + + +async def async_check_output(*args, **kwargs): + async with async_process( + *args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs + ) as process: + stdout, stderr = await process.communicate() + if process.returncode == 0: + return stdout.decode(*DECODE_ARGS) + else: + raise CalledProcessError( + process.returncode, args, + stdout.decode(*DECODE_ARGS), stderr.decode(*DECODE_ARGS) + ) + + +# Return a list of the serial numbers of connected devices. Emulators will have +# serials of the form "emulator-5678". +async def list_devices(): + serials = [] + header_found = False + + lines = (await async_check_output(adb, "devices")).splitlines() + for line in lines: + # Ignore blank lines, and all lines before the header. + line = line.strip() + if line == "List of devices attached": + header_found = True + elif header_found and line: + try: + serial, status = line.split() + except ValueError: + raise ValueError(f"failed to parse {line!r}") + if status == "device": + serials.append(serial) + + if not header_found: + raise ValueError(f"failed to parse {lines}") + return serials + + +async def find_device(context, initial_devices): + if context.managed: + print("Waiting for managed device - this may take several minutes") + while True: + new_devices = set(await list_devices()).difference(initial_devices) + if len(new_devices) == 0: + await asyncio.sleep(1) + elif len(new_devices) == 1: + serial = new_devices.pop() + print(f"Serial: {serial}") + return serial + else: + exit(f"Found more than one new device: {new_devices}") + else: + return context.connected + + +# An older version of this script in #121595 filtered the logs by UID instead. +# But logcat can't filter by UID until API level 31. If we ever switch back to +# filtering by UID, we'll also have to filter by time so we only show messages +# produced after the initial call to `stop_app`. +# +# We're more likely to miss the PID because it's shorter-lived, so there's a +# workaround in PythonSuite.kt to stop it being *too* short-lived. +async def find_pid(serial): + print("Waiting for app to start - this may take several minutes") + shown_error = False + while True: + try: + pid = (await async_check_output( + adb, "-s", serial, "shell", "pidof", "-s", APP_ID + )).strip() + except CalledProcessError as e: + # If the app isn't running yet, pidof gives no output. So if there + # is output, there must have been some other error. However, this + # sometimes happens transiently, especially when running a managed + # emulator for the first time, so don't make it fatal. + if (e.stdout or e.stderr) and not shown_error: + print_called_process_error(e) + print("This may be transient, so continuing to wait") + shown_error = True + else: + # Some older devices (e.g. Nexus 4) return zero even when no process + # was found, so check whether we actually got any output. + if pid: + print(f"PID: {pid}") + return pid + + # Loop fairly rapidly to avoid missing a short-lived process. + await asyncio.sleep(0.2) + + +async def logcat_task(context, initial_devices): + # Gradle may need to do some large downloads of libraries and emulator + # images. This will happen during find_device in --managed mode, or find_pid + # in --connected mode. + startup_timeout = 600 + serial = await wait_for(find_device(context, initial_devices), startup_timeout) + pid = await wait_for(find_pid(serial), startup_timeout) + + args = [adb, "-s", serial, "logcat", "--pid", pid, "--format", "tag"] + hidden_output = [] + async with async_process( + *args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + ) as process: + while line := (await process.stdout.readline()).decode(*DECODE_ARGS): + if match := re.fullmatch(r"([A-Z])/(.*)", line, re.DOTALL): + level, message = match.groups() + else: + # If the regex doesn't match, this is probably the second or + # subsequent line of a multi-line message. Python won't produce + # such messages, but other components might. + level, message = None, line + + # Put high-level messages on stderr so they're highlighted in the + # buildbot logs. This will include Python's own stderr. + stream = ( + sys.stderr + if level in ["E", "F"] # ERROR and FATAL (aka ASSERT) + else sys.stdout + ) + + # To simplify automated processing of the output, e.g. a buildbot + # posting a failure notice on a GitHub PR, we strip the level and + # tag indicators from Python's stdout and stderr. + for prefix in ["python.stdout: ", "python.stderr: "]: + if message.startswith(prefix): + global logcat_started + logcat_started = True + stream.write(message.removeprefix(prefix)) + break + else: + if context.verbose: + # Non-Python messages add a lot of noise, but they may + # sometimes help explain a failure. + stream.write(line) + else: + hidden_output.append(line) + + # If the device disconnects while logcat is running, which always + # happens in --managed mode, some versions of adb return non-zero. + # Distinguish this from a logcat startup error by checking whether we've + # received a message from Python yet. + status = await wait_for(process.wait(), timeout=1) + if status != 0 and not logcat_started: + raise CalledProcessError(status, args, "".join(hidden_output)) + + +def stop_app(serial): + run([adb, "-s", serial, "shell", "am", "force-stop", APP_ID], log=False) + + +async def gradle_task(context): + env = os.environ.copy() + if context.managed: + task_prefix = context.managed + else: + task_prefix = "connected" + env["ANDROID_SERIAL"] = context.connected + + args = [ + gradlew, "--console", "plain", f"{task_prefix}DebugAndroidTest", + "-Pandroid.testInstrumentationRunnerArguments.pythonArgs=" + + shlex.join(context.args), + ] + hidden_output = [] + try: + async with async_process( + *args, cwd=TESTBED_DIR, env=env, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + ) as process: + while line := (await process.stdout.readline()).decode(*DECODE_ARGS): + # Gradle may take several minutes to install SDK packages, so + # it's worth showing those messages even in non-verbose mode. + if context.verbose or line.startswith('Preparing "Install'): + sys.stdout.write(line) + else: + hidden_output.append(line) + + status = await wait_for(process.wait(), timeout=1) + if status == 0: + exit(0) + else: + raise CalledProcessError(status, args) + finally: + # If logcat never started, then something has gone badly wrong, so the + # user probably wants to see the Gradle output even in non-verbose mode. + if hidden_output and not logcat_started: + sys.stdout.write("".join(hidden_output)) + + # Gradle does not stop the tests when interrupted. + if context.connected: + stop_app(context.connected) + + +async def run_testbed(context): + setup_sdk() + setup_testbed() + + if context.managed: + # In this mode, Gradle will create a device with an unpredictable name. + # So we save a list of the running devices before starting Gradle, and + # find_device then waits for a new device to appear. + initial_devices = await list_devices() + else: + # In case the previous shutdown was unclean, make sure the app isn't + # running, otherwise we might show logs from a previous run. This is + # unnecessary in --managed mode, because Gradle creates a new emulator + # every time. + stop_app(context.connected) + initial_devices = None + + try: + async with asyncio.TaskGroup() as tg: + tg.create_task(logcat_task(context, initial_devices)) + tg.create_task(gradle_task(context)) + except* MySystemExit as e: + raise SystemExit(*e.exceptions[0].args) from None + except* CalledProcessError as e: + # Extract it from the ExceptionGroup so it can be handled by `main`. + raise e.exceptions[0] + + +# Handle SIGTERM the same way as SIGINT. This ensures that if we're terminated +# by the buildbot worker, we'll make an attempt to clean up our subprocesses. +def install_signal_handler(): + def signal_handler(*args): + os.kill(os.getpid(), signal.SIGINT) + + signal.signal(signal.SIGTERM, signal_handler) + + +def parse_args(): parser = argparse.ArgumentParser() subcommands = parser.add_subparsers(dest="subcommand") build = subcommands.add_parser("build", help="Build everything") @@ -219,8 +554,6 @@ def main(): help="Run `make` for Android") subcommands.add_parser( "clean", help="Delete the cross-build directory") - subcommands.add_parser( - "setup-testbed", help="Download the testbed Gradle wrapper") for subcommand in build, configure_build, configure_host: subcommand.add_argument( @@ -235,15 +568,66 @@ def main(): subcommand.add_argument("args", nargs="*", help="Extra arguments to pass to `configure`") - context = parser.parse_args() + subcommands.add_parser( + "build-testbed", help="Build the testbed app") + test = subcommands.add_parser( + "test", help="Run the test suite") + test.add_argument( + "-v", "--verbose", action="store_true", + help="Show Gradle output, and non-Python logcat messages") + device_group = test.add_mutually_exclusive_group(required=True) + device_group.add_argument( + "--connected", metavar="SERIAL", help="Run on a connected device. " + "Connect it yourself, then get its serial from `adb devices`.") + device_group.add_argument( + "--managed", metavar="NAME", help="Run on a Gradle-managed device. " + "These are defined in `managedDevices` in testbed/app/build.gradle.kts.") + test.add_argument( + "args", nargs="*", help=f"Arguments for `python -m test`. " + f"Separate them from {SCRIPT_NAME}'s own arguments with `--`.") + + return parser.parse_args() + + +def main(): + install_signal_handler() + context = parse_args() dispatch = {"configure-build": configure_build_python, "make-build": make_build_python, "configure-host": configure_host_python, "make-host": make_host_python, "build": build_all, "clean": clean_all, - "setup-testbed": setup_testbed} - dispatch[context.subcommand](context) + "build-testbed": build_testbed, + "test": run_testbed} + + try: + result = dispatch[context.subcommand](context) + if asyncio.iscoroutine(result): + asyncio.run(result) + except CalledProcessError as e: + print_called_process_error(e) + sys.exit(1) + + +def print_called_process_error(e): + for stream_name in ["stdout", "stderr"]: + content = getattr(e, stream_name) + stream = getattr(sys, stream_name) + if content: + stream.write(content) + if not content.endswith("\n"): + stream.write("\n") + + # Format the command so it can be copied into a shell. shlex uses single + # quotes, so we surround the whole command with double quotes. + args_joined = ( + e.cmd if isinstance(e.cmd, str) + else " ".join(shlex.quote(str(arg)) for arg in e.cmd) + ) + print( + f'Command "{args_joined}" returned exit status {e.returncode}' + ) if __name__ == "__main__": diff --git a/Android/testbed/app/build.gradle.kts b/Android/testbed/app/build.gradle.kts index 7320b21e98bbd1..7e0bef58ed88eb 100644 --- a/Android/testbed/app/build.gradle.kts +++ b/Android/testbed/app/build.gradle.kts @@ -1,17 +1,18 @@ import com.android.build.api.variant.* +import kotlin.math.max plugins { id("com.android.application") id("org.jetbrains.kotlin.android") } -val PYTHON_DIR = File(projectDir, "../../..").canonicalPath +val PYTHON_DIR = file("../../..").canonicalPath val PYTHON_CROSS_DIR = "$PYTHON_DIR/cross-build" val ABIS = mapOf( "arm64-v8a" to "aarch64-linux-android", "x86_64" to "x86_64-linux-android", -).filter { File("$PYTHON_CROSS_DIR/${it.value}").exists() } +).filter { file("$PYTHON_CROSS_DIR/${it.value}").exists() } if (ABIS.isEmpty()) { throw GradleException( "No Android ABIs found in $PYTHON_CROSS_DIR: see Android/README.md " + @@ -19,7 +20,7 @@ if (ABIS.isEmpty()) { ) } -val PYTHON_VERSION = File("$PYTHON_DIR/Include/patchlevel.h").useLines { +val PYTHON_VERSION = file("$PYTHON_DIR/Include/patchlevel.h").useLines { for (line in it) { val match = """#define PY_VERSION\s+"(\d+\.\d+)""".toRegex().find(line) if (match != null) { @@ -29,6 +30,16 @@ val PYTHON_VERSION = File("$PYTHON_DIR/Include/patchlevel.h").useLines { throw GradleException("Failed to find Python version") } +android.ndkVersion = file("../../android-env.sh").useLines { + for (line in it) { + val match = """ndk_version=(\S+)""".toRegex().find(line) + if (match != null) { + return@useLines match.groupValues[1] + } + } + throw GradleException("Failed to find NDK version") +} + android { namespace = "org.python.testbed" @@ -45,6 +56,8 @@ android { externalNativeBuild.cmake.arguments( "-DPYTHON_CROSS_DIR=$PYTHON_CROSS_DIR", "-DPYTHON_VERSION=$PYTHON_VERSION") + + testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" } externalNativeBuild.cmake { @@ -62,41 +75,81 @@ android { kotlinOptions { jvmTarget = "1.8" } + + testOptions { + managedDevices { + localDevices { + create("minVersion") { + device = "Small Phone" + + // Managed devices have a minimum API level of 27. + apiLevel = max(27, defaultConfig.minSdk!!) + + // ATD devices are smaller and faster, but have a minimum + // API level of 30. + systemImageSource = if (apiLevel >= 30) "aosp-atd" else "aosp" + } + + create("maxVersion") { + device = "Small Phone" + apiLevel = defaultConfig.targetSdk!! + systemImageSource = "aosp-atd" + } + } + + // If the previous test run succeeded and nothing has changed, + // Gradle thinks there's no need to run it again. Override that. + afterEvaluate { + (localDevices.names + listOf("connected")).forEach { + tasks.named("${it}DebugAndroidTest") { + outputs.upToDateWhen { false } + } + } + } + } + } } dependencies { implementation("androidx.appcompat:appcompat:1.6.1") implementation("com.google.android.material:material:1.11.0") implementation("androidx.constraintlayout:constraintlayout:2.1.4") + androidTestImplementation("androidx.test.ext:junit:1.1.5") + androidTestImplementation("androidx.test:rules:1.5.0") } // Create some custom tasks to copy Python and its standard library from // elsewhere in the repository. androidComponents.onVariants { variant -> + val pyPlusVer = "python$PYTHON_VERSION" generateTask(variant, variant.sources.assets!!) { into("python") { - for (triplet in ABIS.values) { - for (subDir in listOf("include", "lib")) { - into(subDir) { - from("$PYTHON_CROSS_DIR/$triplet/prefix/$subDir") - include("python$PYTHON_VERSION/**") - duplicatesStrategy = DuplicatesStrategy.EXCLUDE - } + into("include/$pyPlusVer") { + for (triplet in ABIS.values) { + from("$PYTHON_CROSS_DIR/$triplet/prefix/include/$pyPlusVer") } + duplicatesStrategy = DuplicatesStrategy.EXCLUDE } - into("lib/python$PYTHON_VERSION") { - // Uncomment this to pick up edits from the source directory - // without having to rerun `make install`. - // from("$PYTHON_DIR/Lib") - // duplicatesStrategy = DuplicatesStrategy.INCLUDE + + into("lib/$pyPlusVer") { + // To aid debugging, the source directory takes priority. + from("$PYTHON_DIR/Lib") + + // The cross-build directory provides ABI-specific files such as + // sysconfigdata. + for (triplet in ABIS.values) { + from("$PYTHON_CROSS_DIR/$triplet/prefix/lib/$pyPlusVer") + } into("site-packages") { from("$projectDir/src/main/python") } + + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + exclude("**/__pycache__") } } - exclude("**/__pycache__") } generateTask(variant, variant.sources.jniLibs!!) { diff --git a/Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt b/Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt new file mode 100644 index 00000000000000..0e888ab71d87da --- /dev/null +++ b/Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt @@ -0,0 +1,35 @@ +package org.python.testbed + +import androidx.test.annotation.UiThreadTest +import androidx.test.platform.app.InstrumentationRegistry +import androidx.test.ext.junit.runners.AndroidJUnit4 + +import org.junit.Test +import org.junit.runner.RunWith + +import org.junit.Assert.* + + +@RunWith(AndroidJUnit4::class) +class PythonSuite { + @Test + @UiThreadTest + fun testPython() { + val start = System.currentTimeMillis() + try { + val context = + InstrumentationRegistry.getInstrumentation().targetContext + val args = + InstrumentationRegistry.getArguments().getString("pythonArgs", "") + val status = PythonTestRunner(context).run(args) + assertEquals(0, status) + } finally { + // Make sure the process lives long enough for the test script to + // detect it (see `find_pid` in android.py). + val delay = 2000 - (System.currentTimeMillis() - start) + if (delay > 0) { + Thread.sleep(delay) + } + } + } +} diff --git a/Android/testbed/app/src/main/c/main_activity.c b/Android/testbed/app/src/main/c/main_activity.c index 73aba4164d000f..534709048990c6 100644 --- a/Android/testbed/app/src/main/c/main_activity.c +++ b/Android/testbed/app/src/main/c/main_activity.c @@ -84,7 +84,7 @@ static char *redirect_stream(StreamInfo *si) { return 0; } -JNIEXPORT void JNICALL Java_org_python_testbed_MainActivity_redirectStdioToLogcat( +JNIEXPORT void JNICALL Java_org_python_testbed_PythonTestRunner_redirectStdioToLogcat( JNIEnv *env, jobject obj ) { for (StreamInfo *si = STREAMS; si->file; si++) { @@ -115,7 +115,7 @@ static void throw_status(JNIEnv *env, PyStatus status) { throw_runtime_exception(env, status.err_msg ? status.err_msg : ""); } -JNIEXPORT void JNICALL Java_org_python_testbed_MainActivity_runPython( +JNIEXPORT int JNICALL Java_org_python_testbed_PythonTestRunner_runPython( JNIEnv *env, jobject obj, jstring home, jstring runModule ) { PyConfig config; @@ -125,13 +125,13 @@ JNIEXPORT void JNICALL Java_org_python_testbed_MainActivity_runPython( status = set_config_string(env, &config, &config.home, home); if (PyStatus_Exception(status)) { throw_status(env, status); - return; + return 1; } status = set_config_string(env, &config, &config.run_module, runModule); if (PyStatus_Exception(status)) { throw_status(env, status); - return; + return 1; } // Some tests generate SIGPIPE and SIGXFSZ, which should be ignored. @@ -140,8 +140,8 @@ JNIEXPORT void JNICALL Java_org_python_testbed_MainActivity_runPython( status = Py_InitializeFromConfig(&config); if (PyStatus_Exception(status)) { throw_status(env, status); - return; + return 1; } - Py_RunMain(); + return Py_RunMain(); } diff --git a/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt b/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt index 5a590d5d04e954..c4bf6cbe83d8cd 100644 --- a/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt +++ b/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt @@ -1,38 +1,56 @@ package org.python.testbed +import android.content.Context import android.os.* import android.system.Os import android.widget.TextView import androidx.appcompat.app.* import java.io.* + +// Launching the tests from an activity is OK for a quick check, but for +// anything more complicated it'll be more convenient to use `android.py test` +// to launch the tests via PythonSuite. class MainActivity : AppCompatActivity() { override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) setContentView(R.layout.activity_main) + val status = PythonTestRunner(this).run("-W -uall") + findViewById(R.id.tvHello).text = "Exit status $status" + } +} + + +class PythonTestRunner(val context: Context) { + /** @param args Extra arguments for `python -m test`. + * @return The Python exit status: zero if the tests passed, nonzero if + * they failed. */ + fun run(args: String = "") : Int { + Os.setenv("PYTHON_ARGS", args, true) // Python needs this variable to help it find the temporary directory, // but Android only sets it on API level 33 and later. - Os.setenv("TMPDIR", cacheDir.toString(), false) + Os.setenv("TMPDIR", context.cacheDir.toString(), false) val pythonHome = extractAssets() System.loadLibrary("main_activity") redirectStdioToLogcat() - runPython(pythonHome.toString(), "main") - findViewById(R.id.tvHello).text = "Python complete" + + // The main module is in src/main/python/main.py. + return runPython(pythonHome.toString(), "main") } private fun extractAssets() : File { - val pythonHome = File(filesDir, "python") + val pythonHome = File(context.filesDir, "python") if (pythonHome.exists() && !pythonHome.deleteRecursively()) { throw RuntimeException("Failed to delete $pythonHome") } - extractAssetDir("python", filesDir) + extractAssetDir("python", context.filesDir) return pythonHome } private fun extractAssetDir(path: String, targetDir: File) { - val names = assets.list(path) + val names = context.assets.list(path) ?: throw RuntimeException("Failed to list $path") val targetSubdir = File(targetDir, path) if (!targetSubdir.mkdirs()) { @@ -43,7 +61,7 @@ class MainActivity : AppCompatActivity() { val subPath = "$path/$name" val input: InputStream try { - input = assets.open(subPath) + input = context.assets.open(subPath) } catch (e: FileNotFoundException) { extractAssetDir(subPath, targetDir) continue @@ -57,5 +75,5 @@ class MainActivity : AppCompatActivity() { } private external fun redirectStdioToLogcat() - private external fun runPython(home: String, runModule: String) -} \ No newline at end of file + private external fun runPython(home: String, runModule: String) : Int +} diff --git a/Android/testbed/app/src/main/python/main.py b/Android/testbed/app/src/main/python/main.py index a1b6def34ede81..c7314b500bf821 100644 --- a/Android/testbed/app/src/main/python/main.py +++ b/Android/testbed/app/src/main/python/main.py @@ -1,4 +1,6 @@ +import os import runpy +import shlex import signal import sys @@ -8,10 +10,7 @@ # profile save"), so disabling it should not weaken the tests. signal.pthread_sigmask(signal.SIG_UNBLOCK, [signal.SIGUSR1]) -# To run specific tests, or pass any other arguments to the test suite, edit -# this command line. -sys.argv[1:] = [ - "--use", "all,-cpu", - "--verbose3", -] +sys.argv[1:] = shlex.split(os.environ["PYTHON_ARGS"]) + +# The test module will call sys.exit to indicate whether the tests passed. runpy.run_module("test") diff --git a/Android/testbed/build.gradle.kts b/Android/testbed/build.gradle.kts index 53f4a67287fcc5..2dad1501c2422f 100644 --- a/Android/testbed/build.gradle.kts +++ b/Android/testbed/build.gradle.kts @@ -1,5 +1,5 @@ // Top-level build file where you can add configuration options common to all sub-projects/modules. plugins { - id("com.android.application") version "8.2.2" apply false + id("com.android.application") version "8.4.2" apply false id("org.jetbrains.kotlin.android") version "1.9.22" apply false -} \ No newline at end of file +} diff --git a/Android/testbed/gradle.properties b/Android/testbed/gradle.properties index 3c5031eb7d63f7..e9f345c8c26250 100644 --- a/Android/testbed/gradle.properties +++ b/Android/testbed/gradle.properties @@ -20,4 +20,9 @@ kotlin.code.style=official # Enables namespacing of each library's R class so that its R class includes only the # resources declared in the library itself and none from the library's dependencies, # thereby reducing the size of the R class for that library -android.nonTransitiveRClass=true \ No newline at end of file +android.nonTransitiveRClass=true + +# By default, the app will be uninstalled after the tests finish (apparently +# after 10 seconds in case of an unclean shutdown). We disable this, because +# when using android.py it can conflict with the installation of the next run. +android.injected.androidTest.leaveApksInstalledAfterRun=true diff --git a/Android/testbed/gradle/wrapper/gradle-wrapper.properties b/Android/testbed/gradle/wrapper/gradle-wrapper.properties index 2dc3339a3ef213..57b2f57cc86b51 100644 --- a/Android/testbed/gradle/wrapper/gradle-wrapper.properties +++ b/Android/testbed/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ #Mon Feb 19 20:29:06 GMT 2024 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.6-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/Doc/Makefile b/Doc/Makefile index 9ddf97fd775dec..8020884447983c 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -188,54 +188,69 @@ dist: mkdir -p dist # archive the HTML - make html + @echo "Building HTML..." + $(MAKE) html cp -pPR build/html dist/python-$(DISTVERSION)-docs-html tar -C dist -cf dist/python-$(DISTVERSION)-docs-html.tar python-$(DISTVERSION)-docs-html bzip2 -9 -k dist/python-$(DISTVERSION)-docs-html.tar (cd dist; zip -q -r -9 python-$(DISTVERSION)-docs-html.zip python-$(DISTVERSION)-docs-html) rm -r dist/python-$(DISTVERSION)-docs-html rm dist/python-$(DISTVERSION)-docs-html.tar + @echo "Build finished and archived!" # archive the text build - make text + @echo "Building text..." + $(MAKE) text cp -pPR build/text dist/python-$(DISTVERSION)-docs-text tar -C dist -cf dist/python-$(DISTVERSION)-docs-text.tar python-$(DISTVERSION)-docs-text bzip2 -9 -k dist/python-$(DISTVERSION)-docs-text.tar (cd dist; zip -q -r -9 python-$(DISTVERSION)-docs-text.zip python-$(DISTVERSION)-docs-text) rm -r dist/python-$(DISTVERSION)-docs-text rm dist/python-$(DISTVERSION)-docs-text.tar + @echo "Build finished and archived!" # archive the A4 latex + @echo "Building LaTeX (A4 paper)..." rm -rf build/latex - make latex PAPER=a4 - -sed -i 's/makeindex/makeindex -q/' build/latex/Makefile - (cd build/latex; make clean && make all-pdf && make FMT=pdf zip bz2) + $(MAKE) latex PAPER=a4 + # remove zip & bz2 dependency on all-pdf, + # as otherwise the full latexmk process is run twice. + # ($$ is needed to escape the $; https://www.gnu.org/software/make/manual/make.html#Basics-of-Variable-References) + -sed -i 's/: all-$$(FMT)/:/' build/latex/Makefile + (cd build/latex; $(MAKE) clean && $(MAKE) --jobs=$((`nproc`+1)) --output-sync LATEXMKOPTS='-quiet' all-pdf && $(MAKE) FMT=pdf zip bz2) cp build/latex/docs-pdf.zip dist/python-$(DISTVERSION)-docs-pdf-a4.zip cp build/latex/docs-pdf.tar.bz2 dist/python-$(DISTVERSION)-docs-pdf-a4.tar.bz2 + @echo "Build finished and archived!" # archive the letter latex + @echo "Building LaTeX (US paper)..." rm -rf build/latex - make latex PAPER=letter - -sed -i 's/makeindex/makeindex -q/' build/latex/Makefile - (cd build/latex; make clean && make all-pdf && make FMT=pdf zip bz2) + $(MAKE) latex PAPER=letter + -sed -i 's/: all-$$(FMT)/:/' build/latex/Makefile + (cd build/latex; $(MAKE) clean && $(MAKE) --jobs=$((`nproc`+1)) --output-sync LATEXMKOPTS='-quiet' all-pdf && $(MAKE) FMT=pdf zip bz2) cp build/latex/docs-pdf.zip dist/python-$(DISTVERSION)-docs-pdf-letter.zip cp build/latex/docs-pdf.tar.bz2 dist/python-$(DISTVERSION)-docs-pdf-letter.tar.bz2 + @echo "Build finished and archived!" # copy the epub build + @echo "Building EPUB..." rm -rf build/epub - make epub + $(MAKE) epub cp -pPR build/epub/Python.epub dist/python-$(DISTVERSION)-docs.epub + @echo "Build finished and archived!" # archive the texinfo build + @echo "Building Texinfo..." rm -rf build/texinfo - make texinfo - make info --directory=build/texinfo + $(MAKE) texinfo + $(MAKE) info --directory=build/texinfo cp -pPR build/texinfo dist/python-$(DISTVERSION)-docs-texinfo tar -C dist -cf dist/python-$(DISTVERSION)-docs-texinfo.tar python-$(DISTVERSION)-docs-texinfo bzip2 -9 -k dist/python-$(DISTVERSION)-docs-texinfo.tar (cd dist; zip -q -r -9 python-$(DISTVERSION)-docs-texinfo.zip python-$(DISTVERSION)-docs-texinfo) rm -r dist/python-$(DISTVERSION)-docs-texinfo rm dist/python-$(DISTVERSION)-docs-texinfo.tar + @echo "Build finished and archived!" .PHONY: _ensure-package _ensure-package: venv @@ -247,11 +262,11 @@ _ensure-package: venv .PHONY: _ensure-pre-commit _ensure-pre-commit: - make _ensure-package PACKAGE=pre-commit + $(MAKE) _ensure-package PACKAGE=pre-commit .PHONY: _ensure-sphinx-autobuild _ensure-sphinx-autobuild: - make _ensure-package PACKAGE=sphinx-autobuild + $(MAKE) _ensure-package PACKAGE=sphinx-autobuild .PHONY: check check: _ensure-pre-commit @@ -271,12 +286,12 @@ serve: # for development releases: always build .PHONY: autobuild-dev autobuild-dev: - make dist SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1' + $(MAKE) dist SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1' # for quick rebuilds (HTML only) .PHONY: autobuild-dev-html autobuild-dev-html: - make html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1' + $(MAKE) html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1' # for stable releases: only build if not in pre-release stage (alpha, beta) # release candidate downloads are okay, since the stable tree can be in that stage @@ -286,7 +301,7 @@ autobuild-stable: echo "Not building; $(DISTVERSION) is not a release version."; \ exit 1;; \ esac - @make autobuild-dev + @$(MAKE) autobuild-dev .PHONY: autobuild-stable-html autobuild-stable-html: @@ -294,4 +309,4 @@ autobuild-stable-html: echo "Not building; $(DISTVERSION) is not a release version."; \ exit 1;; \ esac - @make autobuild-dev-html + @$(MAKE) autobuild-dev-html diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 1fab3f577f2f89..567b5f6855a48a 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -394,8 +394,7 @@ Initializing and finalizing the interpreter Undo all initializations made by :c:func:`Py_Initialize` and subsequent use of Python/C API functions, and destroy all sub-interpreters (see :c:func:`Py_NewInterpreter` below) that were created and not yet destroyed since - the last call to :c:func:`Py_Initialize`. Ideally, this frees all memory - allocated by the Python interpreter. This is a no-op when called for a second + the last call to :c:func:`Py_Initialize`. This is a no-op when called for a second time (without calling :c:func:`Py_Initialize` again first). Since this is the reverse of :c:func:`Py_Initialize`, it should be called @@ -407,6 +406,12 @@ Initializing and finalizing the interpreter If there were errors during finalization (flushing buffered data), ``-1`` is returned. + Note that Python will do a best effort at freeing all memory allocated by the Python + interpreter. Therefore, any C-Extension should make sure to correctly clean up all + of the preveiously allocated PyObjects before using them in subsequent calls to + :c:func:`Py_Initialize`. Otherwise it could introduce vulnerabilities and incorrect + behavior. + This function is provided for a number of reasons. An embedding application might want to restart Python without having to restart the application itself. An application that has loaded the Python interpreter from a dynamically @@ -421,10 +426,11 @@ Initializing and finalizing the interpreter loaded extension modules loaded by Python are not unloaded. Small amounts of memory allocated by the Python interpreter may not be freed (if you find a leak, please report it). Memory tied up in circular references between objects is not - freed. Some memory allocated by extension modules may not be freed. Some - extensions may not work properly if their initialization routine is called more - than once; this can happen if an application calls :c:func:`Py_Initialize` and - :c:func:`Py_FinalizeEx` more than once. + freed. Interned strings will all be deallocated regarldess of their reference count. + Some memory allocated by extension modules may not be freed. Some extensions may not + work properly if their initialization routine is called more than once; this can + happen if an application calls :c:func:`Py_Initialize` and :c:func:`Py_FinalizeEx` + more than once. .. audit-event:: cpython._PySys_ClearAuditHooks "" c.Py_FinalizeEx diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 9f2c48d98b8344..c7c6fe9f942bd8 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -94,9 +94,9 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. ignored. If there are no digits or *str* is not NULL-terminated following the digits and trailing whitespace, :exc:`ValueError` will be raised. - .. seealso:: Python methods :meth:`int.to_bytes` and :meth:`int.from_bytes` - to convert a :c:type:`PyLongObject` to/from an array of bytes in base - ``256``. You can call those from C using :c:func:`PyObject_CallMethod`. + .. seealso:: :c:func:`PyLong_AsNativeBytes()` and + :c:func:`PyLong_FromNativeBytes()` functions can be used to convert + a :c:type:`PyLongObject` to/from an array of bytes in base ``256``. .. c:function:: PyObject* PyLong_FromUnicodeObject(PyObject *u, int base) diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst index 7aaad932c0104a..587fe50fd51164 100644 --- a/Doc/library/configparser.rst +++ b/Doc/library/configparser.rst @@ -1314,13 +1314,19 @@ RawConfigParser Objects .. method:: add_section(section) - Add a section named *section* to the instance. If a section by the given - name already exists, :exc:`DuplicateSectionError` is raised. If the - *default section* name is passed, :exc:`ValueError` is raised. + Add a section named *section* or :const:`UNNAMED_SECTION` to the instance. + + If the given section already exists, :exc:`DuplicateSectionError` is + raised. If the *default section* name is passed, :exc:`ValueError` is + raised. If :const:`UNNAMED_SECTION` is passed and support is disabled, + :exc:`UnnamedSectionDisabledError` is raised. Type of *section* is not checked which lets users create non-string named sections. This behaviour is unsupported and may cause internal errors. + .. versionchanged:: 3.14 + Added support for :const:`UNNAMED_SECTION`. + .. method:: set(section, option, value) @@ -1405,7 +1411,6 @@ Exceptions Exception raised when attempting to parse a file which has no section headers. - .. exception:: ParsingError Exception raised when errors occur attempting to parse a file. @@ -1421,6 +1426,13 @@ Exceptions .. versionadded:: 3.13 +.. exception:: UnnamedSectionDisabledError + + Exception raised when attempting to use the + :const:`UNNAMED_SECTION` without enabling it. + + .. versionadded:: 3.14 + .. rubric:: Footnotes .. [1] Config parsers allow for heavy customization. If you are interested in diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 440ca233584e57..a770ad87ab02d3 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1555,7 +1555,7 @@ iterations of the loop. end = STACK.pop() start = STACK.pop() - STACK.append(slice(start, stop)) + STACK.append(slice(start, end)) if it is 3, implements:: diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index b1e35e68b132e2..253a120e5c0f51 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -704,6 +704,13 @@ Constants .. versionadded:: 3.12 +.. data:: SHUT_RD + SHUT_WR + SHUT_RDWR + + These constants are used by the :meth:`~socket.socket.shutdown` method of socket objects. + + .. availability:: not WASI. Functions ^^^^^^^^^ diff --git a/Doc/library/struct.rst b/Doc/library/struct.rst index a2c293443e23d3..4769affdf1d666 100644 --- a/Doc/library/struct.rst +++ b/Doc/library/struct.rst @@ -279,9 +279,9 @@ Notes: (1) .. index:: single: ? (question mark); in struct format strings - The ``'?'`` conversion code corresponds to the :c:expr:`_Bool` type defined by - C99. If this type is not available, it is simulated using a :c:expr:`char`. In - standard mode, it is always represented by one byte. + The ``'?'`` conversion code corresponds to the :c:expr:`_Bool` type + defined by C standards since C99. In standard mode, it is + represented by one byte. (2) When attempting to pack a non-integer using any of the integer conversion diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index ed809d04167ffd..99252a82d9e9af 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -1712,7 +1712,7 @@ always available. To enable, pass a *depth* value greater than zero; this sets the number of frames whose information will be captured. To disable, - pass set *depth* to zero. + set *depth* to zero. This setting is thread-specific. diff --git a/Doc/tutorial/appendix.rst b/Doc/tutorial/appendix.rst index b8faf756698097..da664f2f360ff1 100644 --- a/Doc/tutorial/appendix.rst +++ b/Doc/tutorial/appendix.rst @@ -14,8 +14,8 @@ There are two variants of the interactive :term:`REPL`. The classic basic interpreter is supported on all platforms with minimal line control capabilities. -On Unix-like systems (e.g. Linux or macOS) with :mod:`curses` and -:mod:`readline` support, a new interactive shell is used by default. +On Windows, or Unix-like systems with :mod:`curses` support, +a new interactive shell is used by default. This one supports color, multiline editing, history browsing, and paste mode. To disable color, see :ref:`using-on-controlling-color` for details. Function keys provide some additional functionality. diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index bd3c45c8ab503b..c5e11458733508 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -88,7 +88,7 @@ Several legacy standard library modules have now `been removed This article doesn't attempt to provide a complete specification of all new features, but instead gives a convenient overview. For full details refer to the documentation, -such as the :ref:`Library Reference ` +such as the :ref:`Library Reference ` and :ref:`Language Reference `. To understand the complete implementation and design rationale for a change, refer to the PEP for a particular new feature; @@ -173,10 +173,10 @@ New typing features: Platform support: -* :pep:`730`: Apple's iOS is now an officially supported platform, - at :pep:`tier 3 <11#tier-3>`. +* :pep:`730`: Apple's iOS is now an :ref:`officially supported platform + `, at :pep:`tier 3 <11#tier-3>`. Official Android support (:pep:`738`) is in the works as well. -* ``wasm32-wasi`` is now a supported as a :pep:`tier 2 <11#tier-2>` platform. +* ``wasm32-wasi`` is now supported as a :pep:`tier 2 <11#tier-2>` platform. * ``wasm32-emscripten`` is no longer an officially supported platform. Important removals: @@ -187,7 +187,7 @@ Important removals: :mod:`!crypt`, :mod:`!imghdr`, :mod:`!mailcap`, :mod:`!msilib`, :mod:`!nis`, :mod:`!nntplib`, :mod:`!ossaudiodev`, :mod:`!pipes`, :mod:`!sndhdr`, :mod:`!spwd`, :mod:`!sunau`, :mod:`!telnetlib`, :mod:`!uu` and :mod:`!xdrlib`. -* Remove the :program:`!2to3` tool and :mod:`!lib2to3` module +* Remove the :program:`2to3` tool and :mod:`!lib2to3` module (deprecated in Python 3.11). * Remove the :mod:`!tkinter.tix` module (deprecated in Python 3.6). * Remove :func:`!locale.resetlocale()`. @@ -209,51 +209,53 @@ This updated policy means that: New Features ============ + .. _whatsnew313-better-interactive-interpreter: -A Better Interactive Interpreter +A better interactive interpreter -------------------------------- -On Unix-like systems like Linux or macOS as well as Windows, Python now -uses a new :term:`interactive` shell. When the user starts the -:term:`REPL` from an interactive terminal the interactive shell now -supports the following new features: +Python now uses a new :term:`interactive` shell by default, based on code +from the `PyPy project`_. +When the user starts the :term:`REPL` from an interactive terminal, +the following new features are now supported: -* Colorized prompts. * Multiline editing with history preservation. +* Direct support for REPL-specific commands like :kbd:`help`, :kbd:`exit`, + and :kbd:`quit`, without the need to call them as functions. +* Prompts and tracebacks with :ref:`color enabled by default + `. * Interactive help browsing using :kbd:`F1` with a separate command history. * History browsing using :kbd:`F2` that skips output as well as the :term:`>>>` and :term:`...` prompts. * "Paste mode" with :kbd:`F3` that makes pasting larger blocks of code easier (press :kbd:`F3` again to return to the regular prompt). -* The ability to issue REPL-specific commands like :kbd:`help`, :kbd:`exit`, - and :kbd:`quit` without the need to use call parentheses after the command - name. - -If the new interactive shell is not desired, it can be disabled via -the :envvar:`PYTHON_BASIC_REPL` environment variable. - -The new shell requires :mod:`curses` on Unix-like systems. +To disable the new interactive shell, +set the :envvar:`PYTHON_BASIC_REPL` environment variable. For more on interactive mode, see :ref:`tut-interac`. (Contributed by Pablo Galindo Salgado, Łukasz Langa, and Lysandros Nikolaou in :gh:`111201` based on code from the PyPy project. Windows support contributed by Dino Viehland and Anthony Shaw.) +.. _`PyPy project`: https://pypy.org/ + + .. _whatsnew313-improved-error-messages: -Improved Error Messages +Improved error messages ----------------------- -* The interpreter now colorizes error messages when displaying tracebacks by default. - This feature can be controlled via the new :envvar:`PYTHON_COLORS` environment - variable as well as the canonical |NO_COLOR|_ and |FORCE_COLOR|_ environment - variables. See also :ref:`using-on-controlling-color`. +* The interpreter now uses color by default when displaying tracebacks in the + terminal. This feature :ref:`can be controlled ` + via the new :envvar:`PYTHON_COLORS` environment variable as well as + the canonical |NO_COLOR|_ and |FORCE_COLOR|_ environment variables. (Contributed by Pablo Galindo Salgado in :gh:`112730`.) .. Apparently this how you hack together a formatted link: + (https://www.docutils.org/docs/ref/rst/directives.html#replacement-text) .. |FORCE_COLOR| replace:: ``FORCE_COLOR`` .. _FORCE_COLOR: https://force-color.org/ @@ -265,345 +267,349 @@ Improved Error Messages standard library module. When this results in errors, we now display a more helpful error message: - .. code-block:: shell-session + .. code-block:: pytb - $ python random.py - Traceback (most recent call last): - File "/home/random.py", line 1, in - import random; print(random.randint(5)) - ^^^^^^^^^^^^^ - File "/home/random.py", line 1, in - import random; print(random.randint(5)) - ^^^^^^^^^^^^^^ - AttributeError: module 'random' has no attribute 'randint' (consider renaming '/home/random.py' since it has the same name as the standard library module named 'random' and the import system gives it precedence) + $ python random.py + Traceback (most recent call last): + File "/home/me/random.py", line 1, in + import random + File "/home/me/random.py", line 3, in + print(random.randint(5)) + ^^^^^^^^^^^^^^ + AttributeError: module 'random' has no attribute 'randint' (consider renaming '/home/me/random.py' since it has the same name as the standard library module named 'random' and the import system gives it precedence) Similarly, if a script has the same name as a third-party - module it attempts to import, and this results in errors, + module that it attempts to import and this results in errors, we also display a more helpful error message: - .. code-block:: shell-session + .. code-block:: pytb - $ python numpy.py - Traceback (most recent call last): - File "/home/numpy.py", line 1, in - import numpy as np; np.array([1,2,3]) - ^^^^^^^^^^^^^^^^^^ - File "/home/numpy.py", line 1, in - import numpy as np; np.array([1,2,3]) - ^^^^^^^^ - AttributeError: module 'numpy' has no attribute 'array' (consider renaming '/home/numpy.py' if it has the same name as a third-party module you intended to import) + $ python numpy.py + Traceback (most recent call last): + File "/home/me/numpy.py", line 1, in + import numpy as np + File "/home/me/numpy.py", line 3, in + np.array([1, 2, 3]) + ^^^^^^^^ + AttributeError: module 'numpy' has no attribute 'array' (consider renaming '/home/me/numpy.py' if it has the same name as a third-party module you intended to import) (Contributed by Shantanu Jain in :gh:`95754`.) -* When an incorrect keyword argument is passed to a function, the error message - now potentially suggests the correct keyword argument. - (Contributed by Pablo Galindo Salgado and Shantanu Jain in :gh:`107944`.) - - >>> "better error messages!".split(max_split=1) - Traceback (most recent call last): - File "", line 1, in - "better error messages!".split(max_split=1) - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^ - TypeError: split() got an unexpected keyword argument 'max_split'. Did you mean 'maxsplit'? +* The error message now tries to suggest the correct keyword argument + when an incorrect keyword argument is passed to a function. -* Classes have a new :attr:`~class.__static_attributes__` attribute, populated by the compiler, - with a tuple of names of attributes of this class which are assigned - through ``self.X`` from any function in its body. (Contributed by Irit Katriel - in :gh:`115775`.) + .. code-block:: pycon -.. _whatsnew313-locals-semantics: + >>> "Better error messages!".split(max_split=1) + Traceback (most recent call last): + File "", line 1, in + "Better error messages!".split(max_split=1) + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^ + TypeError: split() got an unexpected keyword argument 'max_split'. Did you mean 'maxsplit'? -Defined mutation semantics for :py:func:`locals` ------------------------------------------------- + (Contributed by Pablo Galindo Salgado and Shantanu Jain in :gh:`107944`.) -Historically, the expected result of mutating the return value of :func:`locals` -has been left to individual Python implementations to define. - -Through :pep:`667`, Python 3.13 standardises the historical behaviour of CPython -for most code execution scopes, but changes -:term:`optimized scopes ` (functions, generators, coroutines, -comprehensions, and generator expressions) to explicitly return independent -snapshots of the currently assigned local variables, including locally -referenced nonlocal variables captured in closures. - -This change to the semantics of :func:`locals` in optimized scopes also affects the default -behaviour of code execution functions that implicitly target ``locals()`` if no explicit -namespace is provided (such as :func:`exec` and :func:`eval`). In previous versions, whether -or not changes could be accessed by calling ``locals()`` after calling the code execution -function was implementation dependent. In CPython specifically, such code would typically -appear to work as desired, but could sometimes fail in optimized scopes based on other code -(including debuggers and code execution tracing tools) potentially resetting the shared -snapshot in that scope. Now, the code will always run against an independent snapshot of the -local variables in optimized scopes, and hence the changes will never be visible in -subsequent calls to ``locals()``. To access the changes made in these cases, an explicit -namespace reference must now be passed to the relevant function. Alternatively, it may make -sense to update affected code to use a higher level code execution API that returns the -resulting code execution namespace (e.g. :func:`runpy.run_path` when executing Python -files from disk). -To ensure debuggers and similar tools can reliably update local variables in -scopes affected by this change, :attr:`FrameType.f_locals ` now -returns a write-through proxy to the frame's local and locally referenced -nonlocal variables in these scopes, rather than returning an inconsistently -updated shared ``dict`` instance with undefined runtime semantics. +.. _whatsnew313-free-threaded-cpython: -See :pep:`667` for more details, including related C API changes and deprecations. Porting -notes are also provided below for the affected :ref:`Python APIs ` -and :ref:`C APIs `. +Free-threaded CPython +--------------------- -(PEP and implementation contributed by Mark Shannon and Tian Gao in -:gh:`74929`. Documentation updates provided by Guido van Rossum and -Alyssa Coghlan.) +CPython now has experimental support for running in a free-threaded mode, +with the :term:`global interpreter lock` (GIL) disabled. +This is an experimental feature and therefore is not enabled by default. +The free-threaded mode requires a different executable, +usually called ``python3.13t`` or ``python3.13t.exe``. +Pre-built binaries marked as *free-threaded* can be installed as part of +the official :ref:`Windows ` +and :ref:`macOS ` installers, +or CPython can be built from source with the :option:`--disable-gil` option. -Incremental Garbage Collection ------------------------------- +.. better macOS link pending + https://github.com/python/cpython/issues/109975#issuecomment-2286391179 -* The cycle garbage collector is now incremental. - This means that maximum pause times are reduced - by an order of magnitude or more for larger heaps. +Free-threaded execution allows for full utilization of the available +processing power by running threads in parallel on available CPU cores. +While not all software will benefit from this automatically, programs +designed with threading in mind will run faster on multi-core hardware. +**The free-threaded mode is experimental** and work is ongoing to improve it: +expect some bugs and a substantial single-threaded performance hit. +Free-threaded builds of CPython support optionally running with the GIL +enabled at runtime using the environment variable :envvar:`PYTHON_GIL` or +the command-line option :option:`-X gil`. -Support For Mobile Platforms ----------------------------- +To check if the current interpreter supports free-threading, :option:`python -VV <-V>` +and :attr:`sys.version` contain "experimental free-threading build". +The new :func:`!sys._is_gil_enabled` function can be used to check whether +the GIL is actually disabled in the running process. -* iOS is now a :pep:`11` supported platform. ``arm64-apple-ios`` - (iPhone and iPad devices released after 2013) and - ``arm64-apple-ios-simulator`` (Xcode iOS simulator running on Apple Silicon - hardware) are now tier 3 platforms. +C-API extension modules need to be built specifically for the free-threaded +build. Extensions that support running with the :term:`GIL` disabled should +use the :c:data:`Py_mod_gil` slot. Extensions using single-phase init should +use :c:func:`PyUnstable_Module_SetGIL` to indicate whether they support +running with the GIL disabled. Importing C extensions that don't use these +mechanisms will cause the GIL to be enabled, unless the GIL was explicitly +disabled with the :envvar:`PYTHON_GIL` environment variable or the +:option:`-X gil=0` option. +pip 24.1 or newer is required to install packages with C extensions in the +free-threaded build. - ``x86_64-apple-ios-simulator`` (Xcode iOS simulator running on older x86_64 - hardware) is not a tier 3 supported platform, but will be supported on a - best-effort basis. +.. seealso:: - See :pep:`730`: for more details. + :pep:`703` "Making the Global Interpreter Lock Optional in CPython" + contains rationale and information surrounding this work. - (PEP written and implementation contributed by Russell Keith-Magee in - :gh:`114099`.) .. _whatsnew313-jit-compiler: -Experimental JIT Compiler -------------------------- +An experimental just-in-time (JIT) compiler +------------------------------------------- -When CPython is configured using the ``--enable-experimental-jit`` option, -a just-in-time compiler is added which may speed up some Python programs. +When CPython is configured and built using +the :option:`!--enable-experimental-jit` option, +a just-in-time (JIT) compiler is added which may speed up some Python programs. +On Windows, use ``PCbuild/build.bat --experimental-jit`` to enable the JIT +or ``--experimental-jit-interpreter`` to enable the Tier 2 interpreter. +Build requirements and further supporting information `are contained at`__ +:file:`Tools/jit/README.md`. -The internal architecture is roughly as follows. +__ https://github.com/python/cpython/blob/main/Tools/jit/README.md -* We start with specialized *Tier 1 bytecode*. - See :ref:`What's new in 3.11 ` for details. +The :option:`!--enable-experimental-jit` option takes these (optional) values, +defaulting to ``yes`` if :option:`!--enable-experimental-jit` is present +without the optional value. -* When the Tier 1 bytecode gets hot enough, it gets translated - to a new, purely internal *Tier 2 IR*, a.k.a. micro-ops ("uops"). +* ``no``: Disable the entire Tier 2 and JIT pipeline. +* ``yes``: Enable the JIT. + To disable the JIT at runtime, pass the environment variable ``PYTHON_JIT=0``. +* ``yes-off``: Build the JIT but disable it by default. + To enable the JIT at runtime, pass the environment variable ``PYTHON_JIT=1``. +* ``interpreter``: Enable the Tier 2 interpreter but disable the JIT. + The interpreter can be disabled by running with ``PYTHON_JIT=0``. -* The Tier 2 IR uses the same stack-based VM as Tier 1, but the - instruction format is better suited to translation to machine code. +The internal architecture is roughly as follows: +* We start with specialized *Tier 1 bytecode*. + See :ref:`What's new in 3.11 ` for details. +* When the Tier 1 bytecode gets hot enough, it gets translated + to a new purely internal intermediate representation (IR), + called the *Tier 2 IR*, and sometimes referred to as micro-ops ("uops"). +* The Tier 2 IR uses the same stack-based virtual machine as Tier 1, + but the instruction format is better suited to translation to machine code. * We have several optimization passes for Tier 2 IR, which are applied before it is interpreted or translated to machine code. - * There is a Tier 2 interpreter, but it is mostly intended for debugging the earlier stages of the optimization pipeline. The Tier 2 interpreter can be enabled by configuring Python with ``--enable-experimental-jit=interpreter``. - * When the JIT is enabled, the optimized Tier 2 IR is translated to machine code, which is then executed. - * The machine code translation process uses a technique called *copy-and-patch*. It has no runtime dependencies, but there is a new build-time dependency on LLVM. -The ``--enable-experimental-jit`` flag has the following optional values: - -* ``no`` (default) -- Disable the entire Tier 2 and JIT pipeline. - -* ``yes`` (default if the flag is present without optional value) - -- Enable the JIT. To disable the JIT at runtime, - pass the environment variable ``PYTHON_JIT=0``. - -* ``yes-off`` -- Build the JIT but disable it by default. - To enable the JIT at runtime, pass the environment variable - ``PYTHON_JIT=1``. - -* ``interpreter`` -- Enable the Tier 2 interpreter but disable the JIT. - The interpreter can be disabled by running with - ``PYTHON_JIT=0``. - -(On Windows, use ``PCbuild/build.bat --experimental-jit`` to enable the JIT -or ``--experimental-jit-interpreter`` to enable the Tier 2 interpreter.) - -See :pep:`744` for more details. +.. seealso:: :pep:`744` (JIT by Brandt Bucher, inspired by a paper by Haoran Xu and Fredrik Kjolstad. Tier 2 IR by Mark Shannon and Guido van Rossum. Tier 2 optimizer by Ken Jin.) -.. _whatsnew313-free-threaded-cpython: -Free-threaded CPython ---------------------- +.. _whatsnew313-locals-semantics: -CPython will run with the :term:`global interpreter lock` (GIL) disabled when -configured using the ``--disable-gil`` option at build time. This is an -experimental feature and therefore isn't used by default. Users need to -either compile their own interpreter, or install one of the experimental -builds that are marked as *free-threaded*. See :pep:`703` "Making the Global -Interpreter Lock Optional in CPython" for more detail. +Defined mutation semantics for :py:func:`locals` +------------------------------------------------ -Free-threaded execution allows for full utilization of the available -processing power by running threads in parallel on available CPU cores. -While not all software will benefit from this automatically, programs -designed with threading in mind will run faster on multicore hardware. +Historically, the expected result of mutating the return value of +:func:`locals` has been left to individual Python implementations to define. +Starting from Python 3.13, :pep:`667` standardises +the historical behaviour of CPython for most code execution scopes, +but changes :term:`optimized scopes ` +(functions, generators, coroutines, comprehensions, and generator expressions) +to explicitly return independent snapshots of the currently assigned local +variables, including locally referenced nonlocal variables captured in closures. + +This change to the semantics of :func:`locals` in optimized scopes also +affects the default behaviour of code execution functions that implicitly +target :func:`!locals` if no explicit namespace is provided +(such as :func:`exec` and :func:`eval`). +In previous versions, whether or not changes could be accessed by calling +:func:`!locals` after calling the code execution function was +implementation-dependent. In CPython specifically, such code would typically +appear to work as desired, but could sometimes fail in optimized scopes based +on other code (including debuggers and code execution tracing tools) +potentially resetting the shared snapshot in that scope. +Now, the code will always run against an independent snapshot of +the local variables in optimized scopes, and hence the changes will never +be visible in subsequent calls to :func:`!locals`. +To access the changes made in these cases, an explicit namespace reference +must now be passed to the relevant function. +Alternatively, it may make sense to update affected code to use a higher level +code execution API that returns the resulting code execution namespace +(e.g. :func:`runpy.run_path` when executing Python files from disk). -Work is still ongoing: expect some bugs and a substantial single-threaded -performance hit. +To ensure debuggers and similar tools can reliably update local variables in +scopes affected by this change, :attr:`FrameType.f_locals ` now +returns a write-through proxy to the frame's local and locally referenced +nonlocal variables in these scopes, rather than returning an inconsistently +updated shared ``dict`` instance with undefined runtime semantics. -The free-threaded build still supports optionally running with the GIL -enabled at runtime using the environment variable :envvar:`PYTHON_GIL` or -the command line option :option:`-X gil`. +See :pep:`667` for more details, including related C API changes +and deprecations. Porting notes are also provided below for the affected +:ref:`Python APIs ` and :ref:`C APIs +`. -To check if the current interpreter is configured with ``--disable-gil``, -use ``sysconfig.get_config_var("Py_GIL_DISABLED")``. To check if the :term:`GIL` -is actually disabled in the running process, the :func:`!sys._is_gil_enabled` -function can be used. +(PEP and implementation contributed by Mark Shannon and Tian Gao in +:gh:`74929`. Documentation updates provided by Guido van Rossum and +Alyssa Coghlan.) -C-API extension modules need to be built specifically for the free-threaded -build. Extensions that support running with the :term:`GIL` disabled should -use the :c:data:`Py_mod_gil` slot. Extensions using single-phase init should -use :c:func:`PyUnstable_Module_SetGIL` to indicate whether they support -running with the GIL disabled. Importing C extensions that don't use these -mechanisms will cause the GIL to be enabled, unless the GIL was explicitly -disabled with the :envvar:`PYTHON_GIL` environment variable or the -:option:`-X gil=0` option. -pip 24.1b1 or newer is required to install packages with C extensions in the -free-threaded build. +.. _whatsnew313-platform-support: -Other Language Changes -====================== +Support for mobile platforms +---------------------------- -* The :func:`exec` and :func:`eval` built-ins now accept their ``globals`` - and ``locals`` namespace arguments as keywords. - (Contributed by Raphael Gaschignard in :gh:`105879`) +:pep:`730`: iOS is now a :pep:`11` supported platform, with the +``arm64-apple-ios`` and ``arm64-apple-ios-simulator`` targets at tier 3 +(iPhone and iPad devices released after 2013 and the Xcode iOS simulator +running on Apple silicon hardware, respectively). +``x86_64-apple-ios-simulator`` +(the Xcode iOS simulator running on older ``x86_64`` hardware) +is not a tier 3 supported platform, but will have best-effort support. +(PEP written and implementation contributed by Russell Keith-Magee in +:gh:`114099`.) -* Allow the *count* argument of :meth:`str.replace` to be a keyword. - (Contributed by Hugo van Kemenade in :gh:`106487`.) +:pep:`738`: Android support is being actively worked on, +but the platform is not yet officially supported. +(PEP written and implementation contributed by Malcolm Smith in +:gh:`116622`.) -* Compiler now strip indents from docstrings. - This will reduce the size of :term:`bytecode cache ` (e.g. ``.pyc`` file). - For example, cache file size for ``sqlalchemy.orm.session`` in SQLAlchemy 2.0 - is reduced by about 5%. - This change will affect tools using docstrings, like :mod:`doctest`. - (Contributed by Inada Naoki in :gh:`81283`.) +.. seealso:: :pep:`730`, :pep:`738` -* The :func:`compile` built-in can now accept a new flag, - ``ast.PyCF_OPTIMIZED_AST``, which is similar to ``ast.PyCF_ONLY_AST`` - except that the returned ``AST`` is optimized according to the value - of the ``optimize`` argument. - (Contributed by Irit Katriel in :gh:`108113`). -* :mod:`multiprocessing`, :mod:`concurrent.futures`, :mod:`compileall`: - Replace :func:`os.cpu_count` with :func:`os.process_cpu_count` to select the - default number of worker threads and processes. Get the CPU affinity - if supported. - (Contributed by Victor Stinner in :gh:`109649`.) +Incremental garbage collection +------------------------------ -* :func:`os.path.realpath` now resolves MS-DOS style file names even if - the file is not accessible. - (Contributed by Moonsik Park in :gh:`82367`.) +The cycle garbage collector is now incremental. +This means that maximum pause times are reduced +by an order of magnitude or more for larger heaps. +(Contributed by Mark Shannon in :gh:`108362`.) -* Fixed a bug where a :keyword:`global` declaration in an :keyword:`except` block - is rejected when the global is used in the :keyword:`else` block. - (Contributed by Irit Katriel in :gh:`111123`.) -* Many functions now emit a warning if a boolean value is passed as - a file descriptor argument. - This can help catch some errors earlier. - (Contributed by Serhiy Storchaka in :gh:`82626`.) +Other Language Changes +====================== -* Added a new environment variable :envvar:`PYTHON_FROZEN_MODULES`. It - determines whether or not frozen modules are ignored by the import machinery, - equivalent of the :option:`-X frozen_modules <-X>` command-line option. - (Contributed by Yilei Yang in :gh:`111374`.) +* The compiler now strips common leading whitespace + from every line in a docstring. + This reduces the size of the :term:`bytecode cache ` + (such as ``.pyc`` files), with reductions in file size of around 5%, + for example in :mod:`!sqlalchemy.orm.session` from SQLAlchemy 2.0. + This change affects tools that use docstrings, such as :mod:`doctest`. + + .. doctest:: + + >>> def spam(): + ... """ + ... This is a docstring with + ... leading whitespace. + ... + ... It even has multiple paragraphs! + ... """ + ... + >>> spam.__doc__ + '\nThis is a docstring with\n leading whitespace.\n\nIt even has multiple paragraphs!\n' -* Add :ref:`support for the perf profiler ` working without - frame pointers through the new environment variable - :envvar:`PYTHON_PERF_JIT_SUPPORT` and command-line option :option:`-X perf_jit - <-X>` (Contributed by Pablo Galindo in :gh:`118518`.) + (Contributed by Inada Naoki in :gh:`81283`.) -* The new :envvar:`PYTHON_HISTORY` environment variable can be used to change - the location of a ``.python_history`` file. - (Contributed by Levi Sabah, Zackery Spytz and Hugo van Kemenade in - :gh:`73965`.) +* :ref:`Annotation scopes ` within class scopes + can now contain lambdas and comprehensions. + Comprehensions that are located within class scopes + are not inlined into their parent scope. -* Add :exc:`PythonFinalizationError` exception. This exception derived from - :exc:`RuntimeError` is raised when an operation is blocked during - the :term:`Python finalization `. + .. code-block:: python - The following functions now raise PythonFinalizationError, instead of - :exc:`RuntimeError`: + class C[T]: + type Alias = lambda: T - * :func:`_thread.start_new_thread`. - * :class:`subprocess.Popen`. - * :func:`os.fork`. - * :func:`os.forkpty`. + (Contributed by Jelle Zijlstra in :gh:`109118` and :gh:`118160`.) - (Contributed by Victor Stinner in :gh:`114570`.) +* :ref:`Future statements ` are no longer triggered by + relative imports of the :mod:`__future__` module, + meaning that statements of the form ``from .__future__ import ...`` + are now simply standard relative imports, with no special features activated. + (Contributed by Jeremiah Gabriel Pascual in :gh:`118216`.) -* Added :attr:`!name` and :attr:`!mode` attributes for compressed - and archived file-like objects in modules :mod:`bz2`, :mod:`lzma`, - :mod:`tarfile` and :mod:`zipfile`. - (Contributed by Serhiy Storchaka in :gh:`115961`.) +* :keyword:`global` declarations are now permitted in :keyword:`except` blocks + when that global is used in the :keyword:`else` block. + Previously this raised an erroneous :exc:`SyntaxError`. + (Contributed by Irit Katriel in :gh:`111123`.) -* Allow controlling Expat >=2.6.0 reparse deferral (:cve:`2023-52425`) - by adding five new methods: +* Add :envvar:`PYTHON_FROZEN_MODULES`, a new environment variable that + determines whether frozen modules are ignored by the import machinery, + equivalent to the :option:`-X frozen_modules <-X>` command-line option. + (Contributed by Yilei Yang in :gh:`111374`.) - * :meth:`xml.etree.ElementTree.XMLParser.flush` - * :meth:`xml.etree.ElementTree.XMLPullParser.flush` - * :meth:`xml.parsers.expat.xmlparser.GetReparseDeferralEnabled` - * :meth:`xml.parsers.expat.xmlparser.SetReparseDeferralEnabled` - * :meth:`!xml.sax.expatreader.ExpatParser.flush` +* Add :ref:`support for the perf profiler ` working + without `frame pointers `_ through + the new environment variable :envvar:`PYTHON_PERF_JIT_SUPPORT` + and command-line option :option:`-X perf_jit <-X>`. + (Contributed by Pablo Galindo in :gh:`118518`.) - (Contributed by Sebastian Pipping in :gh:`115623`.) +* The location of a :file:`.python_history` file can be changed via the + new :envvar:`PYTHON_HISTORY` environment variable. + (Contributed by Levi Sabah, Zackery Spytz and Hugo van Kemenade + in :gh:`73965`.) -* The :func:`ssl.create_default_context` API now includes - :data:`ssl.VERIFY_X509_PARTIAL_CHAIN` and :data:`ssl.VERIFY_X509_STRICT` - in its default flags. +* Classes have a new :attr:`~class.__static_attributes__` attribute. + This is populated by the compiler with a tuple of the class's attribute names + which are assigned through ``self.`` from any function in its body. + (Contributed by Irit Katriel in :gh:`115775`.) - .. note:: +* The compiler now creates a :attr:`!__firstlineno__` attribute on classes + with the line number of the first line of the class definition. + (Contributed by Serhiy Storchaka in :gh:`118465`.) - :data:`ssl.VERIFY_X509_STRICT` may reject pre-:rfc:`5280` or malformed - certificates that the underlying OpenSSL implementation otherwise would - accept. While disabling this is not recommended, you can do so using:: +* The :func:`exec` and :func:`eval` builtins now accept + the *globals* and *locals* arguments as keywords. + (Contributed by Raphael Gaschignard in :gh:`105879`) - ctx = ssl.create_default_context() - ctx.verify_flags &= ~ssl.VERIFY_X509_STRICT +* The :func:`compile` builtin now accepts a new flag, + ``ast.PyCF_OPTIMIZED_AST``, which is similar to ``ast.PyCF_ONLY_AST`` + except that the returned AST is optimized according to + the value of the *optimize* argument. + (Contributed by Irit Katriel in :gh:`108113`). - (Contributed by William Woodruff in :gh:`112389`.) +* Add :exc:`PythonFinalizationError`, a new exception derived from + :exc:`RuntimeError` and used to signal when operations are blocked + during :term:`finalization `. + The following callables now raise :exc:`!PythonFinalizationError`, + instead of :exc:`RuntimeError`: -* The :class:`configparser.ConfigParser` now accepts unnamed sections before named - ones if configured to do so. - (Contributed by Pedro Sousa Lacerda in :gh:`66449`.) + * :func:`_thread.start_new_thread` + * :func:`os.fork` + * :func:`os.forkpty` + * :class:`subprocess.Popen` -* :ref:`annotation scope ` within class scopes can now - contain lambdas and comprehensions. Comprehensions that are located within - class scopes are not inlined into their parent scope. (Contributed by - Jelle Zijlstra in :gh:`109118` and :gh:`118160`.) + (Contributed by Victor Stinner in :gh:`114570`.) -* Classes have a new :attr:`!__firstlineno__` attribute, - populated by the compiler, with the line number of the first line - of the class definition. - (Contributed by Serhiy Storchaka in :gh:`118465`.) +* Allow the *count* argument of :meth:`str.replace` to be a keyword. + (Contributed by Hugo van Kemenade in :gh:`106487`.) -* ``from __future__ import ...`` statements are now just normal - relative imports if dots are present before the module name. - (Contributed by Jeremiah Gabriel Pascual in :gh:`118216`.) +* Many functions now emit a warning if a boolean value is passed as + a file descriptor argument. + This can help catch some errors earlier. + (Contributed by Serhiy Storchaka in :gh:`82626`.) + +* Added :attr:`!name` and :attr:`!mode` attributes + for compressed and archived file-like objects in + the :mod:`bz2`, :mod:`lzma`, :mod:`tarfile`, and :mod:`zipfile` modules. + (Contributed by Serhiy Storchaka in :gh:`115961`.) New Modules =========== -* :mod:`dbm.sqlite3`: SQLite backend for :mod:`dbm`. +* :mod:`dbm.sqlite3`: An SQLite backend for :mod:`dbm`. (Contributed by Raymond Hettinger and Erlend E. Aasland in :gh:`100414`.) @@ -626,7 +632,7 @@ array It can be used instead of ``'u'`` type code, which is deprecated. (Contributed by Inada Naoki in :gh:`80480`.) -* Add ``clear()`` method in order to implement ``MutableSequence``. +* Add ``clear()`` method in order to implement :class:`~collections.abc.MutableSequence`. (Contributed by Mike Zimin in :gh:`114894`.) ast @@ -728,6 +734,23 @@ base64 See the `Z85 specification `_ for more information. (Contributed by Matan Perelman in :gh:`75299`.) + +compileall +---------- + +* Select the default number of worker threads and processes using + :func:`os.process_cpu_count` instead of :func:`os.cpu_count`. + (Contributed by Victor Stinner in :gh:`109649`.) + + +concurrent.futures +------------------ + +* Select the default number of worker threads and processes using + :func:`os.process_cpu_count` instead of :func:`os.cpu_count`. + (Contributed by Victor Stinner in :gh:`109649`.) + + copy ---- @@ -881,7 +904,7 @@ importlib io -- -* The :class:`io.IOBase` finalizer now logs the ``close()`` method errors with +* The :class:`io.IOBase` finalizer now logs errors raised by the ``close()`` method with :data:`sys.unraisablehook`. Previously, errors were ignored silently by default, and only logged in :ref:`Python Development Mode ` or on :ref:`Python built on debug mode `. @@ -934,7 +957,7 @@ mimetypes mmap ---- -* The :class:`mmap.mmap` class now has an :meth:`~mmap.mmap.seekable` method +* The :class:`mmap.mmap` class now has a :meth:`~mmap.mmap.seekable` method that can be used when a seekable file-like object is required. The :meth:`~mmap.mmap.seek` method now returns the new absolute position. (Contributed by Donghee Na and Sylvie Liberman in :gh:`111835`.) @@ -945,6 +968,15 @@ mmap is inaccessible due to file system errors or access violations. (Contributed by Jannis Weigend in :gh:`118209`.) + +multiprocessing +--------------- + +* Select the default number of worker threads and processes using + :func:`os.process_cpu_count` instead of :func:`os.cpu_count`. + (Contributed by Victor Stinner in :gh:`109649`.) + + opcode ------ @@ -1009,10 +1041,15 @@ os.path * Add :func:`os.path.isreserved` to check if a path is reserved on the current system. This function is only available on Windows. (Contributed by Barney Gale in :gh:`88569`.) + * On Windows, :func:`os.path.isabs` no longer considers paths starting with exactly one (back)slash to be absolute. (Contributed by Barney Gale and Jon Foster in :gh:`44626`.) +* :func:`os.path.realpath` now resolves MS-DOS style file names even if + the file is not accessible. + (Contributed by Moonsik Park in :gh:`82367`.) + * Add support of *dir_fd* and *follow_symlinks* keyword arguments in :func:`shutil.chown`. (Contributed by Berker Peksag and Tahia K in :gh:`62308`) @@ -1054,7 +1091,7 @@ pathlib pdb --- -* Add ability to move between chained exceptions during post mortem debugging in :func:`~pdb.pm` using +* Add ability to move between chained exceptions during post-mortem debugging in :func:`~pdb.pm` using the new ``exceptions [exc_number]`` command for Pdb. (Contributed by Matthias Bussonnier in :gh:`106676`.) @@ -1112,6 +1149,26 @@ sqlite3 for filtering database objects to dump. (Contributed by Mariusz Felisiak in :gh:`91602`.) + +ssl +--- + +* The :func:`ssl.create_default_context` API now includes + :data:`ssl.VERIFY_X509_PARTIAL_CHAIN` and :data:`ssl.VERIFY_X509_STRICT` + in its default flags. + + .. note:: + + :data:`ssl.VERIFY_X509_STRICT` may reject pre-:rfc:`5280` or malformed + certificates that the underlying OpenSSL implementation otherwise would + accept. While disabling this is not recommended, you can do so using:: + + ctx = ssl.create_default_context() + ctx.verify_flags &= ~ssl.VERIFY_X509_STRICT + + (Contributed by William Woodruff in :gh:`112389`.) + + statistics ---------- @@ -1130,7 +1187,7 @@ subprocess more situations. Notably in the default case of ``close_fds=True`` on more recent versions of platforms including Linux, FreeBSD, and Solaris where the C library provides :c:func:`!posix_spawn_file_actions_addclosefrom_np`. - On Linux this should perform similar to our existing Linux :c:func:`!vfork` + On Linux this should perform similarly to the existing Linux :c:func:`!vfork` based code. A private control knob :attr:`!subprocess._USE_POSIX_SPAWN` can be set to ``False`` if you need to force :mod:`subprocess` not to ever use :func:`os.posix_spawn`. Please report your reason and platform details in @@ -1141,7 +1198,7 @@ subprocess sys --- -* Add the :func:`sys._is_interned` function to test if the string was interned. +* Add the :func:`sys._is_interned` function to test if a string was interned. This function is not guaranteed to exist in all implementations of Python. (Contributed by Serhiy Storchaka in :gh:`78573`.) @@ -1157,7 +1214,7 @@ time ---- * On Windows, :func:`time.monotonic()` now uses the - ``QueryPerformanceCounter()`` clock to have a resolution better than 1 us, + ``QueryPerformanceCounter()`` clock to have a resolution better than 1 μs, instead of the ``GetTickCount64()`` clock which has a resolution of 15.6 ms. (Contributed by Victor Stinner in :gh:`88494`.) @@ -1188,7 +1245,7 @@ tkinter * Add new optional keyword-only parameter *return_ints* in the :meth:`!Text.count` method. - Passing ``return_ints=True`` makes it always returning the single count + Passing ``return_ints=True`` makes it always return the single count as an integer instead of a 1-tuple or ``None``. (Contributed by Serhiy Storchaka in :gh:`97928`.) @@ -1231,8 +1288,8 @@ traceback types ----- -* :class:`~types.SimpleNamespace` constructor now allows specifying initial - values of attributes as a positional argument which must be a mapping or +* The :class:`~types.SimpleNamespace` constructor now allows specifying initial + values of attributes as a positional argument, which must be a mapping or an iterable of key-value pairs. (Contributed by Serhiy Storchaka in :gh:`108191`.) @@ -1277,13 +1334,26 @@ warnings warning may also be emitted when a decorated function or class is used at runtime. See :pep:`702`. (Contributed by Jelle Zijlstra in :gh:`104003`.) -xml.etree.ElementTree ---------------------- + +xml +--- + +* Allow controlling Expat >=2.6.0 reparse deferral (:cve:`2023-52425`) + by adding five new methods: + + * :meth:`xml.etree.ElementTree.XMLParser.flush` + * :meth:`xml.etree.ElementTree.XMLPullParser.flush` + * :meth:`xml.parsers.expat.xmlparser.GetReparseDeferralEnabled` + * :meth:`xml.parsers.expat.xmlparser.SetReparseDeferralEnabled` + * :meth:`!xml.sax.expatreader.ExpatParser.flush` + + (Contributed by Sebastian Pipping in :gh:`115623`.) * Add the :meth:`!close` method for the iterator returned by :func:`~xml.etree.ElementTree.iterparse` for explicit cleaning up. (Contributed by Serhiy Storchaka in :gh:`69893`.) + zipimport --------- @@ -1439,6 +1509,10 @@ PEP 594: dead batteries (and other module removals) configparser ------------ +* The :class:`configparser.ConfigParser` now accepts unnamed sections + before named ones if configured to do so. + (Contributed by Pedro Sousa Lacerda in :gh:`66449`.) + * Remove the undocumented :class:`!configparser.LegacyInterpolation` class, deprecated in the docstring since Python 3.2, and with a deprecation warning since Python 3.11. @@ -1641,7 +1715,7 @@ New Deprecations (Contributed by Erlend E. Aasland in :gh:`107948` and :gh:`108278`.) -* :mod:`sys`: :func:`sys._enablelegacywindowsfsencoding` function. +* :mod:`sys`: The :func:`sys._enablelegacywindowsfsencoding` function is deprecated. Replace it with the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment variable. (Contributed by Inada Naoki in :gh:`73427`.) @@ -1665,8 +1739,8 @@ New Deprecations deprecated. Passing ``None`` to the 'fields' parameter (``NT = NamedTuple("NT", None)`` or ``TD = TypedDict("TD", None)``) is also deprecated. Both will be disallowed in Python 3.15. To create a NamedTuple - class with 0 fields, use ``class NT(NamedTuple): pass`` or - ``NT = NamedTuple("NT", [])``. To create a TypedDict class with 0 fields, use + class with zero fields, use ``class NT(NamedTuple): pass`` or + ``NT = NamedTuple("NT", [])``. To create a TypedDict class with zero fields, use ``class TD(TypedDict): pass`` or ``TD = TypedDict("TD", {})``. (Contributed by Alex Waygood in :gh:`105566` and :gh:`105570`.) @@ -1683,7 +1757,7 @@ New Deprecations * :ref:`user-defined-funcs`: Assignment to a function's :attr:`~function.__code__` attribute where the new code - object's type does not match the function's type, is deprecated. The + object's type does not match the function's type is deprecated. The different types are: plain function, generator, async generator and coroutine. (Contributed by Irit Katriel in :gh:`81137`.) @@ -1924,7 +1998,7 @@ New Features (Contributed by Victor Stinner in :gh:`116936`.) * Add two new functions to the C-API, :c:func:`PyRefTracer_SetTracer` and - :c:func:`PyRefTracer_GetTracer`, that allows to track object creation and + :c:func:`PyRefTracer_GetTracer`, that allow to track object creation and destruction the same way the :mod:`tracemalloc` module does. (Contributed by Pablo Galindo in :gh:`93502`.) @@ -2033,8 +2107,8 @@ Changes in the Python API independent snapshot on each call, and hence no longer implicitly updates previously returned references. Obtaining the legacy CPython behaviour now requires explicit calls to update the initially returned dictionary with the - results of subsequent calls to ``locals()``. Code execution functions that - implicitly target ``locals()`` (such as ``exec`` and ``eval``) must be + results of subsequent calls to :func:`!locals`. Code execution functions that + implicitly target :func:`!locals` (such as ``exec`` and ``eval``) must be passed an explicit namespace to access their results in an optimized scope. (Changed as part of :pep:`667`.) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 267860712b9967..088f70d9e9fad4 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -419,6 +419,16 @@ New Features which has an ambiguous return value. (Contributed by Irit Katriel and Erlend Aasland in :gh:`105201`.) +* :c:func:`Py_Finalize` now deletes all interned strings. This + is backwards incompatible to any C-Extension that holds onto an interned + string after a call to :c:func:`Py_Finalize` and is then reused after a + call to :c:func:`Py_Initialize`. Any issues arising from this behavior will + normally result in crashes during the exectuion of the subsequent call to + :c:func:`Py_Initialize` from accessing uninitialized memory. To fix, use + an address sanitizer to identify any use-after-free coming from + an interned string and deallocate it during module shutdown. + (Contribued by Eddie Elizondo in :gh:`113601`.) + Porting to Python 3.14 ---------------------- diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 67aeab24db2347..a722a2e2527862 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -156,6 +156,7 @@ typedef struct { } _PyCallCache; #define INLINE_CACHE_ENTRIES_CALL CACHE_ENTRIES(_PyCallCache) +#define INLINE_CACHE_ENTRIES_CALL_KW CACHE_ENTRIES(_PyCallCache) typedef struct { _Py_BackoffCounter counter; @@ -335,6 +336,8 @@ extern void _Py_Specialize_StoreSubscr(_PyStackRef container, _PyStackRef sub, _Py_CODEUNIT *instr); extern void _Py_Specialize_Call(_PyStackRef callable, _Py_CODEUNIT *instr, int nargs); +extern void _Py_Specialize_CallKw(_PyStackRef callable, _Py_CODEUNIT *instr, + int nargs); extern void _Py_Specialize_BinaryOp(_PyStackRef lhs, _PyStackRef rhs, _Py_CODEUNIT *instr, int oparg, _PyStackRef *locals); extern void _Py_Specialize_CompareOp(_PyStackRef lhs, _PyStackRef rhs, @@ -590,6 +593,10 @@ extern _Py_CODEUNIT _Py_GetBaseCodeUnit(PyCodeObject *code, int offset); extern int _PyInstruction_GetLength(PyCodeObject *code, int offset); +struct _PyCode8 _PyCode_DEF(8); + +PyAPI_DATA(const struct _PyCode8) _Py_InitCleanup; + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index d3a5be000fbce7..a6f7c1735b349f 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -128,6 +128,13 @@ static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame * // Don't leave a dangling pointer to the old frame when creating generators // and coroutines: dest->previous = NULL; + +#ifdef Py_GIL_DISABLED + PyCodeObject *co = (PyCodeObject *)dest->f_executable; + for (int i = stacktop; i < co->co_nlocalsplus + co->co_stacksize; i++) { + dest->localsplus[i] = PyStackRef_NULL; + } +#endif } /* Consumes reference to func and locals. @@ -137,8 +144,9 @@ static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame * static inline void _PyFrame_Initialize( _PyInterpreterFrame *frame, PyFunctionObject *func, - PyObject *locals, PyCodeObject *code, int null_locals_from) + PyObject *locals, PyCodeObject *code, int null_locals_from, _PyInterpreterFrame *previous) { + frame->previous = previous; frame->f_funcobj = (PyObject *)func; frame->f_executable = Py_NewRef(code); frame->f_builtins = func->func_builtins; @@ -153,6 +161,16 @@ _PyFrame_Initialize( for (int i = null_locals_from; i < code->co_nlocalsplus; i++) { frame->localsplus[i] = PyStackRef_NULL; } + +#ifdef Py_GIL_DISABLED + // On GIL disabled, we walk the entire stack in GC. Since stacktop + // is not always in sync with the real stack pointer, we have + // no choice but to traverse the entire stack. + // This just makes sure we don't pass the GC invalid stack values. + for (int i = code->co_nlocalsplus; i < code->co_nlocalsplus + code->co_stacksize; i++) { + frame->localsplus[i] = PyStackRef_NULL; + } +#endif } /* Gets the pointer to the locals array @@ -281,26 +299,27 @@ PyAPI_FUNC(void) _PyThreadState_PopFrame(PyThreadState *tstate, _PyInterpreterFr * Must be guarded by _PyThreadState_HasStackSpace() * Consumes reference to func. */ static inline _PyInterpreterFrame * -_PyFrame_PushUnchecked(PyThreadState *tstate, PyFunctionObject *func, int null_locals_from) +_PyFrame_PushUnchecked(PyThreadState *tstate, PyFunctionObject *func, int null_locals_from, _PyInterpreterFrame * previous) { CALL_STAT_INC(frames_pushed); PyCodeObject *code = (PyCodeObject *)func->func_code; _PyInterpreterFrame *new_frame = (_PyInterpreterFrame *)tstate->datastack_top; tstate->datastack_top += code->co_framesize; assert(tstate->datastack_top < tstate->datastack_limit); - _PyFrame_Initialize(new_frame, func, NULL, code, null_locals_from); + _PyFrame_Initialize(new_frame, func, NULL, code, null_locals_from, previous); return new_frame; } /* Pushes a trampoline frame without checking for space. * Must be guarded by _PyThreadState_HasStackSpace() */ static inline _PyInterpreterFrame * -_PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int stackdepth) +_PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int stackdepth, _PyInterpreterFrame * previous) { CALL_STAT_INC(frames_pushed); _PyInterpreterFrame *frame = (_PyInterpreterFrame *)tstate->datastack_top; tstate->datastack_top += code->co_framesize; assert(tstate->datastack_top < tstate->datastack_limit); + frame->previous = previous; frame->f_funcobj = Py_None; frame->f_executable = Py_NewRef(code); #ifdef Py_DEBUG @@ -314,13 +333,21 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int frame->instr_ptr = _PyCode_CODE(code); frame->owner = FRAME_OWNED_BY_THREAD; frame->return_offset = 0; + +#ifdef Py_GIL_DISABLED + assert(code->co_nlocalsplus == 0); + for (int i = 0; i < code->co_stacksize; i++) { + frame->localsplus[i] = PyStackRef_NULL; + } +#endif return frame; } PyAPI_FUNC(_PyInterpreterFrame *) _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, PyObject *locals, _PyStackRef const* args, - size_t argcount, PyObject *kwnames); + size_t argcount, PyObject *kwnames, + _PyInterpreterFrame *previous); #ifdef __cplusplus } diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index 5dd5b0c78d42fa..89f6017bacc525 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -381,6 +381,8 @@ extern void _PyGC_ClearAllFreeLists(PyInterpreterState *interp); extern void _Py_ScheduleGC(PyThreadState *tstate); extern void _Py_RunGC(PyThreadState *tstate); +// GC visit callback for tracked interpreter frames +extern int _PyGC_VisitFrameStack(struct _PyInterpreterFrame *frame, visitproc visit, void *arg); #ifdef __cplusplus } diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index 0af6e3f3cd4c92..095eb0f8a89b79 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -257,6 +257,7 @@ Known values: Python 3.14a1 3603 (Remove BUILD_CONST_KEY_MAP) Python 3.14a1 3604 (Do not duplicate test at end of while statements) Python 3.14a1 3605 (Move ENTER_EXECUTOR to opcode 255) + Python 3.14a1 3606 (Specialize CALL_KW) Python 3.15 will start with 3650 @@ -269,7 +270,7 @@ PC/launcher.c must also be updated. */ -#define PYC_MAGIC_NUMBER 3605 +#define PYC_MAGIC_NUMBER 3606 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Include/internal/pycore_mimalloc.h b/Include/internal/pycore_mimalloc.h index d10b01d5b49b19..d870d01beb702c 100644 --- a/Include/internal/pycore_mimalloc.h +++ b/Include/internal/pycore_mimalloc.h @@ -36,9 +36,18 @@ typedef enum { # define MI_TSAN 1 #endif +#ifdef __cplusplus +extern "C++" { +#endif + #include "mimalloc/mimalloc.h" #include "mimalloc/mimalloc/types.h" #include "mimalloc/mimalloc/internal.h" + +#ifdef __cplusplus +} +#endif + #endif #ifdef Py_GIL_DISABLED diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index a5640b7bcb7d60..ee33da77f01f2d 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -301,6 +301,14 @@ _Py_INCREF_TYPE(PyTypeObject *type) return; } + // gh-122974: GCC 11 warns about the access to PyHeapTypeObject fields when + // _Py_INCREF_TYPE() is called on a statically allocated type. The + // _PyType_HasFeature check above ensures that the type is a heap type. +#if defined(__GNUC__) && __GNUC__ >= 11 +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Warray-bounds" +#endif + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); PyHeapTypeObject *ht = (PyHeapTypeObject *)type; @@ -319,6 +327,10 @@ _Py_INCREF_TYPE(PyTypeObject *type) // It handles the unique_id=-1 case to keep the inlinable function smaller. _PyType_IncrefSlow(ht); } + +#if defined(__GNUC__) && __GNUC__ >= 11 +# pragma GCC diagnostic pop +#endif } static inline void @@ -749,7 +761,7 @@ static inline int _PyType_SUPPORTS_WEAKREFS(PyTypeObject *type) { } extern PyObject* _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems); -extern PyObject *_PyType_NewManagedObject(PyTypeObject *type); +PyAPI_FUNC(PyObject *) _PyType_NewManagedObject(PyTypeObject *type); extern PyTypeObject* _PyType_CalculateMetaclass(PyTypeObject *, PyObject *); extern PyObject* _PyType_GetDocFromInternalDoc(const char *, const char *); diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index a39d2390b74f2d..236e6b414611f3 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -105,6 +105,12 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 2 + oparg; case CALL_KW: return 3 + oparg; + case CALL_KW_BOUND_METHOD: + return 3 + oparg; + case CALL_KW_NON_PY: + return 3 + oparg; + case CALL_KW_PY: + return 3 + oparg; case CALL_LEN: return 2 + oparg; case CALL_LIST_APPEND: @@ -531,7 +537,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case CALL: return 1; case CALL_ALLOC_AND_ENTER_INIT: - return 1; + return 0; case CALL_BOUND_METHOD_EXACT_ARGS: return 0; case CALL_BOUND_METHOD_GENERAL: @@ -554,6 +560,12 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1; case CALL_KW: return 1; + case CALL_KW_BOUND_METHOD: + return 0; + case CALL_KW_NON_PY: + return 1; + case CALL_KW_PY: + return 0; case CALL_LEN: return 1; case CALL_LIST_APPEND: @@ -1027,7 +1039,10 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [CALL_INTRINSIC_1] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_INTRINSIC_2] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_ISINSTANCE] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_KW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW_BOUND_METHOD] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW_NON_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_LEN] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, [CALL_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1084,7 +1099,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [IMPORT_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, 0 }, - [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, @@ -1246,6 +1261,7 @@ _PyOpcode_macro_expansion[256] = { [BUILD_SLICE] = { .nuops = 1, .uops = { { _BUILD_SLICE, 0, 0 } } }, [BUILD_STRING] = { .nuops = 1, .uops = { { _BUILD_STRING, 0, 0 } } }, [BUILD_TUPLE] = { .nuops = 1, .uops = { { _BUILD_TUPLE, 0, 0 } } }, + [CALL_ALLOC_AND_ENTER_INIT] = { .nuops = 4, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_AND_ALLOCATE_OBJECT, 2, 1 }, { _CREATE_INIT_FRAME, 0, 0 }, { _PUSH_FRAME, 0, 0 } } }, [CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 9, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, 0, 0 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, [CALL_BOUND_METHOD_GENERAL] = { .nuops = 6, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_METHOD_VERSION, 2, 1 }, { _EXPAND_METHOD, 0, 0 }, { _PY_FRAME_GENERAL, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, [CALL_BUILTIN_CLASS] = { .nuops = 2, .uops = { { _CALL_BUILTIN_CLASS, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, @@ -1255,6 +1271,9 @@ _PyOpcode_macro_expansion[256] = { [CALL_INTRINSIC_1] = { .nuops = 1, .uops = { { _CALL_INTRINSIC_1, 0, 0 } } }, [CALL_INTRINSIC_2] = { .nuops = 1, .uops = { { _CALL_INTRINSIC_2, 0, 0 } } }, [CALL_ISINSTANCE] = { .nuops = 1, .uops = { { _CALL_ISINSTANCE, 0, 0 } } }, + [CALL_KW_BOUND_METHOD] = { .nuops = 6, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_METHOD_VERSION_KW, 2, 1 }, { _EXPAND_METHOD_KW, 0, 0 }, { _PY_FRAME_KW, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, + [CALL_KW_NON_PY] = { .nuops = 3, .uops = { { _CHECK_IS_NOT_PY_CALLABLE_KW, 0, 0 }, { _CALL_KW_NON_PY, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_KW_PY] = { .nuops = 5, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_FUNCTION_VERSION_KW, 2, 1 }, { _PY_FRAME_KW, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, [CALL_LEN] = { .nuops = 1, .uops = { { _CALL_LEN, 0, 0 } } }, [CALL_LIST_APPEND] = { .nuops = 1, .uops = { { _CALL_LIST_APPEND, 0, 0 } } }, [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, @@ -1436,6 +1455,9 @@ const char *_PyOpcode_OpName[264] = { [CALL_INTRINSIC_2] = "CALL_INTRINSIC_2", [CALL_ISINSTANCE] = "CALL_ISINSTANCE", [CALL_KW] = "CALL_KW", + [CALL_KW_BOUND_METHOD] = "CALL_KW_BOUND_METHOD", + [CALL_KW_NON_PY] = "CALL_KW_NON_PY", + [CALL_KW_PY] = "CALL_KW_PY", [CALL_LEN] = "CALL_LEN", [CALL_LIST_APPEND] = "CALL_LIST_APPEND", [CALL_METHOD_DESCRIPTOR_FAST] = "CALL_METHOD_DESCRIPTOR_FAST", @@ -1643,6 +1665,7 @@ const uint8_t _PyOpcode_Caches[256] = { [POP_JUMP_IF_NOT_NONE] = 1, [FOR_ITER] = 1, [CALL] = 3, + [CALL_KW] = 3, [BINARY_OP] = 1, }; #endif @@ -1686,6 +1709,9 @@ const uint8_t _PyOpcode_Deopt[256] = { [CALL_INTRINSIC_2] = CALL_INTRINSIC_2, [CALL_ISINSTANCE] = CALL, [CALL_KW] = CALL_KW, + [CALL_KW_BOUND_METHOD] = CALL_KW, + [CALL_KW_NON_PY] = CALL_KW, + [CALL_KW_PY] = CALL_KW, [CALL_LEN] = CALL, [CALL_LIST_APPEND] = CALL, [CALL_METHOD_DESCRIPTOR_FAST] = CALL, @@ -1898,9 +1924,6 @@ const uint8_t _PyOpcode_Deopt[256] = { case 146: \ case 147: \ case 148: \ - case 223: \ - case 224: \ - case 225: \ case 226: \ case 227: \ case 228: \ diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index d4ffd977940a02..d4291b87261ae0 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -44,6 +44,15 @@ struct _gilstate_runtime_state { /* Runtime audit hook state */ +#define _Py_Debug_Cookie "xdebugpy" + +#ifdef Py_GIL_DISABLED +# define _Py_Debug_gilruntimestate_enabled offsetof(struct _gil_runtime_state, enabled) +# define _Py_Debug_Free_Threaded 1 +#else +# define _Py_Debug_gilruntimestate_enabled 0 +# define _Py_Debug_Free_Threaded 0 +#endif typedef struct _Py_AuditHookEntry { struct _Py_AuditHookEntry *next; Py_AuditHookFunction hookCFunction; @@ -53,6 +62,7 @@ typedef struct _Py_AuditHookEntry { typedef struct _Py_DebugOffsets { char cookie[8]; uint64_t version; + uint64_t free_threaded; // Runtime state offset; struct _runtime_state { uint64_t size; @@ -71,6 +81,8 @@ typedef struct _Py_DebugOffsets { uint64_t sysdict; uint64_t builtins; uint64_t ceval_gil; + uint64_t gil_runtime_state; + uint64_t gil_runtime_state_enabled; uint64_t gil_runtime_state_locked; uint64_t gil_runtime_state_holder; } interpreter_state; @@ -122,20 +134,57 @@ typedef struct _Py_DebugOffsets { struct _type_object { uint64_t size; uint64_t tp_name; + uint64_t tp_repr; + uint64_t tp_flags; } type_object; // PyTuple object offset; struct _tuple_object { uint64_t size; uint64_t ob_item; + uint64_t ob_size; } tuple_object; + // PyList object offset; + struct _list_object { + uint64_t size; + uint64_t ob_item; + uint64_t ob_size; + } list_object; + + // PyDict object offset; + struct _dict_object { + uint64_t size; + uint64_t ma_keys; + uint64_t ma_values; + } dict_object; + + // PyFloat object offset; + struct _float_object { + uint64_t size; + uint64_t ob_fval; + } float_object; + + // PyLong object offset; + struct _long_object { + uint64_t size; + uint64_t lv_tag; + uint64_t ob_digit; + } long_object; + + // PyBytes object offset; + struct _bytes_object { + uint64_t size; + uint64_t ob_size; + uint64_t ob_sval; + } bytes_object; + // Unicode object offset; struct _unicode_object { uint64_t size; uint64_t state; uint64_t length; - size_t asciiobject_size; + uint64_t asciiobject_size; } unicode_object; // GC runtime state offset; diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index da2b8d5570de62..1746bbf0eec8b5 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -29,11 +29,12 @@ extern PyTypeObject _PyExc_MemoryError; /* The static initializers defined here should only be used in the runtime init code (in pystate.c and pylifecycle.c). */ -#define _PyRuntimeState_INIT(runtime) \ +#define _PyRuntimeState_INIT(runtime, debug_cookie) \ { \ .debug_offsets = { \ - .cookie = "xdebugpy", \ + .cookie = debug_cookie, \ .version = PY_VERSION_HEX, \ + .free_threaded = _Py_Debug_Free_Threaded, \ .runtime_state = { \ .size = sizeof(_PyRuntimeState), \ .finalizing = offsetof(_PyRuntimeState, _finalizing), \ @@ -49,6 +50,8 @@ extern PyTypeObject _PyExc_MemoryError; .sysdict = offsetof(PyInterpreterState, sysdict), \ .builtins = offsetof(PyInterpreterState, builtins), \ .ceval_gil = offsetof(PyInterpreterState, ceval.gil), \ + .gil_runtime_state = offsetof(PyInterpreterState, _gil), \ + .gil_runtime_state_enabled = _Py_Debug_gilruntimestate_enabled, \ .gil_runtime_state_locked = offsetof(PyInterpreterState, _gil.locked), \ .gil_runtime_state_holder = offsetof(PyInterpreterState, _gil.last_holder), \ }, \ @@ -90,10 +93,37 @@ extern PyTypeObject _PyExc_MemoryError; .type_object = { \ .size = sizeof(PyTypeObject), \ .tp_name = offsetof(PyTypeObject, tp_name), \ + .tp_repr = offsetof(PyTypeObject, tp_repr), \ + .tp_flags = offsetof(PyTypeObject, tp_flags), \ }, \ .tuple_object = { \ .size = sizeof(PyTupleObject), \ .ob_item = offsetof(PyTupleObject, ob_item), \ + .ob_size = offsetof(PyTupleObject, ob_base.ob_size), \ + }, \ + .list_object = { \ + .size = sizeof(PyListObject), \ + .ob_item = offsetof(PyListObject, ob_item), \ + .ob_size = offsetof(PyListObject, ob_base.ob_size), \ + }, \ + .dict_object = { \ + .size = sizeof(PyDictObject), \ + .ma_keys = offsetof(PyDictObject, ma_keys), \ + .ma_values = offsetof(PyDictObject, ma_values), \ + }, \ + .float_object = { \ + .size = sizeof(PyFloatObject), \ + .ob_fval = offsetof(PyFloatObject, ob_fval), \ + }, \ + .long_object = { \ + .size = sizeof(PyLongObject), \ + .lv_tag = offsetof(_PyLongValue, lv_tag), \ + .ob_digit = offsetof(_PyLongValue, ob_digit), \ + }, \ + .bytes_object = { \ + .size = sizeof(PyBytesObject), \ + .ob_size = offsetof(PyBytesObject, ob_base.ob_size), \ + .ob_sval = offsetof(PyBytesObject, ob_sval), \ }, \ .unicode_object = { \ .size = sizeof(PyUnicodeObject), \ diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index 1b35a3e3269257..8c9bb1ae8c4908 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -150,8 +150,7 @@ PyStackRef_FromPyObjectNew(PyObject *obj) // Make sure we don't take an already tagged value. assert(((uintptr_t)obj & Py_TAG_BITS) == 0); assert(obj != NULL); - // TODO (gh-117139): Add deferred objects later. - if (_Py_IsImmortal(obj)) { + if (_Py_IsImmortal(obj) || _PyObject_HasDeferredRefcount(obj)) { return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_DEFERRED }; } else { @@ -220,7 +219,8 @@ PyStackRef_DUP(_PyStackRef stackref) { if (PyStackRef_IsDeferred(stackref)) { assert(PyStackRef_IsNull(stackref) || - _Py_IsImmortal(PyStackRef_AsPyObjectBorrow(stackref))); + _Py_IsImmortal(PyStackRef_AsPyObjectBorrow(stackref)) || + _PyObject_HasDeferredRefcount(PyStackRef_AsPyObjectBorrow(stackref))); return stackref; } Py_INCREF(PyStackRef_AsPyObjectBorrow(stackref)); diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 74c7cc9cac64ae..226f4a63460b37 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -34,7 +34,6 @@ extern "C" { #define _BUILD_SLICE BUILD_SLICE #define _BUILD_STRING BUILD_STRING #define _BUILD_TUPLE BUILD_TUPLE -#define _CALL_ALLOC_AND_ENTER_INIT CALL_ALLOC_AND_ENTER_INIT #define _CALL_BUILTIN_CLASS 314 #define _CALL_BUILTIN_FAST 315 #define _CALL_BUILTIN_FAST_WITH_KEYWORDS 316 @@ -42,103 +41,110 @@ extern "C" { #define _CALL_INTRINSIC_1 CALL_INTRINSIC_1 #define _CALL_INTRINSIC_2 CALL_INTRINSIC_2 #define _CALL_ISINSTANCE CALL_ISINSTANCE +#define _CALL_KW_NON_PY 318 #define _CALL_LEN CALL_LEN #define _CALL_LIST_APPEND CALL_LIST_APPEND -#define _CALL_METHOD_DESCRIPTOR_FAST 318 -#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 319 -#define _CALL_METHOD_DESCRIPTOR_NOARGS 320 -#define _CALL_METHOD_DESCRIPTOR_O 321 -#define _CALL_NON_PY_GENERAL 322 -#define _CALL_STR_1 323 -#define _CALL_TUPLE_1 324 +#define _CALL_METHOD_DESCRIPTOR_FAST 319 +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 320 +#define _CALL_METHOD_DESCRIPTOR_NOARGS 321 +#define _CALL_METHOD_DESCRIPTOR_O 322 +#define _CALL_NON_PY_GENERAL 323 +#define _CALL_STR_1 324 +#define _CALL_TUPLE_1 325 #define _CALL_TYPE_1 CALL_TYPE_1 -#define _CHECK_ATTR_CLASS 325 -#define _CHECK_ATTR_METHOD_LAZY_DICT 326 -#define _CHECK_ATTR_MODULE 327 -#define _CHECK_ATTR_WITH_HINT 328 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 329 +#define _CHECK_AND_ALLOCATE_OBJECT 326 +#define _CHECK_ATTR_CLASS 327 +#define _CHECK_ATTR_METHOD_LAZY_DICT 328 +#define _CHECK_ATTR_MODULE 329 +#define _CHECK_ATTR_WITH_HINT 330 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 331 #define _CHECK_EG_MATCH CHECK_EG_MATCH #define _CHECK_EXC_MATCH CHECK_EXC_MATCH -#define _CHECK_FUNCTION 330 -#define _CHECK_FUNCTION_EXACT_ARGS 331 -#define _CHECK_FUNCTION_VERSION 332 -#define _CHECK_IS_NOT_PY_CALLABLE 333 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 334 -#define _CHECK_METHOD_VERSION 335 -#define _CHECK_PEP_523 336 -#define _CHECK_PERIODIC 337 -#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 338 -#define _CHECK_STACK_SPACE 339 -#define _CHECK_STACK_SPACE_OPERAND 340 -#define _CHECK_VALIDITY 341 -#define _CHECK_VALIDITY_AND_SET_IP 342 -#define _COMPARE_OP 343 -#define _COMPARE_OP_FLOAT 344 -#define _COMPARE_OP_INT 345 -#define _COMPARE_OP_STR 346 -#define _CONTAINS_OP 347 +#define _CHECK_FUNCTION 332 +#define _CHECK_FUNCTION_EXACT_ARGS 333 +#define _CHECK_FUNCTION_VERSION 334 +#define _CHECK_FUNCTION_VERSION_KW 335 +#define _CHECK_IS_NOT_PY_CALLABLE 336 +#define _CHECK_IS_NOT_PY_CALLABLE_KW 337 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 338 +#define _CHECK_METHOD_VERSION 339 +#define _CHECK_METHOD_VERSION_KW 340 +#define _CHECK_PEP_523 341 +#define _CHECK_PERIODIC 342 +#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 343 +#define _CHECK_STACK_SPACE 344 +#define _CHECK_STACK_SPACE_OPERAND 345 +#define _CHECK_VALIDITY 346 +#define _CHECK_VALIDITY_AND_SET_IP 347 +#define _COMPARE_OP 348 +#define _COMPARE_OP_FLOAT 349 +#define _COMPARE_OP_INT 350 +#define _COMPARE_OP_STR 351 +#define _CONTAINS_OP 352 #define _CONTAINS_OP_DICT CONTAINS_OP_DICT #define _CONTAINS_OP_SET CONTAINS_OP_SET #define _CONVERT_VALUE CONVERT_VALUE #define _COPY COPY #define _COPY_FREE_VARS COPY_FREE_VARS +#define _CREATE_INIT_FRAME 353 #define _DELETE_ATTR DELETE_ATTR #define _DELETE_DEREF DELETE_DEREF #define _DELETE_FAST DELETE_FAST #define _DELETE_GLOBAL DELETE_GLOBAL #define _DELETE_NAME DELETE_NAME #define _DELETE_SUBSCR DELETE_SUBSCR -#define _DEOPT 348 +#define _DEOPT 354 #define _DICT_MERGE DICT_MERGE #define _DICT_UPDATE DICT_UPDATE -#define _DO_CALL 349 -#define _DO_CALL_KW 350 -#define _DYNAMIC_EXIT 351 +#define _DO_CALL 355 +#define _DO_CALL_KW 356 +#define _DYNAMIC_EXIT 357 #define _END_SEND END_SEND -#define _ERROR_POP_N 352 +#define _ERROR_POP_N 358 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _EXPAND_METHOD 353 -#define _FATAL_ERROR 354 +#define _EXPAND_METHOD 359 +#define _EXPAND_METHOD_KW 360 +#define _FATAL_ERROR 361 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 355 -#define _FOR_ITER_GEN_FRAME 356 -#define _FOR_ITER_TIER_TWO 357 +#define _FOR_ITER 362 +#define _FOR_ITER_GEN_FRAME 363 +#define _FOR_ITER_TIER_TWO 364 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BOTH_FLOAT 358 -#define _GUARD_BOTH_INT 359 -#define _GUARD_BOTH_UNICODE 360 -#define _GUARD_BUILTINS_VERSION 361 -#define _GUARD_DORV_NO_DICT 362 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 363 -#define _GUARD_GLOBALS_VERSION 364 -#define _GUARD_IS_FALSE_POP 365 -#define _GUARD_IS_NONE_POP 366 -#define _GUARD_IS_NOT_NONE_POP 367 -#define _GUARD_IS_TRUE_POP 368 -#define _GUARD_KEYS_VERSION 369 -#define _GUARD_NOS_FLOAT 370 -#define _GUARD_NOS_INT 371 -#define _GUARD_NOT_EXHAUSTED_LIST 372 -#define _GUARD_NOT_EXHAUSTED_RANGE 373 -#define _GUARD_NOT_EXHAUSTED_TUPLE 374 -#define _GUARD_TOS_FLOAT 375 -#define _GUARD_TOS_INT 376 -#define _GUARD_TYPE_VERSION 377 +#define _GUARD_BOTH_FLOAT 365 +#define _GUARD_BOTH_INT 366 +#define _GUARD_BOTH_UNICODE 367 +#define _GUARD_BUILTINS_VERSION 368 +#define _GUARD_DORV_NO_DICT 369 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 370 +#define _GUARD_GLOBALS_VERSION 371 +#define _GUARD_IS_FALSE_POP 372 +#define _GUARD_IS_NONE_POP 373 +#define _GUARD_IS_NOT_NONE_POP 374 +#define _GUARD_IS_TRUE_POP 375 +#define _GUARD_KEYS_VERSION 376 +#define _GUARD_NOS_FLOAT 377 +#define _GUARD_NOS_INT 378 +#define _GUARD_NOT_EXHAUSTED_LIST 379 +#define _GUARD_NOT_EXHAUSTED_RANGE 380 +#define _GUARD_NOT_EXHAUSTED_TUPLE 381 +#define _GUARD_TOS_FLOAT 382 +#define _GUARD_TOS_INT 383 +#define _GUARD_TYPE_VERSION 384 #define _IMPORT_FROM IMPORT_FROM #define _IMPORT_NAME IMPORT_NAME -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 378 -#define _INIT_CALL_PY_EXACT_ARGS 379 -#define _INIT_CALL_PY_EXACT_ARGS_0 380 -#define _INIT_CALL_PY_EXACT_ARGS_1 381 -#define _INIT_CALL_PY_EXACT_ARGS_2 382 -#define _INIT_CALL_PY_EXACT_ARGS_3 383 -#define _INIT_CALL_PY_EXACT_ARGS_4 384 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 385 +#define _INIT_CALL_PY_EXACT_ARGS 386 +#define _INIT_CALL_PY_EXACT_ARGS_0 387 +#define _INIT_CALL_PY_EXACT_ARGS_1 388 +#define _INIT_CALL_PY_EXACT_ARGS_2 389 +#define _INIT_CALL_PY_EXACT_ARGS_3 390 +#define _INIT_CALL_PY_EXACT_ARGS_4 391 #define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX #define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER @@ -150,65 +156,65 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _INTERNAL_INCREMENT_OPT_COUNTER 385 -#define _IS_NONE 386 +#define _INTERNAL_INCREMENT_OPT_COUNTER 392 +#define _IS_NONE 393 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 387 -#define _ITER_CHECK_RANGE 388 -#define _ITER_CHECK_TUPLE 389 -#define _ITER_JUMP_LIST 390 -#define _ITER_JUMP_RANGE 391 -#define _ITER_JUMP_TUPLE 392 -#define _ITER_NEXT_LIST 393 -#define _ITER_NEXT_RANGE 394 -#define _ITER_NEXT_TUPLE 395 -#define _JUMP_TO_TOP 396 +#define _ITER_CHECK_LIST 394 +#define _ITER_CHECK_RANGE 395 +#define _ITER_CHECK_TUPLE 396 +#define _ITER_JUMP_LIST 397 +#define _ITER_JUMP_RANGE 398 +#define _ITER_JUMP_TUPLE 399 +#define _ITER_NEXT_LIST 400 +#define _ITER_NEXT_RANGE 401 +#define _ITER_NEXT_TUPLE 402 +#define _JUMP_TO_TOP 403 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 397 -#define _LOAD_ATTR_CLASS 398 -#define _LOAD_ATTR_CLASS_0 399 -#define _LOAD_ATTR_CLASS_1 400 +#define _LOAD_ATTR 404 +#define _LOAD_ATTR_CLASS 405 +#define _LOAD_ATTR_CLASS_0 406 +#define _LOAD_ATTR_CLASS_1 407 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 401 -#define _LOAD_ATTR_INSTANCE_VALUE_0 402 -#define _LOAD_ATTR_INSTANCE_VALUE_1 403 -#define _LOAD_ATTR_METHOD_LAZY_DICT 404 -#define _LOAD_ATTR_METHOD_NO_DICT 405 -#define _LOAD_ATTR_METHOD_WITH_VALUES 406 -#define _LOAD_ATTR_MODULE 407 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 408 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 409 -#define _LOAD_ATTR_PROPERTY_FRAME 410 -#define _LOAD_ATTR_SLOT 411 -#define _LOAD_ATTR_SLOT_0 412 -#define _LOAD_ATTR_SLOT_1 413 -#define _LOAD_ATTR_WITH_HINT 414 +#define _LOAD_ATTR_INSTANCE_VALUE 408 +#define _LOAD_ATTR_INSTANCE_VALUE_0 409 +#define _LOAD_ATTR_INSTANCE_VALUE_1 410 +#define _LOAD_ATTR_METHOD_LAZY_DICT 411 +#define _LOAD_ATTR_METHOD_NO_DICT 412 +#define _LOAD_ATTR_METHOD_WITH_VALUES 413 +#define _LOAD_ATTR_MODULE 414 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 415 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 416 +#define _LOAD_ATTR_PROPERTY_FRAME 417 +#define _LOAD_ATTR_SLOT 418 +#define _LOAD_ATTR_SLOT_0 419 +#define _LOAD_ATTR_SLOT_1 420 +#define _LOAD_ATTR_WITH_HINT 421 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 415 -#define _LOAD_CONST_INLINE_BORROW 416 -#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 417 -#define _LOAD_CONST_INLINE_WITH_NULL 418 +#define _LOAD_CONST_INLINE 422 +#define _LOAD_CONST_INLINE_BORROW 423 +#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 424 +#define _LOAD_CONST_INLINE_WITH_NULL 425 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 419 -#define _LOAD_FAST_0 420 -#define _LOAD_FAST_1 421 -#define _LOAD_FAST_2 422 -#define _LOAD_FAST_3 423 -#define _LOAD_FAST_4 424 -#define _LOAD_FAST_5 425 -#define _LOAD_FAST_6 426 -#define _LOAD_FAST_7 427 +#define _LOAD_FAST 426 +#define _LOAD_FAST_0 427 +#define _LOAD_FAST_1 428 +#define _LOAD_FAST_2 429 +#define _LOAD_FAST_3 430 +#define _LOAD_FAST_4 431 +#define _LOAD_FAST_5 432 +#define _LOAD_FAST_6 433 +#define _LOAD_FAST_7 434 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 428 -#define _LOAD_GLOBAL_BUILTINS 429 -#define _LOAD_GLOBAL_MODULE 430 +#define _LOAD_GLOBAL 435 +#define _LOAD_GLOBAL_BUILTINS 436 +#define _LOAD_GLOBAL_MODULE 437 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME #define _LOAD_SPECIAL LOAD_SPECIAL @@ -221,58 +227,59 @@ extern "C" { #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 431 -#define _MONITOR_CALL 432 -#define _MONITOR_JUMP_BACKWARD 433 -#define _MONITOR_RESUME 434 +#define _MAYBE_EXPAND_METHOD 438 +#define _MONITOR_CALL 439 +#define _MONITOR_JUMP_BACKWARD 440 +#define _MONITOR_RESUME 441 #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT -#define _POP_JUMP_IF_FALSE 435 -#define _POP_JUMP_IF_TRUE 436 +#define _POP_JUMP_IF_FALSE 442 +#define _POP_JUMP_IF_TRUE 443 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 437 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 444 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 438 +#define _PUSH_FRAME 445 #define _PUSH_NULL PUSH_NULL -#define _PY_FRAME_GENERAL 439 -#define _QUICKEN_RESUME 440 -#define _REPLACE_WITH_TRUE 441 +#define _PY_FRAME_GENERAL 446 +#define _PY_FRAME_KW 447 +#define _QUICKEN_RESUME 448 +#define _REPLACE_WITH_TRUE 449 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 442 -#define _SEND 443 -#define _SEND_GEN_FRAME 444 +#define _SAVE_RETURN_OFFSET 450 +#define _SEND 451 +#define _SEND_GEN_FRAME 452 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 445 -#define _STORE_ATTR 446 -#define _STORE_ATTR_INSTANCE_VALUE 447 -#define _STORE_ATTR_SLOT 448 -#define _STORE_ATTR_WITH_HINT 449 +#define _START_EXECUTOR 453 +#define _STORE_ATTR 454 +#define _STORE_ATTR_INSTANCE_VALUE 455 +#define _STORE_ATTR_SLOT 456 +#define _STORE_ATTR_WITH_HINT 457 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 450 -#define _STORE_FAST_0 451 -#define _STORE_FAST_1 452 -#define _STORE_FAST_2 453 -#define _STORE_FAST_3 454 -#define _STORE_FAST_4 455 -#define _STORE_FAST_5 456 -#define _STORE_FAST_6 457 -#define _STORE_FAST_7 458 +#define _STORE_FAST 458 +#define _STORE_FAST_0 459 +#define _STORE_FAST_1 460 +#define _STORE_FAST_2 461 +#define _STORE_FAST_3 462 +#define _STORE_FAST_4 463 +#define _STORE_FAST_5 464 +#define _STORE_FAST_6 465 +#define _STORE_FAST_7 466 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME #define _STORE_SLICE STORE_SLICE -#define _STORE_SUBSCR 459 +#define _STORE_SUBSCR 467 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _TIER2_RESUME_CHECK 460 -#define _TO_BOOL 461 +#define _TIER2_RESUME_CHECK 468 +#define _TO_BOOL 469 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST @@ -282,14 +289,14 @@ extern "C" { #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 462 +#define _UNPACK_SEQUENCE 470 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE #define __DO_CALL_FUNCTION_EX _DO_CALL_FUNCTION_EX -#define MAX_UOP_ID 462 +#define MAX_UOP_ID 470 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 69833e484e9915..15c0ac110301c7 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -225,6 +225,8 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CALL_TYPE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_CALL_STR_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_TUPLE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_AND_ALLOCATE_OBJECT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CREATE_INIT_FRAME] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_EXIT_INIT_CHECK] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_CALL_BUILTIN_CLASS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_BUILTIN_O] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -237,6 +239,12 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_METHOD_DESCRIPTOR_NOARGS] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_PY_FRAME_KW] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_FUNCTION_VERSION_KW] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CHECK_METHOD_VERSION_KW] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_EXPAND_METHOD_KW] = HAS_ARG_FLAG, + [_CHECK_IS_NOT_PY_CALLABLE_KW] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CALL_KW_NON_PY] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_MAKE_FUNCTION] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_SET_FUNCTION_ATTRIBUTE] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_RETURN_GENERATOR] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, @@ -310,6 +318,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_CALL_INTRINSIC_1] = "_CALL_INTRINSIC_1", [_CALL_INTRINSIC_2] = "_CALL_INTRINSIC_2", [_CALL_ISINSTANCE] = "_CALL_ISINSTANCE", + [_CALL_KW_NON_PY] = "_CALL_KW_NON_PY", [_CALL_LEN] = "_CALL_LEN", [_CALL_LIST_APPEND] = "_CALL_LIST_APPEND", [_CALL_METHOD_DESCRIPTOR_FAST] = "_CALL_METHOD_DESCRIPTOR_FAST", @@ -320,6 +329,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_CALL_STR_1] = "_CALL_STR_1", [_CALL_TUPLE_1] = "_CALL_TUPLE_1", [_CALL_TYPE_1] = "_CALL_TYPE_1", + [_CHECK_AND_ALLOCATE_OBJECT] = "_CHECK_AND_ALLOCATE_OBJECT", [_CHECK_ATTR_CLASS] = "_CHECK_ATTR_CLASS", [_CHECK_ATTR_METHOD_LAZY_DICT] = "_CHECK_ATTR_METHOD_LAZY_DICT", [_CHECK_ATTR_MODULE] = "_CHECK_ATTR_MODULE", @@ -330,9 +340,12 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_CHECK_FUNCTION] = "_CHECK_FUNCTION", [_CHECK_FUNCTION_EXACT_ARGS] = "_CHECK_FUNCTION_EXACT_ARGS", [_CHECK_FUNCTION_VERSION] = "_CHECK_FUNCTION_VERSION", + [_CHECK_FUNCTION_VERSION_KW] = "_CHECK_FUNCTION_VERSION_KW", [_CHECK_IS_NOT_PY_CALLABLE] = "_CHECK_IS_NOT_PY_CALLABLE", + [_CHECK_IS_NOT_PY_CALLABLE_KW] = "_CHECK_IS_NOT_PY_CALLABLE_KW", [_CHECK_MANAGED_OBJECT_HAS_VALUES] = "_CHECK_MANAGED_OBJECT_HAS_VALUES", [_CHECK_METHOD_VERSION] = "_CHECK_METHOD_VERSION", + [_CHECK_METHOD_VERSION_KW] = "_CHECK_METHOD_VERSION_KW", [_CHECK_PEP_523] = "_CHECK_PEP_523", [_CHECK_PERIODIC] = "_CHECK_PERIODIC", [_CHECK_PERIODIC_IF_NOT_YIELD_FROM] = "_CHECK_PERIODIC_IF_NOT_YIELD_FROM", @@ -350,6 +363,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_CONVERT_VALUE] = "_CONVERT_VALUE", [_COPY] = "_COPY", [_COPY_FREE_VARS] = "_COPY_FREE_VARS", + [_CREATE_INIT_FRAME] = "_CREATE_INIT_FRAME", [_DELETE_ATTR] = "_DELETE_ATTR", [_DELETE_DEREF] = "_DELETE_DEREF", [_DELETE_FAST] = "_DELETE_FAST", @@ -365,6 +379,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_EXIT_INIT_CHECK] = "_EXIT_INIT_CHECK", [_EXIT_TRACE] = "_EXIT_TRACE", [_EXPAND_METHOD] = "_EXPAND_METHOD", + [_EXPAND_METHOD_KW] = "_EXPAND_METHOD_KW", [_FATAL_ERROR] = "_FATAL_ERROR", [_FORMAT_SIMPLE] = "_FORMAT_SIMPLE", [_FORMAT_WITH_SPEC] = "_FORMAT_WITH_SPEC", @@ -480,6 +495,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_PUSH_FRAME] = "_PUSH_FRAME", [_PUSH_NULL] = "_PUSH_NULL", [_PY_FRAME_GENERAL] = "_PY_FRAME_GENERAL", + [_PY_FRAME_KW] = "_PY_FRAME_KW", [_REPLACE_WITH_TRUE] = "_REPLACE_WITH_TRUE", [_RESUME_CHECK] = "_RESUME_CHECK", [_RETURN_GENERATOR] = "_RETURN_GENERATOR", @@ -948,6 +964,10 @@ int _PyUop_num_popped(int opcode, int oparg) return 3; case _CALL_TUPLE_1: return 3; + case _CHECK_AND_ALLOCATE_OBJECT: + return 2 + oparg; + case _CREATE_INIT_FRAME: + return 2 + oparg; case _EXIT_INIT_CHECK: return 1; case _CALL_BUILTIN_CLASS: @@ -972,6 +992,18 @@ int _PyUop_num_popped(int opcode, int oparg) return 2 + oparg; case _CALL_METHOD_DESCRIPTOR_FAST: return 2 + oparg; + case _PY_FRAME_KW: + return 3 + oparg; + case _CHECK_FUNCTION_VERSION_KW: + return 3 + oparg; + case _CHECK_METHOD_VERSION_KW: + return 3 + oparg; + case _EXPAND_METHOD_KW: + return 3 + oparg; + case _CHECK_IS_NOT_PY_CALLABLE_KW: + return 3 + oparg; + case _CALL_KW_NON_PY: + return 3 + oparg; case _MAKE_FUNCTION: return 1; case _SET_FUNCTION_ATTRIBUTE: diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index 95984a98df84c6..1189712e12e21e 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -148,59 +148,62 @@ extern "C" { #define CALL_BUILTIN_FAST_WITH_KEYWORDS 167 #define CALL_BUILTIN_O 168 #define CALL_ISINSTANCE 169 -#define CALL_LEN 170 -#define CALL_LIST_APPEND 171 -#define CALL_METHOD_DESCRIPTOR_FAST 172 -#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 173 -#define CALL_METHOD_DESCRIPTOR_NOARGS 174 -#define CALL_METHOD_DESCRIPTOR_O 175 -#define CALL_NON_PY_GENERAL 176 -#define CALL_PY_EXACT_ARGS 177 -#define CALL_PY_GENERAL 178 -#define CALL_STR_1 179 -#define CALL_TUPLE_1 180 -#define CALL_TYPE_1 181 -#define COMPARE_OP_FLOAT 182 -#define COMPARE_OP_INT 183 -#define COMPARE_OP_STR 184 -#define CONTAINS_OP_DICT 185 -#define CONTAINS_OP_SET 186 -#define FOR_ITER_GEN 187 -#define FOR_ITER_LIST 188 -#define FOR_ITER_RANGE 189 -#define FOR_ITER_TUPLE 190 -#define LOAD_ATTR_CLASS 191 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 192 -#define LOAD_ATTR_INSTANCE_VALUE 193 -#define LOAD_ATTR_METHOD_LAZY_DICT 194 -#define LOAD_ATTR_METHOD_NO_DICT 195 -#define LOAD_ATTR_METHOD_WITH_VALUES 196 -#define LOAD_ATTR_MODULE 197 -#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 198 -#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 199 -#define LOAD_ATTR_PROPERTY 200 -#define LOAD_ATTR_SLOT 201 -#define LOAD_ATTR_WITH_HINT 202 -#define LOAD_GLOBAL_BUILTIN 203 -#define LOAD_GLOBAL_MODULE 204 -#define LOAD_SUPER_ATTR_ATTR 205 -#define LOAD_SUPER_ATTR_METHOD 206 -#define RESUME_CHECK 207 -#define SEND_GEN 208 -#define STORE_ATTR_INSTANCE_VALUE 209 -#define STORE_ATTR_SLOT 210 -#define STORE_ATTR_WITH_HINT 211 -#define STORE_SUBSCR_DICT 212 -#define STORE_SUBSCR_LIST_INT 213 -#define TO_BOOL_ALWAYS_TRUE 214 -#define TO_BOOL_BOOL 215 -#define TO_BOOL_INT 216 -#define TO_BOOL_LIST 217 -#define TO_BOOL_NONE 218 -#define TO_BOOL_STR 219 -#define UNPACK_SEQUENCE_LIST 220 -#define UNPACK_SEQUENCE_TUPLE 221 -#define UNPACK_SEQUENCE_TWO_TUPLE 222 +#define CALL_KW_BOUND_METHOD 170 +#define CALL_KW_NON_PY 171 +#define CALL_KW_PY 172 +#define CALL_LEN 173 +#define CALL_LIST_APPEND 174 +#define CALL_METHOD_DESCRIPTOR_FAST 175 +#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 176 +#define CALL_METHOD_DESCRIPTOR_NOARGS 177 +#define CALL_METHOD_DESCRIPTOR_O 178 +#define CALL_NON_PY_GENERAL 179 +#define CALL_PY_EXACT_ARGS 180 +#define CALL_PY_GENERAL 181 +#define CALL_STR_1 182 +#define CALL_TUPLE_1 183 +#define CALL_TYPE_1 184 +#define COMPARE_OP_FLOAT 185 +#define COMPARE_OP_INT 186 +#define COMPARE_OP_STR 187 +#define CONTAINS_OP_DICT 188 +#define CONTAINS_OP_SET 189 +#define FOR_ITER_GEN 190 +#define FOR_ITER_LIST 191 +#define FOR_ITER_RANGE 192 +#define FOR_ITER_TUPLE 193 +#define LOAD_ATTR_CLASS 194 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 195 +#define LOAD_ATTR_INSTANCE_VALUE 196 +#define LOAD_ATTR_METHOD_LAZY_DICT 197 +#define LOAD_ATTR_METHOD_NO_DICT 198 +#define LOAD_ATTR_METHOD_WITH_VALUES 199 +#define LOAD_ATTR_MODULE 200 +#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 201 +#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 202 +#define LOAD_ATTR_PROPERTY 203 +#define LOAD_ATTR_SLOT 204 +#define LOAD_ATTR_WITH_HINT 205 +#define LOAD_GLOBAL_BUILTIN 206 +#define LOAD_GLOBAL_MODULE 207 +#define LOAD_SUPER_ATTR_ATTR 208 +#define LOAD_SUPER_ATTR_METHOD 209 +#define RESUME_CHECK 210 +#define SEND_GEN 211 +#define STORE_ATTR_INSTANCE_VALUE 212 +#define STORE_ATTR_SLOT 213 +#define STORE_ATTR_WITH_HINT 214 +#define STORE_SUBSCR_DICT 215 +#define STORE_SUBSCR_LIST_INT 216 +#define TO_BOOL_ALWAYS_TRUE 217 +#define TO_BOOL_BOOL 218 +#define TO_BOOL_INT 219 +#define TO_BOOL_LIST 220 +#define TO_BOOL_NONE 221 +#define TO_BOOL_STR 222 +#define UNPACK_SEQUENCE_LIST 223 +#define UNPACK_SEQUENCE_TUPLE 224 +#define UNPACK_SEQUENCE_TWO_TUPLE 225 #define INSTRUMENTED_END_FOR 236 #define INSTRUMENTED_END_SEND 237 #define INSTRUMENTED_LOAD_SUPER_ATTR 238 diff --git a/Lib/_android_support.py b/Lib/_android_support.py index d5d13ec6a48e14..066e54708fb75c 100644 --- a/Lib/_android_support.py +++ b/Lib/_android_support.py @@ -31,15 +31,17 @@ def init_streams(android_log_write, stdout_prio, stderr_prio): logcat = Logcat(android_log_write) sys.stdout = TextLogStream( - stdout_prio, "python.stdout", errors=sys.stdout.errors) + stdout_prio, "python.stdout", sys.stdout.fileno(), + errors=sys.stdout.errors) sys.stderr = TextLogStream( - stderr_prio, "python.stderr", errors=sys.stderr.errors) + stderr_prio, "python.stderr", sys.stderr.fileno(), + errors=sys.stderr.errors) class TextLogStream(io.TextIOWrapper): - def __init__(self, prio, tag, **kwargs): + def __init__(self, prio, tag, fileno=None, **kwargs): kwargs.setdefault("encoding", "UTF-8") - super().__init__(BinaryLogStream(prio, tag), **kwargs) + super().__init__(BinaryLogStream(prio, tag, fileno), **kwargs) self._lock = RLock() self._pending_bytes = [] self._pending_bytes_count = 0 @@ -98,9 +100,10 @@ def line_buffering(self): class BinaryLogStream(io.RawIOBase): - def __init__(self, prio, tag): + def __init__(self, prio, tag, fileno=None): self.prio = prio self.tag = tag + self._fileno = fileno def __repr__(self): return f"" @@ -122,6 +125,12 @@ def write(self, b): logcat.write(self.prio, self.tag, b) return len(b) + # This is needed by the test suite --timeout option, which uses faulthandler. + def fileno(self): + if self._fileno is None: + raise io.UnsupportedOperation("fileno") + return self._fileno + # When a large volume of data is written to logcat at once, e.g. when a test # module fails in --verbose3 mode, there's a risk of overflowing logcat's own diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index 94c8a0accf6d62..0f3d9734897559 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -107,6 +107,11 @@ "CALL_BOUND_METHOD_GENERAL", "CALL_NON_PY_GENERAL", ], + "CALL_KW": [ + "CALL_KW_BOUND_METHOD", + "CALL_KW_PY", + "CALL_KW_NON_PY", + ], } _specialized_opmap = { @@ -131,59 +136,62 @@ 'CALL_BUILTIN_FAST_WITH_KEYWORDS': 167, 'CALL_BUILTIN_O': 168, 'CALL_ISINSTANCE': 169, - 'CALL_LEN': 170, - 'CALL_LIST_APPEND': 171, - 'CALL_METHOD_DESCRIPTOR_FAST': 172, - 'CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS': 173, - 'CALL_METHOD_DESCRIPTOR_NOARGS': 174, - 'CALL_METHOD_DESCRIPTOR_O': 175, - 'CALL_NON_PY_GENERAL': 176, - 'CALL_PY_EXACT_ARGS': 177, - 'CALL_PY_GENERAL': 178, - 'CALL_STR_1': 179, - 'CALL_TUPLE_1': 180, - 'CALL_TYPE_1': 181, - 'COMPARE_OP_FLOAT': 182, - 'COMPARE_OP_INT': 183, - 'COMPARE_OP_STR': 184, - 'CONTAINS_OP_DICT': 185, - 'CONTAINS_OP_SET': 186, - 'FOR_ITER_GEN': 187, - 'FOR_ITER_LIST': 188, - 'FOR_ITER_RANGE': 189, - 'FOR_ITER_TUPLE': 190, - 'LOAD_ATTR_CLASS': 191, - 'LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN': 192, - 'LOAD_ATTR_INSTANCE_VALUE': 193, - 'LOAD_ATTR_METHOD_LAZY_DICT': 194, - 'LOAD_ATTR_METHOD_NO_DICT': 195, - 'LOAD_ATTR_METHOD_WITH_VALUES': 196, - 'LOAD_ATTR_MODULE': 197, - 'LOAD_ATTR_NONDESCRIPTOR_NO_DICT': 198, - 'LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES': 199, - 'LOAD_ATTR_PROPERTY': 200, - 'LOAD_ATTR_SLOT': 201, - 'LOAD_ATTR_WITH_HINT': 202, - 'LOAD_GLOBAL_BUILTIN': 203, - 'LOAD_GLOBAL_MODULE': 204, - 'LOAD_SUPER_ATTR_ATTR': 205, - 'LOAD_SUPER_ATTR_METHOD': 206, - 'RESUME_CHECK': 207, - 'SEND_GEN': 208, - 'STORE_ATTR_INSTANCE_VALUE': 209, - 'STORE_ATTR_SLOT': 210, - 'STORE_ATTR_WITH_HINT': 211, - 'STORE_SUBSCR_DICT': 212, - 'STORE_SUBSCR_LIST_INT': 213, - 'TO_BOOL_ALWAYS_TRUE': 214, - 'TO_BOOL_BOOL': 215, - 'TO_BOOL_INT': 216, - 'TO_BOOL_LIST': 217, - 'TO_BOOL_NONE': 218, - 'TO_BOOL_STR': 219, - 'UNPACK_SEQUENCE_LIST': 220, - 'UNPACK_SEQUENCE_TUPLE': 221, - 'UNPACK_SEQUENCE_TWO_TUPLE': 222, + 'CALL_KW_BOUND_METHOD': 170, + 'CALL_KW_NON_PY': 171, + 'CALL_KW_PY': 172, + 'CALL_LEN': 173, + 'CALL_LIST_APPEND': 174, + 'CALL_METHOD_DESCRIPTOR_FAST': 175, + 'CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS': 176, + 'CALL_METHOD_DESCRIPTOR_NOARGS': 177, + 'CALL_METHOD_DESCRIPTOR_O': 178, + 'CALL_NON_PY_GENERAL': 179, + 'CALL_PY_EXACT_ARGS': 180, + 'CALL_PY_GENERAL': 181, + 'CALL_STR_1': 182, + 'CALL_TUPLE_1': 183, + 'CALL_TYPE_1': 184, + 'COMPARE_OP_FLOAT': 185, + 'COMPARE_OP_INT': 186, + 'COMPARE_OP_STR': 187, + 'CONTAINS_OP_DICT': 188, + 'CONTAINS_OP_SET': 189, + 'FOR_ITER_GEN': 190, + 'FOR_ITER_LIST': 191, + 'FOR_ITER_RANGE': 192, + 'FOR_ITER_TUPLE': 193, + 'LOAD_ATTR_CLASS': 194, + 'LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN': 195, + 'LOAD_ATTR_INSTANCE_VALUE': 196, + 'LOAD_ATTR_METHOD_LAZY_DICT': 197, + 'LOAD_ATTR_METHOD_NO_DICT': 198, + 'LOAD_ATTR_METHOD_WITH_VALUES': 199, + 'LOAD_ATTR_MODULE': 200, + 'LOAD_ATTR_NONDESCRIPTOR_NO_DICT': 201, + 'LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES': 202, + 'LOAD_ATTR_PROPERTY': 203, + 'LOAD_ATTR_SLOT': 204, + 'LOAD_ATTR_WITH_HINT': 205, + 'LOAD_GLOBAL_BUILTIN': 206, + 'LOAD_GLOBAL_MODULE': 207, + 'LOAD_SUPER_ATTR_ATTR': 208, + 'LOAD_SUPER_ATTR_METHOD': 209, + 'RESUME_CHECK': 210, + 'SEND_GEN': 211, + 'STORE_ATTR_INSTANCE_VALUE': 212, + 'STORE_ATTR_SLOT': 213, + 'STORE_ATTR_WITH_HINT': 214, + 'STORE_SUBSCR_DICT': 215, + 'STORE_SUBSCR_LIST_INT': 216, + 'TO_BOOL_ALWAYS_TRUE': 217, + 'TO_BOOL_BOOL': 218, + 'TO_BOOL_INT': 219, + 'TO_BOOL_LIST': 220, + 'TO_BOOL_NONE': 221, + 'TO_BOOL_STR': 222, + 'UNPACK_SEQUENCE_LIST': 223, + 'UNPACK_SEQUENCE_TUPLE': 224, + 'UNPACK_SEQUENCE_TWO_TUPLE': 225, } opmap = { diff --git a/Lib/_pyrepl/console.py b/Lib/_pyrepl/console.py index a8d3f520340dcf..330ebbd6185679 100644 --- a/Lib/_pyrepl/console.py +++ b/Lib/_pyrepl/console.py @@ -161,17 +161,22 @@ def __init__( super().__init__(locals=locals, filename=filename, local_exit=local_exit) # type: ignore[call-arg] self.can_colorize = _colorize.can_colorize() - def showsyntaxerror(self, filename=None): - super().showsyntaxerror(colorize=self.can_colorize) + def showsyntaxerror(self, filename=None, **kwargs): + super().showsyntaxerror(**kwargs) - def showtraceback(self): - super().showtraceback(colorize=self.can_colorize) + def _excepthook(self, typ, value, tb): + import traceback + lines = traceback.format_exception( + typ, value, tb, + colorize=self.can_colorize, + limit=traceback.BUILTIN_EXCEPTION_LIMIT) + self.write(''.join(lines)) def runsource(self, source, filename="", symbol="single"): try: tree = ast.parse(source) except (SyntaxError, OverflowError, ValueError): - self.showsyntaxerror(filename) + self.showsyntaxerror(filename, source=source) return False if tree.body: *_, last_stmt = tree.body @@ -188,10 +193,10 @@ def runsource(self, source, filename="", symbol="single"): f"Try the asyncio REPL ({python} -m asyncio) to use" f" top-level 'await' and run background asyncio tasks." ) - self.showsyntaxerror(filename) + self.showsyntaxerror(filename, source=source) return False except (OverflowError, ValueError): - self.showsyntaxerror(filename) + self.showsyntaxerror(filename, source=source) return False if code is None: diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py index 3d94f91753587e..143770a885a2c2 100644 --- a/Lib/_pyrepl/readline.py +++ b/Lib/_pyrepl/readline.py @@ -342,10 +342,10 @@ def do(self) -> None: class _ReadlineWrapper: f_in: int = -1 f_out: int = -1 - reader: ReadlineAlikeReader | None = None + reader: ReadlineAlikeReader | None = field(default=None, repr=False) saved_history_length: int = -1 startup_hook: Callback | None = None - config: ReadlineConfig = field(default_factory=ReadlineConfig) + config: ReadlineConfig = field(default_factory=ReadlineConfig, repr=False) def __post_init__(self) -> None: if self.f_in == -1: diff --git a/Lib/code.py b/Lib/code.py index a1fd389b5a1015..b1079824a75414 100644 --- a/Lib/code.py +++ b/Lib/code.py @@ -64,7 +64,7 @@ def runsource(self, source, filename="", symbol="single"): code = self.compile(source, filename, symbol) except (OverflowError, SyntaxError, ValueError): # Case 1 - self.showsyntaxerror(filename) + self.showsyntaxerror(filename, source=source) return False if code is None: @@ -106,7 +106,6 @@ def showsyntaxerror(self, filename=None, **kwargs): The output is written by self.write(), below. """ - colorize = kwargs.pop('colorize', False) try: typ, value, tb = sys.exc_info() if filename and typ is SyntaxError: @@ -119,11 +118,12 @@ def showsyntaxerror(self, filename=None, **kwargs): else: # Stuff in the right filename value = SyntaxError(msg, (filename, lineno, offset, line)) - self._showtraceback(typ, value, None, colorize) + source = kwargs.pop('source', "") + self._showtraceback(typ, value, None, source) finally: typ = value = tb = None - def showtraceback(self, **kwargs): + def showtraceback(self): """Display the exception that just occurred. We remove the first stack item because it is our own code. @@ -131,21 +131,24 @@ def showtraceback(self, **kwargs): The output is written by self.write(), below. """ - colorize = kwargs.pop('colorize', False) try: typ, value, tb = sys.exc_info() - self._showtraceback(typ, value, tb.tb_next, colorize) + self._showtraceback(typ, value, tb.tb_next, "") finally: typ = value = tb = None - def _showtraceback(self, typ, value, tb, colorize): + def _showtraceback(self, typ, value, tb, source): sys.last_type = typ sys.last_traceback = tb - sys.last_exc = sys.last_value = value = value.with_traceback(tb) + value = value.with_traceback(tb) + # Set the line of text that the exception refers to + lines = source.splitlines() + if (source and typ is SyntaxError + and not value.text and len(lines) >= value.lineno): + value.text = lines[value.lineno - 1] + sys.last_exc = sys.last_value = value if sys.excepthook is sys.__excepthook__: - lines = traceback.format_exception(typ, value, tb, - colorize=colorize) - self.write(''.join(lines)) + self._excepthook(typ, value, tb) else: # If someone has set sys.excepthook, we let that take precedence # over self.write @@ -162,6 +165,12 @@ def _showtraceback(self, typ, value, tb, colorize): print('Original exception was:', file=sys.stderr) sys.__excepthook__(typ, value, tb) + def _excepthook(self, typ, value, tb): + # This method is being overwritten in + # _pyrepl.console.InteractiveColoredConsole + lines = traceback.format_exception(typ, value, tb) + self.write(''.join(lines)) + def write(self, data): """Write a string. diff --git a/Lib/configparser.py b/Lib/configparser.py index 4344a9e8baca44..420dce77c234e1 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -160,7 +160,7 @@ "NoOptionError", "InterpolationError", "InterpolationDepthError", "InterpolationMissingOptionError", "InterpolationSyntaxError", "ParsingError", "MissingSectionHeaderError", - "MultilineContinuationError", + "MultilineContinuationError", "UnnamedSectionDisabledError", "ConfigParser", "RawConfigParser", "Interpolation", "BasicInterpolation", "ExtendedInterpolation", "SectionProxy", "ConverterMapping", @@ -362,6 +362,14 @@ def __init__(self, filename, lineno, line): self.line = line self.args = (filename, lineno, line) + +class UnnamedSectionDisabledError(Error): + """Raised when an attempt to use UNNAMED_SECTION is made with the + feature disabled.""" + def __init__(self): + Error.__init__(self, "Support for UNNAMED_SECTION is disabled.") + + class _UnnamedSection: def __repr__(self): @@ -692,6 +700,10 @@ def add_section(self, section): if section == self.default_section: raise ValueError('Invalid section name: %r' % section) + if section is UNNAMED_SECTION: + if not self._allow_unnamed_section: + raise UnnamedSectionDisabledError + if section in self._sections: raise DuplicateSectionError(section) self._sections[section] = self._dict() @@ -1203,20 +1215,20 @@ def _convert_to_boolean(self, value): return self.BOOLEAN_STATES[value.lower()] def _validate_value_types(self, *, section="", option="", value=""): - """Raises a TypeError for non-string values. + """Raises a TypeError for illegal non-string values. - The only legal non-string value if we allow valueless - options is None, so we need to check if the value is a - string if: - - we do not allow valueless options, or - - we allow valueless options but the value is not None + Legal non-string values are UNNAMED_SECTION and falsey values if + they are allowed. For compatibility reasons this method is not used in classic set() for RawConfigParsers. It is invoked in every case for mapping protocol access and in ConfigParser.set(). """ - if not isinstance(section, str): - raise TypeError("section names must be strings") + if section is UNNAMED_SECTION: + if not self._allow_unnamed_section: + raise UnnamedSectionDisabledError + elif not isinstance(section, str): + raise TypeError("section names must be strings or UNNAMED_SECTION") if not isinstance(option, str): raise TypeError("option keys must be strings") if not self._allow_no_value or value: diff --git a/Lib/http/cookies.py b/Lib/http/cookies.py index 351faf428a20cd..6b9ed24ad8ec78 100644 --- a/Lib/http/cookies.py +++ b/Lib/http/cookies.py @@ -184,8 +184,13 @@ def _quote(str): return '"' + str.translate(_Translator) + '"' -_OctalPatt = re.compile(r"\\[0-3][0-7][0-7]") -_QuotePatt = re.compile(r"[\\].") +_unquote_sub = re.compile(r'\\(?:([0-3][0-7][0-7])|(.))').sub + +def _unquote_replace(m): + if m[1]: + return chr(int(m[1], 8)) + else: + return m[2] def _unquote(str): # If there aren't any doublequotes, @@ -205,30 +210,7 @@ def _unquote(str): # \012 --> \n # \" --> " # - i = 0 - n = len(str) - res = [] - while 0 <= i < n: - o_match = _OctalPatt.search(str, i) - q_match = _QuotePatt.search(str, i) - if not o_match and not q_match: # Neither matched - res.append(str[i:]) - break - # else: - j = k = -1 - if o_match: - j = o_match.start(0) - if q_match: - k = q_match.start(0) - if q_match and (not o_match or k < j): # QuotePatt matched - res.append(str[i:k]) - res.append(str[k+1]) - i = k + 2 - else: # OctalPatt matched - res.append(str[i:j]) - res.append(chr(int(str[j+1:j+4], 8))) - i = j + 4 - return _nulljoin(res) + return _unquote_sub(_unquote_replace, str) # The _getdate() routine is used to set the expiration time in the cookie's HTTP # header. By default, _getdate() returns the current time in the appropriate diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py index d8b2652d5d7979..e882c6cb3b8d19 100755 --- a/Lib/idlelib/pyshell.py +++ b/Lib/idlelib/pyshell.py @@ -706,7 +706,7 @@ def prepend_syspath(self, filename): del _filename, _sys, _dirname, _dir \n""".format(filename)) - def showsyntaxerror(self, filename=None): + def showsyntaxerror(self, filename=None, **kwargs): """Override Interactive Interpreter method: Use Colorizing Color the offending position instead of printing it and pointing at it diff --git a/Lib/importlib/resources/_functional.py b/Lib/importlib/resources/_functional.py index 9e3ea1547d486a..f59416f2dd627d 100644 --- a/Lib/importlib/resources/_functional.py +++ b/Lib/importlib/resources/_functional.py @@ -57,11 +57,7 @@ def contents(anchor, *path_names): DeprecationWarning, stacklevel=1, ) - return ( - resource.name - for resource - in _get_resource(anchor, path_names).iterdir() - ) + return (resource.name for resource in _get_resource(anchor, path_names).iterdir()) def _get_encoding_arg(path_names, encoding): diff --git a/Lib/inspect.py b/Lib/inspect.py index 0a50189ade41c1..90c44cf74007a8 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -970,10 +970,10 @@ def findsource(object): if isclass(object): try: - firstlineno = object.__firstlineno__ - except AttributeError: + firstlineno = vars(object)['__firstlineno__'] + except (TypeError, KeyError): raise OSError('source code not available') - return lines, object.__firstlineno__ - 1 + return lines, firstlineno - 1 if ismethod(object): object = object.__func__ diff --git a/Lib/opcode.py b/Lib/opcode.py index 2698609cd5636d..974f4d35e2a524 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -85,6 +85,10 @@ "counter": 1, "func_version": 2, }, + "CALL_KW": { + "counter": 1, + "func_version": 2, + }, "STORE_SUBSCR": { "counter": 1, }, diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index 500846d19cf811..720756cac66f68 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -585,7 +585,7 @@ def read_bytes(self): """ Open the file in bytes mode, read it, and close the file. """ - with self.open(mode='rb') as f: + with self.open(mode='rb', buffering=0) as f: return f.read() def read_text(self, encoding=None, errors=None, newline=None): diff --git a/Lib/ssl.py b/Lib/ssl.py index a3ecf5380e4e30..c8703b046cfd4b 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -1164,11 +1164,21 @@ def getpeercert(self, binary_form=False): @_sslcopydoc def get_verified_chain(self): - return self._sslobj.get_verified_chain() + chain = self._sslobj.get_verified_chain() + + if chain is None: + return [] + + return [cert.public_bytes(_ssl.ENCODING_DER) for cert in chain] @_sslcopydoc def get_unverified_chain(self): - return self._sslobj.get_unverified_chain() + chain = self._sslobj.get_unverified_chain() + + if chain is None: + return [] + + return [cert.public_bytes(_ssl.ENCODING_DER) for cert in chain] @_sslcopydoc def selected_npn_protocol(self): diff --git a/Lib/test/_test_embed_structseq.py b/Lib/test/_test_embed_structseq.py index 834daa4df55fec..868f9f83e8be77 100644 --- a/Lib/test/_test_embed_structseq.py +++ b/Lib/test/_test_embed_structseq.py @@ -1,31 +1,27 @@ import sys import types +import unittest -# Note: This test file can't import `unittest` since the runtime can't -# currently guarantee that it will not leak memory. Doing so will mark -# the test as passing but with reference leaks. This can safely import -# the `unittest` library once there's a strict guarantee of no leaks -# during runtime shutdown. # bpo-46417: Test that structseq types used by the sys module are still # valid when Py_Finalize()/Py_Initialize() are called multiple times. -class TestStructSeq: +class TestStructSeq(unittest.TestCase): # test PyTypeObject members - def _check_structseq(self, obj_type): + def check_structseq(self, obj_type): # ob_refcnt - assert sys.getrefcount(obj_type) > 1 + self.assertGreaterEqual(sys.getrefcount(obj_type), 1) # tp_base - assert issubclass(obj_type, tuple) + self.assertTrue(issubclass(obj_type, tuple)) # tp_bases - assert obj_type.__bases__ == (tuple,) + self.assertEqual(obj_type.__bases__, (tuple,)) # tp_dict - assert isinstance(obj_type.__dict__, types.MappingProxyType) + self.assertIsInstance(obj_type.__dict__, types.MappingProxyType) # tp_mro - assert obj_type.__mro__ == (obj_type, tuple, object) + self.assertEqual(obj_type.__mro__, (obj_type, tuple, object)) # tp_name - assert isinstance(type.__name__, str) + self.assertIsInstance(type.__name__, str) # tp_subclasses - assert obj_type.__subclasses__() == [] + self.assertEqual(obj_type.__subclasses__(), []) def test_sys_attrs(self): for attr_name in ( @@ -36,23 +32,23 @@ def test_sys_attrs(self): 'thread_info', # ThreadInfoType 'version_info', # VersionInfoType ): - attr = getattr(sys, attr_name) - self._check_structseq(type(attr)) + with self.subTest(attr=attr_name): + attr = getattr(sys, attr_name) + self.check_structseq(type(attr)) def test_sys_funcs(self): func_names = ['get_asyncgen_hooks'] # AsyncGenHooksType if hasattr(sys, 'getwindowsversion'): func_names.append('getwindowsversion') # WindowsVersionType for func_name in func_names: - func = getattr(sys, func_name) - obj = func() - self._check_structseq(type(obj)) + with self.subTest(func=func_name): + func = getattr(sys, func_name) + obj = func() + self.check_structseq(type(obj)) try: - tests = TestStructSeq() - tests.test_sys_attrs() - tests.test_sys_funcs() + unittest.main() except SystemExit as exc: if exc.args[0] != 0: raise diff --git a/Lib/test/certdata/cert3.pem b/Lib/test/certdata/cert3.pem new file mode 100644 index 00000000000000..034bc43ff1974e --- /dev/null +++ b/Lib/test/certdata/cert3.pem @@ -0,0 +1,34 @@ +-----BEGIN CERTIFICATE----- +MIIF8TCCBFmgAwIBAgIJAMstgJlaaVJcMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx +NDIzMTZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj +MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMMCWxv +Y2FsaG9zdDCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAKAqKHEL7aDt +3swl8hQF8VaK4zDGDRaF3E/IZTMwCN7FsQ4ejSiOe3E90f0phHCIpEpv2OebNenY +IpOGoFgkh62r/cthmnhu8Mn+FUIv17iOq7WX7B30OSqEpnr1voLX93XYkAq8LlMh +P79vsSCVhTwow3HZY7krEgl5WlfryOfj1i1TODSFPRCJePh66BsOTUvV/33GC+Qd +pVZVDGLowU1Ycmr/FdRvwT+F39Dehp03UFcxaX0/joPhH5gYpBB1kWTAQmxuqKMW +9ZZs6hrPtMXF/yfSrrXrzTdpct9paKR8RcufOcS8qju/ISK+1P/LXg2b5KJHedLo +TTIO3yCZ4d1odyuZBP7JDrI05gMJx95gz6sG685Qc+52MzLSTwr/Qg+MOjQoBy0o +8fRRVvIMEwoN0ZDb4uFEUuwZceUP1vTk/GGpNQt7ct4ropn6K4Zta3BUtovlLjZa +IIBhc1KETUqjRDvC6ACKmlcJ/5pY/dbH1lOux+IMFsh+djmaV90b3QIDAQABo4IB +wDCCAbwwFAYDVR0RBA0wC4IJbG9jYWxob3N0MA4GA1UdDwEB/wQEAwIFoDAdBgNV +HSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4E +FgQUP7HpT6C+MGY+ChjID0caTzRqD0IwfQYDVR0jBHYwdIAU8+yUjvKOMMSOaMK/ +jmoZwMGfdmWhUaRPME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29m +dHdhcmUgRm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcoIJAMst +gJlaaVJbMIGDBggrBgEFBQcBAQR3MHUwPAYIKwYBBQUHMAKGMGh0dHA6Ly90ZXN0 +Y2EucHl0aG9udGVzdC5uZXQvdGVzdGNhL3B5Y2FjZXJ0LmNlcjA1BggrBgEFBQcw +AYYpaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2Evb2NzcC8wQwYD +VR0fBDwwOjA4oDagNIYyaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0 +Y2EvcmV2b2NhdGlvbi5jcmwwDQYJKoZIhvcNAQELBQADggGBAMo0usXQzycxMtYN +JzC42xfftzmnu7E7hsQx/fur22MazJCruU6rNEkMXow+cKOnay+nmiV7AVoYlkh2 ++DZ4dPq8fWh/5cqmnXvccr2jJVEXaOjp1wKGLH0WfLXcRLIK4/fJM6NRNoO81HDN +hJGfBrot0gUKZcPZVQmouAlpu5OGwrfCkHR8v/BdvA5jE4zr+g/x+uUScE0M64wu +okJCAAQP/PkfQZxjePBmk7KPLuiTHFDLLX+2uldvUmLXOQsJgqumU03MBT4Z8NTA +zqmtEM65ceSP8lo8Zbrcy+AEkCulFaZ92tyjtbe8oN4wTmTLFw06oFLSZzuiOgDV +OaphdVKf/pvA6KBpr6izox0KQFIE5z3AAJZfKzMGDDD20xhy7jjQZNMAhjfsT+k4 +SeYB/6KafNxq08uoulj7w4Z4R/EGpkXnU96ZHYHmvGN0RnxwI1cpYHCazG8AjsK/ +anN9brBi5twTGrn+D8LRBqF5Yn+2MKkD0EdXJdtIENHP+32sPQ== +-----END CERTIFICATE----- \ No newline at end of file diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 2ff4715e82a41b..8bef04cba81138 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -428,9 +428,7 @@ def _parse_args(args, **kwargs): # Continuous Integration (CI): common options for fast/slow CI modes if ns.slow_ci or ns.fast_ci: # Similar to options: - # - # -j0 --randomize --fail-env-changed --fail-rerun --rerun - # --slowest --verbose3 + # -j0 --randomize --fail-env-changed --rerun --slowest --verbose3 if ns.use_mp is None: ns.use_mp = 0 ns.randomize = True diff --git a/Lib/test/test_android.py b/Lib/test/test_android.py index 82035061bb6fdd..09fa21cea877c2 100644 --- a/Lib/test/test_android.py +++ b/Lib/test/test_android.py @@ -19,6 +19,9 @@ api_level = platform.android_ver().api_level +# (name, level, fileno) +STREAM_INFO = [("stdout", "I", 1), ("stderr", "W", 2)] + # Test redirection of stdout and stderr to the Android log. @unittest.skipIf( @@ -94,19 +97,21 @@ def stream_context(self, stream_name, level): stack = ExitStack() stack.enter_context(self.subTest(stream_name)) stream = getattr(sys, stream_name) + native_stream = getattr(sys, f"__{stream_name}__") if isinstance(stream, io.StringIO): stack.enter_context( patch( f"sys.{stream_name}", TextLogStream( - prio, f"python.{stream_name}", errors="backslashreplace" + prio, f"python.{stream_name}", native_stream.fileno(), + errors="backslashreplace" ), ) ) return stack def test_str(self): - for stream_name, level in [("stdout", "I"), ("stderr", "W")]: + for stream_name, level, fileno in STREAM_INFO: with self.stream_context(stream_name, level): stream = getattr(sys, stream_name) tag = f"python.{stream_name}" @@ -114,6 +119,7 @@ def test_str(self): self.assertIs(stream.writable(), True) self.assertIs(stream.readable(), False) + self.assertEqual(stream.fileno(), fileno) self.assertEqual("UTF-8", stream.encoding) self.assertIs(stream.line_buffering, True) self.assertIs(stream.write_through, False) @@ -257,13 +263,14 @@ def __str__(self): write("\n", [s * 51]) # 0 bytes in, 510 bytes out def test_bytes(self): - for stream_name, level in [("stdout", "I"), ("stderr", "W")]: + for stream_name, level, fileno in STREAM_INFO: with self.stream_context(stream_name, level): stream = getattr(sys, stream_name).buffer tag = f"python.{stream_name}" self.assertEqual(f"", repr(stream)) self.assertIs(stream.writable(), True) self.assertIs(stream.readable(), False) + self.assertEqual(stream.fileno(), fileno) def write(b, lines=None, *, write_len=None): if write_len is None: diff --git a/Lib/test/test_configparser.py b/Lib/test/test_configparser.py index a934e493a76391..e3c5d08dd1e7d1 100644 --- a/Lib/test/test_configparser.py +++ b/Lib/test/test_configparser.py @@ -2161,6 +2161,19 @@ def test_no_section(self): self.assertEqual('1', cfg2[configparser.UNNAMED_SECTION]['a']) self.assertEqual('2', cfg2[configparser.UNNAMED_SECTION]['b']) + def test_add_section(self): + cfg = configparser.ConfigParser(allow_unnamed_section=True) + cfg.add_section(configparser.UNNAMED_SECTION) + cfg.set(configparser.UNNAMED_SECTION, 'a', '1') + self.assertEqual('1', cfg[configparser.UNNAMED_SECTION]['a']) + + def test_disabled_error(self): + with self.assertRaises(configparser.MissingSectionHeaderError): + configparser.ConfigParser().read_string("a = 1") + + with self.assertRaises(configparser.UnnamedSectionDisabledError): + configparser.ConfigParser().add_section(configparser.UNNAMED_SECTION) + class MiscTestCase(unittest.TestCase): def test__all__(self): diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index aab43338ece02d..7860c67f082b1f 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -1,6 +1,6 @@ # Run the tests in Programs/_testembed.c (tests for the CPython embedding APIs) from test import support -from test.support import import_helper, os_helper, MS_WINDOWS +from test.support import import_helper, os_helper, threading_helper, MS_WINDOWS import unittest from collections import namedtuple @@ -1802,6 +1802,13 @@ def test_init_main_interpreter_settings(self): self.assertEqual(out, expected) + @threading_helper.requires_working_threading() + def test_init_in_background_thread(self): + # gh-123022: Check that running Py_Initialize() in a background + # thread doesn't crash. + out, err = self.run_embedded_interpreter("test_init_in_background_thread") + self.assertEqual(err, "") + class SetConfigTests(unittest.TestCase): def test_set_config(self): diff --git a/Lib/test/test_http_cookies.py b/Lib/test/test_http_cookies.py index 925c8697f60de6..8879902a6e2f41 100644 --- a/Lib/test/test_http_cookies.py +++ b/Lib/test/test_http_cookies.py @@ -5,6 +5,7 @@ import doctest from http import cookies import pickle +from test import support class CookieTests(unittest.TestCase): @@ -58,6 +59,43 @@ def test_basic(self): for k, v in sorted(case['dict'].items()): self.assertEqual(C[k].value, v) + def test_unquote(self): + cases = [ + (r'a="b=\""', 'b="'), + (r'a="b=\\"', 'b=\\'), + (r'a="b=\="', 'b=='), + (r'a="b=\n"', 'b=n'), + (r'a="b=\042"', 'b="'), + (r'a="b=\134"', 'b=\\'), + (r'a="b=\377"', 'b=\xff'), + (r'a="b=\400"', 'b=400'), + (r'a="b=\42"', 'b=42'), + (r'a="b=\\042"', 'b=\\042'), + (r'a="b=\\134"', 'b=\\134'), + (r'a="b=\\\""', 'b=\\"'), + (r'a="b=\\\042"', 'b=\\"'), + (r'a="b=\134\""', 'b=\\"'), + (r'a="b=\134\042"', 'b=\\"'), + ] + for encoded, decoded in cases: + with self.subTest(encoded): + C = cookies.SimpleCookie() + C.load(encoded) + self.assertEqual(C['a'].value, decoded) + + @support.requires_resource('cpu') + def test_unquote_large(self): + n = 10**6 + for encoded in r'\\', r'\134': + with self.subTest(encoded): + data = 'a="b=' + encoded*n + ';"' + C = cookies.SimpleCookie() + C.load(data) + value = C['a'].value + self.assertEqual(value[:3], 'b=\\') + self.assertEqual(value[-2:], '\\;') + self.assertEqual(len(value), n + 3) + def test_load(self): C = cookies.SimpleCookie() C.load('Customer="WILE_E_COYOTE"; Version=1; Path=/acme') diff --git a/Lib/test/test_importlib/resources/test_functional.py b/Lib/test/test_importlib/resources/test_functional.py index d60a2bedd89798..f65ffe646cf0d4 100644 --- a/Lib/test/test_importlib/resources/test_functional.py +++ b/Lib/test/test_importlib/resources/test_functional.py @@ -1,9 +1,9 @@ import unittest import os -from test.support.warnings_helper import ignore_warnings, check_warnings +from test.support import warnings_helper -import importlib.resources as resources +from importlib import resources # Since the functional API forwards to Traversable, we only test # filesystem resources here -- not zip files, namespace packages etc. @@ -22,8 +22,7 @@ class ModuleAnchorMixin: class FunctionalAPIBase: def _gen_resourcetxt_path_parts(self): - """Yield various names of a text file in anchor02, each in a subTest - """ + """Yield various names of a text file in anchor02, each in a subTest""" for path_parts in ( ('subdirectory', 'subsubdir', 'resource.txt'), ('subdirectory/subsubdir/resource.txt',), @@ -36,7 +35,7 @@ def assertEndsWith(self, string, suffix): """Assert that `string` ends with `suffix`. Used to ignore an architecture-specific UTF-16 byte-order mark.""" - self.assertEqual(string[-len(suffix):], suffix) + self.assertEqual(string[-len(suffix) :], suffix) def test_read_text(self): self.assertEqual( @@ -45,7 +44,10 @@ def test_read_text(self): ) self.assertEqual( resources.read_text( - self.anchor02, 'subdirectory', 'subsubdir', 'resource.txt', + self.anchor02, + 'subdirectory', + 'subsubdir', + 'resource.txt', encoding='utf-8', ), 'a resource', @@ -53,7 +55,9 @@ def test_read_text(self): for path_parts in self._gen_resourcetxt_path_parts(): self.assertEqual( resources.read_text( - self.anchor02, *path_parts, encoding='utf-8', + self.anchor02, + *path_parts, + encoding='utf-8', ), 'a resource', ) @@ -67,13 +71,16 @@ def test_read_text(self): resources.read_text(self.anchor01, 'utf-16.file') self.assertEqual( resources.read_text( - self.anchor01, 'binary.file', encoding='latin1', + self.anchor01, + 'binary.file', + encoding='latin1', ), '\x00\x01\x02\x03', ) self.assertEndsWith( # ignore the BOM resources.read_text( - self.anchor01, 'utf-16.file', + self.anchor01, + 'utf-16.file', errors='backslashreplace', ), 'Hello, UTF-16 world!\n'.encode('utf-16-le').decode( @@ -97,7 +104,8 @@ def test_open_text(self): self.assertEqual(f.read(), 'Hello, UTF-8 world!\n') for path_parts in self._gen_resourcetxt_path_parts(): with resources.open_text( - self.anchor02, *path_parts, + self.anchor02, + *path_parts, encoding='utf-8', ) as f: self.assertEqual(f.read(), 'a resource') @@ -111,11 +119,14 @@ def test_open_text(self): with self.assertRaises(UnicodeDecodeError): f.read() with resources.open_text( - self.anchor01, 'binary.file', encoding='latin1', + self.anchor01, + 'binary.file', + encoding='latin1', ) as f: self.assertEqual(f.read(), '\x00\x01\x02\x03') with resources.open_text( - self.anchor01, 'utf-16.file', + self.anchor01, + 'utf-16.file', errors='backslashreplace', ) as f: self.assertEndsWith( # ignore the BOM @@ -130,16 +141,17 @@ def test_open_binary(self): self.assertEqual(f.read(), b'Hello, UTF-8 world!\n') for path_parts in self._gen_resourcetxt_path_parts(): with resources.open_binary( - self.anchor02, *path_parts, + self.anchor02, + *path_parts, ) as f: self.assertEqual(f.read(), b'a resource') def test_path(self): with resources.path(self.anchor01, 'utf-8.file') as path: - with open(str(path)) as f: + with open(str(path), encoding='utf-8') as f: self.assertEqual(f.read(), 'Hello, UTF-8 world!\n') with resources.path(self.anchor01) as path: - with open(os.path.join(path, 'utf-8.file')) as f: + with open(os.path.join(path, 'utf-8.file'), encoding='utf-8') as f: self.assertEqual(f.read(), 'Hello, UTF-8 world!\n') def test_is_resource(self): @@ -152,32 +164,32 @@ def test_is_resource(self): self.assertTrue(is_resource(self.anchor02, *path_parts)) def test_contents(self): - is_resource = resources.is_resource - with check_warnings((".*contents.*", DeprecationWarning)): + with warnings_helper.check_warnings((".*contents.*", DeprecationWarning)): c = resources.contents(self.anchor01) self.assertGreaterEqual( set(c), {'utf-8.file', 'utf-16.file', 'binary.file', 'subdirectory'}, ) - with ( - self.assertRaises(OSError), - check_warnings((".*contents.*", DeprecationWarning)), - ): + with self.assertRaises(OSError), warnings_helper.check_warnings(( + ".*contents.*", + DeprecationWarning, + )): list(resources.contents(self.anchor01, 'utf-8.file')) + for path_parts in self._gen_resourcetxt_path_parts(): - with ( - self.assertRaises(OSError), - check_warnings((".*contents.*", DeprecationWarning)), - ): + with self.assertRaises(OSError), warnings_helper.check_warnings(( + ".*contents.*", + DeprecationWarning, + )): list(resources.contents(self.anchor01, *path_parts)) - with check_warnings((".*contents.*", DeprecationWarning)): + with warnings_helper.check_warnings((".*contents.*", DeprecationWarning)): c = resources.contents(self.anchor01, 'subdirectory') self.assertGreaterEqual( set(c), {'binary.file'}, ) - @ignore_warnings(category=DeprecationWarning) + @warnings_helper.ignore_warnings(category=DeprecationWarning) def test_common_errors(self): for func in ( resources.read_text, @@ -208,18 +220,24 @@ def test_text_errors(self): # Multiple path arguments need explicit encoding argument. with self.assertRaises(TypeError): func( - self.anchor02, 'subdirectory', - 'subsubdir', 'resource.txt', + self.anchor02, + 'subdirectory', + 'subsubdir', + 'resource.txt', ) class FunctionalAPITest_StringAnchor( - unittest.TestCase, FunctionalAPIBase, StringAnchorMixin, + unittest.TestCase, + FunctionalAPIBase, + StringAnchorMixin, ): pass class FunctionalAPITest_ModuleAnchor( - unittest.TestCase, FunctionalAPIBase, ModuleAnchorMixin, + unittest.TestCase, + FunctionalAPIBase, + ModuleAnchorMixin, ): pass diff --git a/Lib/test/test_inspect/inspect_fodder2.py b/Lib/test/test_inspect/inspect_fodder2.py index bb9d3e88cfbee1..43e9f852022934 100644 --- a/Lib/test/test_inspect/inspect_fodder2.py +++ b/Lib/test/test_inspect/inspect_fodder2.py @@ -315,3 +315,45 @@ def g(): class ClassWithCodeObject: import sys code = sys._getframe(0).f_code + +import enum + +# line 321 +class enum322(enum.Enum): + A = 'a' + +# line 325 +class enum326(enum.IntEnum): + A = 1 + +# line 329 +class flag330(enum.Flag): + A = 1 + +# line 333 +class flag334(enum.IntFlag): + A = 1 + +# line 337 +simple_enum338 = enum.Enum('simple_enum338', 'A') +simple_enum339 = enum.IntEnum('simple_enum339', 'A') +simple_flag340 = enum.Flag('simple_flag340', 'A') +simple_flag341 = enum.IntFlag('simple_flag341', 'A') + +import typing + +# line 345 +class nt346(typing.NamedTuple): + x: int + y: int + +# line 350 +nt351 = typing.NamedTuple('nt351', (('x', int), ('y', int))) + +# line 353 +class td354(typing.TypedDict): + x: int + y: int + +# line 358 +td359 = typing.TypedDict('td359', (('x', int), ('y', int))) diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 730c192d1aa260..81188ad4d1fbe1 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -821,7 +821,7 @@ def test_getsource_on_code_object(self): self.assertSourceEqual(mod.eggs.__code__, 12, 18) def test_getsource_on_generated_class(self): - A = type('A', (), {}) + A = type('A', (unittest.TestCase,), {}) self.assertEqual(inspect.getsourcefile(A), __file__) self.assertEqual(inspect.getfile(A), __file__) self.assertIs(inspect.getmodule(A), sys.modules[__name__]) @@ -929,6 +929,24 @@ def test_anonymous(self): # as argument to another function. self.assertSourceEqual(mod2.anonymous, 55, 55) + def test_enum(self): + self.assertSourceEqual(mod2.enum322, 322, 323) + self.assertSourceEqual(mod2.enum326, 326, 327) + self.assertSourceEqual(mod2.flag330, 330, 331) + self.assertSourceEqual(mod2.flag334, 334, 335) + self.assertRaises(OSError, inspect.getsource, mod2.simple_enum338) + self.assertRaises(OSError, inspect.getsource, mod2.simple_enum339) + self.assertRaises(OSError, inspect.getsource, mod2.simple_flag340) + self.assertRaises(OSError, inspect.getsource, mod2.simple_flag341) + + def test_namedtuple(self): + self.assertSourceEqual(mod2.nt346, 346, 348) + self.assertRaises(OSError, inspect.getsource, mod2.nt351) + + def test_typeddict(self): + self.assertSourceEqual(mod2.td354, 354, 356) + self.assertRaises(OSError, inspect.getsource, mod2.td359) + class TestBlockComments(GetSourceBase): fodderModule = mod @@ -1550,6 +1568,56 @@ def f(self): self.assertIn(('f', b.f), inspect.getmembers(b)) self.assertIn(('f', b.f), inspect.getmembers(b, inspect.ismethod)) + def test_getmembers_custom_dir(self): + class CorrectDir: + def __init__(self, attr): + self.attr = attr + def method(self): + return self.attr + 1 + def __dir__(self): + return ['attr', 'method'] + + cd = CorrectDir(5) + self.assertEqual(inspect.getmembers(cd), [ + ('attr', 5), + ('method', cd.method), + ]) + self.assertEqual(inspect.getmembers(cd, inspect.ismethod), [ + ('method', cd.method), + ]) + + def test_getmembers_custom_broken_dir(self): + # inspect.getmembers calls `dir()` on the passed object inside. + # if `__dir__` mentions some non-existent attribute, + # we still need to return others correctly. + class BrokenDir: + existing = 1 + def method(self): + return self.existing + 1 + def __dir__(self): + return ['method', 'missing', 'existing'] + + bd = BrokenDir() + self.assertEqual(inspect.getmembers(bd), [ + ('existing', 1), + ('method', bd.method), + ]) + self.assertEqual(inspect.getmembers(bd, inspect.ismethod), [ + ('method', bd.method), + ]) + + def test_getmembers_custom_duplicated_dir(self): + # Duplicates in `__dir__` must not fail and return just one result. + class DuplicatedDir: + attr = 1 + def __dir__(self): + return ['attr', 'attr'] + + dd = DuplicatedDir() + self.assertEqual(inspect.getmembers(dd), [ + ('attr', 1), + ]) + def test_getmembers_VirtualAttribute(self): class M(type): def __getattr__(cls, name): diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py index 629a1d4bdeb4de..f222fd5b1ec082 100644 --- a/Lib/test/test_pathlib/test_pathlib_abc.py +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -1464,7 +1464,7 @@ def stat(self, *, follow_symlinks=True): def open(self, mode='r', buffering=-1, encoding=None, errors=None, newline=None): - if buffering != -1: + if buffering != -1 and not (buffering == 0 and 'b' in mode): raise NotImplementedError path_obj = self.resolve() path = str(path_obj) diff --git a/Lib/test/test_patma.py b/Lib/test/test_patma.py index 1bdab125dc6ef0..8325b83a5932b9 100644 --- a/Lib/test/test_patma.py +++ b/Lib/test/test_patma.py @@ -1,6 +1,7 @@ import array import collections import dataclasses +import dis import enum import inspect import sys @@ -3377,6 +3378,24 @@ class Keys: self.assertIs(y, None) self.assertIs(z, None) +class TestSourceLocations(unittest.TestCase): + def test_jump_threading(self): + # See gh-123048 + def f(): + x = 0 + v = 1 + match v: + case 1: + if x < 0: + x = 1 + case 2: + if x < 0: + x = 1 + x += 1 + + for inst in dis.get_instructions(f): + if inst.opcode in dis.hasjump: + self.assertIsNotNone(inst.positions.lineno, "jump without location") class TestTracing(unittest.TestCase): diff --git a/Lib/test/test_pyrepl/test_interact.py b/Lib/test/test_pyrepl/test_interact.py index 369dab316af132..b7adaffbac0e22 100644 --- a/Lib/test/test_pyrepl/test_interact.py +++ b/Lib/test/test_pyrepl/test_interact.py @@ -88,6 +88,20 @@ def test_runsource_returns_false_for_failed_compilation(self): self.assertFalse(result) self.assertIn('SyntaxError', f.getvalue()) + @force_not_colorized + def test_runsource_show_syntax_error_location(self): + console = InteractiveColoredConsole() + source = "def f(x, x): ..." + f = io.StringIO() + with contextlib.redirect_stderr(f): + result = console.runsource(source) + self.assertFalse(result) + r = """ + def f(x, x): ... + ^ +SyntaxError: duplicate argument 'x' in function definition""" + self.assertIn(r, f.getvalue()) + def test_runsource_shows_syntax_error_for_failed_compilation(self): console = InteractiveColoredConsole() source = "print('Hello, world!'" diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py index d5eafc5eb58cac..e11ec74aa14058 100644 --- a/Lib/test/test_pyrepl/test_pyrepl.py +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -1020,7 +1020,7 @@ def test_dumb_terminal_exits_cleanly(self): env.update({"TERM": "dumb"}) output, exit_code = self.run_repl("exit()\n", env=env) self.assertEqual(exit_code, 0) - self.assertIn("warning: can\'t use pyrepl", output) + self.assertIn("warning: can't use pyrepl", output) self.assertNotIn("Exception", output) self.assertNotIn("Traceback", output) @@ -1100,6 +1100,38 @@ def test_not_wiping_history_file(self): self.assertIn("spam", output) self.assertNotEqual(pathlib.Path(hfile.name).stat().st_size, 0) + @force_not_colorized + def test_proper_tracebacklimit(self): + env = os.environ.copy() + for set_tracebacklimit in [True, False]: + commands = ("import sys\n" + + ("sys.tracebacklimit = 1\n" if set_tracebacklimit else "") + + "def x1(): 1/0\n\n" + "def x2(): x1()\n\n" + "def x3(): x2()\n\n" + "x3()\n" + "exit()\n") + + for basic_repl in [True, False]: + if basic_repl: + env["PYTHON_BASIC_REPL"] = "1" + else: + env.pop("PYTHON_BASIC_REPL", None) + with self.subTest(set_tracebacklimit=set_tracebacklimit, + basic_repl=basic_repl): + output, exit_code = self.run_repl(commands, env=env) + if "can't use pyrepl" in output: + self.skipTest("pyrepl not available") + self.assertIn("in x1", output) + if set_tracebacklimit: + self.assertNotIn("in x2", output) + self.assertNotIn("in x3", output) + self.assertNotIn("in ", output) + else: + self.assertIn("in x2", output) + self.assertIn("in x3", output) + self.assertIn("in ", output) + def run_repl( self, repl_input: str | list[str], diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 6ec010d13f9e7e..9c415bd7d1c4e4 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -103,6 +103,7 @@ def data_file(*name): # Two keys and certs signed by the same CA (for SNI tests) SIGNED_CERTFILE = data_file("keycert3.pem") +SINGED_CERTFILE_ONLY = data_file("cert3.pem") SIGNED_CERTFILE_HOSTNAME = 'localhost' SIGNED_CERTFILE_INFO = { @@ -4720,6 +4721,40 @@ def test_internal_chain_client(self): ssl.PEM_cert_to_DER_cert(pem), der ) + def test_certificate_chain(self): + client_context, server_context, hostname = testing_context( + server_chain=False + ) + server = ThreadedEchoServer(context=server_context, chatty=False) + + with open(SIGNING_CA) as f: + expected_ca_cert = ssl.PEM_cert_to_DER_cert(f.read()) + + with open(SINGED_CERTFILE_ONLY) as f: + expected_ee_cert = ssl.PEM_cert_to_DER_cert(f.read()) + + with server: + with client_context.wrap_socket( + socket.socket(), + server_hostname=hostname + ) as s: + s.connect((HOST, server.port)) + vc = s.get_verified_chain() + self.assertEqual(len(vc), 2) + + ee, ca = vc + self.assertIsInstance(ee, bytes) + self.assertIsInstance(ca, bytes) + self.assertEqual(expected_ca_cert, ca) + self.assertEqual(expected_ee_cert, ee) + + uvc = s.get_unverified_chain() + self.assertEqual(len(uvc), 1) + self.assertIsInstance(uvc[0], bytes) + + self.assertEqual(ee, uvc[0]) + self.assertNotEqual(ee, ca) + def test_internal_chain_server(self): client_context, server_context, hostname = testing_context() client_context.load_cert_chain(SIGNED_CERTFILE) diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 1895c88d23b70d..b568221212d71f 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -699,6 +699,35 @@ def f_with_multiline(): result_lines = self.get_exception(f_with_multiline) self.assertEqual(result_lines, expected_f.splitlines()) + # Check custom error messages covering multiple lines + code = textwrap.dedent(""" + dummy_call( + "dummy value" + foo="bar", + ) + """) + + def f_with_multiline(): + # Need to defer the compilation until in self.get_exception(..) + return compile(code, "?", "exec") + + lineno_f = f_with_multiline.__code__.co_firstlineno + + expected_f = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+2}, in f_with_multiline\n' + ' return compile(code, "?", "exec")\n' + ' File "?", line 3\n' + ' "dummy value"\n' + ' ^^^^^^^^^^^^^' + ) + + result_lines = self.get_exception(f_with_multiline) + self.assertEqual(result_lines, expected_f.splitlines()) + def test_caret_multiline_expression_bin_op(self): # Make sure no carets are printed for expressions spanning multiple # lines. @@ -2312,19 +2341,22 @@ def test_message_none(self): def test_syntax_error_various_offsets(self): for offset in range(-5, 10): for add in [0, 2]: - text = " "*add + "text%d" % offset + text = " " * add + "text%d" % offset expected = [' File "file.py", line 1'] if offset < 1: expected.append(" %s" % text.lstrip()) elif offset <= 6: expected.append(" %s" % text.lstrip()) - expected.append(" %s^" % (" "*(offset-1))) + # Set the caret length to match the length of the text minus the offset. + caret_length = max(1, len(text.lstrip()) - offset + 1) + expected.append(" %s%s" % (" " * (offset - 1), "^" * caret_length)) else: + caret_length = max(1, len(text.lstrip()) - 4) expected.append(" %s" % text.lstrip()) - expected.append(" %s^" % (" "*5)) + expected.append(" %s%s" % (" " * 5, "^" * caret_length)) expected.append("SyntaxError: msg") expected.append("") - err = self.get_report(SyntaxError("msg", ("file.py", 1, offset+add, text))) + err = self.get_report(SyntaxError("msg", ("file.py", 1, offset + add, text))) exp = "\n".join(expected) self.assertEqual(exp, err) diff --git a/Lib/test/test_unittest/testmock/testasync.py b/Lib/test/test_unittest/testmock/testasync.py index f57b83f457f279..73f04291373f91 100644 --- a/Lib/test/test_unittest/testmock/testasync.py +++ b/Lib/test/test_unittest/testmock/testasync.py @@ -8,7 +8,7 @@ support.requires_working_socket(module=True) -from asyncio import run, iscoroutinefunction +from asyncio import run from unittest import IsolatedAsyncioTestCase from unittest.mock import (ANY, call, AsyncMock, patch, MagicMock, Mock, create_autospec, sentinel, _CallList, seal) @@ -60,7 +60,7 @@ class AsyncPatchDecoratorTest(unittest.TestCase): def test_is_coroutine_function_patch(self): @patch.object(AsyncClass, 'async_method') def test_async(mock_method): - self.assertTrue(iscoroutinefunction(mock_method)) + self.assertTrue(inspect.iscoroutinefunction(mock_method)) test_async() def test_is_async_patch(self): @@ -121,7 +121,7 @@ class AsyncPatchCMTest(unittest.TestCase): def test_is_async_function_cm(self): def test_async(): with patch.object(AsyncClass, 'async_method') as mock_method: - self.assertTrue(iscoroutinefunction(mock_method)) + self.assertTrue(inspect.iscoroutinefunction(mock_method)) test_async() @@ -155,7 +155,7 @@ def test_patch_dict_async_def(self): async def test_async(): self.assertEqual(foo['a'], 'b') - self.assertTrue(iscoroutinefunction(test_async)) + self.assertTrue(inspect.iscoroutinefunction(test_async)) run(test_async()) def test_patch_dict_async_def_context(self): @@ -170,12 +170,11 @@ async def test_async(): class AsyncMockTest(unittest.TestCase): def test_iscoroutinefunction_default(self): mock = AsyncMock() - self.assertTrue(iscoroutinefunction(mock)) + self.assertTrue(inspect.iscoroutinefunction(mock)) def test_iscoroutinefunction_function(self): async def foo(): pass mock = AsyncMock(foo) - self.assertTrue(iscoroutinefunction(mock)) self.assertTrue(inspect.iscoroutinefunction(mock)) def test_isawaitable(self): @@ -188,7 +187,6 @@ def test_isawaitable(self): def test_iscoroutinefunction_normal_function(self): def foo(): pass mock = AsyncMock(foo) - self.assertTrue(iscoroutinefunction(mock)) self.assertTrue(inspect.iscoroutinefunction(mock)) def test_future_isfuture(self): @@ -231,7 +229,6 @@ async def main(): run(main()) - self.assertTrue(iscoroutinefunction(spec)) self.assertTrue(inspect.iscoroutinefunction(spec)) self.assertTrue(asyncio.iscoroutine(awaitable)) self.assertTrue(inspect.iscoroutine(awaitable)) @@ -273,7 +270,6 @@ async def test_async(): awaitable = mock_method(1, 2, c=3) self.assertIsInstance(mock_method.mock, AsyncMock) - self.assertTrue(iscoroutinefunction(mock_method)) self.assertTrue(inspect.iscoroutinefunction(mock_method)) self.assertTrue(asyncio.iscoroutine(awaitable)) self.assertTrue(inspect.iscoroutine(awaitable)) @@ -430,13 +426,13 @@ def test_async(async_method): def test_is_async_AsyncMock(self): mock = AsyncMock(spec_set=AsyncClass.async_method) - self.assertTrue(iscoroutinefunction(mock)) + self.assertTrue(inspect.iscoroutinefunction(mock)) self.assertIsInstance(mock, AsyncMock) def test_is_child_AsyncMock(self): mock = MagicMock(spec_set=AsyncClass) - self.assertTrue(iscoroutinefunction(mock.async_method)) - self.assertFalse(iscoroutinefunction(mock.normal_method)) + self.assertTrue(inspect.iscoroutinefunction(mock.async_method)) + self.assertFalse(inspect.iscoroutinefunction(mock.normal_method)) self.assertIsInstance(mock.async_method, AsyncMock) self.assertIsInstance(mock.normal_method, MagicMock) self.assertIsInstance(mock, MagicMock) @@ -606,8 +602,8 @@ def test_magic_methods_are_async_functions(self): self.assertIsInstance(m_mock.__aenter__, AsyncMock) self.assertIsInstance(m_mock.__aexit__, AsyncMock) # AsyncMocks are also coroutine functions - self.assertTrue(iscoroutinefunction(m_mock.__aenter__)) - self.assertTrue(iscoroutinefunction(m_mock.__aexit__)) + self.assertTrue(inspect.iscoroutinefunction(m_mock.__aenter__)) + self.assertTrue(inspect.iscoroutinefunction(m_mock.__aexit__)) class AsyncContextManagerTest(unittest.TestCase): @@ -746,11 +742,11 @@ def inner_test(mock_type): mock_instance = mock_type(instance) # Check that the mock and the real thing bahave the same # __aiter__ is not actually async, so not a coroutinefunction - self.assertFalse(iscoroutinefunction(instance.__aiter__)) - self.assertFalse(iscoroutinefunction(mock_instance.__aiter__)) + self.assertFalse(inspect.iscoroutinefunction(instance.__aiter__)) + self.assertFalse(inspect.iscoroutinefunction(mock_instance.__aiter__)) # __anext__ is async - self.assertTrue(iscoroutinefunction(instance.__anext__)) - self.assertTrue(iscoroutinefunction(mock_instance.__anext__)) + self.assertTrue(inspect.iscoroutinefunction(instance.__anext__)) + self.assertTrue(inspect.iscoroutinefunction(mock_instance.__anext__)) for mock_type in [AsyncMock, MagicMock]: with self.subTest(f"test aiter and anext corourtine with {mock_type}"): @@ -806,7 +802,7 @@ def test_assert_called_but_not_awaited(self): mock = AsyncMock(AsyncClass) with assertNeverAwaited(self): mock.async_method() - self.assertTrue(iscoroutinefunction(mock.async_method)) + self.assertTrue(inspect.iscoroutinefunction(mock.async_method)) mock.async_method.assert_called() mock.async_method.assert_called_once() mock.async_method.assert_called_once_with() diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index eed0599642edfb..19179fdc9508ca 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -8,6 +8,7 @@ import os import io +import ftplib import socket import array import sys @@ -754,7 +755,6 @@ def connect_ftp(self, user, passwd, host, port, dirs, self.ftpwrapper = MockFTPWrapper(self.data) return self.ftpwrapper - import ftplib data = "rheum rhaponicum" h = NullFTPHandler(data) h.parent = MockOpener() @@ -794,6 +794,27 @@ def connect_ftp(self, user, passwd, host, port, dirs, self.assertEqual(headers.get("Content-type"), mimetype) self.assertEqual(int(headers["Content-length"]), len(data)) + def test_ftp_error(self): + class ErrorFTPHandler(urllib.request.FTPHandler): + def __init__(self, exception): + self._exception = exception + + def connect_ftp(self, user, passwd, host, port, dirs, + timeout=socket._GLOBAL_DEFAULT_TIMEOUT): + raise self._exception + + exception = ftplib.error_perm( + "500 OOPS: cannot change directory:/nonexistent") + h = ErrorFTPHandler(exception) + urlopen = urllib.request.build_opener(h).open + try: + urlopen("ftp://www.pythontest.net/") + except urllib.error.URLError as raised: + self.assertEqual(raised.reason, + f"ftp error: {exception.args[0]}") + else: + self.fail("Did not raise ftplib exception") + def test_file(self): import email.utils h = urllib.request.FileHandler() diff --git a/Lib/test/test_weakref.py b/Lib/test/test_weakref.py index 2b9b2a04db8298..023df68fca7356 100644 --- a/Lib/test/test_weakref.py +++ b/Lib/test/test_weakref.py @@ -123,10 +123,12 @@ def test_basic_ref(self): def test_ref_repr(self): obj = C() ref = weakref.ref(obj) - self.assertRegex(repr(ref), - rf"") + regex = ( + rf"" + ) + self.assertRegex(repr(ref), regex) obj = None gc_collect() @@ -141,10 +143,13 @@ def __name__(self): obj2 = WithName() ref2 = weakref.ref(obj2) - self.assertRegex(repr(ref2), - rf"") + regex = ( + rf"" + ) + self.assertRegex(repr(ref2), regex) def test_repr_failure_gh99184(self): class MyConfig(dict): @@ -229,10 +234,12 @@ def check(proxy): def test_proxy_repr(self): obj = C() ref = weakref.proxy(obj, self.callback) - self.assertRegex(repr(ref), - rf"") + regex = ( + rf"" + ) + self.assertRegex(repr(ref), regex) obj = None gc_collect() diff --git a/Lib/traceback.py b/Lib/traceback.py index 6ee1a50ca6804a..3e708c6f86a4c5 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -1292,11 +1292,15 @@ def _format_syntax_error(self, stype, **kwargs): yield ' {}\n'.format(ltext) else: offset = self.offset - end_offset = self.end_offset if self.end_offset not in {None, 0} else offset + if self.lineno == self.end_lineno: + end_offset = self.end_offset if self.end_offset not in {None, 0} else offset + else: + end_offset = len(rtext) + 1 + if self.text and offset > len(self.text): - offset = len(self.text) + 1 + offset = len(rtext) + 1 if self.text and end_offset > len(self.text): - end_offset = len(self.text) + 1 + end_offset = len(rtext) + 1 if offset >= end_offset or end_offset < 0: end_offset = offset + 1 diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index 58b0cb574a764a..bc35d8a80e5d03 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -1555,7 +1555,7 @@ def ftp_open(self, req): headers = email.message_from_string(headers) return addinfourl(fp, headers, req.full_url) except ftplib.all_errors as exp: - raise URLError(exp) from exp + raise URLError(f"ftp error: {exp}") from exp def connect_ftp(self, user, passwd, host, port, dirs, timeout): return ftpwrapper(user, passwd, host, port, dirs, timeout, diff --git a/Makefile.pre.in b/Makefile.pre.in index 3a1c1d3d21762d..1b34cb4da2ac45 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -222,6 +222,7 @@ LIBMPDEC_A= Modules/_decimal/libmpdec/libmpdec.a LIBEXPAT_A= Modules/expat/libexpat.a LIBHACL_SHA2_A= Modules/_hacl/libHacl_Hash_SHA2.a LIBHACL_BLAKE2_A= Modules/_hacl/libHacl_Hash_Blake2.a +LIBHACL_CFLAGS=@LIBHACL_CFLAGS@ LIBHACL_SIMD128_FLAGS=@LIBHACL_SIMD128_FLAGS@ LIBHACL_SIMD256_FLAGS=@LIBHACL_SIMD256_FLAGS@ LIBHACL_SIMD128_OBJS=@LIBHACL_SIMD128_OBJS@ @@ -1368,7 +1369,6 @@ $(LIBEXPAT_A): $(LIBEXPAT_OBJS) # Build HACL* static libraries for hashlib: libHacl_Hash_SHA2.a, and # libHacl_Blake2.a -- the contents of the latter vary depending on whether we # have the ability to compile vectorized versions -LIBHACL_CFLAGS=-I$(srcdir)/Modules/_hacl -I$(srcdir)/Modules/_hacl/include -D_BSD_SOURCE -D_DEFAULT_SOURCE $(PY_STDMODULE_CFLAGS) $(CCSHARED) Modules/_hacl/Hacl_Hash_SHA2.o: $(srcdir)/Modules/_hacl/Hacl_Hash_SHA2.c $(LIBHACL_SHA2_HEADERS) $(CC) -c $(LIBHACL_CFLAGS) -o $@ $(srcdir)/Modules/_hacl/Hacl_Hash_SHA2.c diff --git a/Misc/NEWS.d/next/Build/2024-08-14-19-17-34.gh-issue-121634.eOMfHG.rst b/Misc/NEWS.d/next/Build/2024-08-14-19-17-34.gh-issue-121634.eOMfHG.rst new file mode 100644 index 00000000000000..025b6bca809898 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-08-14-19-17-34.gh-issue-121634.eOMfHG.rst @@ -0,0 +1 @@ +Allow for specifying the target compile triple for WASI. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-15-18-11-48.gh-issue-113190.OwQX64.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-15-18-11-48.gh-issue-113190.OwQX64.rst new file mode 100644 index 00000000000000..4c12870c3df548 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-01-15-18-11-48.gh-issue-113190.OwQX64.rst @@ -0,0 +1 @@ +:c:func:`Py_Finalize` now deletes all interned strings. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-08-18-18-25-54.gh-issue-123123.0ZcaEB.rst b/Misc/NEWS.d/next/Core and Builtins/2024-08-18-18-25-54.gh-issue-123123.0ZcaEB.rst new file mode 100644 index 00000000000000..824d307bb270a2 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-08-18-18-25-54.gh-issue-123123.0ZcaEB.rst @@ -0,0 +1,2 @@ +Fix displaying :exc:`SyntaxError` exceptions covering multiple lines. Patch +by Pablo Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-08-20-11-09-16.gh-issue-123048.2TISpv.rst b/Misc/NEWS.d/next/Core and Builtins/2024-08-20-11-09-16.gh-issue-123048.2TISpv.rst new file mode 100644 index 00000000000000..f0b756febbc1b8 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-08-20-11-09-16.gh-issue-123048.2TISpv.rst @@ -0,0 +1,2 @@ +Fix a bug where pattern matching code could emit a :opcode:`JUMP_FORWARD` +with no source location. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-08-14-11-38-56.gh-issue-118093.3BywDP.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-08-14-11-38-56.gh-issue-118093.3BywDP.rst new file mode 100644 index 00000000000000..4a3a094a6fc074 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-08-14-11-38-56.gh-issue-118093.3BywDP.rst @@ -0,0 +1,5 @@ +Add three specializations for :opcode:`CALL_KW`: + +* :opcode:`!CALL_KW_PY` for calls to Python functions +* :opcode:`!CALL_KW_BOUND_METHOD` for calls to bound methods +* :opcode:`!CALL_KW_NON_PY` for all other calls diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-08-15-19-28-43.gh-issue-123022.m3EF9E.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-08-15-19-28-43.gh-issue-123022.m3EF9E.rst new file mode 100644 index 00000000000000..47107dee44eec3 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-08-15-19-28-43.gh-issue-123022.m3EF9E.rst @@ -0,0 +1,2 @@ +Fix crash in free-threaded build when calling :c:func:`Py_Initialize` from +a non-main thread. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-08-19-15-13-13.gh-issue-118093.dLZ8qS.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-08-19-15-13-13.gh-issue-118093.dLZ8qS.rst new file mode 100644 index 00000000000000..d8127d8b5054fe --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-08-19-15-13-13.gh-issue-118093.dLZ8qS.rst @@ -0,0 +1,3 @@ +Break up ``CALL_ALLOC_AND_ENTER_INIT`` into micro-ops and relax +requirement for exact args, in order to increase the amount of code +supported by tier 2. diff --git a/Misc/NEWS.d/next/Library/2023-09-19-17-56-24.gh-issue-109109.WJvvX2.rst b/Misc/NEWS.d/next/Library/2023-09-19-17-56-24.gh-issue-109109.WJvvX2.rst new file mode 100644 index 00000000000000..e741e60ff41a9b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-09-19-17-56-24.gh-issue-109109.WJvvX2.rst @@ -0,0 +1,5 @@ +You can now get the raw TLS certificate chains from TLS connections via +:meth:`ssl.SSLSocket.get_verified_chain` and +:meth:`ssl.SSLSocket.get_unverified_chain` methods. + +Contributed by Mateusz Nowak. diff --git a/Misc/NEWS.d/next/Library/2024-07-16-20-49-07.gh-issue-121804.gYN-In.rst b/Misc/NEWS.d/next/Library/2024-07-16-20-49-07.gh-issue-121804.gYN-In.rst new file mode 100644 index 00000000000000..1cc1cde7c22704 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-16-20-49-07.gh-issue-121804.gYN-In.rst @@ -0,0 +1,2 @@ +Correctly show error locations, when :exc:`SyntaxError` raised in new repl. +Patch by Sergey B Kirpichev. diff --git a/Misc/NEWS.d/next/Library/2024-07-21-10-45-24.gh-issue-122081.dNrYMq.rst b/Misc/NEWS.d/next/Library/2024-07-21-10-45-24.gh-issue-122081.dNrYMq.rst new file mode 100644 index 00000000000000..4c988b16168047 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-21-10-45-24.gh-issue-122081.dNrYMq.rst @@ -0,0 +1,2 @@ +Fix a crash in the :func:`!decimal.IEEEContext` optional function +available via the ``EXTRA_FUNCTIONALITY`` configuration flag. diff --git a/Misc/NEWS.d/next/Library/2024-07-22-08-57-28.gh-issue-120754.Eo5puP.rst b/Misc/NEWS.d/next/Library/2024-07-22-08-57-28.gh-issue-120754.Eo5puP.rst new file mode 100644 index 00000000000000..daf184153646cd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-22-08-57-28.gh-issue-120754.Eo5puP.rst @@ -0,0 +1 @@ +``Pathlib.read_bytes`` no longer opens the file in Python's buffered I/O mode. This reduces overheads as the code reads a file in whole leading to a modest speedup. diff --git a/Misc/NEWS.d/next/Library/2024-07-31-14-55-41.gh-issue-82378.eZvYmR.rst b/Misc/NEWS.d/next/Library/2024-07-31-14-55-41.gh-issue-82378.eZvYmR.rst new file mode 100644 index 00000000000000..8af016e7c82fcb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-31-14-55-41.gh-issue-82378.eZvYmR.rst @@ -0,0 +1,2 @@ +Make sure that the new :term:`REPL` interprets :data:`sys.tracebacklimit` in +the same way that the classic REPL did. diff --git a/Misc/NEWS.d/next/Library/2024-08-14-10-41-11.gh-issue-122981.BHV0Z9.rst b/Misc/NEWS.d/next/Library/2024-08-14-10-41-11.gh-issue-122981.BHV0Z9.rst new file mode 100644 index 00000000000000..7713d805155f9a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-08-14-10-41-11.gh-issue-122981.BHV0Z9.rst @@ -0,0 +1,2 @@ +Fix :func:`inspect.getsource` for generated classes with Python base classes +(e.g. enums). diff --git a/Misc/NEWS.d/next/Library/2024-08-16-16-53-52.gh-issue-123049.izx_fH.rst b/Misc/NEWS.d/next/Library/2024-08-16-16-53-52.gh-issue-123049.izx_fH.rst new file mode 100644 index 00000000000000..2faf85092a0f8a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-08-16-16-53-52.gh-issue-123049.izx_fH.rst @@ -0,0 +1,2 @@ +Add support for :const:`~configparser.UNNAMED_SECTION` +in :meth:`configparser.ConfigParser.add_section`. diff --git a/Misc/NEWS.d/next/Library/2024-08-16-19-13-21.gh-issue-123067.Nx9O4R.rst b/Misc/NEWS.d/next/Library/2024-08-16-19-13-21.gh-issue-123067.Nx9O4R.rst new file mode 100644 index 00000000000000..6a234561fe31a3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-08-16-19-13-21.gh-issue-123067.Nx9O4R.rst @@ -0,0 +1 @@ +Fix quadratic complexity in parsing ``"``-quoted cookie values with backslashes by :mod:`http.cookies`. diff --git a/Misc/NEWS.d/next/Library/2024-08-19-17-37-18.gh-issue-122909.kP12SK.rst b/Misc/NEWS.d/next/Library/2024-08-19-17-37-18.gh-issue-122909.kP12SK.rst new file mode 100644 index 00000000000000..50eb4afd10791b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-08-19-17-37-18.gh-issue-122909.kP12SK.rst @@ -0,0 +1,3 @@ +In urllib.request when URLError is raised opening an ftp URL, the exception +argument is now consistently a string. Earlier versions passed either a +string or an ftplib exception instance as the argument to URLError. diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index 94a2cc2c8e5f8a..15855e2cf1bb32 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -1519,7 +1519,7 @@ init_extended_context(PyObject *v) #ifdef EXTRA_FUNCTIONALITY /* Factory function for creating IEEE interchange format contexts */ static PyObject * -ieee_context(PyObject *dummy UNUSED, PyObject *v) +ieee_context(PyObject *module, PyObject *v) { PyObject *context; mpd_ssize_t bits; @@ -1536,7 +1536,7 @@ ieee_context(PyObject *dummy UNUSED, PyObject *v) goto error; } - decimal_state *state = get_module_state_by_def(Py_TYPE(v)); + decimal_state *state = get_module_state(module); context = PyObject_CallObject((PyObject *)state->PyDecContext_Type, NULL); if (context == NULL) { return NULL; diff --git a/Modules/_math.h b/Modules/_math.h index b8477d2752b7cc..2285b64747c0bd 100644 --- a/Modules/_math.h +++ b/Modules/_math.h @@ -23,42 +23,3 @@ _Py_log1p(double x) } #define m_log1p _Py_log1p - -/* - wrapper for atan2 that deals directly with special cases before - delegating to the platform libm for the remaining cases. This - is necessary to get consistent behaviour across platforms. - Windows, FreeBSD and alpha Tru64 are amongst platforms that don't - always follow C99. Windows screws up atan2 for inf and nan, and - alpha Tru64 5.1 doesn't follow C99 for atan2(0., 0.). -*/ - -static double -_Py_atan2(double y, double x) -{ - if (isnan(x) || isnan(y)) - return Py_NAN; - if (isinf(y)) { - if (isinf(x)) { - if (copysign(1., x) == 1.) - /* atan2(+-inf, +inf) == +-pi/4 */ - return copysign(0.25*Py_MATH_PI, y); - else - /* atan2(+-inf, -inf) == +-pi*3/4 */ - return copysign(0.75*Py_MATH_PI, y); - } - /* atan2(+-inf, x) == +-pi/2 for finite x */ - return copysign(0.5*Py_MATH_PI, y); - } - if (isinf(x) || y == 0.) { - if (copysign(1., x) == 1.) - /* atan2(+-y, +inf) = atan2(+-0, +x) = +-0. */ - return copysign(0., y); - else - /* atan2(+-y, -inf) = atan2(+-0., -x) = +-pi. */ - return copysign(Py_MATH_PI, y); - } - return atan2(y, x); -} - -#define m_atan2 _Py_atan2 diff --git a/Modules/cmathmodule.c b/Modules/cmathmodule.c index 49ff093871d156..e07c2dbd262354 100644 --- a/Modules/cmathmodule.c +++ b/Modules/cmathmodule.c @@ -925,7 +925,7 @@ cmath_phase_impl(PyObject *module, Py_complex z) double phi; errno = 0; - phi = m_atan2(z.imag, z.real); /* should not cause any exception */ + phi = atan2(z.imag, z.real); /* should not cause any exception */ if (errno != 0) return math_error(); else @@ -950,7 +950,7 @@ cmath_polar_impl(PyObject *module, Py_complex z) double r, phi; errno = 0; - phi = m_atan2(z.imag, z.real); /* should not cause any exception */ + phi = atan2(z.imag, z.real); /* should not cause any exception */ r = _Py_c_abs(z); /* sets errno to ERANGE on overflow */ if (errno != 0) return math_error(); diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index d6d0702169e186..d91fa137314dae 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -1066,7 +1066,7 @@ FUNC1(atan, atan, 0, "atan($module, x, /)\n--\n\n" "Return the arc tangent (measured in radians) of x.\n\n" "The result is between -pi/2 and pi/2.") -FUNC2(atan2, m_atan2, +FUNC2(atan2, atan2, "atan2($module, y, x, /)\n--\n\n" "Return the arc tangent (measured in radians) of y/x.\n\n" "Unlike atan(y/x), the signs of both x and y are considered.") diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 621f8996ccbb8e..85c24550d0b409 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1796,8 +1796,7 @@ init_frame(_PyInterpreterFrame *frame, PyFunctionObject *func, PyObject *locals) { PyCodeObject *code = (PyCodeObject *)func->func_code; _PyFrame_Initialize(frame, (PyFunctionObject*)Py_NewRef(func), - Py_XNewRef(locals), code, 0); - frame->previous = NULL; + Py_XNewRef(locals), code, 0, NULL); } PyFrameObject* diff --git a/Objects/mimalloc/os.c b/Objects/mimalloc/os.c index f3bc7184c41c5b..c9103168c12507 100644 --- a/Objects/mimalloc/os.c +++ b/Objects/mimalloc/os.c @@ -115,8 +115,12 @@ void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size) if (hint == 0 || hint > MI_HINT_MAX) { // wrap or initialize uintptr_t init = MI_HINT_BASE; #if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of aligned allocations unless in debug mode - uintptr_t r = _mi_heap_random_next(mi_prim_get_default_heap()); - init = init + ((MI_SEGMENT_SIZE * ((r>>17) & 0xFFFFF)) % MI_HINT_AREA); // (randomly 20 bits)*4MiB == 0 to 4TiB + mi_heap_t* heap = mi_prim_get_default_heap(); + // gh-123022: default heap may not be initialized in CPython in background threads + if (mi_heap_is_initialized(heap)) { + uintptr_t r = _mi_heap_random_next(heap); + init = init + ((MI_SEGMENT_SIZE * ((r>>17) & 0xFFFFF)) % MI_HINT_AREA); // (randomly 20 bits)*4MiB == 0 to 4TiB + } #endif uintptr_t expected = hint + size; mi_atomic_cas_strong_acq_rel(&aligned_base, &expected, init); @@ -553,8 +557,12 @@ static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) { // Initialize the start address after the 32TiB area start = ((uintptr_t)32 << 40); // 32TiB virtual start address #if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of huge pages unless in debug mode - uintptr_t r = _mi_heap_random_next(mi_prim_get_default_heap()); - start = start + ((uintptr_t)MI_HUGE_OS_PAGE_SIZE * ((r>>17) & 0x0FFF)); // (randomly 12bits)*1GiB == between 0 to 4TiB + mi_heap_t* heap = mi_prim_get_default_heap(); + // gh-123022: default heap may not be initialized in CPython in background threads + if (mi_heap_is_initialized(heap)) { + uintptr_t r = _mi_heap_random_next(heap); + start = start + ((uintptr_t)MI_HUGE_OS_PAGE_SIZE * ((r>>17) & 0x0FFF)); // (randomly 12bits)*1GiB == between 0 to 4TiB + } #endif } end = start + size; diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index da9c5857cc3bee..148d3e55bf830e 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -15623,19 +15623,7 @@ _PyUnicode_ClearInterned(PyInterpreterState *interp) int shared = 0; switch (PyUnicode_CHECK_INTERNED(s)) { case SSTATE_INTERNED_IMMORTAL: - /* Make immortal interned strings mortal again. - * - * Currently, the runtime is not able to guarantee that it can exit - * without allocations that carry over to a future initialization - * of Python within the same process. i.e: - * ./python -X showrefcount -c 'import itertools' - * [237 refs, 237 blocks] - * - * This should remain disabled (`Py_DEBUG` only) until there is a - * strict guarantee that no memory will be left after - * `Py_Finalize`. - */ -#ifdef Py_DEBUG + /* Make immortal interned strings mortal again. */ // Skip the Immortal Instance check and restore // the two references (key and value) ignored // by PyUnicode_InternInPlace(). @@ -15648,7 +15636,6 @@ _PyUnicode_ClearInterned(PyInterpreterState *interp) #ifdef INTERNED_STATS total_length += PyUnicode_GET_LENGTH(s); #endif -#endif // Py_DEBUG break; case SSTATE_INTERNED_IMMORTAL_STATIC: /* It is shared between interpreters, so we should unmark it diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 2c726c915c9df5..e341f0c6bfc595 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -8,6 +8,7 @@ #include #include "pycore_initconfig.h" // _PyConfig_InitCompatConfig() #include "pycore_runtime.h" // _PyRuntime +#include "pycore_pythread.h" // PyThread_start_joinable_thread() #include "pycore_import.h" // _PyImport_FrozenBootstrap #include #include @@ -2022,6 +2023,22 @@ static int test_init_main_interpreter_settings(void) return 0; } +static void do_init(void *unused) +{ + _testembed_Py_Initialize(); + Py_Finalize(); +} + +static int test_init_in_background_thread(void) +{ + PyThread_handle_t handle; + PyThread_ident_t ident; + if (PyThread_start_joinable_thread(&do_init, NULL, &ident, &handle) < 0) { + return -1; + } + return PyThread_join_thread(handle); +} + #ifndef MS_WINDOWS #include "test_frozenmain.h" // M_test_frozenmain @@ -2211,6 +2228,7 @@ static struct TestCase TestCases[] = { {"test_get_argc_argv", test_get_argc_argv}, {"test_init_use_frozen_modules", test_init_use_frozen_modules}, {"test_init_main_interpreter_settings", test_init_main_interpreter_settings}, + {"test_init_in_background_thread", test_init_in_background_thread}, // Audit {"test_open_code_hook", test_open_code_hook}, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 97e37bc28b8cfd..ec0f6ab8fde980 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -797,7 +797,7 @@ dummy_func( PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; PyObject *getitem = ht->_spec_cache.getitem; - new_frame = _PyFrame_PushUnchecked(tstate, (PyFunctionObject *)getitem, 2); + new_frame = _PyFrame_PushUnchecked(tstate, (PyFunctionObject *)getitem, 2, frame); SYNC_SP(); new_frame->localsplus[0] = container; new_frame->localsplus[1] = sub; @@ -1071,6 +1071,8 @@ dummy_func( tstate->exc_info = &gen->gi_exc_state; assert(next_instr - this_instr + oparg <= UINT16_MAX); frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); + assert(gen_frame->previous == NULL); + gen_frame->previous = frame; DISPATCH_INLINED(gen_frame); } if (PyStackRef_Is(v, PyStackRef_None) && PyIter_Check(receiver_o)) { @@ -1113,6 +1115,7 @@ dummy_func( tstate->exc_info = &gen->gi_exc_state; assert(1 + INLINE_CACHE_ENTRIES_SEND + oparg <= UINT16_MAX); frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + oparg); + gen_frame->previous = frame; } macro(SEND_GEN) = @@ -2143,7 +2146,7 @@ dummy_func( DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize)); STAT_INC(LOAD_ATTR, hit); Py_INCREF(fget); - new_frame = _PyFrame_PushUnchecked(tstate, f, 1); + new_frame = _PyFrame_PushUnchecked(tstate, f, 1, frame); new_frame->localsplus[0] = owner; } @@ -2175,7 +2178,7 @@ dummy_func( PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); Py_INCREF(f); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2); + _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2, frame); // Manipulate stack directly because we exit with DISPATCH_INLINED(). STACK_SHRINK(1); new_frame->localsplus[0] = owner; @@ -2956,6 +2959,7 @@ dummy_func( gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; + gen_frame->previous = frame; // oparg is the return offset from the next instruction. frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg); } @@ -3225,7 +3229,7 @@ dummy_func( PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, - args, total_args, NULL + args, total_args, NULL, frame ); // Manipulate stack directly since we leave using DISPATCH_INLINED(). STACK_SHRINK(oparg + 2); @@ -3315,7 +3319,7 @@ dummy_func( PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, - args, total_args, NULL + args, total_args, NULL, frame ); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. @@ -3454,7 +3458,7 @@ dummy_func( int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); PyFunctionObject *func = (PyFunctionObject *)callable_o; - new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { @@ -3468,7 +3472,7 @@ dummy_func( assert(tstate->interp->eval_frame == NULL); SYNC_SP(); _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame->previous = frame; + assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; @@ -3550,58 +3554,51 @@ dummy_func( _CALL_TUPLE_1 + _CHECK_PERIODIC; - inst(CALL_ALLOC_AND_ENTER_INIT, (unused/1, unused/2, callable, null, args[oparg] -- unused)) { + op(_CHECK_AND_ALLOCATE_OBJECT, (type_version/2, callable, null, args[oparg] -- self, init, args[oparg])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - /* This instruction does the following: - * 1. Creates the object (by calling ``object.__new__``) - * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) - * 3. Pushes the frame for ``__init__`` to the frame stack - * */ - _PyCallCache *cache = (_PyCallCache *)&this_instr[1]; DEOPT_IF(!PyStackRef_IsNull(null)); DEOPT_IF(!PyType_Check(callable_o)); PyTypeObject *tp = (PyTypeObject *)callable_o; - DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version)); + DEOPT_IF(tp->tp_version_tag != type_version); assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES); PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; - PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init; - PyCodeObject *code = (PyCodeObject *)init->func_code; - DEOPT_IF(code->co_argcount != oparg+1); + PyFunctionObject *init_func = (PyFunctionObject *)cls->_spec_cache.init; + PyCodeObject *code = (PyCodeObject *)init_func->func_code; DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)); STAT_INC(CALL, hit); - PyObject *self = _PyType_NewManagedObject(tp); - if (self == NULL) { + self = PyStackRef_FromPyObjectSteal(_PyType_NewManagedObject(tp)); + if (PyStackRef_IsNull(self)) { ERROR_NO_POP(); } PyStackRef_CLOSE(callable); + init = PyStackRef_FromPyObjectNew(init_func); + } + + op(_CREATE_INIT_FRAME, (self, init, args[oparg] -- init_frame: _PyInterpreterFrame *)) { _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( - tstate, (PyCodeObject *)&_Py_InitCleanup, 1); + tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK); /* Push self onto stack of shim */ - Py_INCREF(self); - shim->localsplus[0] = PyStackRef_FromPyObjectSteal(self); - Py_INCREF(init); - _PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init, oparg+1); - /* Copy self followed by args to __init__ frame */ - init_frame->localsplus[0] = PyStackRef_FromPyObjectSteal(self); - for (int i = 0; i < oparg; i++) { - init_frame->localsplus[i+1] = args[i]; - } - frame->return_offset = (uint16_t)(next_instr - this_instr); - STACK_SHRINK(oparg+2); - _PyFrame_SetStackPointer(frame, stack_pointer); - /* Link frames */ - init_frame->previous = shim; - shim->previous = frame; - frame = tstate->current_frame = init_frame; - CALL_STAT_INC(inlined_py_calls); + shim->localsplus[0] = PyStackRef_DUP(self); + PyFunctionObject *init_func = (PyFunctionObject *)PyStackRef_AsPyObjectSteal(init); + args[-1] = self; + init_frame = _PyEvalFramePushAndInit( + tstate, init_func, NULL, args-1, oparg+1, NULL, shim); + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; + SYNC_SP(); /* Account for pushing the extra frame. * We don't check recursion depth here, * as it will be checked after start_frame */ tstate->py_recursion_remaining--; - goto start_frame; } + macro(CALL_ALLOC_AND_ENTER_INIT) = + unused/1 + + _CHECK_PEP_523 + + _CHECK_AND_ALLOCATE_OBJECT + + _CREATE_INIT_FRAME + + _PUSH_FRAME; + inst(EXIT_INIT_CHECK, (should_be_none -- )) { assert(STACK_LEVEL() == 2); if (!PyStackRef_Is(should_be_none, PyStackRef_None)) { @@ -4007,7 +4004,14 @@ dummy_func( _CALL_METHOD_DESCRIPTOR_FAST + _CHECK_PERIODIC; - inst(INSTRUMENTED_CALL_KW, ( -- )) { + // Cache layout: counter/1, func_version/2 + family(CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW) = { + CALL_KW_BOUND_METHOD, + CALL_KW_PY, + CALL_KW_NON_PY, + }; + + inst(INSTRUMENTED_CALL_KW, (counter/1, version/2 -- )) { int is_meth = !PyStackRef_IsNull(PEEK(oparg + 2)); int total_args = oparg + is_meth; PyObject *function = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 3)); @@ -4017,6 +4021,7 @@ dummy_func( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, function, arg); ERROR_IF(err, error); + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); GO_TO_INSTRUCTION(CALL_KW); } @@ -4052,7 +4057,7 @@ dummy_func( PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, - args, positional_args, kwnames_o + args, positional_args, kwnames_o, frame ); PyStackRef_CLOSE(kwnames); // Manipulate stack directly since we leave using DISPATCH_INLINED(). @@ -4062,8 +4067,8 @@ dummy_func( if (new_frame == NULL) { ERROR_NO_POP(); } - assert(next_instr - this_instr == 1); - frame->return_offset = 1; + assert(next_instr - this_instr == 1 + INLINE_CACHE_ENTRIES_CALL_KW); + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL_KW; DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ @@ -4104,8 +4109,144 @@ dummy_func( res = PyStackRef_FromPyObjectSteal(res_o); } + op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame: _PyInterpreterFrame*)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); + + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (self_or_null_o != NULL) { + args--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + args, positional_args, kwnames_o, frame + ); + PyStackRef_CLOSE(kwnames); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + SYNC_SP(); + if (new_frame == NULL) { + ERROR_NO_POP(); + } + } + + op(_CHECK_FUNCTION_VERSION_KW, (func_version/2, callable, self_or_null, unused[oparg], kwnames -- callable, self_or_null, unused[oparg], kwnames)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + EXIT_IF(!PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + EXIT_IF(func->func_version != func_version); + } + + macro(CALL_KW_PY) = + unused/1 + // Skip over the counter + _CHECK_PEP_523 + + _CHECK_FUNCTION_VERSION_KW + + _PY_FRAME_KW + + _SAVE_RETURN_OFFSET + + _PUSH_FRAME; + + op(_CHECK_METHOD_VERSION_KW, (func_version/2, callable, null, unused[oparg], kwnames -- callable, null, unused[oparg], kwnames)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + + EXIT_IF(Py_TYPE(callable_o) != &PyMethod_Type); + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + EXIT_IF(!PyFunction_Check(func)); + EXIT_IF(((PyFunctionObject *)func)->func_version != func_version); + EXIT_IF(!PyStackRef_IsNull(null)); + } + + op(_EXPAND_METHOD_KW, (callable, null, unused[oparg], kwnames -- method, self, unused[oparg], kwnames)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + + assert(PyStackRef_IsNull(null)); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + method = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(method)); + PyStackRef_CLOSE(callable); + } + + macro(CALL_KW_BOUND_METHOD) = + unused/1 + // Skip over the counter + _CHECK_PEP_523 + + _CHECK_METHOD_VERSION_KW + + _EXPAND_METHOD_KW + + flush + // so that self is in the argument array + _PY_FRAME_KW + + _SAVE_RETURN_OFFSET + + _PUSH_FRAME; + + specializing op(_SPECIALIZE_CALL_KW, (counter/1, callable, self_or_null, args[oparg], kwnames -- callable, self_or_null, args[oparg], kwnames)) { + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + DISPATCH_SAME_OPARG(); + } + STAT_INC(CALL, deferred); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + macro(CALL_KW) = - _DO_CALL_KW + + _SPECIALIZE_CALL_KW + + unused/2 + + _DO_CALL_KW; + + op(_CHECK_IS_NOT_PY_CALLABLE_KW, (callable, unused, unused[oparg], kwnames -- callable, unused, unused[oparg], kwnames)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + EXIT_IF(PyFunction_Check(callable_o)); + EXIT_IF(Py_TYPE(callable_o) == &PyMethod_Type); + } + + + op(_CALL_KW_NON_PY, (callable, self_or_null, args[oparg], kwnames -- res)) { +#if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); +#endif + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); + + int total_args = oparg; + if (self_or_null_o != NULL) { + args--; + total_args++; + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + DECREF_INPUTS(); + ERROR_IF(true, error); + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); + PyStackRef_CLOSE(kwnames); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable); + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); + } + + macro(CALL_KW_NON_PY) = + unused/1 + // Skip over the counter + unused/2 + + _CHECK_IS_NOT_PY_CALLABLE_KW + + _CALL_KW_NON_PY + _CHECK_PERIODIC; inst(INSTRUMENTED_CALL_FUNCTION_EX, ( -- )) { @@ -4171,7 +4312,7 @@ dummy_func( _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(func_st), locals, - nargs, callargs, kwargs); + nargs, callargs, kwargs, frame); // Need to manually shrink the stack since we exit with DISPATCH_INLINED. STACK_SHRINK(oparg + 3); if (new_frame == NULL) { diff --git a/Python/ceval.c b/Python/ceval.c index c685a95b2ef088..b615deec3c4242 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -246,7 +246,7 @@ static int check_args_iterable(PyThreadState *, PyObject *func, PyObject *vararg static int get_exception_handler(PyCodeObject *, int, int*, int*, int*); static _PyInterpreterFrame * _PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func, - PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs); + PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs, _PyInterpreterFrame *previous); #ifdef HAVE_ERRNO_H #include @@ -671,8 +671,6 @@ static const _Py_CODEUNIT _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS[] = { { .op.code = RESUME, .op.arg = RESUME_OPARG_DEPTH1_MASK | RESUME_AT_FUNC_START } }; -extern const struct _PyCode_DEF(8) _Py_InitCleanup; - #ifdef Py_DEBUG extern void _PyUOpPrint(const _PyUOpInstruction *uop); #endif @@ -1691,7 +1689,7 @@ _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame * frame) _PyInterpreterFrame * _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, PyObject *locals, _PyStackRef const* args, - size_t argcount, PyObject *kwnames) + size_t argcount, PyObject *kwnames, _PyInterpreterFrame *previous) { PyCodeObject * code = (PyCodeObject *)func->func_code; CALL_STAT_INC(frames_pushed); @@ -1699,7 +1697,7 @@ _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, if (frame == NULL) { goto fail; } - _PyFrame_Initialize(frame, func, locals, code, 0); + _PyFrame_Initialize(frame, func, locals, code, 0, previous); if (initialize_locals(tstate, func, frame->localsplus, args, argcount, kwnames)) { assert(frame->owner == FRAME_OWNED_BY_THREAD); clear_thread_frame(tstate, frame); @@ -1726,7 +1724,7 @@ _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, static _PyInterpreterFrame * _PyEvalFramePushAndInit_UnTagged(PyThreadState *tstate, PyFunctionObject *func, PyObject *locals, PyObject *const* args, - size_t argcount, PyObject *kwnames) + size_t argcount, PyObject *kwnames, _PyInterpreterFrame *previous) { #if defined(Py_GIL_DISABLED) size_t kw_count = kwnames == NULL ? 0 : PyTuple_GET_SIZE(kwnames); @@ -1742,11 +1740,11 @@ _PyEvalFramePushAndInit_UnTagged(PyThreadState *tstate, PyFunctionObject *func, for (size_t i = 0; i < kw_count; i++) { tagged_args_buffer[argcount + i] = PyStackRef_FromPyObjectSteal(args[argcount + i]); } - _PyInterpreterFrame *res = _PyEvalFramePushAndInit(tstate, func, locals, (_PyStackRef const *)tagged_args_buffer, argcount, kwnames); + _PyInterpreterFrame *res = _PyEvalFramePushAndInit(tstate, func, locals, (_PyStackRef const *)tagged_args_buffer, argcount, kwnames, previous); PyMem_Free(tagged_args_buffer); return res; #else - return _PyEvalFramePushAndInit(tstate, func, locals, (_PyStackRef const *)args, argcount, kwnames); + return _PyEvalFramePushAndInit(tstate, func, locals, (_PyStackRef const *)args, argcount, kwnames, previous); #endif } @@ -1755,7 +1753,7 @@ _PyEvalFramePushAndInit_UnTagged(PyThreadState *tstate, PyFunctionObject *func, */ static _PyInterpreterFrame * _PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func, - PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs) + PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs, _PyInterpreterFrame *previous) { bool has_dict = (kwargs != NULL && PyDict_GET_SIZE(kwargs) > 0); PyObject *kwnames = NULL; @@ -1776,7 +1774,7 @@ _PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func, } _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_UnTagged( tstate, (PyFunctionObject *)func, locals, - newargs, nargs, kwnames + newargs, nargs, kwnames, previous ); if (has_dict) { _PyStack_UnpackDict_FreeNoDecRef(newargs, kwnames); @@ -1813,7 +1811,7 @@ _PyEval_Vector(PyThreadState *tstate, PyFunctionObject *func, } } _PyInterpreterFrame *frame = _PyEvalFramePushAndInit_UnTagged( - tstate, func, locals, args, argcount, kwnames); + tstate, func, locals, args, argcount, kwnames, NULL); if (frame == NULL) { return NULL; } diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index ed146a10b2af4b..9e1540674d4219 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -124,7 +124,7 @@ do { \ do { \ assert(tstate->interp->eval_frame == NULL); \ _PyFrame_SetStackPointer(frame, stack_pointer); \ - (NEW_FRAME)->previous = frame; \ + assert((NEW_FRAME)->previous == frame); \ frame = tstate->current_frame = (NEW_FRAME); \ CALL_STAT_INC(inlined_py_calls); \ goto start_frame; \ diff --git a/Python/compile.c b/Python/compile.c index 237b5dba0497fd..c369202d53b384 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -67,6 +67,13 @@ return ERROR; \ } +#define RETURN_IF_ERROR_IN_SCOPE(C, CALL) { \ + if ((CALL) < 0) { \ + compiler_exit_scope((C)); \ + return ERROR; \ + } \ +} + #define IS_TOP_LEVEL_AWAIT(C) ( \ ((C)->c_flags.cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT) \ && ((C)->u->u_ste->ste_type == ModuleBlock)) @@ -291,8 +298,6 @@ typedef struct { Py_ssize_t on_top; } pattern_context; -static int codegen_addop_i(instr_sequence *seq, int opcode, Py_ssize_t oparg, location loc); - static void compiler_free(struct compiler *); static int compiler_error(struct compiler *, location loc, const char *, ...); static int compiler_warn(struct compiler *, location loc, const char *, ...); @@ -302,26 +307,26 @@ static PyCodeObject *compiler_mod(struct compiler *, mod_ty); static int compiler_visit_stmt(struct compiler *, stmt_ty); static int compiler_visit_keyword(struct compiler *, keyword_ty); static int compiler_visit_expr(struct compiler *, expr_ty); -static int compiler_augassign(struct compiler *, stmt_ty); +static int codegen_augassign(struct compiler *, stmt_ty); static int compiler_annassign(struct compiler *, stmt_ty); -static int compiler_subscript(struct compiler *, expr_ty); -static int compiler_slice(struct compiler *, expr_ty); +static int codegen_subscript(struct compiler *, expr_ty); +static int codegen_slice(struct compiler *, expr_ty); static bool are_all_items_const(asdl_expr_seq *, Py_ssize_t, Py_ssize_t); -static int compiler_with(struct compiler *, stmt_ty, int); -static int compiler_async_with(struct compiler *, stmt_ty, int); -static int compiler_async_for(struct compiler *, stmt_ty); -static int compiler_call_simple_kw_helper(struct compiler *c, - location loc, - asdl_keyword_seq *keywords, - Py_ssize_t nkwelts); -static int compiler_call_helper(struct compiler *c, location loc, - int n, asdl_expr_seq *args, - asdl_keyword_seq *keywords); -static int compiler_try_except(struct compiler *, stmt_ty); -static int compiler_try_star_except(struct compiler *, stmt_ty); +static int codegen_with(struct compiler *, stmt_ty, int); +static int codegen_async_with(struct compiler *, stmt_ty, int); +static int codegen_async_for(struct compiler *, stmt_ty); +static int codegen_call_simple_kw_helper(struct compiler *c, + location loc, + asdl_keyword_seq *keywords, + Py_ssize_t nkwelts); +static int codegen_call_helper(struct compiler *c, location loc, + int n, asdl_expr_seq *args, + asdl_keyword_seq *keywords); +static int codegen_try_except(struct compiler *, stmt_ty); +static int codegen_try_star_except(struct compiler *, stmt_ty); static int compiler_sync_comprehension_generator( struct compiler *c, location loc, @@ -337,12 +342,12 @@ static int compiler_async_comprehension_generator( expr_ty elt, expr_ty val, int type, int iter_on_stack); -static int compiler_pattern(struct compiler *, pattern_ty, pattern_context *); -static int compiler_match(struct compiler *, stmt_ty); -static int compiler_pattern_subpattern(struct compiler *, - pattern_ty, pattern_context *); -static int compiler_make_closure(struct compiler *c, location loc, - PyCodeObject *co, Py_ssize_t flags); +static int codegen_pattern(struct compiler *, pattern_ty, pattern_context *); +static int codegen_match(struct compiler *, stmt_ty); +static int codegen_pattern_subpattern(struct compiler *, + pattern_ty, pattern_context *); +static int codegen_make_closure(struct compiler *c, location loc, + PyCodeObject *co, Py_ssize_t flags); static PyCodeObject *optimize_and_assemble(struct compiler *, int addNone); @@ -685,6 +690,29 @@ compiler_set_qualname(struct compiler *c) return SUCCESS; } +/* Add an opcode with an integer argument */ +static int +codegen_addop_i(instr_sequence *seq, int opcode, Py_ssize_t oparg, location loc) +{ + /* oparg value is unsigned, but a signed C int is usually used to store + it in the C code (like Python/ceval.c). + + Limit to 32-bit signed C int (rather than INT_MAX) for portability. + + The argument of a concrete bytecode instruction is limited to 8-bit. + EXTENDED_ARG is used for 16, 24, and 32-bit arguments. */ + + int oparg_ = Py_SAFE_DOWNCAST(oparg, Py_ssize_t, int); + assert(!IS_ASSEMBLER_OPCODE(opcode)); + return _PyInstructionSequence_Addop(seq, opcode, oparg_, loc); +} + +#define ADDOP_I(C, LOC, OP, O) \ + RETURN_IF_ERROR(codegen_addop_i(INSTR_SEQUENCE(C), (OP), (O), (LOC))) + +#define ADDOP_I_IN_SCOPE(C, LOC, OP, O) \ + RETURN_IF_ERROR_IN_SCOPE(C, codegen_addop_i(INSTR_SEQUENCE(C), (OP), (O), (LOC))); + static int codegen_addop_noarg(instr_sequence *seq, int opcode, location loc) { @@ -693,6 +721,12 @@ codegen_addop_noarg(instr_sequence *seq, int opcode, location loc) return _PyInstructionSequence_Addop(seq, opcode, 0, loc); } +#define ADDOP(C, LOC, OP) \ + RETURN_IF_ERROR(codegen_addop_noarg(INSTR_SEQUENCE(C), (OP), (LOC))) + +#define ADDOP_IN_SCOPE(C, LOC, OP) \ + RETURN_IF_ERROR_IN_SCOPE((C), codegen_addop_noarg(INSTR_SEQUENCE(C), (OP), (LOC))) + static Py_ssize_t dict_add_o(PyObject *dict, PyObject *o) { @@ -854,24 +888,57 @@ compiler_add_const(struct compiler *c, PyObject *o) } static int -compiler_addop_load_const(struct compiler *c, location loc, PyObject *o) +codegen_addop_load_const(struct compiler *c, location loc, PyObject *o) { Py_ssize_t arg = compiler_add_const(c, o); if (arg < 0) { return ERROR; } - return codegen_addop_i(INSTR_SEQUENCE(c), LOAD_CONST, arg, loc); + ADDOP_I(c, loc, LOAD_CONST, arg); + return SUCCESS; +} + +#define ADDOP_LOAD_CONST(C, LOC, O) \ + RETURN_IF_ERROR(codegen_addop_load_const((C), (LOC), (O))) + +#define ADDOP_LOAD_CONST_IN_SCOPE(C, LOC, O) \ + RETURN_IF_ERROR_IN_SCOPE((C), codegen_addop_load_const((C), (LOC), (O))) + +/* Same as ADDOP_LOAD_CONST, but steals a reference. */ +#define ADDOP_LOAD_CONST_NEW(C, LOC, O) { \ + PyObject *__new_const = (O); \ + if (__new_const == NULL) { \ + return ERROR; \ + } \ + if (codegen_addop_load_const((C), (LOC), __new_const) < 0) { \ + Py_DECREF(__new_const); \ + return ERROR; \ + } \ + Py_DECREF(__new_const); \ } static int -compiler_addop_o(struct compiler *c, location loc, - int opcode, PyObject *dict, PyObject *o) +codegen_addop_o(struct compiler *c, location loc, + int opcode, PyObject *dict, PyObject *o) { Py_ssize_t arg = dict_add_o(dict, o); - if (arg < 0) { - return ERROR; - } - return codegen_addop_i(INSTR_SEQUENCE(c), opcode, arg, loc); + RETURN_IF_ERROR(arg); + ADDOP_I(c, loc, opcode, arg); + return SUCCESS; +} + +#define ADDOP_N(C, LOC, OP, O, TYPE) { \ + assert(!OPCODE_HAS_CONST(OP)); /* use ADDOP_LOAD_CONST_NEW */ \ + int ret = codegen_addop_o((C), (LOC), (OP), (C)->u->u_metadata.u_ ## TYPE, (O)); \ + Py_DECREF((O)); \ + RETURN_IF_ERROR(ret); \ +} + +#define ADDOP_N_IN_SCOPE(C, LOC, OP, O, TYPE) { \ + assert(!OPCODE_HAS_CONST(OP)); /* use ADDOP_LOAD_CONST_NEW */ \ + int ret = codegen_addop_o((C), (LOC), (OP), (C)->u->u_metadata.u_ ## TYPE, (O)); \ + Py_DECREF((O)); \ + RETURN_IF_ERROR_IN_SCOPE((C), ret); \ } #define LOAD_METHOD -1 @@ -880,8 +947,8 @@ compiler_addop_o(struct compiler *c, location loc, #define LOAD_ZERO_SUPER_METHOD -4 static int -compiler_addop_name(struct compiler *c, location loc, - int opcode, PyObject *dict, PyObject *o) +codegen_addop_name(struct compiler *c, location loc, + int opcode, PyObject *dict, PyObject *o) { PyObject *mangled = compiler_maybe_mangle(c, o); if (!mangled) { @@ -918,25 +985,12 @@ compiler_addop_name(struct compiler *c, location loc, arg <<= 2; arg |= 1; } - return codegen_addop_i(INSTR_SEQUENCE(c), opcode, arg, loc); + ADDOP_I(c, loc, opcode, arg); + return SUCCESS; } -/* Add an opcode with an integer argument */ -static int -codegen_addop_i(instr_sequence *seq, int opcode, Py_ssize_t oparg, location loc) -{ - /* oparg value is unsigned, but a signed C int is usually used to store - it in the C code (like Python/ceval.c). - - Limit to 32-bit signed C int (rather than INT_MAX) for portability. - - The argument of a concrete bytecode instruction is limited to 8-bit. - EXTENDED_ARG is used for 16, 24, and 32-bit arguments. */ - - int oparg_ = Py_SAFE_DOWNCAST(oparg, Py_ssize_t, int); - assert(!IS_ASSEMBLER_OPCODE(opcode)); - return _PyInstructionSequence_Addop(seq, opcode, oparg_, loc); -} +#define ADDOP_NAME(C, LOC, OP, O, TYPE) \ + RETURN_IF_ERROR(codegen_addop_name((C), (LOC), (OP), (C)->u->u_metadata.u_ ## TYPE, (O))) static int codegen_addop_j(instr_sequence *seq, location loc, @@ -948,54 +1002,11 @@ codegen_addop_j(instr_sequence *seq, location loc, return _PyInstructionSequence_Addop(seq, opcode, target.id, loc); } -#define RETURN_IF_ERROR_IN_SCOPE(C, CALL) { \ - if ((CALL) < 0) { \ - compiler_exit_scope((C)); \ - return ERROR; \ - } \ -} - -#define ADDOP(C, LOC, OP) \ - RETURN_IF_ERROR(codegen_addop_noarg(INSTR_SEQUENCE(C), (OP), (LOC))) - -#define ADDOP_IN_SCOPE(C, LOC, OP) RETURN_IF_ERROR_IN_SCOPE((C), codegen_addop_noarg(INSTR_SEQUENCE(C), (OP), (LOC))) - -#define ADDOP_LOAD_CONST(C, LOC, O) \ - RETURN_IF_ERROR(compiler_addop_load_const((C), (LOC), (O))) - -/* Same as ADDOP_LOAD_CONST, but steals a reference. */ -#define ADDOP_LOAD_CONST_NEW(C, LOC, O) { \ - PyObject *__new_const = (O); \ - if (__new_const == NULL) { \ - return ERROR; \ - } \ - if (compiler_addop_load_const((C), (LOC), __new_const) < 0) { \ - Py_DECREF(__new_const); \ - return ERROR; \ - } \ - Py_DECREF(__new_const); \ -} - -#define ADDOP_N(C, LOC, OP, O, TYPE) { \ - assert(!OPCODE_HAS_CONST(OP)); /* use ADDOP_LOAD_CONST_NEW */ \ - if (compiler_addop_o((C), (LOC), (OP), (C)->u->u_metadata.u_ ## TYPE, (O)) < 0) { \ - Py_DECREF((O)); \ - return ERROR; \ - } \ - Py_DECREF((O)); \ -} - -#define ADDOP_NAME(C, LOC, OP, O, TYPE) \ - RETURN_IF_ERROR(compiler_addop_name((C), (LOC), (OP), (C)->u->u_metadata.u_ ## TYPE, (O))) - -#define ADDOP_I(C, LOC, OP, O) \ - RETURN_IF_ERROR(codegen_addop_i(INSTR_SEQUENCE(C), (OP), (O), (LOC))) - #define ADDOP_JUMP(C, LOC, OP, O) \ RETURN_IF_ERROR(codegen_addop_j(INSTR_SEQUENCE(C), (LOC), (OP), (O))) #define ADDOP_COMPARE(C, LOC, CMP) \ - RETURN_IF_ERROR(compiler_addcompare((C), (LOC), (cmpop_ty)(CMP))) + RETURN_IF_ERROR(codegen_addcompare((C), (LOC), (cmpop_ty)(CMP))) #define ADDOP_BINARY(C, LOC, BINOP) \ RETURN_IF_ERROR(addop_binary((C), (LOC), (BINOP), false)) @@ -1004,13 +1015,13 @@ codegen_addop_j(instr_sequence *seq, location loc, RETURN_IF_ERROR(addop_binary((C), (LOC), (BINOP), true)) #define ADD_YIELD_FROM(C, LOC, await) \ - RETURN_IF_ERROR(compiler_add_yield_from((C), (LOC), (await))) + RETURN_IF_ERROR(codegen_add_yield_from((C), (LOC), (await))) #define POP_EXCEPT_AND_RERAISE(C, LOC) \ - RETURN_IF_ERROR(compiler_pop_except_and_reraise((C), (LOC))) + RETURN_IF_ERROR(codegen_pop_except_and_reraise((C), (LOC))) #define ADDOP_YIELD(C, LOC) \ - RETURN_IF_ERROR(addop_yield((C), (LOC))) + RETURN_IF_ERROR(codegen_addop_yield((C), (LOC))) /* VISIT and VISIT_SEQ takes an ASDL type as their second argument. They use the ASDL name to synthesize the name of the C type and the visit function. @@ -1046,7 +1057,8 @@ codegen_addop_j(instr_sequence *seq, location loc, static int compiler_enter_scope(struct compiler *c, identifier name, int scope_type, - void *key, int lineno, PyObject *private) + void *key, int lineno, PyObject *private, + _PyCompile_CodeUnitMetadata *umd) { location loc = LOCATION(lineno, lineno, 0, 0); @@ -1058,9 +1070,14 @@ compiler_enter_scope(struct compiler *c, identifier name, int scope_type, return ERROR; } u->u_scope_type = scope_type; - u->u_metadata.u_argcount = 0; - u->u_metadata.u_posonlyargcount = 0; - u->u_metadata.u_kwonlyargcount = 0; + if (umd != NULL) { + u->u_metadata = *umd; + } + else { + u->u_metadata.u_argcount = 0; + u->u_metadata.u_posonlyargcount = 0; + u->u_metadata.u_kwonlyargcount = 0; + } u->u_ste = _PySymtable_Lookup(c->c_st, key); if (!u->u_ste) { compiler_unit_free(u); @@ -1246,7 +1263,7 @@ compiler_pop_fblock(struct compiler *c, enum fblocktype t, jump_target_label blo } static int -compiler_call_exit_with_nones(struct compiler *c, location loc) +codegen_call_exit_with_nones(struct compiler *c, location loc) { ADDOP_LOAD_CONST(c, loc, Py_None); ADDOP_LOAD_CONST(c, loc, Py_None); @@ -1256,7 +1273,7 @@ compiler_call_exit_with_nones(struct compiler *c, location loc) } static int -compiler_add_yield_from(struct compiler *c, location loc, int await) +codegen_add_yield_from(struct compiler *c, location loc, int await) { NEW_JUMP_TARGET_LABEL(c, send); NEW_JUMP_TARGET_LABEL(c, fail); @@ -1281,7 +1298,7 @@ compiler_add_yield_from(struct compiler *c, location loc, int await) } static int -compiler_pop_except_and_reraise(struct compiler *c, location loc) +codegen_pop_except_and_reraise(struct compiler *c, location loc) { /* Stack contents * [exc_info, lasti, exc] COPY 3 @@ -1363,7 +1380,7 @@ compiler_unwind_fblock(struct compiler *c, location *ploc, ADDOP_I(c, *ploc, SWAP, 3); ADDOP_I(c, *ploc, SWAP, 2); } - RETURN_IF_ERROR(compiler_call_exit_with_nones(c, *ploc)); + RETURN_IF_ERROR(codegen_call_exit_with_nones(c, *ploc)); if (info->fb_type == ASYNC_WITH) { ADDOP_I(c, *ploc, GET_AWAITABLE, 2); ADDOP_LOAD_CONST(c, *ploc, Py_None); @@ -1430,14 +1447,19 @@ compiler_unwind_fblock_stack(struct compiler *c, location *ploc, } static int -compiler_setup_annotations_scope(struct compiler *c, location loc, - void *key, PyObject *name) +codegen_setup_annotations_scope(struct compiler *c, location loc, + void *key, PyObject *name) { - if (compiler_enter_scope(c, name, COMPILER_SCOPE_ANNOTATIONS, - key, loc.lineno, NULL) == -1) { - return ERROR; - } - c->u->u_metadata.u_posonlyargcount = 1; + _PyCompile_CodeUnitMetadata umd = { + .u_posonlyargcount = 1, + }; + RETURN_IF_ERROR( + compiler_enter_scope(c, name, COMPILER_SCOPE_ANNOTATIONS, + key, loc.lineno, NULL, &umd)); + assert(c->u->u_metadata.u_posonlyargcount == 1); + assert(c->u->u_metadata.u_argcount == 0); + assert(c->u->u_metadata.u_kwonlyargcount == 0); + // if .format != 1: raise NotImplementedError _Py_DECLARE_STR(format, ".format"); ADDOP_I(c, loc, LOAD_FAST, 0); @@ -1452,8 +1474,8 @@ compiler_setup_annotations_scope(struct compiler *c, location loc, } static int -compiler_leave_annotations_scope(struct compiler *c, location loc, - Py_ssize_t annotations_len) +codegen_leave_annotations_scope(struct compiler *c, location loc, + Py_ssize_t annotations_len) { ADDOP_I(c, loc, BUILD_MAP, annotations_len); ADDOP_IN_SCOPE(c, loc, RETURN_VALUE); @@ -1462,7 +1484,7 @@ compiler_leave_annotations_scope(struct compiler *c, location loc, if (co == NULL) { return ERROR; } - if (compiler_make_closure(c, loc, co, 0) < 0) { + if (codegen_make_closure(c, loc, co, 0) < 0) { Py_DECREF(co); return ERROR; } @@ -1523,8 +1545,8 @@ compiler_body(struct compiler *c, location loc, asdl_stmt_seq *stmts) assert(ste->ste_annotation_block != NULL); PyObject *deferred_anno = Py_NewRef(c->u->u_deferred_annotations); void *key = (void *)((uintptr_t)ste->ste_id + 1); - if (compiler_setup_annotations_scope(c, loc, key, - ste->ste_annotation_block->ste_name) == -1) { + if (codegen_setup_annotations_scope(c, loc, key, + ste->ste_annotation_block->ste_name) == -1) { Py_DECREF(deferred_anno); return ERROR; } @@ -1544,7 +1566,7 @@ compiler_body(struct compiler *c, location loc, asdl_stmt_seq *stmts) Py_DECREF(deferred_anno); RETURN_IF_ERROR( - compiler_leave_annotations_scope(c, loc, annotations_len) + codegen_leave_annotations_scope(c, loc, annotations_len) ); RETURN_IF_ERROR( compiler_nameop(c, loc, &_Py_ID(__annotate__), Store) @@ -1602,12 +1624,12 @@ compiler_codegen(struct compiler *c, mod_ty mod) } static int -compiler_enter_anonymous_scope(struct compiler* c, mod_ty mod) +codegen_enter_anonymous_scope(struct compiler* c, mod_ty mod) { _Py_DECLARE_STR(anon_module, ""); RETURN_IF_ERROR( compiler_enter_scope(c, &_Py_STR(anon_module), COMPILER_SCOPE_MODULE, - mod, 1, NULL)); + mod, 1, NULL, NULL)); return SUCCESS; } @@ -1616,7 +1638,7 @@ compiler_mod(struct compiler *c, mod_ty mod) { PyCodeObject *co = NULL; int addNone = mod->kind != Expression_kind; - if (compiler_enter_anonymous_scope(c, mod) < 0) { + if (codegen_enter_anonymous_scope(c, mod) < 0) { return NULL; } if (compiler_codegen(c, mod) < 0) { @@ -1703,8 +1725,8 @@ compiler_lookup_arg(struct compiler *c, PyCodeObject *co, PyObject *name) } static int -compiler_make_closure(struct compiler *c, location loc, - PyCodeObject *co, Py_ssize_t flags) +codegen_make_closure(struct compiler *c, location loc, + PyCodeObject *co, Py_ssize_t flags) { if (co->co_nfreevars) { int i = PyUnstable_Code_GetFirstFree(co); @@ -1743,7 +1765,7 @@ compiler_make_closure(struct compiler *c, location loc, } static int -compiler_decorators(struct compiler *c, asdl_expr_seq* decos) +codegen_decorators(struct compiler *c, asdl_expr_seq* decos) { if (!decos) { return SUCCESS; @@ -1756,7 +1778,7 @@ compiler_decorators(struct compiler *c, asdl_expr_seq* decos) } static int -compiler_apply_decorators(struct compiler *c, asdl_expr_seq* decos) +codegen_apply_decorators(struct compiler *c, asdl_expr_seq* decos) { if (!decos) { return SUCCESS; @@ -1770,8 +1792,8 @@ compiler_apply_decorators(struct compiler *c, asdl_expr_seq* decos) } static int -compiler_kwonlydefaults(struct compiler *c, location loc, - asdl_arg_seq *kwonlyargs, asdl_expr_seq *kw_defaults) +codegen_kwonlydefaults(struct compiler *c, location loc, + asdl_arg_seq *kwonlyargs, asdl_expr_seq *kw_defaults) { /* Push a dict of keyword-only default values. @@ -1817,7 +1839,7 @@ compiler_visit_annexpr(struct compiler *c, expr_ty annotation) } static int -compiler_argannotation(struct compiler *c, identifier id, +codegen_argannotation(struct compiler *c, identifier id, expr_ty annotation, Py_ssize_t *annotations_len, location loc) { if (!annotation) { @@ -1851,14 +1873,14 @@ compiler_argannotation(struct compiler *c, identifier id, } static int -compiler_argannotations(struct compiler *c, asdl_arg_seq* args, - Py_ssize_t *annotations_len, location loc) +codegen_argannotations(struct compiler *c, asdl_arg_seq* args, + Py_ssize_t *annotations_len, location loc) { int i; for (i = 0; i < asdl_seq_LEN(args); i++) { arg_ty arg = (arg_ty)asdl_seq_GET(args, i); RETURN_IF_ERROR( - compiler_argannotation( + codegen_argannotation( c, arg->arg, arg->annotation, @@ -1869,40 +1891,40 @@ compiler_argannotations(struct compiler *c, asdl_arg_seq* args, } static int -compiler_annotations_in_scope(struct compiler *c, location loc, - arguments_ty args, expr_ty returns, - Py_ssize_t *annotations_len) +codegen_annotations_in_scope(struct compiler *c, location loc, + arguments_ty args, expr_ty returns, + Py_ssize_t *annotations_len) { RETURN_IF_ERROR( - compiler_argannotations(c, args->args, annotations_len, loc)); + codegen_argannotations(c, args->args, annotations_len, loc)); RETURN_IF_ERROR( - compiler_argannotations(c, args->posonlyargs, annotations_len, loc)); + codegen_argannotations(c, args->posonlyargs, annotations_len, loc)); if (args->vararg && args->vararg->annotation) { RETURN_IF_ERROR( - compiler_argannotation(c, args->vararg->arg, - args->vararg->annotation, annotations_len, loc)); + codegen_argannotation(c, args->vararg->arg, + args->vararg->annotation, annotations_len, loc)); } RETURN_IF_ERROR( - compiler_argannotations(c, args->kwonlyargs, annotations_len, loc)); + codegen_argannotations(c, args->kwonlyargs, annotations_len, loc)); if (args->kwarg && args->kwarg->annotation) { RETURN_IF_ERROR( - compiler_argannotation(c, args->kwarg->arg, - args->kwarg->annotation, annotations_len, loc)); + codegen_argannotation(c, args->kwarg->arg, + args->kwarg->annotation, annotations_len, loc)); } RETURN_IF_ERROR( - compiler_argannotation(c, &_Py_ID(return), returns, annotations_len, loc)); + codegen_argannotation(c, &_Py_ID(return), returns, annotations_len, loc)); return 0; } static int -compiler_annotations(struct compiler *c, location loc, - arguments_ty args, expr_ty returns) +codegen_annotations(struct compiler *c, location loc, + arguments_ty args, expr_ty returns) { /* Push arg annotation names and values. The expressions are evaluated separately from the rest of the source code. @@ -1919,15 +1941,15 @@ compiler_annotations(struct compiler *c, location loc, bool annotations_used = ste->ste_annotations_used; if (annotations_used) { - if (compiler_setup_annotations_scope(c, loc, (void *)args, - ste->ste_name) < 0) { + if (codegen_setup_annotations_scope(c, loc, (void *)args, + ste->ste_name) < 0) { Py_DECREF(ste); return ERROR; } } Py_DECREF(ste); - if (compiler_annotations_in_scope(c, loc, args, returns, &annotations_len) < 0) { + if (codegen_annotations_in_scope(c, loc, args, returns, &annotations_len) < 0) { if (annotations_used) { compiler_exit_scope(c); } @@ -1936,7 +1958,7 @@ compiler_annotations(struct compiler *c, location loc, if (annotations_used) { RETURN_IF_ERROR( - compiler_leave_annotations_scope(c, loc, annotations_len) + codegen_leave_annotations_scope(c, loc, annotations_len) ); return MAKE_FUNCTION_ANNOTATE; } @@ -1945,7 +1967,7 @@ compiler_annotations(struct compiler *c, location loc, } static int -compiler_defaults(struct compiler *c, arguments_ty args, +codegen_defaults(struct compiler *c, arguments_ty args, location loc) { VISIT_SEQ(c, expr, args->defaults); @@ -1954,18 +1976,18 @@ compiler_defaults(struct compiler *c, arguments_ty args, } static Py_ssize_t -compiler_default_arguments(struct compiler *c, location loc, - arguments_ty args) +codegen_default_arguments(struct compiler *c, location loc, + arguments_ty args) { Py_ssize_t funcflags = 0; if (args->defaults && asdl_seq_LEN(args->defaults) > 0) { - RETURN_IF_ERROR(compiler_defaults(c, args, loc)); + RETURN_IF_ERROR(codegen_defaults(c, args, loc)); funcflags |= MAKE_FUNCTION_DEFAULTS; } if (args->kwonlyargs) { - int res = compiler_kwonlydefaults(c, loc, - args->kwonlyargs, - args->kw_defaults); + int res = codegen_kwonlydefaults(c, loc, + args->kwonlyargs, + args->kw_defaults); RETURN_IF_ERROR(res); if (res > 0) { funcflags |= MAKE_FUNCTION_KWDEFAULTS; @@ -1975,7 +1997,7 @@ compiler_default_arguments(struct compiler *c, location loc, } static int -wrap_in_stopiteration_handler(struct compiler *c) +codegen_wrap_in_stopiteration_handler(struct compiler *c) { NEW_JUMP_TARGET_LABEL(c, handler); @@ -1994,13 +2016,13 @@ wrap_in_stopiteration_handler(struct compiler *c) } static int -compiler_type_param_bound_or_default(struct compiler *c, expr_ty e, - identifier name, void *key, - bool allow_starred) +codegen_type_param_bound_or_default(struct compiler *c, expr_ty e, + identifier name, void *key, + bool allow_starred) { PyObject *defaults = PyTuple_Pack(1, _PyLong_GetOne()); ADDOP_LOAD_CONST_NEW(c, LOC(e), defaults); - if (compiler_setup_annotations_scope(c, LOC(e), key, name) == -1) { + if (codegen_setup_annotations_scope(c, LOC(e), key, name) == -1) { return ERROR; } if (allow_starred && e->kind == Starred_kind) { @@ -2016,7 +2038,7 @@ compiler_type_param_bound_or_default(struct compiler *c, expr_ty e, if (co == NULL) { return ERROR; } - if (compiler_make_closure(c, LOC(e), co, MAKE_FUNCTION_DEFAULTS) < 0) { + if (codegen_make_closure(c, LOC(e), co, MAKE_FUNCTION_DEFAULTS) < 0) { Py_DECREF(co); return ERROR; } @@ -2025,7 +2047,7 @@ compiler_type_param_bound_or_default(struct compiler *c, expr_ty e, } static int -compiler_type_params(struct compiler *c, asdl_type_param_seq *type_params) +codegen_type_params(struct compiler *c, asdl_type_param_seq *type_params) { if (!type_params) { return SUCCESS; @@ -2041,10 +2063,9 @@ compiler_type_params(struct compiler *c, asdl_type_param_seq *type_params) ADDOP_LOAD_CONST(c, loc, typeparam->v.TypeVar.name); if (typeparam->v.TypeVar.bound) { expr_ty bound = typeparam->v.TypeVar.bound; - if (compiler_type_param_bound_or_default(c, bound, typeparam->v.TypeVar.name, - (void *)typeparam, false) < 0) { - return ERROR; - } + RETURN_IF_ERROR( + codegen_type_param_bound_or_default(c, bound, typeparam->v.TypeVar.name, + (void *)typeparam, false)); int intrinsic = bound->kind == Tuple_kind ? INTRINSIC_TYPEVAR_WITH_CONSTRAINTS @@ -2057,10 +2078,9 @@ compiler_type_params(struct compiler *c, asdl_type_param_seq *type_params) if (typeparam->v.TypeVar.default_value) { seen_default = true; expr_ty default_ = typeparam->v.TypeVar.default_value; - if (compiler_type_param_bound_or_default(c, default_, typeparam->v.TypeVar.name, - (void *)((uintptr_t)typeparam + 1), false) < 0) { - return ERROR; - } + RETURN_IF_ERROR( + codegen_type_param_bound_or_default(c, default_, typeparam->v.TypeVar.name, + (void *)((uintptr_t)typeparam + 1), false)); ADDOP_I(c, loc, CALL_INTRINSIC_2, INTRINSIC_SET_TYPEPARAM_DEFAULT); } else if (seen_default) { @@ -2076,10 +2096,9 @@ compiler_type_params(struct compiler *c, asdl_type_param_seq *type_params) ADDOP_I(c, loc, CALL_INTRINSIC_1, INTRINSIC_TYPEVARTUPLE); if (typeparam->v.TypeVarTuple.default_value) { expr_ty default_ = typeparam->v.TypeVarTuple.default_value; - if (compiler_type_param_bound_or_default(c, default_, typeparam->v.TypeVarTuple.name, - (void *)typeparam, true) < 0) { - return ERROR; - } + RETURN_IF_ERROR( + codegen_type_param_bound_or_default(c, default_, typeparam->v.TypeVarTuple.name, + (void *)typeparam, true)); ADDOP_I(c, loc, CALL_INTRINSIC_2, INTRINSIC_SET_TYPEPARAM_DEFAULT); seen_default = true; } @@ -2096,10 +2115,9 @@ compiler_type_params(struct compiler *c, asdl_type_param_seq *type_params) ADDOP_I(c, loc, CALL_INTRINSIC_1, INTRINSIC_PARAMSPEC); if (typeparam->v.ParamSpec.default_value) { expr_ty default_ = typeparam->v.ParamSpec.default_value; - if (compiler_type_param_bound_or_default(c, default_, typeparam->v.ParamSpec.name, - (void *)typeparam, false) < 0) { - return ERROR; - } + RETURN_IF_ERROR( + codegen_type_param_bound_or_default(c, default_, typeparam->v.ParamSpec.name, + (void *)typeparam, false)); ADDOP_I(c, loc, CALL_INTRINSIC_2, INTRINSIC_SET_TYPEPARAM_DEFAULT); seen_default = true; } @@ -2144,8 +2162,13 @@ compiler_function_body(struct compiler *c, stmt_ty s, int is_async, Py_ssize_t f scope_type = COMPILER_SCOPE_FUNCTION; } + _PyCompile_CodeUnitMetadata umd = { + .u_argcount = asdl_seq_LEN(args->args), + .u_posonlyargcount = asdl_seq_LEN(args->posonlyargs), + .u_kwonlyargcount = asdl_seq_LEN(args->kwonlyargs), + }; RETURN_IF_ERROR( - compiler_enter_scope(c, name, scope_type, (void *)s, firstlineno, NULL)); + compiler_enter_scope(c, name, scope_type, (void *)s, firstlineno, NULL, &umd)); Py_ssize_t first_instr = 0; PyObject *docstring = _PyAST_GetDocString(body); @@ -2163,23 +2186,20 @@ compiler_function_body(struct compiler *c, stmt_ty s, int is_async, Py_ssize_t f docstring = NULL; } } - if (compiler_add_const(c, docstring ? docstring : Py_None) < 0) { - Py_XDECREF(docstring); - compiler_exit_scope(c); - return ERROR; - } - Py_CLEAR(docstring); + Py_ssize_t idx = compiler_add_const(c, docstring ? docstring : Py_None); + Py_XDECREF(docstring); + RETURN_IF_ERROR_IN_SCOPE(c, idx < 0 ? ERROR : SUCCESS); - c->u->u_metadata.u_argcount = asdl_seq_LEN(args->args); - c->u->u_metadata.u_posonlyargcount = asdl_seq_LEN(args->posonlyargs); - c->u->u_metadata.u_kwonlyargcount = asdl_seq_LEN(args->kwonlyargs); + assert(c->u->u_metadata.u_argcount == asdl_seq_LEN(args->args)); + assert(c->u->u_metadata.u_posonlyargcount == asdl_seq_LEN(args->posonlyargs)); + assert(c->u->u_metadata.u_kwonlyargcount == asdl_seq_LEN(args->kwonlyargs)); NEW_JUMP_TARGET_LABEL(c, start); USE_LABEL(c, start); PySTEntryObject *ste = SYMTABLE_ENTRY(c); bool add_stopiteration_handler = ste->ste_coroutine || ste->ste_generator; if (add_stopiteration_handler) { - /* wrap_in_stopiteration_handler will push a block, so we need to account for that */ + /* codegen_wrap_in_stopiteration_handler will push a block, so we need to account for that */ RETURN_IF_ERROR( compiler_push_fblock(c, NO_LOCATION, STOP_ITERATION, start, NO_LABEL, NULL)); @@ -2189,10 +2209,7 @@ compiler_function_body(struct compiler *c, stmt_ty s, int is_async, Py_ssize_t f VISIT_IN_SCOPE(c, stmt, (stmt_ty)asdl_seq_GET(body, i)); } if (add_stopiteration_handler) { - if (wrap_in_stopiteration_handler(c) < 0) { - compiler_exit_scope(c); - return ERROR; - } + RETURN_IF_ERROR_IN_SCOPE(c, codegen_wrap_in_stopiteration_handler(c)); compiler_pop_fblock(c, STOP_ITERATION, start); } PyCodeObject *co = optimize_and_assemble(c, 1); @@ -2201,17 +2218,13 @@ compiler_function_body(struct compiler *c, stmt_ty s, int is_async, Py_ssize_t f Py_XDECREF(co); return ERROR; } - location loc = LOC(s); - if (compiler_make_closure(c, loc, co, funcflags) < 0) { - Py_DECREF(co); - return ERROR; - } + int ret = codegen_make_closure(c, LOC(s), co, funcflags); Py_DECREF(co); - return SUCCESS; + return ret; } static int -compiler_function(struct compiler *c, stmt_ty s, int is_async) +codegen_function(struct compiler *c, stmt_ty s, int is_async) { arguments_ty args; expr_ty returns; @@ -2239,7 +2252,7 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) type_params = s->v.FunctionDef.type_params; } - RETURN_IF_ERROR(compiler_decorators(c, decos)); + RETURN_IF_ERROR(codegen_decorators(c, decos)); firstlineno = s->lineno; if (asdl_seq_LEN(decos)) { @@ -2250,7 +2263,7 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) int is_generic = asdl_seq_LEN(type_params) > 0; - funcflags = compiler_default_arguments(c, loc, args); + funcflags = codegen_default_arguments(c, loc, args); if (funcflags == -1) { return ERROR; } @@ -2271,19 +2284,22 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) if (!type_params_name) { return ERROR; } + _PyCompile_CodeUnitMetadata umd = { + .u_argcount = num_typeparam_args, + }; if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_ANNOTATIONS, - (void *)type_params, firstlineno, NULL) == -1) { + (void *)type_params, firstlineno, NULL, &umd) == -1) { Py_DECREF(type_params_name); return ERROR; } Py_DECREF(type_params_name); - RETURN_IF_ERROR_IN_SCOPE(c, compiler_type_params(c, type_params)); + RETURN_IF_ERROR_IN_SCOPE(c, codegen_type_params(c, type_params)); for (int i = 0; i < num_typeparam_args; i++) { - RETURN_IF_ERROR_IN_SCOPE(c, codegen_addop_i(INSTR_SEQUENCE(c), LOAD_FAST, i, loc)); + ADDOP_I_IN_SCOPE(c, loc, LOAD_FAST, i); } } - int annotations_flag = compiler_annotations(c, loc, args, returns); + int annotations_flag = codegen_annotations(c, loc, args, returns); if (annotations_flag < 0) { if (is_generic) { compiler_exit_scope(c); @@ -2300,18 +2316,16 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) } if (is_generic) { - RETURN_IF_ERROR_IN_SCOPE(c, codegen_addop_i( - INSTR_SEQUENCE(c), SWAP, 2, loc)); - RETURN_IF_ERROR_IN_SCOPE(c, codegen_addop_i( - INSTR_SEQUENCE(c), CALL_INTRINSIC_2, INTRINSIC_SET_FUNCTION_TYPE_PARAMS, loc)); + ADDOP_I_IN_SCOPE(c, loc, SWAP, 2); + ADDOP_I_IN_SCOPE(c, loc, CALL_INTRINSIC_2, INTRINSIC_SET_FUNCTION_TYPE_PARAMS); - c->u->u_metadata.u_argcount = num_typeparam_args; + assert(c->u->u_metadata.u_argcount == num_typeparam_args); PyCodeObject *co = optimize_and_assemble(c, 0); compiler_exit_scope(c); if (co == NULL) { return ERROR; } - if (compiler_make_closure(c, loc, co, 0) < 0) { + if (codegen_make_closure(c, loc, co, 0) < 0) { Py_DECREF(co); return ERROR; } @@ -2326,17 +2340,17 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) } } - RETURN_IF_ERROR(compiler_apply_decorators(c, decos)); + RETURN_IF_ERROR(codegen_apply_decorators(c, decos)); return compiler_nameop(c, loc, name, Store); } static int -compiler_set_type_params_in_class(struct compiler *c, location loc) +codegen_set_type_params_in_class(struct compiler *c, location loc) { _Py_DECLARE_STR(type_params, ".type_params"); RETURN_IF_ERROR(compiler_nameop(c, loc, &_Py_STR(type_params), Load)); RETURN_IF_ERROR(compiler_nameop(c, loc, &_Py_ID(__type_params__), Store)); - return 1; + return SUCCESS; } static int @@ -2350,42 +2364,27 @@ compiler_class_body(struct compiler *c, stmt_ty s, int firstlineno) is the class name is the positional arguments and *varargs argument is the keyword arguments and **kwds argument - This borrows from compiler_call. + This borrows from codegen_call. */ /* 1. compile the class body into a code object */ RETURN_IF_ERROR( compiler_enter_scope(c, s->v.ClassDef.name, COMPILER_SCOPE_CLASS, - (void *)s, firstlineno, s->v.ClassDef.name)); + (void *)s, firstlineno, s->v.ClassDef.name, NULL)); location loc = LOCATION(firstlineno, firstlineno, 0, 0); /* load (global) __name__ ... */ - if (compiler_nameop(c, loc, &_Py_ID(__name__), Load) < 0) { - compiler_exit_scope(c); - return ERROR; - } + RETURN_IF_ERROR_IN_SCOPE(c, compiler_nameop(c, loc, &_Py_ID(__name__), Load)); /* ... and store it as __module__ */ - if (compiler_nameop(c, loc, &_Py_ID(__module__), Store) < 0) { - compiler_exit_scope(c); - return ERROR; - } + RETURN_IF_ERROR_IN_SCOPE(c, compiler_nameop(c, loc, &_Py_ID(__module__), Store)); assert(c->u->u_metadata.u_qualname); ADDOP_LOAD_CONST(c, loc, c->u->u_metadata.u_qualname); - if (compiler_nameop(c, loc, &_Py_ID(__qualname__), Store) < 0) { - compiler_exit_scope(c); - return ERROR; - } + RETURN_IF_ERROR_IN_SCOPE(c, compiler_nameop(c, loc, &_Py_ID(__qualname__), Store)); ADDOP_LOAD_CONST_NEW(c, loc, PyLong_FromLong(c->u->u_metadata.u_firstlineno)); - if (compiler_nameop(c, loc, &_Py_ID(__firstlineno__), Store) < 0) { - compiler_exit_scope(c); - return ERROR; - } + RETURN_IF_ERROR_IN_SCOPE(c, compiler_nameop(c, loc, &_Py_ID(__firstlineno__), Store)); asdl_type_param_seq *type_params = s->v.ClassDef.type_params; if (asdl_seq_LEN(type_params) > 0) { - if (!compiler_set_type_params_in_class(c, loc)) { - compiler_exit_scope(c); - return ERROR; - } + RETURN_IF_ERROR_IN_SCOPE(c, codegen_set_type_params_in_class(c, loc)); } if (SYMTABLE_ENTRY(c)->ste_needs_classdict) { ADDOP(c, loc, LOAD_LOCALS); @@ -2393,18 +2392,10 @@ compiler_class_body(struct compiler *c, stmt_ty s, int firstlineno) // We can't use compiler_nameop here because we need to generate a // STORE_DEREF in a class namespace, and compiler_nameop() won't do // that by default. - PyObject *cellvars = c->u->u_metadata.u_cellvars; - if (compiler_addop_o(c, loc, STORE_DEREF, cellvars, - &_Py_ID(__classdict__)) < 0) { - compiler_exit_scope(c); - return ERROR; - } + ADDOP_N_IN_SCOPE(c, loc, STORE_DEREF, &_Py_ID(__classdict__), cellvars); } /* compile the body proper */ - if (compiler_body(c, loc, s->v.ClassDef.body) < 0) { - compiler_exit_scope(c); - return ERROR; - } + RETURN_IF_ERROR_IN_SCOPE(c, compiler_body(c, loc, s->v.ClassDef.body)); assert(c->u->u_static_attributes); PyObject *static_attributes = PySequence_Tuple(c->u->u_static_attributes); if (static_attributes == NULL) { @@ -2413,39 +2404,27 @@ compiler_class_body(struct compiler *c, stmt_ty s, int firstlineno) } ADDOP_LOAD_CONST(c, NO_LOCATION, static_attributes); Py_CLEAR(static_attributes); - if (compiler_nameop(c, NO_LOCATION, &_Py_ID(__static_attributes__), Store) < 0) { - compiler_exit_scope(c); - return ERROR; - } + RETURN_IF_ERROR_IN_SCOPE( + c, compiler_nameop(c, NO_LOCATION, &_Py_ID(__static_attributes__), Store)); /* The following code is artificial */ /* Set __classdictcell__ if necessary */ if (SYMTABLE_ENTRY(c)->ste_needs_classdict) { /* Store __classdictcell__ into class namespace */ int i = dict_lookup_arg(c->u->u_metadata.u_cellvars, &_Py_ID(__classdict__)); - if (i < 0) { - compiler_exit_scope(c); - return ERROR; - } + RETURN_IF_ERROR_IN_SCOPE(c, i); ADDOP_I(c, NO_LOCATION, LOAD_CLOSURE, i); - if (compiler_nameop(c, NO_LOCATION, &_Py_ID(__classdictcell__), Store) < 0) { - compiler_exit_scope(c); - return ERROR; - } + RETURN_IF_ERROR_IN_SCOPE( + c, compiler_nameop(c, NO_LOCATION, &_Py_ID(__classdictcell__), Store)); } /* Return __classcell__ if it is referenced, otherwise return None */ if (SYMTABLE_ENTRY(c)->ste_needs_class_closure) { /* Store __classcell__ into class namespace & return it */ int i = dict_lookup_arg(c->u->u_metadata.u_cellvars, &_Py_ID(__class__)); - if (i < 0) { - compiler_exit_scope(c); - return ERROR; - } + RETURN_IF_ERROR_IN_SCOPE(c, i); ADDOP_I(c, NO_LOCATION, LOAD_CLOSURE, i); ADDOP_I(c, NO_LOCATION, COPY, 1); - if (compiler_nameop(c, NO_LOCATION, &_Py_ID(__classcell__), Store) < 0) { - compiler_exit_scope(c); - return ERROR; - } + RETURN_IF_ERROR_IN_SCOPE( + c, compiler_nameop(c, NO_LOCATION, &_Py_ID(__classcell__), Store)); } else { /* No methods referenced __class__, so just return None */ @@ -2470,7 +2449,7 @@ compiler_class_body(struct compiler *c, stmt_ty s, int firstlineno) ADDOP(c, loc, PUSH_NULL); /* 3. load a function (or closure) made from the code object */ - if (compiler_make_closure(c, loc, co, 0) < 0) { + if (codegen_make_closure(c, loc, co, 0) < 0) { Py_DECREF(co); return ERROR; } @@ -2483,11 +2462,11 @@ compiler_class_body(struct compiler *c, stmt_ty s, int firstlineno) } static int -compiler_class(struct compiler *c, stmt_ty s) +codegen_class(struct compiler *c, stmt_ty s) { asdl_expr_seq *decos = s->v.ClassDef.decorator_list; - RETURN_IF_ERROR(compiler_decorators(c, decos)); + RETURN_IF_ERROR(codegen_decorators(c, decos)); int firstlineno = s->lineno; if (asdl_seq_LEN(decos)) { @@ -2504,12 +2483,12 @@ compiler_class(struct compiler *c, stmt_ty s) return ERROR; } if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_ANNOTATIONS, - (void *)type_params, firstlineno, s->v.ClassDef.name) == -1) { + (void *)type_params, firstlineno, s->v.ClassDef.name, NULL) == -1) { Py_DECREF(type_params_name); return ERROR; } Py_DECREF(type_params_name); - RETURN_IF_ERROR_IN_SCOPE(c, compiler_type_params(c, type_params)); + RETURN_IF_ERROR_IN_SCOPE(c, codegen_type_params(c, type_params)); _Py_DECLARE_STR(type_params, ".type_params"); RETURN_IF_ERROR_IN_SCOPE(c, compiler_nameop(c, loc, &_Py_STR(type_params), Store)); } @@ -2527,9 +2506,7 @@ compiler_class(struct compiler *c, stmt_ty s) _Py_DECLARE_STR(type_params, ".type_params"); _Py_DECLARE_STR(generic_base, ".generic_base"); RETURN_IF_ERROR_IN_SCOPE(c, compiler_nameop(c, loc, &_Py_STR(type_params), Load)); - RETURN_IF_ERROR_IN_SCOPE( - c, codegen_addop_i(INSTR_SEQUENCE(c), CALL_INTRINSIC_1, INTRINSIC_SUBSCRIPT_GENERIC, loc) - ) + ADDOP_I_IN_SCOPE(c, loc, CALL_INTRINSIC_1, INTRINSIC_SUBSCRIPT_GENERIC); RETURN_IF_ERROR_IN_SCOPE(c, compiler_nameop(c, loc, &_Py_STR(generic_base), Store)); Py_ssize_t original_len = asdl_seq_LEN(s->v.ClassDef.bases); @@ -2551,9 +2528,9 @@ compiler_class(struct compiler *c, stmt_ty s) return ERROR; } asdl_seq_SET(bases, original_len, name_node); - RETURN_IF_ERROR_IN_SCOPE(c, compiler_call_helper(c, loc, 2, - bases, - s->v.ClassDef.keywords)); + RETURN_IF_ERROR_IN_SCOPE(c, codegen_call_helper(c, loc, 2, + bases, + s->v.ClassDef.keywords)); PyCodeObject *co = optimize_and_assemble(c, 0); @@ -2561,7 +2538,7 @@ compiler_class(struct compiler *c, stmt_ty s) if (co == NULL) { return ERROR; } - if (compiler_make_closure(c, loc, co, 0) < 0) { + if (codegen_make_closure(c, loc, co, 0) < 0) { Py_DECREF(co); return ERROR; } @@ -2569,13 +2546,13 @@ compiler_class(struct compiler *c, stmt_ty s) ADDOP(c, loc, PUSH_NULL); ADDOP_I(c, loc, CALL, 0); } else { - RETURN_IF_ERROR(compiler_call_helper(c, loc, 2, + RETURN_IF_ERROR(codegen_call_helper(c, loc, 2, s->v.ClassDef.bases, s->v.ClassDef.keywords)); } /* 6. apply decorators */ - RETURN_IF_ERROR(compiler_apply_decorators(c, decos)); + RETURN_IF_ERROR(codegen_apply_decorators(c, decos)); /* 7. store into */ RETURN_IF_ERROR(compiler_nameop(c, loc, s->v.ClassDef.name, Store)); @@ -2583,14 +2560,14 @@ compiler_class(struct compiler *c, stmt_ty s) } static int -compiler_typealias_body(struct compiler *c, stmt_ty s) +codegen_typealias_body(struct compiler *c, stmt_ty s) { location loc = LOC(s); PyObject *name = s->v.TypeAlias.name->v.Name.id; PyObject *defaults = PyTuple_Pack(1, _PyLong_GetOne()); ADDOP_LOAD_CONST_NEW(c, loc, defaults); RETURN_IF_ERROR( - compiler_setup_annotations_scope(c, LOC(s), s, name)); + codegen_setup_annotations_scope(c, LOC(s), s, name)); /* Make None the first constant, so the evaluate function can't have a docstring. */ RETURN_IF_ERROR(compiler_add_const(c, Py_None)); @@ -2601,7 +2578,7 @@ compiler_typealias_body(struct compiler *c, stmt_ty s) if (co == NULL) { return ERROR; } - if (compiler_make_closure(c, loc, co, MAKE_FUNCTION_DEFAULTS) < 0) { + if (codegen_make_closure(c, loc, co, MAKE_FUNCTION_DEFAULTS) < 0) { Py_DECREF(co); return ERROR; } @@ -2612,7 +2589,7 @@ compiler_typealias_body(struct compiler *c, stmt_ty s) } static int -compiler_typealias(struct compiler *c, stmt_ty s) +codegen_typealias(struct compiler *c, stmt_ty s) { location loc = LOC(s); asdl_type_param_seq *type_params = s->v.TypeAlias.type_params; @@ -2625,22 +2602,20 @@ compiler_typealias(struct compiler *c, stmt_ty s) return ERROR; } if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_ANNOTATIONS, - (void *)type_params, loc.lineno, NULL) == -1) { + (void *)type_params, loc.lineno, NULL, NULL) == -1) { Py_DECREF(type_params_name); return ERROR; } Py_DECREF(type_params_name); - RETURN_IF_ERROR_IN_SCOPE( - c, compiler_addop_load_const(c, loc, name) - ); - RETURN_IF_ERROR_IN_SCOPE(c, compiler_type_params(c, type_params)); + ADDOP_LOAD_CONST_IN_SCOPE(c, loc, name); + RETURN_IF_ERROR_IN_SCOPE(c, codegen_type_params(c, type_params)); } else { ADDOP_LOAD_CONST(c, loc, name); ADDOP_LOAD_CONST(c, loc, Py_None); } - if (compiler_typealias_body(c, s) < 0) { + if (codegen_typealias_body(c, s) < 0) { if (is_generic) { compiler_exit_scope(c); } @@ -2653,7 +2628,7 @@ compiler_typealias(struct compiler *c, stmt_ty s) if (co == NULL) { return ERROR; } - if (compiler_make_closure(c, loc, co, 0) < 0) { + if (codegen_make_closure(c, loc, co, 0) < 0) { Py_DECREF(co); return ERROR; } @@ -2686,7 +2661,7 @@ static PyTypeObject * infer_type(expr_ty e); Emit a warning if any operand is a constant except named singletons. */ static int -check_compare(struct compiler *c, expr_ty e) +codegen_check_compare(struct compiler *c, expr_ty e) { Py_ssize_t i, n; bool left = check_is_arg(e->v.Compare.left); @@ -2713,8 +2688,8 @@ check_compare(struct compiler *c, expr_ty e) return SUCCESS; } -static int compiler_addcompare(struct compiler *c, location loc, - cmpop_ty op) +static int +codegen_addcompare(struct compiler *c, location loc, cmpop_ty op) { int cmp; switch (op) { @@ -2759,16 +2734,14 @@ static int compiler_addcompare(struct compiler *c, location loc, return SUCCESS; } - - static int -compiler_jump_if(struct compiler *c, location loc, - expr_ty e, jump_target_label next, int cond) +codegen_jump_if(struct compiler *c, location loc, + expr_ty e, jump_target_label next, int cond) { switch (e->kind) { case UnaryOp_kind: if (e->v.UnaryOp.op == Not) { - return compiler_jump_if(c, loc, e->v.UnaryOp.operand, next, !cond); + return codegen_jump_if(c, loc, e->v.UnaryOp.operand, next, !cond); } /* fallback to general implementation */ break; @@ -2784,10 +2757,10 @@ compiler_jump_if(struct compiler *c, location loc, } for (i = 0; i < n; ++i) { RETURN_IF_ERROR( - compiler_jump_if(c, loc, (expr_ty)asdl_seq_GET(s, i), next2, cond2)); + codegen_jump_if(c, loc, (expr_ty)asdl_seq_GET(s, i), next2, cond2)); } RETURN_IF_ERROR( - compiler_jump_if(c, loc, (expr_ty)asdl_seq_GET(s, n), next, cond)); + codegen_jump_if(c, loc, (expr_ty)asdl_seq_GET(s, n), next, cond)); if (!SAME_LABEL(next2, next)) { USE_LABEL(c, next2); } @@ -2797,14 +2770,14 @@ compiler_jump_if(struct compiler *c, location loc, NEW_JUMP_TARGET_LABEL(c, end); NEW_JUMP_TARGET_LABEL(c, next2); RETURN_IF_ERROR( - compiler_jump_if(c, loc, e->v.IfExp.test, next2, 0)); + codegen_jump_if(c, loc, e->v.IfExp.test, next2, 0)); RETURN_IF_ERROR( - compiler_jump_if(c, loc, e->v.IfExp.body, next, cond)); + codegen_jump_if(c, loc, e->v.IfExp.body, next, cond)); ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); USE_LABEL(c, next2); RETURN_IF_ERROR( - compiler_jump_if(c, loc, e->v.IfExp.orelse, next, cond)); + codegen_jump_if(c, loc, e->v.IfExp.orelse, next, cond)); USE_LABEL(c, end); return SUCCESS; @@ -2812,7 +2785,7 @@ compiler_jump_if(struct compiler *c, location loc, case Compare_kind: { Py_ssize_t n = asdl_seq_LEN(e->v.Compare.ops) - 1; if (n > 0) { - RETURN_IF_ERROR(check_compare(c, e)); + RETURN_IF_ERROR(codegen_check_compare(c, e)); NEW_JUMP_TARGET_LABEL(c, cleanup); VISIT(c, expr, e->v.Compare.left); for (Py_ssize_t i = 0; i < n; i++) { @@ -2856,14 +2829,14 @@ compiler_jump_if(struct compiler *c, location loc, } static int -compiler_ifexp(struct compiler *c, expr_ty e) +codegen_ifexp(struct compiler *c, expr_ty e) { assert(e->kind == IfExp_kind); NEW_JUMP_TARGET_LABEL(c, end); NEW_JUMP_TARGET_LABEL(c, next); RETURN_IF_ERROR( - compiler_jump_if(c, LOC(e), e->v.IfExp.test, next, 0)); + codegen_jump_if(c, LOC(e), e->v.IfExp.test, next, 0)); VISIT(c, expr, e->v.IfExp.body); ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); @@ -2876,7 +2849,7 @@ compiler_ifexp(struct compiler *c, expr_ty e) } static int -compiler_lambda(struct compiler *c, expr_ty e) +codegen_lambda(struct compiler *c, expr_ty e) { PyCodeObject *co; Py_ssize_t funcflags; @@ -2884,23 +2857,28 @@ compiler_lambda(struct compiler *c, expr_ty e) assert(e->kind == Lambda_kind); location loc = LOC(e); - funcflags = compiler_default_arguments(c, loc, args); + funcflags = codegen_default_arguments(c, loc, args); if (funcflags == -1) { return ERROR; } + _PyCompile_CodeUnitMetadata umd = { + .u_argcount = asdl_seq_LEN(args->args), + .u_posonlyargcount = asdl_seq_LEN(args->posonlyargs), + .u_kwonlyargcount = asdl_seq_LEN(args->kwonlyargs), + }; _Py_DECLARE_STR(anon_lambda, ""); RETURN_IF_ERROR( compiler_enter_scope(c, &_Py_STR(anon_lambda), COMPILER_SCOPE_LAMBDA, - (void *)e, e->lineno, NULL)); + (void *)e, e->lineno, NULL, &umd)); /* Make None the first constant, so the lambda can't have a docstring. */ RETURN_IF_ERROR(compiler_add_const(c, Py_None)); - c->u->u_metadata.u_argcount = asdl_seq_LEN(args->args); - c->u->u_metadata.u_posonlyargcount = asdl_seq_LEN(args->posonlyargs); - c->u->u_metadata.u_kwonlyargcount = asdl_seq_LEN(args->kwonlyargs); + assert(c->u->u_metadata.u_argcount == asdl_seq_LEN(args->args)); + assert(c->u->u_metadata.u_posonlyargcount == asdl_seq_LEN(args->posonlyargs)); + assert(c->u->u_metadata.u_kwonlyargcount == asdl_seq_LEN(args->kwonlyargs)); VISIT_IN_SCOPE(c, expr, e->v.Lambda.body); if (SYMTABLE_ENTRY(c)->ste_generator) { co = optimize_and_assemble(c, 0); @@ -2915,7 +2893,7 @@ compiler_lambda(struct compiler *c, expr_ty e) return ERROR; } - if (compiler_make_closure(c, loc, co, funcflags) < 0) { + if (codegen_make_closure(c, loc, co, funcflags) < 0) { Py_DECREF(co); return ERROR; } @@ -2925,7 +2903,7 @@ compiler_lambda(struct compiler *c, expr_ty e) } static int -compiler_if(struct compiler *c, stmt_ty s) +codegen_if(struct compiler *c, stmt_ty s) { jump_target_label next; assert(s->kind == If_kind); @@ -2938,7 +2916,7 @@ compiler_if(struct compiler *c, stmt_ty s) next = end; } RETURN_IF_ERROR( - compiler_jump_if(c, LOC(s), s->v.If.test, next, 0)); + codegen_jump_if(c, LOC(s), s->v.If.test, next, 0)); VISIT_SEQ(c, stmt, s->v.If.body); if (asdl_seq_LEN(s->v.If.orelse)) { @@ -2953,7 +2931,7 @@ compiler_if(struct compiler *c, stmt_ty s) } static int -compiler_for(struct compiler *c, stmt_ty s) +codegen_for(struct compiler *c, stmt_ty s) { location loc = LOC(s); NEW_JUMP_TARGET_LABEL(c, start); @@ -3000,7 +2978,7 @@ compiler_for(struct compiler *c, stmt_ty s) static int -compiler_async_for(struct compiler *c, stmt_ty s) +codegen_async_for(struct compiler *c, stmt_ty s) { location loc = LOC(s); @@ -3045,7 +3023,7 @@ compiler_async_for(struct compiler *c, stmt_ty s) } static int -compiler_while(struct compiler *c, stmt_ty s) +codegen_while(struct compiler *c, stmt_ty s) { NEW_JUMP_TARGET_LABEL(c, loop); NEW_JUMP_TARGET_LABEL(c, end); @@ -3054,7 +3032,7 @@ compiler_while(struct compiler *c, stmt_ty s) USE_LABEL(c, loop); RETURN_IF_ERROR(compiler_push_fblock(c, LOC(s), WHILE_LOOP, loop, end, NULL)); - RETURN_IF_ERROR(compiler_jump_if(c, LOC(s), s->v.While.test, anchor, 0)); + RETURN_IF_ERROR(codegen_jump_if(c, LOC(s), s->v.While.test, anchor, 0)); VISIT_SEQ(c, stmt, s->v.While.body); ADDOP_JUMP(c, NO_LOCATION, JUMP, loop); @@ -3071,7 +3049,7 @@ compiler_while(struct compiler *c, stmt_ty s) } static int -compiler_return(struct compiler *c, stmt_ty s) +codegen_return(struct compiler *c, stmt_ty s) { location loc = LOC(s); int preserve_tos = ((s->v.Return.value != NULL) && @@ -3112,7 +3090,7 @@ compiler_return(struct compiler *c, stmt_ty s) } static int -compiler_break(struct compiler *c, location loc) +codegen_break(struct compiler *c, location loc) { struct fblockinfo *loop = NULL; location origin_loc = loc; @@ -3128,7 +3106,7 @@ compiler_break(struct compiler *c, location loc) } static int -compiler_continue(struct compiler *c, location loc) +codegen_continue(struct compiler *c, location loc) { struct fblockinfo *loop = NULL; location origin_loc = loc; @@ -3173,7 +3151,7 @@ compiler_continue(struct compiler *c, location loc) */ static int -compiler_try_finally(struct compiler *c, stmt_ty s) +codegen_try_finally(struct compiler *c, stmt_ty s) { location loc = LOC(s); @@ -3191,7 +3169,7 @@ compiler_try_finally(struct compiler *c, stmt_ty s) s->v.Try.finalbody)); if (s->v.Try.handlers && asdl_seq_LEN(s->v.Try.handlers)) { - RETURN_IF_ERROR(compiler_try_except(c, s)); + RETURN_IF_ERROR(codegen_try_except(c, s)); } else { VISIT_SEQ(c, stmt, s->v.Try.body); @@ -3224,7 +3202,7 @@ compiler_try_finally(struct compiler *c, stmt_ty s) } static int -compiler_try_star_finally(struct compiler *c, stmt_ty s) +codegen_try_star_finally(struct compiler *c, stmt_ty s) { location loc = LOC(s); @@ -3241,7 +3219,7 @@ compiler_try_star_finally(struct compiler *c, stmt_ty s) s->v.TryStar.finalbody)); if (s->v.TryStar.handlers && asdl_seq_LEN(s->v.TryStar.handlers)) { - RETURN_IF_ERROR(compiler_try_star_except(c, s)); + RETURN_IF_ERROR(codegen_try_star_except(c, s)); } else { VISIT_SEQ(c, stmt, s->v.TryStar.body); @@ -3304,7 +3282,7 @@ compiler_try_star_finally(struct compiler *c, stmt_ty s) Of course, parts are not generated if Vi or Ei is not present. */ static int -compiler_try_except(struct compiler *c, stmt_ty s) +codegen_try_except(struct compiler *c, stmt_ty s) { location loc = LOC(s); Py_ssize_t i, n; @@ -3486,7 +3464,7 @@ compiler_try_except(struct compiler *c, stmt_ty s) [] L0: */ static int -compiler_try_star_except(struct compiler *c, stmt_ty s) +codegen_try_star_except(struct compiler *c, stmt_ty s) { location loc = LOC(s); @@ -3653,27 +3631,27 @@ compiler_try_star_except(struct compiler *c, stmt_ty s) } static int -compiler_try(struct compiler *c, stmt_ty s) { +codegen_try(struct compiler *c, stmt_ty s) { if (s->v.Try.finalbody && asdl_seq_LEN(s->v.Try.finalbody)) - return compiler_try_finally(c, s); + return codegen_try_finally(c, s); else - return compiler_try_except(c, s); + return codegen_try_except(c, s); } static int -compiler_try_star(struct compiler *c, stmt_ty s) +codegen_try_star(struct compiler *c, stmt_ty s) { if (s->v.TryStar.finalbody && asdl_seq_LEN(s->v.TryStar.finalbody)) { - return compiler_try_star_finally(c, s); + return codegen_try_star_finally(c, s); } else { - return compiler_try_star_except(c, s); + return codegen_try_star_except(c, s); } } static int -compiler_import_as(struct compiler *c, location loc, - identifier name, identifier asname) +codegen_import_as(struct compiler *c, location loc, + identifier name, identifier asname) { /* The IMPORT_NAME opcode was already generated. This function merely needs to bind the result to a name. @@ -3714,7 +3692,7 @@ compiler_import_as(struct compiler *c, location loc, } static int -compiler_import(struct compiler *c, stmt_ty s) +codegen_import(struct compiler *c, stmt_ty s) { location loc = LOC(s); /* The Import node stores a module name like a.b.c as a single @@ -3736,7 +3714,7 @@ compiler_import(struct compiler *c, stmt_ty s) ADDOP_NAME(c, loc, IMPORT_NAME, alias->name, names); if (alias->asname) { - r = compiler_import_as(c, loc, alias->name, alias->asname); + r = codegen_import_as(c, loc, alias->name, alias->asname); RETURN_IF_ERROR(r); } else { @@ -3760,7 +3738,7 @@ compiler_import(struct compiler *c, stmt_ty s) } static int -compiler_from_import(struct compiler *c, stmt_ty s) +codegen_from_import(struct compiler *c, stmt_ty s) { Py_ssize_t n = asdl_seq_LEN(s->v.ImportFrom.names); @@ -3828,7 +3806,7 @@ compiler_assert(struct compiler *c, stmt_ty s) return SUCCESS; } NEW_JUMP_TARGET_LABEL(c, end); - RETURN_IF_ERROR(compiler_jump_if(c, LOC(s), s->v.Assert.test, end, 1)); + RETURN_IF_ERROR(codegen_jump_if(c, LOC(s), s->v.Assert.test, end, 1)); ADDOP_I(c, LOC(s), LOAD_COMMON_CONSTANT, CONSTANT_ASSERTIONERROR); if (s->v.Assert.msg) { VISIT(c, expr, s->v.Assert.msg); @@ -3867,13 +3845,13 @@ compiler_visit_stmt(struct compiler *c, stmt_ty s) switch (s->kind) { case FunctionDef_kind: - return compiler_function(c, s, 0); + return codegen_function(c, s, 0); case ClassDef_kind: - return compiler_class(c, s); + return codegen_class(c, s); case TypeAlias_kind: - return compiler_typealias(c, s); + return codegen_typealias(c, s); case Return_kind: - return compiler_return(c, s); + return codegen_return(c, s); case Delete_kind: VISIT_SEQ(c, expr, s->v.Delete.targets) break; @@ -3891,17 +3869,17 @@ compiler_visit_stmt(struct compiler *c, stmt_ty s) break; } case AugAssign_kind: - return compiler_augassign(c, s); + return codegen_augassign(c, s); case AnnAssign_kind: return compiler_annassign(c, s); case For_kind: - return compiler_for(c, s); + return codegen_for(c, s); case While_kind: - return compiler_while(c, s); + return codegen_while(c, s); case If_kind: - return compiler_if(c, s); + return codegen_if(c, s); case Match_kind: - return compiler_match(c, s); + return codegen_match(c, s); case Raise_kind: { Py_ssize_t n = 0; @@ -3917,15 +3895,15 @@ compiler_visit_stmt(struct compiler *c, stmt_ty s) break; } case Try_kind: - return compiler_try(c, s); + return codegen_try(c, s); case TryStar_kind: - return compiler_try_star(c, s); + return codegen_try_star(c, s); case Assert_kind: return compiler_assert(c, s); case Import_kind: - return compiler_import(c, s); + return codegen_import(c, s); case ImportFrom_kind: - return compiler_from_import(c, s); + return codegen_from_import(c, s); case Global_kind: case Nonlocal_kind: break; @@ -3940,20 +3918,20 @@ compiler_visit_stmt(struct compiler *c, stmt_ty s) } case Break_kind: { - return compiler_break(c, LOC(s)); + return codegen_break(c, LOC(s)); } case Continue_kind: { - return compiler_continue(c, LOC(s)); + return codegen_continue(c, LOC(s)); } case With_kind: - return compiler_with(c, s, 0); + return codegen_with(c, s, 0); case AsyncFunctionDef_kind: - return compiler_function(c, s, 1); + return codegen_function(c, s, 1); case AsyncWith_kind: - return compiler_async_with(c, s, 0); + return codegen_async_with(c, s, 0); case AsyncFor_kind: - return compiler_async_for(c, s); + return codegen_async_for(c, s); } return SUCCESS; @@ -4030,7 +4008,7 @@ addop_binary(struct compiler *c, location loc, operator_ty binop, static int -addop_yield(struct compiler *c, location loc) { +codegen_addop_yield(struct compiler *c, location loc) { PySTEntryObject *ste = SYMTABLE_ENTRY(c); if (ste->ste_generator && ste->ste_coroutine) { ADDOP_I(c, loc, CALL_INTRINSIC_1, INTRINSIC_ASYNC_GEN_WRAP); @@ -4040,6 +4018,13 @@ addop_yield(struct compiler *c, location loc) { return SUCCESS; } +static int +codegen_load_classdict_freevar(struct compiler *c, location loc) +{ + ADDOP_N(c, loc, LOAD_DEREF, &_Py_ID(__classdict__), freevars); + return SUCCESS; +} + static int compiler_nameop(struct compiler *c, location loc, identifier name, expr_context_ty ctx) @@ -4119,8 +4104,7 @@ compiler_nameop(struct compiler *c, location loc, else if (SYMTABLE_ENTRY(c)->ste_can_see_class_scope) { op = LOAD_FROM_DICT_OR_DEREF; // First load the classdict - if (compiler_addop_o(c, loc, LOAD_DEREF, - c->u->u_metadata.u_freevars, &_Py_ID(__classdict__)) < 0) { + if (codegen_load_classdict_freevar(c, loc) < 0) { goto error; } } @@ -4146,8 +4130,7 @@ compiler_nameop(struct compiler *c, location loc, if (SYMTABLE_ENTRY(c)->ste_can_see_class_scope && scope == GLOBAL_IMPLICIT) { op = LOAD_FROM_DICT_OR_GLOBALS; // First load the classdict - if (compiler_addop_o(c, loc, LOAD_DEREF, - c->u->u_metadata.u_freevars, &_Py_ID(__classdict__)) < 0) { + if (codegen_load_classdict_freevar(c, loc) < 0) { goto error; } } else { @@ -4181,7 +4164,8 @@ compiler_nameop(struct compiler *c, location loc, if (op == LOAD_GLOBAL) { arg <<= 1; } - return codegen_addop_i(INSTR_SEQUENCE(c), op, arg, loc); + ADDOP_I(c, loc, op, arg); + return SUCCESS; error: Py_DECREF(mangled); @@ -4189,7 +4173,7 @@ compiler_nameop(struct compiler *c, location loc, } static int -compiler_boolop(struct compiler *c, expr_ty e) +codegen_boolop(struct compiler *c, expr_ty e) { int jumpi; Py_ssize_t i, n; @@ -4344,7 +4328,7 @@ assignment_helper(struct compiler *c, location loc, asdl_expr_seq *elts) } static int -compiler_list(struct compiler *c, expr_ty e) +codegen_list(struct compiler *c, expr_ty e) { location loc = LOC(e); asdl_expr_seq *elts = e->v.List.elts; @@ -4362,7 +4346,7 @@ compiler_list(struct compiler *c, expr_ty e) } static int -compiler_tuple(struct compiler *c, expr_ty e) +codegen_tuple(struct compiler *c, expr_ty e) { location loc = LOC(e); asdl_expr_seq *elts = e->v.Tuple.elts; @@ -4380,7 +4364,7 @@ compiler_tuple(struct compiler *c, expr_ty e) } static int -compiler_set(struct compiler *c, expr_ty e) +codegen_set(struct compiler *c, expr_ty e) { location loc = LOC(e); return starunpack_helper(c, loc, e->v.Set.elts, 0, @@ -4400,7 +4384,7 @@ are_all_items_const(asdl_expr_seq *seq, Py_ssize_t begin, Py_ssize_t end) } static int -compiler_subdict(struct compiler *c, expr_ty e, Py_ssize_t begin, Py_ssize_t end) +codegen_subdict(struct compiler *c, expr_ty e, Py_ssize_t begin, Py_ssize_t end) { Py_ssize_t i, n = end - begin; int big = n*2 > STACK_USE_GUIDELINE; @@ -4422,7 +4406,7 @@ compiler_subdict(struct compiler *c, expr_ty e, Py_ssize_t begin, Py_ssize_t end } static int -compiler_dict(struct compiler *c, expr_ty e) +codegen_dict(struct compiler *c, expr_ty e) { location loc = LOC(e); Py_ssize_t i, n, elements; @@ -4435,7 +4419,7 @@ compiler_dict(struct compiler *c, expr_ty e) is_unpacking = (expr_ty)asdl_seq_GET(e->v.Dict.keys, i) == NULL; if (is_unpacking) { if (elements) { - RETURN_IF_ERROR(compiler_subdict(c, e, i - elements, i)); + RETURN_IF_ERROR(codegen_subdict(c, e, i - elements, i)); if (have_dict) { ADDOP_I(c, loc, DICT_UPDATE, 1); } @@ -4451,7 +4435,7 @@ compiler_dict(struct compiler *c, expr_ty e) } else { if (elements*2 > STACK_USE_GUIDELINE) { - RETURN_IF_ERROR(compiler_subdict(c, e, i - elements, i + 1)); + RETURN_IF_ERROR(codegen_subdict(c, e, i - elements, i + 1)); if (have_dict) { ADDOP_I(c, loc, DICT_UPDATE, 1); } @@ -4464,7 +4448,7 @@ compiler_dict(struct compiler *c, expr_ty e) } } if (elements) { - RETURN_IF_ERROR(compiler_subdict(c, e, n - elements, n)); + RETURN_IF_ERROR(codegen_subdict(c, e, n - elements, n)); if (have_dict) { ADDOP_I(c, loc, DICT_UPDATE, 1); } @@ -4477,12 +4461,12 @@ compiler_dict(struct compiler *c, expr_ty e) } static int -compiler_compare(struct compiler *c, expr_ty e) +codegen_compare(struct compiler *c, expr_ty e) { location loc = LOC(e); Py_ssize_t i, n; - RETURN_IF_ERROR(check_compare(c, e)); + RETURN_IF_ERROR(codegen_check_compare(c, e)); VISIT(c, expr, e->v.Compare.left); assert(asdl_seq_LEN(e->v.Compare.ops) > 0); n = asdl_seq_LEN(e->v.Compare.ops) - 1; @@ -4832,7 +4816,7 @@ maybe_optimize_method_call(struct compiler *c, expr_ty e) if (kwdsl) { VISIT_SEQ(c, keyword, kwds); RETURN_IF_ERROR( - compiler_call_simple_kw_helper(c, loc, kwds, kwdsl)); + codegen_call_simple_kw_helper(c, loc, kwds, kwdsl)); loc = update_start_location_to_match_attr(c, LOC(e), meth); ADDOP_I(c, loc, CALL_KW, argsl + kwdsl); } @@ -4844,7 +4828,7 @@ maybe_optimize_method_call(struct compiler *c, expr_ty e) } static int -validate_keywords(struct compiler *c, asdl_keyword_seq *keywords) +codegen_validate_keywords(struct compiler *c, asdl_keyword_seq *keywords) { Py_ssize_t nkeywords = asdl_seq_LEN(keywords); for (Py_ssize_t i = 0; i < nkeywords; i++) { @@ -4864,9 +4848,9 @@ validate_keywords(struct compiler *c, asdl_keyword_seq *keywords) } static int -compiler_call(struct compiler *c, expr_ty e) +codegen_call(struct compiler *c, expr_ty e) { - RETURN_IF_ERROR(validate_keywords(c, e->v.Call.keywords)); + RETURN_IF_ERROR(codegen_validate_keywords(c, e->v.Call.keywords)); int ret = maybe_optimize_method_call(c, e); if (ret < 0) { return ERROR; @@ -4879,13 +4863,13 @@ compiler_call(struct compiler *c, expr_ty e) location loc = LOC(e->v.Call.func); ADDOP(c, loc, PUSH_NULL); loc = LOC(e); - return compiler_call_helper(c, loc, 0, - e->v.Call.args, - e->v.Call.keywords); + return codegen_call_helper(c, loc, 0, + e->v.Call.args, + e->v.Call.keywords); } static int -compiler_joined_str(struct compiler *c, expr_ty e) +codegen_joined_str(struct compiler *c, expr_ty e) { location loc = LOC(e); Py_ssize_t value_count = asdl_seq_LEN(e->v.JoinedStr.values); @@ -4915,7 +4899,7 @@ compiler_joined_str(struct compiler *c, expr_ty e) /* Used to implement f-strings. Format a single value. */ static int -compiler_formatted_value(struct compiler *c, expr_ty e) +codegen_formatted_value(struct compiler *c, expr_ty e) { /* Our oparg encodes 2 pieces of information: the conversion character, and whether or not a format_spec was provided. @@ -4961,9 +4945,9 @@ compiler_formatted_value(struct compiler *c, expr_ty e) } static int -compiler_subkwargs(struct compiler *c, location loc, - asdl_keyword_seq *keywords, - Py_ssize_t begin, Py_ssize_t end) +codegen_subkwargs(struct compiler *c, location loc, + asdl_keyword_seq *keywords, + Py_ssize_t begin, Py_ssize_t end) { Py_ssize_t i, n = end - begin; keyword_ty kw; @@ -4986,12 +4970,12 @@ compiler_subkwargs(struct compiler *c, location loc, return SUCCESS; } -/* Used by compiler_call_helper and maybe_optimize_method_call to emit +/* Used by codegen_call_helper and maybe_optimize_method_call to emit * a tuple of keyword names before CALL. */ static int -compiler_call_simple_kw_helper(struct compiler *c, location loc, - asdl_keyword_seq *keywords, Py_ssize_t nkwelts) +codegen_call_simple_kw_helper(struct compiler *c, location loc, + asdl_keyword_seq *keywords, Py_ssize_t nkwelts) { PyObject *names; names = PyTuple_New(nkwelts); @@ -5007,16 +4991,16 @@ compiler_call_simple_kw_helper(struct compiler *c, location loc, } -/* shared code between compiler_call and compiler_class */ +/* shared code between codegen_call and codegen_class */ static int -compiler_call_helper(struct compiler *c, location loc, - int n, /* Args already pushed */ - asdl_expr_seq *args, - asdl_keyword_seq *keywords) +codegen_call_helper(struct compiler *c, location loc, + int n, /* Args already pushed */ + asdl_expr_seq *args, + asdl_keyword_seq *keywords) { Py_ssize_t i, nseen, nelts, nkwelts; - RETURN_IF_ERROR(validate_keywords(c, keywords)); + RETURN_IF_ERROR(codegen_validate_keywords(c, keywords)); nelts = asdl_seq_LEN(args); nkwelts = asdl_seq_LEN(keywords); @@ -5046,7 +5030,7 @@ compiler_call_helper(struct compiler *c, location loc, if (nkwelts) { VISIT_SEQ(c, keyword, keywords); RETURN_IF_ERROR( - compiler_call_simple_kw_helper(c, loc, keywords, nkwelts)); + codegen_call_simple_kw_helper(c, loc, keywords, nkwelts)); ADDOP_I(c, loc, CALL_KW, n + nelts + nkwelts); } else { @@ -5075,7 +5059,7 @@ compiler_call_helper(struct compiler *c, location loc, if (kw->arg == NULL) { /* A keyword argument unpacking. */ if (nseen) { - RETURN_IF_ERROR(compiler_subkwargs(c, loc, keywords, i - nseen, i)); + RETURN_IF_ERROR(codegen_subkwargs(c, loc, keywords, i - nseen, i)); if (have_dict) { ADDOP_I(c, loc, DICT_MERGE, 1); } @@ -5095,7 +5079,7 @@ compiler_call_helper(struct compiler *c, location loc, } if (nseen) { /* Pack up any trailing keyword arguments. */ - RETURN_IF_ERROR(compiler_subkwargs(c, loc, keywords, nkwelts - nseen, nkwelts)); + RETURN_IF_ERROR(codegen_subkwargs(c, loc, keywords, nkwelts - nseen, nkwelts)); if (have_dict) { ADDOP_I(c, loc, DICT_MERGE, 1); } @@ -5205,7 +5189,7 @@ compiler_sync_comprehension_generator(struct compiler *c, location loc, Py_ssize_t n = asdl_seq_LEN(gen->ifs); for (Py_ssize_t i = 0; i < n; i++) { expr_ty e = (expr_ty)asdl_seq_GET(gen->ifs, i); - RETURN_IF_ERROR(compiler_jump_if(c, loc, e, if_cleanup, 0)); + RETURN_IF_ERROR(codegen_jump_if(c, loc, e, if_cleanup, 0)); } if (++gen_index < asdl_seq_LEN(generators)) { @@ -5309,7 +5293,7 @@ compiler_async_comprehension_generator(struct compiler *c, location loc, Py_ssize_t n = asdl_seq_LEN(gen->ifs); for (Py_ssize_t i = 0; i < n; i++) { expr_ty e = (expr_ty)asdl_seq_GET(gen->ifs, i); - RETURN_IF_ERROR(compiler_jump_if(c, loc, e, if_cleanup, 0)); + RETURN_IF_ERROR(codegen_jump_if(c, loc, e, if_cleanup, 0)); } depth++; @@ -5371,38 +5355,28 @@ typedef struct { PyObject *temp_symbols; PyObject *fast_hidden; jump_target_label cleanup; - jump_target_label end; } inlined_comprehension_state; static int -push_inlined_comprehension_state(struct compiler *c, location loc, - PySTEntryObject *entry, - inlined_comprehension_state *state) +compiler_tweak_inlined_comprehension_scopes(struct compiler *c, location loc, + PySTEntryObject *entry, + inlined_comprehension_state *state) { int in_class_block = (SYMTABLE_ENTRY(c)->ste_type == ClassBlock) && !c->u->u_in_inlined_comp; c->u->u_in_inlined_comp++; - // iterate over names bound in the comprehension and ensure we isolate - // them from the outer scope as needed + PyObject *k, *v; Py_ssize_t pos = 0; while (PyDict_Next(entry->ste_symbols, &pos, &k, &v)) { long symbol = PyLong_AsLong(v); - if (symbol == -1 && PyErr_Occurred()) { - return ERROR; - } + assert(symbol >= 0 || PyErr_Occurred()); + RETURN_IF_ERROR(symbol); long scope = SYMBOL_TO_SCOPE(symbol); - PyObject *outv = PyDict_GetItemWithError(SYMTABLE_ENTRY(c)->ste_symbols, k); - if (outv == NULL) { - if (PyErr_Occurred()) { - return ERROR; - } - outv = _PyLong_GetZero(); - } - long outsymbol = PyLong_AsLong(outv); - if (outsymbol == -1 && PyErr_Occurred()) { - return ERROR; - } + + long outsymbol = _PyST_GetSymbol(SYMTABLE_ENTRY(c), k); + RETURN_IF_ERROR(outsymbol); long outsc = SYMBOL_TO_SCOPE(outsymbol); + // If a name has different scope inside than outside the comprehension, // we need to temporarily handle it with the right scope while // compiling the comprehension. If it's free in the comprehension @@ -5422,16 +5396,16 @@ push_inlined_comprehension_state(struct compiler *c, location loc, // update the symbol to the in-comprehension version and save // the outer version; we'll restore it after running the // comprehension - Py_INCREF(outv); if (PyDict_SetItem(SYMTABLE_ENTRY(c)->ste_symbols, k, v) < 0) { - Py_DECREF(outv); return ERROR; } - if (PyDict_SetItem(state->temp_symbols, k, outv) < 0) { - Py_DECREF(outv); + PyObject *outv = PyLong_FromLong(outsymbol); + if (outv == NULL) { return ERROR; } + int res = PyDict_SetItem(state->temp_symbols, k, outv); Py_DECREF(outv); + RETURN_IF_ERROR(res); } // locals handling for names bound in comprehension (DEF_LOCAL | // DEF_NONLOCAL occurs in assignment expression to nonlocal) @@ -5442,9 +5416,8 @@ push_inlined_comprehension_state(struct compiler *c, location loc, if (PyDict_GetItemRef(c->u->u_metadata.u_fasthidden, k, &orig) < 0) { return ERROR; } - int orig_is_true = (orig == Py_True); - Py_XDECREF(orig); - if (!orig_is_true) { + assert(orig == NULL || orig == Py_True || orig == Py_False); + if (orig != Py_True) { if (PyDict_SetItem(c->u->u_metadata.u_fasthidden, k, Py_True) < 0) { return ERROR; } @@ -5459,6 +5432,33 @@ push_inlined_comprehension_state(struct compiler *c, location loc, } } } + } + } + return SUCCESS; +} + +static int +codegen_push_inlined_comprehension_locals(struct compiler *c, location loc, + PySTEntryObject *comp, + inlined_comprehension_state *state) +{ + int in_class_block = (SYMTABLE_ENTRY(c)->ste_type == ClassBlock) && !c->u->u_in_inlined_comp; + PySTEntryObject *outer = SYMTABLE_ENTRY(c); + // iterate over names bound in the comprehension and ensure we isolate + // them from the outer scope as needed + PyObject *k, *v; + Py_ssize_t pos = 0; + while (PyDict_Next(comp->ste_symbols, &pos, &k, &v)) { + long symbol = PyLong_AsLong(v); + assert(symbol >= 0 || PyErr_Occurred()); + RETURN_IF_ERROR(symbol); + long scope = SYMBOL_TO_SCOPE(symbol); + + long outsymbol = _PyST_GetSymbol(outer, k); + RETURN_IF_ERROR(outsymbol); + long outsc = SYMBOL_TO_SCOPE(outsymbol); + + if ((symbol & DEF_LOCAL && !(symbol & DEF_NONLOCAL)) || in_class_block) { // local names bound in comprehension must be isolated from // outer scope; push existing value (which may be NULL if // not defined) on stack @@ -5496,31 +5496,40 @@ push_inlined_comprehension_state(struct compiler *c, location loc, // handler or finally block. NEW_JUMP_TARGET_LABEL(c, cleanup); state->cleanup = cleanup; - NEW_JUMP_TARGET_LABEL(c, end); - state->end = end; // no need to push an fblock for this "virtual" try/finally; there can't // be return/continue/break inside a comprehension ADDOP_JUMP(c, loc, SETUP_FINALLY, cleanup); } + return SUCCESS; +} +static int +push_inlined_comprehension_state(struct compiler *c, location loc, + PySTEntryObject *comp, + inlined_comprehension_state *state) +{ + RETURN_IF_ERROR( + compiler_tweak_inlined_comprehension_scopes(c, loc, comp, state)); + RETURN_IF_ERROR( + codegen_push_inlined_comprehension_locals(c, loc, comp, state)); return SUCCESS; } static int restore_inlined_comprehension_locals(struct compiler *c, location loc, - inlined_comprehension_state state) + inlined_comprehension_state *state) { PyObject *k; // pop names we pushed to stack earlier - Py_ssize_t npops = PyList_GET_SIZE(state.pushed_locals); + Py_ssize_t npops = PyList_GET_SIZE(state->pushed_locals); // Preserve the comprehension result (or exception) as TOS. This - // reverses the SWAP we did in push_inlined_comprehension_state to get - // the outermost iterable to TOS, so we can still just iterate + // reverses the SWAP we did in push_inlined_comprehension_state + // to get the outermost iterable to TOS, so we can still just iterate // pushed_locals in simple reverse order ADDOP_I(c, loc, SWAP, npops + 1); for (Py_ssize_t i = npops - 1; i >= 0; --i) { - k = PyList_GetItem(state.pushed_locals, i); + k = PyList_GetItem(state->pushed_locals, i); if (k == NULL) { return ERROR; } @@ -5530,43 +5539,47 @@ restore_inlined_comprehension_locals(struct compiler *c, location loc, } static int -pop_inlined_comprehension_state(struct compiler *c, location loc, - inlined_comprehension_state state) +codegen_pop_inlined_comprehension_locals(struct compiler *c, location loc, + inlined_comprehension_state *state) { - c->u->u_in_inlined_comp--; - PyObject *k, *v; - Py_ssize_t pos = 0; - if (state.temp_symbols) { - while (PyDict_Next(state.temp_symbols, &pos, &k, &v)) { - if (PyDict_SetItem(SYMTABLE_ENTRY(c)->ste_symbols, k, v)) { - return ERROR; - } - } - Py_CLEAR(state.temp_symbols); - } - if (state.pushed_locals) { + if (state->pushed_locals) { ADDOP(c, NO_LOCATION, POP_BLOCK); - ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, state.end); + + NEW_JUMP_TARGET_LABEL(c, end); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); // cleanup from an exception inside the comprehension - USE_LABEL(c, state.cleanup); + USE_LABEL(c, state->cleanup); // discard incomplete comprehension result (beneath exc on stack) ADDOP_I(c, NO_LOCATION, SWAP, 2); ADDOP(c, NO_LOCATION, POP_TOP); - if (restore_inlined_comprehension_locals(c, loc, state) < 0) { - return ERROR; - } + RETURN_IF_ERROR(restore_inlined_comprehension_locals(c, loc, state)); ADDOP_I(c, NO_LOCATION, RERAISE, 0); - USE_LABEL(c, state.end); - if (restore_inlined_comprehension_locals(c, loc, state) < 0) { - return ERROR; + USE_LABEL(c, end); + RETURN_IF_ERROR(restore_inlined_comprehension_locals(c, loc, state)); + Py_CLEAR(state->pushed_locals); + } + return SUCCESS; +} + +static int +compiler_revert_inlined_comprehension_scopes(struct compiler *c, location loc, + inlined_comprehension_state *state) +{ + if (state->temp_symbols) { + PyObject *k, *v; + Py_ssize_t pos = 0; + while (PyDict_Next(state->temp_symbols, &pos, &k, &v)) { + if (PyDict_SetItem(SYMTABLE_ENTRY(c)->ste_symbols, k, v)) { + return ERROR; + } } - Py_CLEAR(state.pushed_locals); + Py_CLEAR(state->temp_symbols); } - if (state.fast_hidden) { - while (PySet_Size(state.fast_hidden) > 0) { - PyObject *k = PySet_Pop(state.fast_hidden); + if (state->fast_hidden) { + while (PySet_Size(state->fast_hidden) > 0) { + PyObject *k = PySet_Pop(state->fast_hidden); if (k == NULL) { return ERROR; } @@ -5578,11 +5591,21 @@ pop_inlined_comprehension_state(struct compiler *c, location loc, } Py_DECREF(k); } - Py_CLEAR(state.fast_hidden); + Py_CLEAR(state->fast_hidden); } return SUCCESS; } +static int +pop_inlined_comprehension_state(struct compiler *c, location loc, + inlined_comprehension_state *state) +{ + c->u->u_in_inlined_comp--; + RETURN_IF_ERROR(codegen_pop_inlined_comprehension_locals(c, loc, state)); + RETURN_IF_ERROR(compiler_revert_inlined_comprehension_scopes(c, loc, state)); + return SUCCESS; +} + static inline int compiler_comprehension_iter(struct compiler *c, location loc, comprehension_ty comp) @@ -5603,7 +5626,7 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, expr_ty val) { PyCodeObject *co = NULL; - inlined_comprehension_state inline_state = {NULL, NULL, NULL, NO_LABEL, NO_LABEL}; + inlined_comprehension_state inline_state = {NULL, NULL, NULL, NO_LABEL}; comprehension_ty outermost; #ifndef NDEBUG int scope_type = c->u->u_scope_type; @@ -5629,7 +5652,7 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, } else { if (compiler_enter_scope(c, name, COMPILER_SCOPE_COMPREHENSION, - (void *)e, e->lineno, NULL) < 0) { + (void *)e, e->lineno, NULL, NULL) < 0) { goto error; } } @@ -5671,7 +5694,7 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, } if (is_inlined) { - if (pop_inlined_comprehension_state(c, loc, inline_state)) { + if (pop_inlined_comprehension_state(c, loc, &inline_state)) { goto error; } return SUCCESS; @@ -5681,7 +5704,7 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, ADDOP(c, LOC(e), RETURN_VALUE); } if (type == COMP_GENEXP) { - if (wrap_in_stopiteration_handler(c) < 0) { + if (codegen_wrap_in_stopiteration_handler(c) < 0) { goto error_in_scope; } } @@ -5693,7 +5716,7 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, } loc = LOC(e); - if (compiler_make_closure(c, loc, co, 0) < 0) { + if (codegen_make_closure(c, loc, co, 0) < 0) { goto error; } Py_CLEAR(co); @@ -5775,7 +5798,7 @@ compiler_visit_keyword(struct compiler *c, keyword_ty k) static int -compiler_with_except_finish(struct compiler *c, jump_target_label cleanup) { +codegen_with_except_finish(struct compiler *c, jump_target_label cleanup) { NEW_JUMP_TARGET_LABEL(c, suppress); ADDOP(c, NO_LOCATION, TO_BOOL); ADDOP_JUMP(c, NO_LOCATION, POP_JUMP_IF_TRUE, suppress); @@ -5823,7 +5846,7 @@ compiler_with_except_finish(struct compiler *c, jump_target_label cleanup) { raise */ static int -compiler_async_with(struct compiler *c, stmt_ty s, int pos) +codegen_async_with(struct compiler *c, stmt_ty s, int pos) { location loc = LOC(s); withitem_ty item = asdl_seq_GET(s->v.AsyncWith.items, pos); @@ -5868,7 +5891,7 @@ compiler_async_with(struct compiler *c, stmt_ty s, int pos) VISIT_SEQ(c, stmt, s->v.AsyncWith.body) } else { - RETURN_IF_ERROR(compiler_async_with(c, s, pos)); + RETURN_IF_ERROR(codegen_async_with(c, s, pos)); } compiler_pop_fblock(c, ASYNC_WITH, block); @@ -5879,7 +5902,7 @@ compiler_async_with(struct compiler *c, stmt_ty s, int pos) /* For successful outcome: * call __exit__(None, None, None) */ - RETURN_IF_ERROR(compiler_call_exit_with_nones(c, loc)); + RETURN_IF_ERROR(codegen_call_exit_with_nones(c, loc)); ADDOP_I(c, loc, GET_AWAITABLE, 2); ADDOP_LOAD_CONST(c, loc, Py_None); ADD_YIELD_FROM(c, loc, 1); @@ -5897,7 +5920,7 @@ compiler_async_with(struct compiler *c, stmt_ty s, int pos) ADDOP_I(c, loc, GET_AWAITABLE, 2); ADDOP_LOAD_CONST(c, loc, Py_None); ADD_YIELD_FROM(c, loc, 1); - RETURN_IF_ERROR(compiler_with_except_finish(c, cleanup)); + RETURN_IF_ERROR(codegen_with_except_finish(c, cleanup)); USE_LABEL(c, exit); return SUCCESS; @@ -5926,7 +5949,7 @@ compiler_async_with(struct compiler *c, stmt_ty s, int pos) */ static int -compiler_with(struct compiler *c, stmt_ty s, int pos) +codegen_with(struct compiler *c, stmt_ty s, int pos) { withitem_ty item = asdl_seq_GET(s->v.With.items, pos); @@ -5967,7 +5990,7 @@ compiler_with(struct compiler *c, stmt_ty s, int pos) VISIT_SEQ(c, stmt, s->v.With.body) } else { - RETURN_IF_ERROR(compiler_with(c, s, pos)); + RETURN_IF_ERROR(codegen_with(c, s, pos)); } ADDOP(c, NO_LOCATION, POP_BLOCK); @@ -5978,7 +6001,7 @@ compiler_with(struct compiler *c, stmt_ty s, int pos) /* For successful outcome: * call __exit__(None, None, None) */ - RETURN_IF_ERROR(compiler_call_exit_with_nones(c, loc)); + RETURN_IF_ERROR(codegen_call_exit_with_nones(c, loc)); ADDOP(c, loc, POP_TOP); ADDOP_JUMP(c, loc, JUMP, exit); @@ -5988,7 +6011,7 @@ compiler_with(struct compiler *c, stmt_ty s, int pos) ADDOP_JUMP(c, loc, SETUP_CLEANUP, cleanup); ADDOP(c, loc, PUSH_EXC_INFO); ADDOP(c, loc, WITH_EXCEPT_START); - RETURN_IF_ERROR(compiler_with_except_finish(c, cleanup)); + RETURN_IF_ERROR(codegen_with_except_finish(c, cleanup)); USE_LABEL(c, exit); return SUCCESS; @@ -6005,7 +6028,7 @@ compiler_visit_expr(struct compiler *c, expr_ty e) VISIT(c, expr, e->v.NamedExpr.target); break; case BoolOp_kind: - return compiler_boolop(c, e); + return codegen_boolop(c, e); case BinOp_kind: VISIT(c, expr, e->v.BinOp.left); VISIT(c, expr, e->v.BinOp.right); @@ -6025,13 +6048,13 @@ compiler_visit_expr(struct compiler *c, expr_ty e) } break; case Lambda_kind: - return compiler_lambda(c, e); + return codegen_lambda(c, e); case IfExp_kind: - return compiler_ifexp(c, e); + return codegen_ifexp(c, e); case Dict_kind: - return compiler_dict(c, e); + return codegen_dict(c, e); case Set_kind: - return compiler_set(c, e); + return codegen_set(c, e); case GeneratorExp_kind: return compiler_genexp(c, e); case ListComp_kind: @@ -6076,16 +6099,16 @@ compiler_visit_expr(struct compiler *c, expr_ty e) ADD_YIELD_FROM(c, loc, 1); break; case Compare_kind: - return compiler_compare(c, e); + return codegen_compare(c, e); case Call_kind: - return compiler_call(c, e); + return codegen_call(c, e); case Constant_kind: ADDOP_LOAD_CONST(c, loc, e->v.Constant.value); break; case JoinedStr_kind: - return compiler_joined_str(c, e); + return codegen_joined_str(c, e); case FormattedValue_kind: - return compiler_formatted_value(c, e); + return codegen_formatted_value(c, e); /* The following exprs can be assignment targets. */ case Attribute_kind: if (e->v.Attribute.ctx == Load) { @@ -6118,12 +6141,12 @@ compiler_visit_expr(struct compiler *c, expr_ty e) } break; case Subscript_kind: - return compiler_subscript(c, e); + return codegen_subscript(c, e); case Starred_kind: switch (e->v.Starred.ctx) { case Store: /* In all legitimate cases, the Starred node was already replaced - * by compiler_list/compiler_tuple. XXX: is that okay? */ + * by codegen_list/codegen_tuple. XXX: is that okay? */ return compiler_error(c, loc, "starred assignment target must be in a list or tuple"); default: @@ -6133,7 +6156,7 @@ compiler_visit_expr(struct compiler *c, expr_ty e) break; case Slice_kind: { - int n = compiler_slice(c, e); + int n = codegen_slice(c, e); RETURN_IF_ERROR(n); ADDOP_I(c, loc, BUILD_SLICE, n); break; @@ -6142,9 +6165,9 @@ compiler_visit_expr(struct compiler *c, expr_ty e) return compiler_nameop(c, loc, e->v.Name.id, e->v.Name.ctx); /* child nodes of List and Tuple will have expr_context set */ case List_kind: - return compiler_list(c, e); + return codegen_list(c, e); case Tuple_kind: - return compiler_tuple(c, e); + return codegen_tuple(c, e); } return SUCCESS; } @@ -6157,7 +6180,7 @@ is_two_element_slice(expr_ty s) } static int -compiler_augassign(struct compiler *c, stmt_ty s) +codegen_augassign(struct compiler *c, stmt_ty s) { assert(s->kind == AugAssign_kind); expr_ty e = s->v.AugAssign.target; @@ -6174,7 +6197,7 @@ compiler_augassign(struct compiler *c, stmt_ty s) case Subscript_kind: VISIT(c, expr, e->v.Subscript.value); if (is_two_element_slice(e->v.Subscript.slice)) { - RETURN_IF_ERROR(compiler_slice(c, e->v.Subscript.slice)); + RETURN_IF_ERROR(codegen_slice(c, e->v.Subscript.slice)); ADDOP_I(c, loc, COPY, 3); ADDOP_I(c, loc, COPY, 3); ADDOP_I(c, loc, COPY, 3); @@ -6232,7 +6255,7 @@ compiler_augassign(struct compiler *c, stmt_ty s) } static int -check_ann_expr(struct compiler *c, expr_ty e) +codegen_check_ann_expr(struct compiler *c, expr_ty e) { VISIT(c, expr, e); ADDOP(c, LOC(e), POP_TOP); @@ -6240,7 +6263,7 @@ check_ann_expr(struct compiler *c, expr_ty e) } static int -check_annotation(struct compiler *c, stmt_ty s) +compiler_check_annotation(struct compiler *c, stmt_ty s) { /* Annotations of complex targets does not produce anything under annotations future */ @@ -6251,24 +6274,24 @@ check_annotation(struct compiler *c, stmt_ty s) /* Annotations are only evaluated in a module or class. */ if (c->u->u_scope_type == COMPILER_SCOPE_MODULE || c->u->u_scope_type == COMPILER_SCOPE_CLASS) { - return check_ann_expr(c, s->v.AnnAssign.annotation); + return codegen_check_ann_expr(c, s->v.AnnAssign.annotation); } return SUCCESS; } static int -check_ann_subscr(struct compiler *c, expr_ty e) +codegen_check_ann_subscr(struct compiler *c, expr_ty e) { /* We check that everything in a subscript is defined at runtime. */ switch (e->kind) { case Slice_kind: - if (e->v.Slice.lower && check_ann_expr(c, e->v.Slice.lower) < 0) { + if (e->v.Slice.lower && codegen_check_ann_expr(c, e->v.Slice.lower) < 0) { return ERROR; } - if (e->v.Slice.upper && check_ann_expr(c, e->v.Slice.upper) < 0) { + if (e->v.Slice.upper && codegen_check_ann_expr(c, e->v.Slice.upper) < 0) { return ERROR; } - if (e->v.Slice.step && check_ann_expr(c, e->v.Slice.step) < 0) { + if (e->v.Slice.step && codegen_check_ann_expr(c, e->v.Slice.step) < 0) { return ERROR; } return SUCCESS; @@ -6277,12 +6300,12 @@ check_ann_subscr(struct compiler *c, expr_ty e) asdl_expr_seq *elts = e->v.Tuple.elts; Py_ssize_t i, n = asdl_seq_LEN(elts); for (i = 0; i < n; i++) { - RETURN_IF_ERROR(check_ann_subscr(c, asdl_seq_GET(elts, i))); + RETURN_IF_ERROR(codegen_check_ann_subscr(c, asdl_seq_GET(elts, i))); } return SUCCESS; } default: - return check_ann_expr(c, e); + return codegen_check_ann_expr(c, e); } } @@ -6335,14 +6358,14 @@ compiler_annassign(struct compiler *c, stmt_ty s) break; case Attribute_kind: if (!s->v.AnnAssign.value && - check_ann_expr(c, targ->v.Attribute.value) < 0) { + codegen_check_ann_expr(c, targ->v.Attribute.value) < 0) { return ERROR; } break; case Subscript_kind: if (!s->v.AnnAssign.value && - (check_ann_expr(c, targ->v.Subscript.value) < 0 || - check_ann_subscr(c, targ->v.Subscript.slice) < 0)) { + (codegen_check_ann_expr(c, targ->v.Subscript.value) < 0 || + codegen_check_ann_subscr(c, targ->v.Subscript.slice) < 0)) { return ERROR; } break; @@ -6353,7 +6376,7 @@ compiler_annassign(struct compiler *c, stmt_ty s) return ERROR; } /* Annotation is evaluated last. */ - if (future_annotations && !s->v.AnnAssign.simple && check_annotation(c, s) < 0) { + if (future_annotations && !s->v.AnnAssign.simple && compiler_check_annotation(c, s) < 0) { return ERROR; } return SUCCESS; @@ -6425,7 +6448,7 @@ compiler_warn(struct compiler *c, location loc, } static int -compiler_subscript(struct compiler *c, expr_ty e) +codegen_subscript(struct compiler *c, expr_ty e) { location loc = LOC(e); expr_context_ty ctx = e->v.Subscript.ctx; @@ -6438,7 +6461,7 @@ compiler_subscript(struct compiler *c, expr_ty e) VISIT(c, expr, e->v.Subscript.value); if (is_two_element_slice(e->v.Subscript.slice) && ctx != Del) { - RETURN_IF_ERROR(compiler_slice(c, e->v.Subscript.slice)); + RETURN_IF_ERROR(codegen_slice(c, e->v.Subscript.slice)); if (ctx == Load) { ADDOP(c, loc, BINARY_SLICE); } @@ -6463,7 +6486,7 @@ compiler_subscript(struct compiler *c, expr_ty e) /* Returns the number of the values emitted, * thus are needed to build the slice, or -1 if there is an error. */ static int -compiler_slice(struct compiler *c, expr_ty s) +codegen_slice(struct compiler *c, expr_ty s) { int n = 2; assert(s->kind == Slice_kind); @@ -6493,9 +6516,9 @@ compiler_slice(struct compiler *c, expr_ty s) // PEP 634: Structural Pattern Matching -// To keep things simple, all compiler_pattern_* and pattern_helper_* routines -// follow the convention of consuming TOS (the subject for the given pattern) -// and calling jump_to_fail_pop on failure (no match). +// To keep things simple, all codegen_pattern_* routines follow the convention +// of consuming TOS (the subject for the given pattern) and calling +// jump_to_fail_pop on failure (no match). // When calling into these routines, it's important that pc->on_top be kept // updated to reflect the current number of items that we are using on the top @@ -6581,7 +6604,7 @@ compiler_error_duplicate_store(struct compiler *c, location loc, identifier n) // Duplicate the effect of 3.10's ROT_* instructions using SWAPs. static int -pattern_helper_rotate(struct compiler *c, location loc, Py_ssize_t count) +codegen_pattern_helper_rotate(struct compiler *c, location loc, Py_ssize_t count) { while (1 < count) { ADDOP_I(c, loc, SWAP, count--); @@ -6590,8 +6613,8 @@ pattern_helper_rotate(struct compiler *c, location loc, Py_ssize_t count) } static int -pattern_helper_store_name(struct compiler *c, location loc, - identifier n, pattern_context *pc) +codegen_pattern_helper_store_name(struct compiler *c, location loc, + identifier n, pattern_context *pc) { if (n == NULL) { ADDOP(c, loc, POP_TOP); @@ -6605,15 +6628,15 @@ pattern_helper_store_name(struct compiler *c, location loc, } // Rotate this object underneath any items we need to preserve: Py_ssize_t rotations = pc->on_top + PyList_GET_SIZE(pc->stores) + 1; - RETURN_IF_ERROR(pattern_helper_rotate(c, loc, rotations)); + RETURN_IF_ERROR(codegen_pattern_helper_rotate(c, loc, rotations)); RETURN_IF_ERROR(PyList_Append(pc->stores, n)); return SUCCESS; } static int -pattern_unpack_helper(struct compiler *c, location loc, - asdl_pattern_seq *elts) +codegen_pattern_unpack_helper(struct compiler *c, location loc, + asdl_pattern_seq *elts) { Py_ssize_t n = asdl_seq_LEN(elts); int seen_star = 0; @@ -6645,7 +6668,7 @@ pattern_helper_sequence_unpack(struct compiler *c, location loc, asdl_pattern_seq *patterns, Py_ssize_t star, pattern_context *pc) { - RETURN_IF_ERROR(pattern_unpack_helper(c, loc, patterns)); + RETURN_IF_ERROR(codegen_pattern_unpack_helper(c, loc, patterns)); Py_ssize_t size = asdl_seq_LEN(patterns); // We've now got a bunch of new subjects on the stack. They need to remain // there after each subpattern match: @@ -6654,7 +6677,7 @@ pattern_helper_sequence_unpack(struct compiler *c, location loc, // One less item to keep track of each time we loop through: pc->on_top--; pattern_ty pattern = asdl_seq_GET(patterns, i); - RETURN_IF_ERROR(compiler_pattern_subpattern(c, pattern, pc)); + RETURN_IF_ERROR(codegen_pattern_subpattern(c, pattern, pc)); } return SUCCESS; } @@ -6691,7 +6714,7 @@ pattern_helper_sequence_subscr(struct compiler *c, location loc, ADDOP_BINARY(c, loc, Sub); } ADDOP(c, loc, BINARY_SUBSCR); - RETURN_IF_ERROR(compiler_pattern_subpattern(c, pattern, pc)); + RETURN_IF_ERROR(codegen_pattern_subpattern(c, pattern, pc)); } // Pop the subject, we're done with it: pc->on_top--; @@ -6699,20 +6722,20 @@ pattern_helper_sequence_subscr(struct compiler *c, location loc, return SUCCESS; } -// Like compiler_pattern, but turn off checks for irrefutability. +// Like codegen_pattern, but turn off checks for irrefutability. static int -compiler_pattern_subpattern(struct compiler *c, +codegen_pattern_subpattern(struct compiler *c, pattern_ty p, pattern_context *pc) { int allow_irrefutable = pc->allow_irrefutable; pc->allow_irrefutable = 1; - RETURN_IF_ERROR(compiler_pattern(c, p, pc)); + RETURN_IF_ERROR(codegen_pattern(c, p, pc)); pc->allow_irrefutable = allow_irrefutable; return SUCCESS; } static int -compiler_pattern_as(struct compiler *c, pattern_ty p, pattern_context *pc) +codegen_pattern_as(struct compiler *c, pattern_ty p, pattern_context *pc) { assert(p->kind == MatchAs_kind); if (p->v.MatchAs.pattern == NULL) { @@ -6725,24 +6748,24 @@ compiler_pattern_as(struct compiler *c, pattern_ty p, pattern_context *pc) const char *e = "wildcard makes remaining patterns unreachable"; return compiler_error(c, LOC(p), e); } - return pattern_helper_store_name(c, LOC(p), p->v.MatchAs.name, pc); + return codegen_pattern_helper_store_name(c, LOC(p), p->v.MatchAs.name, pc); } // Need to make a copy for (possibly) storing later: pc->on_top++; ADDOP_I(c, LOC(p), COPY, 1); - RETURN_IF_ERROR(compiler_pattern(c, p->v.MatchAs.pattern, pc)); + RETURN_IF_ERROR(codegen_pattern(c, p->v.MatchAs.pattern, pc)); // Success! Store it: pc->on_top--; - RETURN_IF_ERROR(pattern_helper_store_name(c, LOC(p), p->v.MatchAs.name, pc)); + RETURN_IF_ERROR(codegen_pattern_helper_store_name(c, LOC(p), p->v.MatchAs.name, pc)); return SUCCESS; } static int -compiler_pattern_star(struct compiler *c, pattern_ty p, pattern_context *pc) +codegen_pattern_star(struct compiler *c, pattern_ty p, pattern_context *pc) { assert(p->kind == MatchStar_kind); RETURN_IF_ERROR( - pattern_helper_store_name(c, LOC(p), p->v.MatchStar.name, pc)); + codegen_pattern_helper_store_name(c, LOC(p), p->v.MatchStar.name, pc)); return SUCCESS; } @@ -6767,7 +6790,7 @@ validate_kwd_attrs(struct compiler *c, asdl_identifier_seq *attrs, asdl_pattern_ } static int -compiler_pattern_class(struct compiler *c, pattern_ty p, pattern_context *pc) +codegen_pattern_class(struct compiler *c, pattern_ty p, pattern_context *pc) { assert(p->kind == MatchClass_kind); asdl_pattern_seq *patterns = p->v.MatchClass.patterns; @@ -6824,15 +6847,15 @@ compiler_pattern_class(struct compiler *c, pattern_ty p, pattern_context *pc) ADDOP(c, LOC(p), POP_TOP); continue; } - RETURN_IF_ERROR(compiler_pattern_subpattern(c, pattern, pc)); + RETURN_IF_ERROR(codegen_pattern_subpattern(c, pattern, pc)); } // Success! Pop the tuple of attributes: return SUCCESS; } static int -compiler_pattern_mapping(struct compiler *c, pattern_ty p, - pattern_context *pc) +codegen_pattern_mapping(struct compiler *c, pattern_ty p, + pattern_context *pc) { assert(p->kind == MatchMapping_kind); asdl_expr_seq *keys = p->v.MatchMapping.keys; @@ -6931,7 +6954,7 @@ compiler_pattern_mapping(struct compiler *c, pattern_ty p, for (Py_ssize_t i = 0; i < size; i++) { pc->on_top--; pattern_ty pattern = asdl_seq_GET(patterns, i); - RETURN_IF_ERROR(compiler_pattern_subpattern(c, pattern, pc)); + RETURN_IF_ERROR(codegen_pattern_subpattern(c, pattern, pc)); } // If we get this far, it's a match! Whatever happens next should consume // the tuple of keys and the subject: @@ -6952,7 +6975,7 @@ compiler_pattern_mapping(struct compiler *c, pattern_ty p, ADDOP_I(c, LOC(p), SWAP, 2); // [copy, keys..., copy, key] ADDOP(c, LOC(p), DELETE_SUBSCR); // [copy, keys...] } - RETURN_IF_ERROR(pattern_helper_store_name(c, LOC(p), star_target, pc)); + RETURN_IF_ERROR(codegen_pattern_helper_store_name(c, LOC(p), star_target, pc)); } else { ADDOP(c, LOC(p), POP_TOP); // Tuple of keys. @@ -6966,7 +6989,7 @@ compiler_pattern_mapping(struct compiler *c, pattern_ty p, } static int -compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) +codegen_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) { assert(p->kind == MatchOr_kind); NEW_JUMP_TARGET_LABEL(c, end); @@ -6993,7 +7016,7 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) pc->fail_pop_size = 0; pc->on_top = 0; if (codegen_addop_i(INSTR_SEQUENCE(c), COPY, 1, LOC(alt)) < 0 || - compiler_pattern(c, alt, pc) < 0) { + codegen_pattern(c, alt, pc) < 0) { goto error; } // Success! @@ -7047,7 +7070,7 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) // Do the same thing to the stack, using several // rotations: while (rotations--) { - if (pattern_helper_rotate(c, LOC(alt), icontrol + 1) < 0) { + if (codegen_pattern_helper_rotate(c, LOC(alt), icontrol + 1) < 0) { goto error; } } @@ -7083,7 +7106,7 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) Py_ssize_t nrots = nstores + 1 + pc->on_top + PyList_GET_SIZE(pc->stores); for (Py_ssize_t i = 0; i < nstores; i++) { // Rotate this capture to its proper place on the stack: - if (pattern_helper_rotate(c, LOC(p), nrots) < 0) { + if (codegen_pattern_helper_rotate(c, LOC(p), nrots) < 0) { goto error; } // Update the list of previous stores with this new name, checking for @@ -7118,8 +7141,8 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) static int -compiler_pattern_sequence(struct compiler *c, pattern_ty p, - pattern_context *pc) +codegen_pattern_sequence(struct compiler *c, pattern_ty p, + pattern_context *pc) { assert(p->kind == MatchSequence_kind); asdl_pattern_seq *patterns = p->v.MatchSequence.patterns; @@ -7176,7 +7199,7 @@ compiler_pattern_sequence(struct compiler *c, pattern_ty p, } static int -compiler_pattern_value(struct compiler *c, pattern_ty p, pattern_context *pc) +codegen_pattern_value(struct compiler *c, pattern_ty p, pattern_context *pc) { assert(p->kind == MatchValue_kind); expr_ty value = p->v.MatchValue.value; @@ -7192,7 +7215,7 @@ compiler_pattern_value(struct compiler *c, pattern_ty p, pattern_context *pc) } static int -compiler_pattern_singleton(struct compiler *c, pattern_ty p, pattern_context *pc) +codegen_pattern_singleton(struct compiler *c, pattern_ty p, pattern_context *pc) { assert(p->kind == MatchSingleton_kind); ADDOP_LOAD_CONST(c, LOC(p), p->v.MatchSingleton.value); @@ -7202,25 +7225,25 @@ compiler_pattern_singleton(struct compiler *c, pattern_ty p, pattern_context *pc } static int -compiler_pattern(struct compiler *c, pattern_ty p, pattern_context *pc) +codegen_pattern(struct compiler *c, pattern_ty p, pattern_context *pc) { switch (p->kind) { case MatchValue_kind: - return compiler_pattern_value(c, p, pc); + return codegen_pattern_value(c, p, pc); case MatchSingleton_kind: - return compiler_pattern_singleton(c, p, pc); + return codegen_pattern_singleton(c, p, pc); case MatchSequence_kind: - return compiler_pattern_sequence(c, p, pc); + return codegen_pattern_sequence(c, p, pc); case MatchMapping_kind: - return compiler_pattern_mapping(c, p, pc); + return codegen_pattern_mapping(c, p, pc); case MatchClass_kind: - return compiler_pattern_class(c, p, pc); + return codegen_pattern_class(c, p, pc); case MatchStar_kind: - return compiler_pattern_star(c, p, pc); + return codegen_pattern_star(c, p, pc); case MatchAs_kind: - return compiler_pattern_as(c, p, pc); + return codegen_pattern_as(c, p, pc); case MatchOr_kind: - return compiler_pattern_or(c, p, pc); + return codegen_pattern_or(c, p, pc); } // AST validator shouldn't let this happen, but if it does, // just fail, don't crash out of the interpreter @@ -7229,7 +7252,7 @@ compiler_pattern(struct compiler *c, pattern_ty p, pattern_context *pc) } static int -compiler_match_inner(struct compiler *c, stmt_ty s, pattern_context *pc) +codegen_match_inner(struct compiler *c, stmt_ty s, pattern_context *pc) { VISIT(c, expr, s->v.Match.subject); NEW_JUMP_TARGET_LABEL(c, end); @@ -7253,7 +7276,7 @@ compiler_match_inner(struct compiler *c, stmt_ty s, pattern_context *pc) pc->fail_pop_size = 0; pc->on_top = 0; // NOTE: Can't use returning macros here (they'll leak pc->stores)! - if (compiler_pattern(c, m->pattern, pc) < 0) { + if (codegen_pattern(c, m->pattern, pc) < 0) { Py_DECREF(pc->stores); return ERROR; } @@ -7271,14 +7294,14 @@ compiler_match_inner(struct compiler *c, stmt_ty s, pattern_context *pc) // NOTE: Returning macros are safe again. if (m->guard) { RETURN_IF_ERROR(ensure_fail_pop(c, pc, 0)); - RETURN_IF_ERROR(compiler_jump_if(c, LOC(m->pattern), m->guard, pc->fail_pop[0], 0)); + RETURN_IF_ERROR(codegen_jump_if(c, LOC(m->pattern), m->guard, pc->fail_pop[0], 0)); } // Success! Pop the subject off, we're done with it: if (i != cases - has_default - 1) { ADDOP(c, LOC(m->pattern), POP_TOP); } VISIT_SEQ(c, stmt, m->body); - ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); + ADDOP_JUMP(c, NO_LOCATION, JUMP, end); // If the pattern fails to match, we want the line number of the // cleanup to be associated with the failed pattern, not the last line // of the body @@ -7297,7 +7320,7 @@ compiler_match_inner(struct compiler *c, stmt_ty s, pattern_context *pc) ADDOP(c, LOC(m->pattern), NOP); } if (m->guard) { - RETURN_IF_ERROR(compiler_jump_if(c, LOC(m->pattern), m->guard, end, 0)); + RETURN_IF_ERROR(codegen_jump_if(c, LOC(m->pattern), m->guard, end, 0)); } VISIT_SEQ(c, stmt, m->body); } @@ -7306,11 +7329,11 @@ compiler_match_inner(struct compiler *c, stmt_ty s, pattern_context *pc) } static int -compiler_match(struct compiler *c, stmt_ty s) +codegen_match(struct compiler *c, stmt_ty s) { pattern_context pc; pc.fail_pop = NULL; - int result = compiler_match_inner(c, s, &pc); + int result = codegen_match_inner(c, s, &pc); PyMem_Free(pc.fail_pop); return result; } @@ -7633,7 +7656,7 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, return NULL; } - if (compiler_enter_anonymous_scope(c, mod) < 0) { + if (codegen_enter_anonymous_scope(c, mod) < 0) { return NULL; } if (compiler_codegen(c, mod) < 0) { diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index b8785381cdd572..9226b7a4e5d4bc 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1031,7 +1031,7 @@ PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; PyObject *getitem = ht->_spec_cache.getitem; - new_frame = _PyFrame_PushUnchecked(tstate, (PyFunctionObject *)getitem, 2); + new_frame = _PyFrame_PushUnchecked(tstate, (PyFunctionObject *)getitem, 2, frame); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); new_frame->localsplus[0] = container; @@ -1319,6 +1319,7 @@ tstate->exc_info = &gen->gi_exc_state; assert(1 + INLINE_CACHE_ENTRIES_SEND + oparg <= UINT16_MAX); frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + oparg); + gen_frame->previous = frame; stack_pointer[-1].bits = (uintptr_t)gen_frame; break; } @@ -2552,7 +2553,7 @@ } STAT_INC(LOAD_ATTR, hit); Py_INCREF(fget); - new_frame = _PyFrame_PushUnchecked(tstate, f, 1); + new_frame = _PyFrame_PushUnchecked(tstate, f, 1, frame); new_frame->localsplus[0] = owner; stack_pointer[-1].bits = (uintptr_t)new_frame; break; @@ -3316,6 +3317,7 @@ gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; + gen_frame->previous = frame; // oparg is the return offset from the next instruction. frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg); stack_pointer[0].bits = (uintptr_t)gen_frame; @@ -3604,7 +3606,7 @@ PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, - args, total_args, NULL + args, total_args, NULL, frame ); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. @@ -3838,7 +3840,7 @@ int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); PyFunctionObject *func = (PyFunctionObject *)callable_o; - new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { @@ -3864,7 +3866,7 @@ int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); PyFunctionObject *func = (PyFunctionObject *)callable_o; - new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { @@ -3890,7 +3892,7 @@ int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); PyFunctionObject *func = (PyFunctionObject *)callable_o; - new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { @@ -3916,7 +3918,7 @@ int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); PyFunctionObject *func = (PyFunctionObject *)callable_o; - new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { @@ -3942,7 +3944,7 @@ int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); PyFunctionObject *func = (PyFunctionObject *)callable_o; - new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { @@ -3967,7 +3969,7 @@ int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); PyFunctionObject *func = (PyFunctionObject *)callable_o; - new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { @@ -3988,7 +3990,7 @@ stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame->previous = frame; + assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; @@ -4087,7 +4089,81 @@ break; } - /* _CALL_ALLOC_AND_ENTER_INIT is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + case _CHECK_AND_ALLOCATE_OBJECT: { + _PyStackRef *args; + _PyStackRef null; + _PyStackRef callable; + _PyStackRef self; + _PyStackRef init; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint32_t type_version = (uint32_t)CURRENT_OPERAND(); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyStackRef_IsNull(null)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyType_Check(callable_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + PyTypeObject *tp = (PyTypeObject *)callable_o; + if (tp->tp_version_tag != type_version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES); + PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; + PyFunctionObject *init_func = (PyFunctionObject *)cls->_spec_cache.init; + PyCodeObject *code = (PyCodeObject *)init_func->func_code; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(CALL, hit); + self = PyStackRef_FromPyObjectSteal(_PyType_NewManagedObject(tp)); + if (PyStackRef_IsNull(self)) { + JUMP_TO_ERROR(); + } + PyStackRef_CLOSE(callable); + init = PyStackRef_FromPyObjectNew(init_func); + stack_pointer[-1 - oparg] = init; + stack_pointer[-2 - oparg] = self; + break; + } + + case _CREATE_INIT_FRAME: { + _PyStackRef *args; + _PyStackRef init; + _PyStackRef self; + _PyInterpreterFrame *init_frame; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + init = stack_pointer[-1 - oparg]; + self = stack_pointer[-2 - oparg]; + _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( + tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); + assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK); + /* Push self onto stack of shim */ + shim->localsplus[0] = PyStackRef_DUP(self); + PyFunctionObject *init_func = (PyFunctionObject *)PyStackRef_AsPyObjectSteal(init); + args[-1] = self; + init_frame = _PyEvalFramePushAndInit( + tstate, init_func, NULL, args-1, oparg+1, NULL, shim); + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + /* Account for pushing the extra frame. + * We don't check recursion depth here, + * as it will be checked after start_frame */ + tstate->py_recursion_remaining--; + stack_pointer[0].bits = (uintptr_t)init_frame; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } case _EXIT_INIT_CHECK: { _PyStackRef should_be_none; @@ -4679,6 +4755,186 @@ /* _DO_CALL_KW is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + case _PY_FRAME_KW: { + _PyStackRef kwnames; + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyInterpreterFrame *new_frame; + oparg = CURRENT_OPARG(); + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (self_or_null_o != NULL) { + args--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + args, positional_args, kwnames_o, frame + ); + PyStackRef_CLOSE(kwnames); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (new_frame == NULL) { + JUMP_TO_ERROR(); + } + stack_pointer[0].bits = (uintptr_t)new_frame; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CHECK_FUNCTION_VERSION_KW: { + _PyStackRef callable; + oparg = CURRENT_OPARG(); + callable = stack_pointer[-3 - oparg]; + uint32_t func_version = (uint32_t)CURRENT_OPERAND(); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _CHECK_METHOD_VERSION_KW: { + _PyStackRef null; + _PyStackRef callable; + oparg = CURRENT_OPARG(); + null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + uint32_t func_version = (uint32_t)CURRENT_OPERAND(); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (Py_TYPE(callable_o) != &PyMethod_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + if (!PyFunction_Check(func)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyStackRef_IsNull(null)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _EXPAND_METHOD_KW: { + _PyStackRef kwnames; + _PyStackRef null; + _PyStackRef callable; + _PyStackRef method; + _PyStackRef self; + oparg = CURRENT_OPARG(); + kwnames = stack_pointer[-1]; + null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyStackRef_IsNull(null)); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + stack_pointer[-2 - oparg] = self; + method = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + stack_pointer[-3 - oparg] = method; + assert(PyStackRef_FunctionCheck(method)); + PyStackRef_CLOSE(callable); + stack_pointer[-1] = kwnames; + break; + } + + case _CHECK_IS_NOT_PY_CALLABLE_KW: { + _PyStackRef callable; + oparg = CURRENT_OPARG(); + callable = stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (PyFunction_Check(callable_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (Py_TYPE(callable_o) == &PyMethod_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _CALL_KW_NON_PY: { + _PyStackRef kwnames; + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; + oparg = CURRENT_OPARG(); + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + #if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); + #endif + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); + int total_args = oparg; + if (self_or_null_o != NULL) { + args--; + total_args++; + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(self_or_null); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + PyStackRef_CLOSE(kwnames); + if (true) JUMP_TO_ERROR(); + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); + PyStackRef_CLOSE(kwnames); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable); + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + /* _INSTRUMENTED_CALL_FUNCTION_EX is not a viable micro-op for tier 2 because it is instrumented */ /* __DO_CALL_FUNCTION_EX is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ diff --git a/Python/frame.c b/Python/frame.c index 25fa2824630f9b..3192968a0fb1b5 100644 --- a/Python/frame.c +++ b/Python/frame.c @@ -15,15 +15,7 @@ _PyFrame_Traverse(_PyInterpreterFrame *frame, visitproc visit, void *arg) Py_VISIT(frame->f_locals); Py_VISIT(frame->f_funcobj); Py_VISIT(_PyFrame_GetCode(frame)); - /* locals */ - _PyStackRef *locals = _PyFrame_GetLocalsArray(frame); - _PyStackRef *sp = frame->stackpointer; - /* locals and stack */ - while (sp > locals) { - sp--; - Py_VISIT(PyStackRef_AsPyObjectBorrow(*sp)); - } - return 0; + return _PyGC_VisitFrameStack(frame, visit, arg); } PyFrameObject * diff --git a/Python/gc.c b/Python/gc.c index 38a0da91a97510..923a79299cab03 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -534,6 +534,17 @@ visit_decref(PyObject *op, void *parent) return 0; } +int +_PyGC_VisitFrameStack(_PyInterpreterFrame *frame, visitproc visit, void *arg) +{ + _PyStackRef *ref = _PyFrame_GetLocalsArray(frame); + /* locals and stack */ + for (; ref < frame->stackpointer; ref++) { + Py_VISIT(PyStackRef_AsPyObjectBorrow(*ref)); + } + return 0; +} + /* Subtract internal references from gc_refs. After this, gc_refs is >= 0 * for all objects in containers, and is GC_REACHABLE for all tracked gc * objects not in containers. The ones with gc_refs > 0 are directly diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index 543bee24652dc9..b95456519dca06 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -164,15 +164,31 @@ gc_decref(PyObject *op) static void disable_deferred_refcounting(PyObject *op) { - if (_PyObject_HasDeferredRefcount(op)) { - op->ob_gc_bits &= ~_PyGC_BITS_DEFERRED; - op->ob_ref_shared -= _Py_REF_SHARED(_Py_REF_DEFERRED, 0); - - if (PyType_Check(op)) { - // Disable thread-local refcounting for heap types - PyTypeObject *type = (PyTypeObject *)op; - if (PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { - _PyType_ReleaseId((PyHeapTypeObject *)op); + if (!_PyObject_HasDeferredRefcount(op)) { + return; + } + + op->ob_gc_bits &= ~_PyGC_BITS_DEFERRED; + op->ob_ref_shared -= _Py_REF_SHARED(_Py_REF_DEFERRED, 0); + + if (PyType_Check(op)) { + // Disable thread-local refcounting for heap types + PyTypeObject *type = (PyTypeObject *)op; + if (PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { + _PyType_ReleaseId((PyHeapTypeObject *)op); + } + } + else if (PyGen_CheckExact(op) || PyCoro_CheckExact(op) || PyAsyncGen_CheckExact(op)) { + // Ensure any non-refcounted pointers in locals are converted to + // strong references. This ensures that the generator/coroutine is not + // freed before its locals. + PyGenObject *gen = (PyGenObject *)op; + struct _PyInterpreterFrame *frame = &gen->gi_iframe; + assert(frame->stackpointer != NULL); + for (_PyStackRef *ref = frame->localsplus; ref < frame->stackpointer; ref++) { + if (!PyStackRef_IsNull(*ref) && PyStackRef_IsDeferred(*ref)) { + // Convert a deferred reference to a strong reference. + *ref = PyStackRef_FromPyObjectSteal(PyStackRef_AsPyObjectSteal(*ref)); } } } @@ -313,6 +329,41 @@ gc_visit_heaps(PyInterpreterState *interp, mi_block_visit_fun *visitor, return err; } +static inline void +gc_visit_stackref(_PyStackRef stackref) +{ + // Note: we MUST check that it is deferred before checking the rest. + // Otherwise we might read into invalid memory due to non-deferred references + // being dead already. + if (PyStackRef_IsDeferred(stackref) && !PyStackRef_IsNull(stackref)) { + PyObject *obj = PyStackRef_AsPyObjectBorrow(stackref); + if (_PyObject_GC_IS_TRACKED(obj)) { + gc_add_refs(obj, 1); + } + } +} + +// Add 1 to the gc_refs for every deferred reference on each thread's stack. +static void +gc_visit_thread_stacks(PyInterpreterState *interp) +{ + HEAD_LOCK(&_PyRuntime); + for (PyThreadState *p = interp->threads.head; p != NULL; p = p->next) { + _PyInterpreterFrame *f = p->current_frame; + while (f != NULL) { + if (f->f_executable != NULL && PyCode_Check(f->f_executable)) { + PyCodeObject *co = (PyCodeObject *)f->f_executable; + int max_stack = co->co_nlocalsplus + co->co_stacksize; + for (int i = 0; i < max_stack; i++) { + gc_visit_stackref(f->localsplus[i]); + } + } + f = f->previous; + } + } + HEAD_UNLOCK(&_PyRuntime); +} + static void merge_queued_objects(_PyThreadStateImpl *tstate, struct collection_state *state) { @@ -617,6 +668,9 @@ deduce_unreachable_heap(PyInterpreterState *interp, gc_visit_heaps(interp, &validate_gc_objects, &state->base); #endif + // Visit the thread stacks to account for any deferred references. + gc_visit_thread_stacks(interp); + // Transitively mark reachable objects by clearing the // _PyGC_BITS_UNREACHABLE flag. if (gc_visit_heaps(interp, &mark_heap_visitor, &state->base) < 0) { @@ -897,6 +951,24 @@ visit_decref_unreachable(PyObject *op, void *data) return 0; } +int +_PyGC_VisitFrameStack(_PyInterpreterFrame *frame, visitproc visit, void *arg) +{ + _PyStackRef *ref = _PyFrame_GetLocalsArray(frame); + /* locals and stack */ + for (; ref < frame->stackpointer; ref++) { + // This is a bit tricky! We want to ignore deferred references when + // computing the incoming references, but otherwise treat them like + // regular references. + if (PyStackRef_IsDeferred(*ref) && + (visit == visit_decref || visit == visit_decref_unreachable)) { + continue; + } + Py_VISIT(PyStackRef_AsPyObjectBorrow(*ref)); + } + return 0; +} + // Handle objects that may have resurrected after a call to 'finalize_garbage'. static int handle_resurrected_objects(struct collection_state *state) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 7582b06a7641aa..27971ce75464f0 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -504,7 +504,7 @@ PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; PyObject *getitem = ht->_spec_cache.getitem; - new_frame = _PyFrame_PushUnchecked(tstate, (PyFunctionObject *)getitem, 2); + new_frame = _PyFrame_PushUnchecked(tstate, (PyFunctionObject *)getitem, 2, frame); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); new_frame->localsplus[0] = container; @@ -517,7 +517,7 @@ // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame->previous = frame; + assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; @@ -888,7 +888,7 @@ PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, - args, total_args, NULL + args, total_args, NULL, frame ); // Manipulate stack directly since we leave using DISPATCH_INLINED(). STACK_SHRINK(oparg + 2); @@ -975,60 +975,75 @@ _PyStackRef callable; _PyStackRef null; _PyStackRef *args; + _PyStackRef self; + _PyStackRef init; + _PyInterpreterFrame *init_frame; + _PyInterpreterFrame *new_frame; /* Skip 1 cache entry */ - /* Skip 2 cache entries */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); + } + // _CHECK_AND_ALLOCATE_OBJECT args = &stack_pointer[-oparg]; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - /* This instruction does the following: - * 1. Creates the object (by calling ``object.__new__``) - * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) - * 3. Pushes the frame for ``__init__`` to the frame stack - * */ - _PyCallCache *cache = (_PyCallCache *)&this_instr[1]; - DEOPT_IF(!PyStackRef_IsNull(null), CALL); - DEOPT_IF(!PyType_Check(callable_o), CALL); - PyTypeObject *tp = (PyTypeObject *)callable_o; - DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version), CALL); - assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES); - PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; - PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init; - PyCodeObject *code = (PyCodeObject *)init->func_code; - DEOPT_IF(code->co_argcount != oparg+1, CALL); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); - STAT_INC(CALL, hit); - PyObject *self = _PyType_NewManagedObject(tp); - if (self == NULL) { - goto error; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + DEOPT_IF(!PyStackRef_IsNull(null), CALL); + DEOPT_IF(!PyType_Check(callable_o), CALL); + PyTypeObject *tp = (PyTypeObject *)callable_o; + DEOPT_IF(tp->tp_version_tag != type_version, CALL); + assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES); + PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; + PyFunctionObject *init_func = (PyFunctionObject *)cls->_spec_cache.init; + PyCodeObject *code = (PyCodeObject *)init_func->func_code; + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); + STAT_INC(CALL, hit); + self = PyStackRef_FromPyObjectSteal(_PyType_NewManagedObject(tp)); + if (PyStackRef_IsNull(self)) { + goto error; + } + PyStackRef_CLOSE(callable); + init = PyStackRef_FromPyObjectNew(init_func); + stack_pointer[-1 - oparg] = init; + } + // _CREATE_INIT_FRAME + { + _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( + tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); + assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK); + /* Push self onto stack of shim */ + shim->localsplus[0] = PyStackRef_DUP(self); + PyFunctionObject *init_func = (PyFunctionObject *)PyStackRef_AsPyObjectSteal(init); + args[-1] = self; + init_frame = _PyEvalFramePushAndInit( + tstate, init_func, NULL, args-1, oparg+1, NULL, shim); + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + /* Account for pushing the extra frame. + * We don't check recursion depth here, + * as it will be checked after start_frame */ + tstate->py_recursion_remaining--; } - PyStackRef_CLOSE(callable); - _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( - tstate, (PyCodeObject *)&_Py_InitCleanup, 1); - assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK); - /* Push self onto stack of shim */ - Py_INCREF(self); - shim->localsplus[0] = PyStackRef_FromPyObjectSteal(self); - Py_INCREF(init); - _PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init, oparg+1); - /* Copy self followed by args to __init__ frame */ - init_frame->localsplus[0] = PyStackRef_FromPyObjectSteal(self); - for (int i = 0; i < oparg; i++) { - init_frame->localsplus[i+1] = args[i]; + // _PUSH_FRAME + new_frame = init_frame; + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); } - frame->return_offset = (uint16_t)(next_instr - this_instr); - STACK_SHRINK(oparg+2); - _PyFrame_SetStackPointer(frame, stack_pointer); - /* Link frames */ - init_frame->previous = shim; - shim->previous = frame; - frame = tstate->current_frame = init_frame; - CALL_STAT_INC(inlined_py_calls); - /* Account for pushing the extra frame. - * We don't check recursion depth here, - * as it will be checked after start_frame */ - tstate->py_recursion_remaining--; - goto start_frame; + DISPATCH(); } TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { @@ -1099,7 +1114,7 @@ int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); PyFunctionObject *func = (PyFunctionObject *)callable_o; - new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { @@ -1123,7 +1138,7 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame->previous = frame; + assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; @@ -1194,7 +1209,7 @@ PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, - args, total_args, NULL + args, total_args, NULL, frame ); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. @@ -1219,7 +1234,7 @@ // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame->previous = frame; + assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; @@ -1595,7 +1610,7 @@ PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(func_st), locals, - nargs, callargs, kwargs); + nargs, callargs, kwargs, frame); // Need to manually shrink the stack since we exit with DISPATCH_INLINED. STACK_SHRINK(oparg + 3); if (new_frame == NULL) { @@ -1720,21 +1735,36 @@ TARGET(CALL_KW) { frame->instr_ptr = next_instr; - next_instr += 1; + next_instr += 4; INSTRUCTION_STATS(CALL_KW); PREDICTED(CALL_KW); - _Py_CODEUNIT *this_instr = next_instr - 1; + _Py_CODEUNIT *this_instr = next_instr - 4; (void)this_instr; _PyStackRef callable; _PyStackRef self_or_null; _PyStackRef *args; _PyStackRef kwnames; _PyStackRef res; + // _SPECIALIZE_CALL_KW + self_or_null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + DISPATCH_SAME_OPARG(); + } + STAT_INC(CALL, deferred); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + /* Skip 2 cache entries */ // _DO_CALL_KW kwnames = stack_pointer[-1]; args = &stack_pointer[-1 - oparg]; - self_or_null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); @@ -1766,7 +1796,7 @@ PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, - args, positional_args, kwnames_o + args, positional_args, kwnames_o, frame ); PyStackRef_CLOSE(kwnames); // Manipulate stack directly since we leave using DISPATCH_INLINED(). @@ -1776,8 +1806,8 @@ if (new_frame == NULL) { goto error; } - assert(next_instr - this_instr == 1); - frame->return_offset = 1; + assert(next_instr - this_instr == 1 + INLINE_CACHE_ENTRIES_CALL_KW); + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL_KW; DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ @@ -1830,6 +1860,183 @@ } res = PyStackRef_FromPyObjectSteal(res_o); } + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(CALL_KW_BOUND_METHOD) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef kwnames; + _PyStackRef method; + _PyStackRef self; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL_KW); + } + // _CHECK_METHOD_VERSION_KW + null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + { + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL_KW); + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + DEOPT_IF(!PyFunction_Check(func), CALL_KW); + DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL_KW); + DEOPT_IF(!PyStackRef_IsNull(null), CALL_KW); + } + // _EXPAND_METHOD_KW + kwnames = stack_pointer[-1]; + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyStackRef_IsNull(null)); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + stack_pointer[-2 - oparg] = self; + method = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + stack_pointer[-3 - oparg] = method; + assert(PyStackRef_FunctionCheck(method)); + PyStackRef_CLOSE(callable); + } + // flush + // _PY_FRAME_KW + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (self_or_null_o != NULL) { + args--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + args, positional_args, kwnames_o, frame + ); + PyStackRef_CLOSE(kwnames); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (new_frame == NULL) { + goto error; + } + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(CALL_KW_NON_PY) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_NON_PY); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef kwnames; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CHECK_IS_NOT_PY_CALLABLE_KW + callable = stack_pointer[-3 - oparg]; + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + DEOPT_IF(PyFunction_Check(callable_o), CALL_KW); + DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL_KW); + } + // _CALL_KW_NON_PY + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + { + #if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); + #endif + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); + int total_args = oparg; + if (self_or_null_o != NULL) { + args--; + total_args++; + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(self_or_null); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + PyStackRef_CLOSE(kwnames); + if (true) { + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); + PyStackRef_CLOSE(kwnames); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable); + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + if (res_o == NULL) { + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + res = PyStackRef_FromPyObjectSteal(res_o); + } // _CHECK_PERIODIC { _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); @@ -1850,6 +2057,87 @@ DISPATCH(); } + TARGET(CALL_KW_PY) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_PY); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef kwnames; + _PyStackRef *args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL_KW); + } + // _CHECK_FUNCTION_VERSION_KW + callable = stack_pointer[-3 - oparg]; + { + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + DEOPT_IF(!PyFunction_Check(callable_o), CALL_KW); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + DEOPT_IF(func->func_version != func_version, CALL_KW); + } + // _PY_FRAME_KW + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (self_or_null_o != NULL) { + args--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + args, positional_args, kwnames_o, frame + ); + PyStackRef_CLOSE(kwnames); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (new_frame == NULL) { + goto error; + } + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + TARGET(CALL_LEN) { frame->instr_ptr = next_instr; next_instr += 4; @@ -2362,7 +2650,7 @@ int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); PyFunctionObject *func = (PyFunctionObject *)callable_o; - new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { @@ -2386,7 +2674,7 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame->previous = frame; + assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; @@ -2437,7 +2725,7 @@ PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, - args, total_args, NULL + args, total_args, NULL, frame ); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. @@ -2462,7 +2750,7 @@ // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame->previous = frame; + assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; @@ -3394,6 +3682,7 @@ gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; + gen_frame->previous = frame; // oparg is the return offset from the next instruction. frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg); } @@ -3404,7 +3693,7 @@ // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame->previous = frame; + assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; @@ -3817,7 +4106,7 @@ PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, - args, total_args, NULL + args, total_args, NULL, frame ); // Manipulate stack directly since we leave using DISPATCH_INLINED(). STACK_SHRINK(oparg + 2); @@ -3906,8 +4195,12 @@ TARGET(INSTRUMENTED_CALL_KW) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; (void)this_instr; - next_instr += 1; + next_instr += 4; INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + uint32_t version = read_u32(&this_instr[2].cache); + (void)version; int is_meth = !PyStackRef_IsNull(PEEK(oparg + 2)); int total_args = oparg + is_meth; PyObject *function = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 3)); @@ -3917,6 +4210,7 @@ tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, function, arg); if (err) goto error; + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); GO_TO_INSTRUCTION(CALL_KW); } @@ -4666,7 +4960,7 @@ STAT_INC(LOAD_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); Py_INCREF(f); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2); + _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2, frame); // Manipulate stack directly because we exit with DISPATCH_INLINED(). STACK_SHRINK(1); new_frame->localsplus[0] = owner; @@ -4996,7 +5290,7 @@ DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); Py_INCREF(fget); - new_frame = _PyFrame_PushUnchecked(tstate, f, 1); + new_frame = _PyFrame_PushUnchecked(tstate, f, 1, frame); new_frame->localsplus[0] = owner; } // _SAVE_RETURN_OFFSET @@ -5016,7 +5310,7 @@ stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame->previous = frame; + assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; @@ -6273,6 +6567,8 @@ tstate->exc_info = &gen->gi_exc_state; assert(next_instr - this_instr + oparg <= UINT16_MAX); frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); + assert(gen_frame->previous == NULL); + gen_frame->previous = frame; DISPATCH_INLINED(gen_frame); } if (PyStackRef_Is(v, PyStackRef_None) && PyIter_Check(receiver_o)) { @@ -6333,6 +6629,7 @@ tstate->exc_info = &gen->gi_exc_state; assert(1 + INLINE_CACHE_ENTRIES_SEND + oparg <= UINT16_MAX); frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + oparg); + gen_frame->previous = frame; } // _PUSH_FRAME new_frame = gen_frame; @@ -6343,7 +6640,7 @@ stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame->previous = frame; + assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; @@ -7372,7 +7669,7 @@ PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(func_st), locals, - nargs, callargs, kwargs); + nargs, callargs, kwargs, frame); // Need to manually shrink the stack since we exit with DISPATCH_INLINED. STACK_SHRINK(oparg + 3); if (new_frame == NULL) { diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 9ea01e26842c75..0418105dee5926 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -169,6 +169,9 @@ static void *opcode_targets[256] = { &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS, &&TARGET_CALL_BUILTIN_O, &&TARGET_CALL_ISINSTANCE, + &&TARGET_CALL_KW_BOUND_METHOD, + &&TARGET_CALL_KW_NON_PY, + &&TARGET_CALL_KW_PY, &&TARGET_CALL_LEN, &&TARGET_CALL_LIST_APPEND, &&TARGET_CALL_METHOD_DESCRIPTOR_FAST, @@ -232,9 +235,6 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_INSTRUMENTED_END_FOR, &&TARGET_INSTRUMENTED_END_SEND, &&TARGET_INSTRUMENTED_LOAD_SUPER_ATTR, diff --git a/Python/optimizer.c b/Python/optimizer.c index dbd5467f0d4653..9198e410627dd4 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -807,7 +807,7 @@ translate_bytecode_to_trace( ADD_TO_TRACE(_DYNAMIC_EXIT, 0, 0, 0); goto done; } - assert(_PyOpcode_Deopt[opcode] == CALL); + assert(_PyOpcode_Deopt[opcode] == CALL || _PyOpcode_Deopt[opcode] == CALL_KW); int func_version_offset = offsetof(_PyCallCache, func_version)/sizeof(_Py_CODEUNIT) // Add one to account for the actual opcode/oparg pair: diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 010733e6ac221a..6e46d9bed11fbc 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -627,6 +627,32 @@ dummy_func(void) { ctx->done = true; } + op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame: _Py_UOpsAbstractFrame *)) { + (void)callable; + (void)self_or_null; + (void)args; + (void)kwnames; + new_frame = NULL; + ctx->done = true; + } + + op(_CHECK_AND_ALLOCATE_OBJECT, (type_version/2, callable, null, args[oparg] -- self, init, args[oparg])) { + (void)type_version; + (void)callable; + (void)null; + (void)args; + self = sym_new_not_null(ctx); + init = sym_new_not_null(ctx); + } + + op(_CREATE_INIT_FRAME, (self, init, args[oparg] -- init_frame: _Py_UOpsAbstractFrame *)) { + (void)self; + (void)init; + (void)args; + init_frame = NULL; + ctx->done = true; + } + op(_RETURN_VALUE, (retval -- res)) { SYNC_SP(); ctx->frame->stack_pointer = stack_pointer; diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 866d7d95b580d4..e5be9d0e3b5ee7 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1866,7 +1866,46 @@ break; } - /* _CALL_ALLOC_AND_ENTER_INIT is not a viable micro-op for tier 2 */ + case _CHECK_AND_ALLOCATE_OBJECT: { + _Py_UopsSymbol **args; + _Py_UopsSymbol *null; + _Py_UopsSymbol *callable; + _Py_UopsSymbol *self; + _Py_UopsSymbol *init; + args = &stack_pointer[-oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-oparg]; + uint32_t type_version = (uint32_t)this_instr->operand; + (void)type_version; + (void)callable; + (void)null; + (void)args; + self = sym_new_not_null(ctx); + init = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = self; + stack_pointer[-1 - oparg] = init; + break; + } + + case _CREATE_INIT_FRAME: { + _Py_UopsSymbol **args; + _Py_UopsSymbol *init; + _Py_UopsSymbol *self; + _Py_UOpsAbstractFrame *init_frame; + args = &stack_pointer[-oparg]; + init = stack_pointer[-1 - oparg]; + self = stack_pointer[-2 - oparg]; + (void)self; + (void)init; + (void)args; + init_frame = NULL; + ctx->done = true; + stack_pointer[-2 - oparg] = (_Py_UopsSymbol *)init_frame; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } case _EXIT_INIT_CHECK: { stack_pointer += -1; @@ -1974,6 +2013,62 @@ /* _DO_CALL_KW is not a viable micro-op for tier 2 */ + case _PY_FRAME_KW: { + _Py_UopsSymbol *kwnames; + _Py_UopsSymbol **args; + _Py_UopsSymbol *self_or_null; + _Py_UopsSymbol *callable; + _Py_UOpsAbstractFrame *new_frame; + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + (void)callable; + (void)self_or_null; + (void)args; + (void)kwnames; + new_frame = NULL; + ctx->done = true; + stack_pointer[-3 - oparg] = (_Py_UopsSymbol *)new_frame; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CHECK_FUNCTION_VERSION_KW: { + break; + } + + case _CHECK_METHOD_VERSION_KW: { + break; + } + + case _EXPAND_METHOD_KW: { + _Py_UopsSymbol *method; + _Py_UopsSymbol *self; + _Py_UopsSymbol *kwnames; + method = sym_new_not_null(ctx); + self = sym_new_not_null(ctx); + kwnames = sym_new_not_null(ctx); + stack_pointer[-3 - oparg] = method; + stack_pointer[-2 - oparg] = self; + stack_pointer[-1] = kwnames; + break; + } + + case _CHECK_IS_NOT_PY_CALLABLE_KW: { + break; + } + + case _CALL_KW_NON_PY: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + /* _INSTRUMENTED_CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ /* __DO_CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 3a41c640fd9599..27faf723745c21 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -104,7 +104,7 @@ _PyRuntimeState _PyRuntime #if defined(__linux__) && (defined(__GNUC__) || defined(__clang__)) __attribute__ ((section (".PyRuntime"))) #endif -= _PyRuntimeState_INIT(_PyRuntime); += _PyRuntimeState_INIT(_PyRuntime, _Py_Debug_Cookie); _Py_COMP_DIAG_POP static int runtime_initialized = 0; diff --git a/Python/pystate.c b/Python/pystate.c index bba88b76088e71..4d7bec65ff5c49 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -390,7 +390,7 @@ _Py_COMP_DIAG_IGNORE_DEPR_DECLS Note that we initialize "initial" relative to _PyRuntime, to ensure pre-initialized pointers point to the active runtime state (and not "initial"). */ -static const _PyRuntimeState initial = _PyRuntimeState_INIT(_PyRuntime); +static const _PyRuntimeState initial = _PyRuntimeState_INIT(_PyRuntime, ""); _Py_COMP_DIAG_POP #define LOCKS_INIT(runtime) \ @@ -455,6 +455,8 @@ _PyRuntimeState_Init(_PyRuntimeState *runtime) // Py_Initialize() must be running again. // Reset to _PyRuntimeState_INIT. memcpy(runtime, &initial, sizeof(*runtime)); + // Preserve the cookie from the original runtime. + memcpy(runtime->debug_offsets.cookie, _Py_Debug_Cookie, 8); assert(!runtime->_initialized); } diff --git a/Python/specialize.c b/Python/specialize.c index 4a227381a8b1d7..b3a2e07c3bbcb8 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1809,10 +1809,6 @@ specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) return -1; } if (init != NULL) { - if (((PyCodeObject *)init->func_code)->co_argcount != nargs+1) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); - return -1; - } _PyCallCache *cache = (_PyCallCache *)(instr + 1); write_u32(cache->func_version, tp->tp_version_tag); _Py_SET_OPCODE(*instr, CALL_ALLOC_AND_ENTER_INIT); @@ -1904,6 +1900,33 @@ specialize_py_call(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, return 0; } + +static int +specialize_py_call_kw(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, + bool bound_method) +{ + _PyCallCache *cache = (_PyCallCache *)(instr + 1); + PyCodeObject *code = (PyCodeObject *)func->func_code; + int kind = function_kind(code); + /* Don't specialize if PEP 523 is active */ + if (_PyInterpreterState_GET()->eval_frame) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_PEP_523); + return -1; + } + if (kind == SPEC_FAIL_CODE_NOT_OPTIMIZED) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CODE_NOT_OPTIMIZED); + return -1; + } + int version = _PyFunction_GetVersionForCurrentState(func); + if (version == 0) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OUT_OF_VERSIONS); + return -1; + } + write_u32(cache->func_version, version); + instr->op.code = bound_method ? CALL_KW_BOUND_METHOD : CALL_KW_PY; + return 0; +} + static int specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) { @@ -1999,6 +2022,46 @@ _Py_Specialize_Call(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs) } } +void +_Py_Specialize_CallKw(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs) +{ + PyObject *callable = PyStackRef_AsPyObjectBorrow(callable_st); + + assert(ENABLE_SPECIALIZATION); + assert(_PyOpcode_Caches[CALL_KW] == INLINE_CACHE_ENTRIES_CALL_KW); + assert(_Py_OPCODE(*instr) != INSTRUMENTED_CALL_KW); + _PyCallCache *cache = (_PyCallCache *)(instr + 1); + int fail; + if (PyFunction_Check(callable)) { + fail = specialize_py_call_kw((PyFunctionObject *)callable, instr, nargs, false); + } + else if (PyMethod_Check(callable)) { + PyObject *func = ((PyMethodObject *)callable)->im_func; + if (PyFunction_Check(func)) { + fail = specialize_py_call_kw((PyFunctionObject *)func, instr, nargs, true); + } + else { + SPECIALIZATION_FAIL(CALL_KW, SPEC_FAIL_CALL_BOUND_METHOD); + fail = -1; + } + } + else { + instr->op.code = CALL_KW_NON_PY; + fail = 0; + } + if (fail) { + STAT_INC(CALL, failure); + assert(!PyErr_Occurred()); + instr->op.code = CALL_KW; + cache->counter = adaptive_counter_backoff(cache->counter); + } + else { + STAT_INC(CALL, success); + assert(!PyErr_Occurred()); + cache->counter = adaptive_counter_cooldown(); + } +} + #ifdef Py_STATS static int binary_op_fail_kind(int oparg, PyObject *lhs, PyObject *rhs) @@ -2587,7 +2650,7 @@ static const PyBytesObject no_location = { .ob_sval = { NO_LOC_4 } }; -const struct _PyCode_DEF(8) _Py_InitCleanup = { +const struct _PyCode8 _Py_InitCleanup = { _PyVarObject_HEAD_INIT(&PyCode_Type, 3), .co_consts = (PyObject *)&_Py_SINGLETON(tuple_empty), .co_names = (PyObject *)&_Py_SINGLETON(tuple_empty), diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 63b640e465ac6b..b91b11763fb56c 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -742,3 +742,7 @@ Modules/clinic/md5module.c.h _md5_md5 _keywords - Modules/clinic/grpmodule.c.h grp_getgrgid _keywords - Modules/clinic/grpmodule.c.h grp_getgrnam _keywords - Objects/object.c - constants static PyObject*[] + + +## False positives +Python/specialize.c - _Py_InitCleanup - diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index a14f58bdac0cb2..050e3723feb247 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -20,8 +20,6 @@ CROSS_BUILD_DIR = CHECKOUT / "cross-build" BUILD_DIR = CROSS_BUILD_DIR / "build" -HOST_TRIPLE = "wasm32-wasi" -HOST_DIR = CROSS_BUILD_DIR / HOST_TRIPLE LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local" LOCAL_SETUP_MARKER = "# Generated by Tools/wasm/wasi.py\n".encode("utf-8") @@ -63,12 +61,17 @@ def subdir(working_dir, *, clean_ok=False): def decorator(func): @functools.wraps(func) def wrapper(context): + nonlocal working_dir + + if callable(working_dir): + working_dir = working_dir(context) try: tput_output = subprocess.check_output(["tput", "cols"], encoding="utf-8") - terminal_width = int(tput_output.strip()) except subprocess.CalledProcessError: terminal_width = 80 + else: + terminal_width = int(tput_output.strip()) print("⎯" * terminal_width) print("📁", working_dir) if (clean_ok and getattr(context, "clean", False) and @@ -193,7 +196,7 @@ def wasi_sdk_env(context): return env -@subdir(HOST_DIR, clean_ok=True) +@subdir(lambda context: CROSS_BUILD_DIR / context.host_triple, clean_ok=True) def configure_wasi_python(context, working_dir): """Configure the WASI/host build.""" if not context.wasi_sdk_path or not context.wasi_sdk_path.exists(): @@ -238,7 +241,7 @@ def configure_wasi_python(context, working_dir): # to find the stdlib due to Python not recognizing that it's being # executed from within a checkout. configure = [os.path.relpath(CHECKOUT / 'configure', working_dir), - f"--host={HOST_TRIPLE}", + f"--host={context.host_triple}", f"--build={build_platform()}", f"--with-build-python={build_python}"] if pydebug: @@ -258,7 +261,7 @@ def configure_wasi_python(context, working_dir): sys.stdout.flush() -@subdir(HOST_DIR) +@subdir(lambda context: CROSS_BUILD_DIR / context.host_triple) def make_wasi_python(context, working_dir): """Run `make` for the WASI/host build.""" call(["make", "--jobs", str(cpu_count()), "all"], @@ -342,6 +345,9 @@ def main(): help="Command template for running the WASI host " "(default designed for wasmtime 14 or newer: " f"`{default_host_runner}`)") + for subcommand in build, configure_host, make_host: + subcommand.add_argument("--host-triple", action="store", default="wasm32-wasi", + help="The target triple for the WASI host build") context = parser.parse_args() diff --git a/configure b/configure index 54f05f4972a8a8..1003c43fd6dfb4 100755 --- a/configure +++ b/configure @@ -717,6 +717,7 @@ LIBHACL_SIMD256_OBJS LIBHACL_SIMD256_FLAGS LIBHACL_SIMD128_OBJS LIBHACL_SIMD128_FLAGS +LIBHACL_CFLAGS MODULE__BLAKE2_FALSE MODULE__BLAKE2_TRUE MODULE__SHA3_FALSE @@ -30618,6 +30619,16 @@ fi printf "%s\n" "$py_cv_module__blake2" >&6; } +LIBHACL_CFLAGS='-I$(srcdir)/Modules/_hacl -I$(srcdir)/Modules/_hacl/include -D_BSD_SOURCE -D_DEFAULT_SOURCE $(PY_STDMODULE_CFLAGS) $(CCSHARED)' +case "$ac_sys_system" in + Linux*) + if test "$ac_cv_func_explicit_bzero" = "no"; then + LIBHACL_CFLAGS="$LIBHACL_CFLAGS -DLINUX_NO_EXPLICIT_BZERO" + fi + ;; +esac + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -msse -msse2 -msse3 -msse4.1 -msse4.2" >&5 printf %s "checking whether C compiler accepts -msse -msse2 -msse3 -msse4.1 -msse4.2... " >&6; } if test ${ax_cv_check_cflags__Werror__msse__msse2__msse3__msse4_1__msse4_2+y} diff --git a/configure.ac b/configure.ac index 37d666b127fd74..0d4f28cde82ba3 100644 --- a/configure.ac +++ b/configure.ac @@ -7769,6 +7769,16 @@ PY_STDLIB_MOD([_sha2], [test "$with_builtin_sha2" = yes]) PY_STDLIB_MOD([_sha3], [test "$with_builtin_sha3" = yes]) PY_STDLIB_MOD([_blake2], [test "$with_builtin_blake2" = yes]) +LIBHACL_CFLAGS='-I$(srcdir)/Modules/_hacl -I$(srcdir)/Modules/_hacl/include -D_BSD_SOURCE -D_DEFAULT_SOURCE $(PY_STDMODULE_CFLAGS) $(CCSHARED)' +case "$ac_sys_system" in + Linux*) + if test "$ac_cv_func_explicit_bzero" = "no"; then + LIBHACL_CFLAGS="$LIBHACL_CFLAGS -DLINUX_NO_EXPLICIT_BZERO" + fi + ;; +esac +AC_SUBST([LIBHACL_CFLAGS]) + dnl This can be extended here to detect e.g. Power8, which HACL* should also support. AX_CHECK_COMPILE_FLAG([-msse -msse2 -msse3 -msse4.1 -msse4.2],[ [LIBHACL_SIMD128_FLAGS="-msse -msse2 -msse3 -msse4.1 -msse4.2"]