Merge branch 'master' into bpo_41832

* master:
  bpo-42260: Add _PyInterpreterState_SetConfig() (pythonGH-23158)
  Disable peg generator tests when building with PGO (pythonGH-23141)
  bpo-1635741: _sqlite3 uses PyModule_AddObjectRef() (pythonGH-23148)
  bpo-1635741: Fix PyInit_pyexpat() error handling (pythonGH-22489)
  bpo-42260: Main init modify sys.flags in-place (pythonGH-23150)
  bpo-1635741: Fix ref leak in _PyWarnings_Init() error path (pythonGH-23151)
  bpo-1635741: _ast uses PyModule_AddObjectRef() (pythonGH-23146)
  bpo-1635741: _contextvars uses PyModule_AddType() (pythonGH-23147)
  bpo-42260: Reorganize PyConfig (pythonGH-23149)
  bpo-1635741: Add PyModule_AddObjectRef() function (pythonGH-23122)
  bpo-42236: os.device_encoding() respects UTF-8 Mode (pythonGH-23119)
  bpo-42251: Add gettrace and getprofile to threading (pythonGH-23125)
  Enable signing of nuget.org packages and update to supported timestamp server (pythonGH-23132)
  Fix incorrect links in ast docs (pythonGH-23017)
  Add _PyType_GetModuleByDef (pythonGH-22835)
  Post 3.10.0a2
  bpo-41796: Call _PyAST_Fini() earlier to fix a leak (pythonGH-23131)
  bpo-42249: Fix writing binary Plist files larger than 4 GiB. (pythonGH-23121)
  bpo-40077: Convert mmap.mmap static type to a heap type (pythonGH-23108)
  Python 3.10.0a2

.azure-pipelines/windows-release/stage-pack-msix.yml

@@ -120,10 +120,11 @@ jobs:
       artifactName: unsigned_msix
       downloadPath: $(Build.BinariesDirectory)
 
+  # MSIX must be signed and timestamped simultaneously
   - powershell: |
       $failed = $true
       foreach ($retry in 1..3) {
-          signtool sign /a /n "$(SigningCertificate)" /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d "$(SigningDescription)" (gi *.msix)
+          signtool sign /a /n "$(SigningCertificate)" /fd sha256 /tr http://timestamp.digicert.com/ /td sha256 /d "$(SigningDescription)" (gi *.msix)
           if ($?) {
               $failed = $false
               break

.azure-pipelines/windows-release/stage-pack-nuget.yml

@@ -4,7 +4,7 @@ jobs:
   condition: and(succeeded(), eq(variables['DoNuget'], 'true'))
 
   pool:
-    vmImage: windows-2019
+    name: 'Windows Release'
 
   workspace:
     clean: all
@@ -36,6 +36,14 @@ jobs:
       nuget pack "$(Build.BinariesDirectory)\layout\python.nuspec" -OutputDirectory $(Build.ArtifactStagingDirectory) -NoPackageAnalysis -NonInteractive
     displayName: 'Create nuget package'
 
+  - powershell: |
+      gci *.nupkg | %{
+        nuget sign "$_" -CertificateSubjectName "$(SigningCertificate)" -Timestamper http://timestamp.digicert.com/ -Overwrite
+      }
+    displayName: 'Sign nuget package'
+    workingDirectory: $(Build.ArtifactStagingDirectory)
+    condition: and(succeeded(), variables['SigningCertificate'])
+
   - task: PublishBuildArtifacts@1
     displayName: 'Publish Artifact: nuget'
     inputs:

.azure-pipelines/windows-release/stage-sign.yml

@@ -57,7 +57,7 @@ jobs:
       $files = (gi ${{ parameters.Include }} -Exclude ${{ parameters.Exclude }})
       $failed = $true
       foreach ($retry in 1..10) {
-          signtool timestamp /t http://timestamp.verisign.com/scripts/timestamp.dll $files
+          signtool timestamp /tr http://timestamp.digicert.com/ /td sha256 $files
           if ($?) {
               $failed = $false
               break

Doc/c-api/init_config.rst

@@ -128,6 +128,8 @@ PyStatus
 
       Initialization error with a message.
 
+      *err_msg* must not be ``NULL``.
+
    .. c:function:: PyStatus PyStatus_NoMemory(void)
 
       Memory allocation failure (out of memory).

Doc/c-api/module.rst

@@ -264,7 +264,7 @@ of the following two module creation functions:
       instead; only use this if you are sure you need it.
 
 Before it is returned from in the initialization function, the resulting module
-object is typically populated using functions like :c:func:`PyModule_AddObject`.
+object is typically populated using functions like :c:func:`PyModule_AddObjectRef`.
 
 .. _multi-phase-initialization:
 
@@ -437,26 +437,102 @@ a function called from a module execution slot (if using multi-phase
 initialization), can use the following functions to help initialize the module
 state:
 
+.. c:function:: int PyModule_AddObjectRef(PyObject *module, const char *name, PyObject *value)
+
+   Add an object to *module* as *name*.  This is a convenience function which
+   can be used from the module's initialization function.
+
+   On success, return ``0``. On error, raise an exception and return ``-1``.
+
+   Return ``NULL`` if *value* is ``NULL``. It must be called with an exception
+   raised in this case.
+
+   Example usage::
+
+       static int
+       add_spam(PyObject *module, int value)
+       {
+           PyObject *obj = PyLong_FromLong(value);
+           if (obj == NULL) {
+               return -1;
+           }
+           int res = PyModule_AddObjectRef(module, "spam", obj);
+           Py_DECREF(obj);
+           return res;
+        }
+
+   The example can also be written without checking explicitly if *obj* is
+   ``NULL``::
+
+       static int
+       add_spam(PyObject *module, int value)
+       {
+           PyObject *obj = PyLong_FromLong(value);
+           int res = PyModule_AddObjectRef(module, "spam", obj);
+           Py_XDECREF(obj);
+           return res;
+        }
+
+   Note that ``Py_XDECREF()`` should be used instead of ``Py_DECREF()`` in
+   this case, since *obj* can be ``NULL``.
+
+   .. versionadded:: 3.10
+
+
 .. c:function:: int PyModule_AddObject(PyObject *module, const char *name, PyObject *value)
 
-   Add an object to *module* as *name*.  This is a convenience function which can
-   be used from the module's initialization function.  This steals a reference to
-   *value* on success. Return ``-1`` on error, ``0`` on success.
+   Similar to :c:func:`PyModule_AddObjectRef`, but steals a reference to
+   *value* on success (if it returns ``0``).
+
+   The new :c:func:`PyModule_AddObjectRef` function is recommended, since it is
+   easy to introduce reference leaks by misusing the
+   :c:func:`PyModule_AddObject` function.
 
    .. note::
 
-      Unlike other functions that steal references, ``PyModule_AddObject()`` only
-      decrements the reference count of *value* **on success**.
+      Unlike other functions that steal references, ``PyModule_AddObject()``
+      only decrements the reference count of *value* **on success**.
 
       This means that its return value must be checked, and calling code must
-      :c:func:`Py_DECREF` *value* manually on error. Example usage::
-
-         Py_INCREF(spam);
-         if (PyModule_AddObject(module, "spam", spam) < 0) {
-             Py_DECREF(module);
-             Py_DECREF(spam);
-             return NULL;
-         }
+      :c:func:`Py_DECREF` *value* manually on error.
+
+   Example usage::
+
+      static int
+      add_spam(PyObject *module, int value)
+      {
+          PyObject *obj = PyLong_FromLong(value);
+          if (obj == NULL) {
+              return -1;
+          }
+          if (PyModule_AddObject(module, "spam", obj) < 0) {
+              Py_DECREF(obj);
+              return -1;
+          }
+          // PyModule_AddObject() stole a reference to obj:
+          // Py_DECREF(obj) is not needed here
+          return 0;
+      }
+
+   The example can also be written without checking explicitly if *obj* is
+   ``NULL``::
+
+      static int
+      add_spam(PyObject *module, int value)
+      {
+          PyObject *obj = PyLong_FromLong(value);
+          if (PyModule_AddObject(module, "spam", obj) < 0) {
+              Py_XDECREF(obj);
+              return -1;
+          }
+          // PyModule_AddObject() stole a reference to obj:
+          // Py_DECREF(obj) is not needed here
+          return 0;
+      }
+
+   Note that ``Py_XDECREF()`` should be used instead of ``Py_DECREF()`` in
+   this case, since *obj* can be ``NULL``.
+
 
 .. c:function:: int PyModule_AddIntConstant(PyObject *module, const char *name, long value)
 

Doc/library/ast.rst

@@ -80,12 +80,13 @@ Node classes
                   end_col_offset
 
       Instances of :class:`ast.expr` and :class:`ast.stmt` subclasses have
-      :attr:`lineno`, :attr:`col_offset`, :attr:`lineno`, and :attr:`col_offset`
-      attributes.  The :attr:`lineno` and :attr:`end_lineno` are the first and
-      last line numbers of source text span (1-indexed so the first line is line 1)
-      and the :attr:`col_offset` and :attr:`end_col_offset` are the corresponding
-      UTF-8 byte offsets of the first and last tokens that generated the node.
-      The UTF-8 offset is recorded because the parser uses UTF-8 internally.
+      :attr:`lineno`, :attr:`col_offset`, :attr:`end_lineno`, and
+      :attr:`end_col_offset` attributes.  The :attr:`lineno` and :attr:`end_lineno`
+      are the first and last line numbers of source text span (1-indexed so the
+      first line is line 1) and the :attr:`col_offset` and :attr:`end_col_offset`
+      are the corresponding UTF-8 byte offsets of the first and last tokens that
+      generated the node. The UTF-8 offset is recorded because the parser uses
+      UTF-8 internally.
 
       Note that the end positions are not required by the compiler and are
       therefore optional. The end offset is *after* the last symbol, for example

Doc/library/os.rst

@@ -113,6 +113,8 @@ of the UTF-8 encoding:
   :ref:`error handler <error-handlers>` being enabled for :data:`sys.stdin`
   and :data:`sys.stdout` (:data:`sys.stderr` continues to use
   ``backslashreplace`` as it does in the default locale-aware mode)
+* On Unix, :func:`os.device_encoding` returns ``'UTF-8'``. rather than the
+  device encoding.
 
 Note that the standard stream settings in UTF-8 mode can be overridden by
 :envvar:`PYTHONIOENCODING` (just as they can be in the default locale-aware
@@ -808,6 +810,12 @@ as internal buffering of data.
    Return a string describing the encoding of the device associated with *fd*
    if it is connected to a terminal; else return :const:`None`.
 
+   On Unix, if the :ref:`Python UTF-8 Mode <utf8-mode>` is enabled, return
+   ``'UTF-8'`` rather than the device encoding.
+
+   .. versionchanged:: 3.10
+      On Unix, the function now implements the Python UTF-8 Mode.
+
 
 .. function:: dup(fd)
 

Doc/library/threading.rst

@@ -121,6 +121,17 @@ This module defines the following functions:
    :meth:`~Thread.run` method is called.
 
 
+.. function:: gettrace()
+
+   .. index::
+      single: trace function
+      single: debugger
+
+   Get the trace function as set by :func:`settrace`.
+
+   .. versionadded:: 3.10
+
+
 .. function:: setprofile(func)
 
    .. index:: single: profile function
@@ -130,6 +141,15 @@ This module defines the following functions:
    :meth:`~Thread.run` method is called.
 
 
+.. function:: getprofile()
+
+   .. index:: single: profile function
+
+   Get the profiler function as set by :func:`setprofile`.
+
+   .. versionadded:: 3.10
+
+
 .. function:: stack_size([size])
 
    Return the thread stack size used when creating new threads.  The optional

Doc/whatsnew/3.10.rst

@@ -224,6 +224,14 @@ Add :data:`sys.orig_argv` attribute: the list of the original command line
 arguments passed to the Python executable.
 (Contributed by Victor Stinner in :issue:`23427`.)
 
+threading
+---------
+
+Added :func:`threading.gettrace` and :func:`threading.getprofile` to
+retrieve the functions set by :func:`threading.settrace` and
+:func:`threading.setprofile` respectively.
+(Contributed by Mario Corchero in :issue:`42251`.)
+
 types
 -----
 
@@ -366,6 +374,11 @@ New Features
 * Added :c:func:`PyUnicode_AsUTF8AndSize` to the limited C API.
   (Contributed by Alex Gaynor in :issue:`41784`.)
 
+* Added :c:func:`PyModule_AddObjectRef` function: similar to
+  :c:func:`PyModule_AddObjectRef` but don't steal a reference to the value on
+  success.
+  (Contributed by Victor Stinner in :issue:`1635741`.)
+
 * The :c:func:`PyType_FromModuleAndSpec` function now accepts NULL ``tp_doc``
   slot.
   (Contributed by Hai Shi in :issue:`41832`.)

Include/cpython/initconfig.h

@@ -41,7 +41,7 @@ PyAPI_FUNC(PyStatus) PyWideStringList_Insert(PyWideStringList *list,
 
 /* --- PyPreConfig ----------------------------------------------- */
 
-typedef struct {
+typedef struct PyPreConfig {
     int _config_init;     /* _PyConfigInitEnum value */
 
     /* Parse Py_PreInitializeFromBytesArgs() arguments?
@@ -128,7 +128,7 @@ PyAPI_FUNC(void) PyPreConfig_InitIsolatedConfig(PyPreConfig *config);
 /* --- PyConfig ---------------------------------------------- */
 
 /* This structure is best documented in the Doc/c-api/init_config.rst file. */
-typedef struct {
+typedef struct PyConfig {
     int _config_init;     /* _PyConfigInitEnum value */
 
     int isolated;
@@ -147,8 +147,8 @@ typedef struct {
     wchar_t *filesystem_errors;
     wchar_t *pycache_prefix;
     int parse_argv;
+    PyWideStringList orig_argv;
     PyWideStringList argv;
-    wchar_t *program_name;
     PyWideStringList xoptions;
     PyWideStringList warnoptions;
     int site_import;
@@ -169,12 +169,13 @@ typedef struct {
     int legacy_windows_stdio;
 #endif
     wchar_t *check_hash_pycs_mode;
-    PyWideStringList orig_argv;
 
     /* --- Path configuration inputs ------------ */
     int pathconfig_warnings;
+    wchar_t *program_name;
     wchar_t *pythonpath_env;
     wchar_t *home;
+    wchar_t *platlibdir;
 
     /* --- Path configuration outputs ----------- */
     int module_search_paths_set;
@@ -185,7 +186,6 @@ typedef struct {
     wchar_t *base_prefix;
     wchar_t *exec_prefix;
     wchar_t *base_exec_prefix;
-    wchar_t *platlibdir;
 
     /* --- Parameter only used by Py_Main() ---------- */
     int skip_source_first_line;

Include/cpython/object.h

@@ -296,6 +296,8 @@ PyAPI_FUNC(PyObject *) _PyObject_LookupSpecial(PyObject *, _Py_Identifier *);
 PyAPI_FUNC(PyTypeObject *) _PyType_CalculateMetaclass(PyTypeObject *, PyObject *);
 PyAPI_FUNC(PyObject *) _PyType_GetDocFromInternalDoc(const char *, const char *);
 PyAPI_FUNC(PyObject *) _PyType_GetTextSignatureFromInternalDoc(const char *, const char *);
+struct PyModuleDef;
+PyAPI_FUNC(PyObject *) _PyType_GetModuleByDef(PyTypeObject *, struct PyModuleDef *);
 
 struct _Py_Identifier;
 PyAPI_FUNC(int) PyObject_Print(PyObject *, FILE *, int);

Include/cpython/pystate.h

@@ -193,6 +193,36 @@ PyAPI_FUNC(void) _PyInterpreterState_SetEvalFrameFunc(
 
 PyAPI_FUNC(const PyConfig*) _PyInterpreterState_GetConfig(PyInterpreterState *interp);
 
+/* Get a copy of the current interpreter configuration.
+
+   Return 0 on success. Raise an exception and return -1 on error.
+
+   The caller must initialize 'config', using PyConfig_InitPythonConfig()
+   for example.
+
+   Python must be preinitialized to call this method.
+   The caller must hold the GIL. */
+PyAPI_FUNC(int) _PyInterpreterState_GetConfigCopy(
+    struct PyConfig *config);
+
+/* Set the configuration of the current interpreter.
+
+   This function should be called during or just after the Python
+   initialization.
+
+   Update the sys module with the new configuration. If the sys module was
+   modified directly after the Python initialization, these changes are lost.
+
+   Some configuration like faulthandler or warnoptions can be updated in the
+   configuration, but don't reconfigure Python (don't enable/disable
+   faulthandler and don't reconfigure warnings filters).
+
+   Return 0 on success. Raise an exception and return -1 on error.
+
+   The configuration should come from _PyInterpreterState_GetConfigCopy(). */
+PyAPI_FUNC(int) _PyInterpreterState_SetConfig(
+    const struct PyConfig *config);
+
 // Get the configuration of the currrent interpreter.
 // The caller must hold the GIL.
 PyAPI_FUNC(const PyConfig*) _Py_GetConfig(void);

Include/internal/pycore_initconfig.h

@@ -44,6 +44,8 @@ struct pyruntimestate;
 #define _PyStatus_UPDATE_FUNC(err) \
     do { err.func = _PyStatus_GET_FUNC(); } while (0)
 
+PyObject* _PyErr_SetFromPyStatus(PyStatus status);
+
 /* --- PyWideStringList ------------------------------------------------ */
 
 #define _PyWideStringList_INIT (PyWideStringList){.length = 0, .items = NULL}