Merge branch 'main' into patch-1

.github/workflows/ci.yml

@@ -12,12 +12,12 @@ jobs:
     name: "Python ${{ matrix.python-version }}"
     runs-on: "ubuntu-latest"
     env:
-      USING_COVERAGE: '3.8,3.9,3.10'
+      USING_COVERAGE: '3.8,3.9,3.10,3.11,3.12'
 
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
 
     steps:
       - uses: "actions/checkout@v3"
@@ -54,7 +54,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: 3.12
       - name: "Install dependencies"
         run: |
           set -xe

CONTRIBUTING.md

@@ -69,7 +69,7 @@ For a full test suite of all envs and linting checks simply run tox without any
 tox
 ```
 
-This will require python3.7, python3.8, python3.9 and python3.10 to be installed. **Note** that python 3.7 has problems with the alpha build which is the available package version on many linux distros. Local build failures with 3.7 can happen as a result (you'll see a seg fault or exist code -11).
+This will require python3.8, 3.9, 3.10, 3.11, and 3.12 to be installed.
 
 Alternavitely pytest can be used if you have an environment already setup which works or has custom packages not present in the tox build.
 

MANIFEST.in

@@ -8,7 +8,9 @@ recursive-include papermill *.txt
 
 include setup.py
 include requirements.txt
+include tox_py_installer
 recursive-include requirements *.txt
+include docs/requirements.txt
 include tox.ini
 include pytest.ini
 include README.md
@@ -29,6 +31,9 @@ prune binder
 graft scripts
 # Test env
 prune .tox
+# Exclude notebooks checkpoints generated by testing
+recursive-exclude papermill/.ipynb_checkpoints *.ipynb
+recursive-exclude papermill/tests/notebooks/.ipynb_checkpoints *.ipynb
 
 # Build files
 exclude .github

README.md

@@ -54,7 +54,7 @@ pip install papermill[all]
 
 ## Python Version Support
 
-This library currently supports Python 3.7+ versions. As minor Python
+This library currently supports Python 3.8+ versions. As minor Python
 versions are officially sunset by the Python org papermill will similarly
 drop support in the future.
 

binder/cli-simple/simple_output.ipynb

@@ -2,12 +2,13 @@
  "cells": [
   {
    "cell_type": "markdown",
+   "id": "df2b0bde",
    "metadata": {
     "papermill": {
-     "duration": 0.00964,
-     "end_time": "2020-08-17T19:57:24.239226",
+     "duration": 0.003083,
+     "end_time": "2023-11-01T18:34:41.235066",
      "exception": false,
-     "start_time": "2020-08-17T19:57:24.229586",
+     "start_time": "2023-11-01T18:34:41.231983",
      "status": "completed"
     },
     "tags": []
@@ -19,18 +20,19 @@
   {
    "cell_type": "code",
    "execution_count": 1,
+   "id": "0d7af2f3",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2020-08-17T19:57:24.263404Z",
-     "iopub.status.busy": "2020-08-17T19:57:24.262659Z",
-     "iopub.status.idle": "2020-08-17T19:57:24.265395Z",
-     "shell.execute_reply": "2020-08-17T19:57:24.264689Z"
+     "iopub.execute_input": "2023-11-01T18:34:41.301099Z",
+     "iopub.status.busy": "2023-11-01T18:34:41.300307Z",
+     "iopub.status.idle": "2023-11-01T18:34:41.306171Z",
+     "shell.execute_reply": "2023-11-01T18:34:41.305038Z"
     },
     "papermill": {
-     "duration": 0.012719,
-     "end_time": "2020-08-17T19:57:24.265567",
+     "duration": 0.012465,
+     "end_time": "2023-11-01T18:34:41.308519",
      "exception": false,
-     "start_time": "2020-08-17T19:57:24.252848",
+     "start_time": "2023-11-01T18:34:41.296054",
      "status": "completed"
     },
     "tags": [
@@ -45,18 +47,19 @@
   {
    "cell_type": "code",
    "execution_count": 2,
+   "id": "a5d4ad35",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2020-08-17T19:57:24.274405Z",
-     "iopub.status.busy": "2020-08-17T19:57:24.273934Z",
-     "iopub.status.idle": "2020-08-17T19:57:24.276194Z",
-     "shell.execute_reply": "2020-08-17T19:57:24.275695Z"
+     "iopub.execute_input": "2023-11-01T18:34:41.314681Z",
+     "iopub.status.busy": "2023-11-01T18:34:41.314338Z",
+     "iopub.status.idle": "2023-11-01T18:34:41.318662Z",
+     "shell.execute_reply": "2023-11-01T18:34:41.317802Z"
     },
     "papermill": {
-     "duration": 0.007713,
-     "end_time": "2020-08-17T19:57:24.276300",
+     "duration": 0.010058,
+     "end_time": "2023-11-01T18:34:41.320883",
      "exception": false,
-     "start_time": "2020-08-17T19:57:24.268587",
+     "start_time": "2023-11-01T18:34:41.310825",
      "status": "completed"
     },
     "tags": [
@@ -72,18 +75,19 @@
   {
    "cell_type": "code",
    "execution_count": 3,
+   "id": "99512b79",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2020-08-17T19:57:24.283119Z",
-     "iopub.status.busy": "2020-08-17T19:57:24.282711Z",
-     "iopub.status.idle": "2020-08-17T19:57:24.284846Z",
-     "shell.execute_reply": "2020-08-17T19:57:24.284482Z"
+     "iopub.execute_input": "2023-11-01T18:34:41.327359Z",
+     "iopub.status.busy": "2023-11-01T18:34:41.326530Z",
+     "iopub.status.idle": "2023-11-01T18:34:41.333035Z",
+     "shell.execute_reply": "2023-11-01T18:34:41.331879Z"
     },
     "papermill": {
-     "duration": 0.006366,
-     "end_time": "2020-08-17T19:57:24.284929",
+     "duration": 0.012117,
+     "end_time": "2023-11-01T18:34:41.335151",
      "exception": false,
-     "start_time": "2020-08-17T19:57:24.278563",
+     "start_time": "2023-11-01T18:34:41.323034",
      "status": "completed"
     },
     "tags": []
@@ -120,22 +124,23 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.2"
+   "version": "3.12.0"
   },
   "papermill": {
-   "duration": 0.727728,
-   "end_time": "2020-08-17T19:57:24.494284",
+   "default_parameters": {},
+   "duration": 1.612177,
+   "end_time": "2023-11-01T18:34:41.659543",
    "environment_variables": {},
    "exception": null,
    "input_path": "binder/cli-simple/simple_input.ipynb",
    "output_path": "binder/cli-simple/simple_output.ipynb",
    "parameters": {
     "msg": "Hello"
    },
-   "start_time": "2020-08-17T19:57:23.766556",
-   "version": "2.1.3"
+   "start_time": "2023-11-01T18:34:40.047366",
+   "version": "2.4.0"
   }
  },
  "nbformat": 4,
- "nbformat_minor": 1
-}
+ "nbformat_minor": 5
+}

docs/changelog.md

@@ -2,6 +2,8 @@
 
 ## Unreleased
 
+- Added support for python 3.11 and 3.12
+
 ## 2.4.0
 
 - Add tracking cell executions with cell descriptions [PR #650](https://github.com/nteract/papermill/pull/650)

docs/conf.py

@@ -73,7 +73,7 @@
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line foexitr these cases.
-language = None
+language = 'python'
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
@@ -163,4 +163,4 @@
 ]
 
 # Example configuration for intersphinx: refer to the Python standard library.
-intersphinx_mapping = {'https://docs.python.org/': None}
+intersphinx_mapping = {'python': ('https://docs.python.org/', None)}

docs/index.rst

@@ -34,7 +34,7 @@ This opens up new opportunities for how notebooks can be used. For example:
 Python Version Support
 ----------------------
 
-This library currently supports python 3.7+ versions. As minor python
+This library currently supports python 3.8+ versions. As minor python
 versions are officially sunset by the python org papermill will similarly
 drop support in the future.
 

docs/requirements.txt

@@ -1,8 +1,8 @@
 # Pin packages for RTD builds
 # https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html#pinning-dependencies
--r ../requirements.txt
-Sphinx==3.5.4
-furo==2021.4.11b34
-myst-parser==0.13.7
-moto==2.0.5
-sphinx-copybutton==0.3.1
+#-r ../requirements.txt
+Sphinx>=3.5.4
+furo>=2021.4.11b34
+myst-parser>=0.13.7
+moto>=2.0.5
+sphinx-copybutton>=0.3.1

docs/usage-inspect.rst

@@ -22,7 +22,7 @@ The `inspect_notebook` function can be called to inspect a notebook:
    pm.inspect_notebook('path/to/input.ipynb')
 
 .. note::
-    If your path is parametrized, you can pass those parameters in a dictionary
+    If your path is parameterized, you can pass those parameters in a dictionary
     as second parameter:
 
     ``inspect_notebook('path/to/input_{month}.ipynb', parameters={month='Feb'})``

papermill/abs.py

@@ -3,6 +3,7 @@
 import io
 
 from azure.storage.blob import BlobServiceClient
+from azure.identity import EnvironmentCredential
 
 
 class AzureBlobStore(object):
@@ -17,11 +18,13 @@ class AzureBlobStore(object):
         - write
     """
 
-    def _blob_service_client(self, account_name, sas_token):
+    def _blob_service_client(self, account_name, sas_token=None):
 
         blob_service_client = BlobServiceClient(
-            "{account}.blob.core.windows.net".format(account=account_name), sas_token
+            account_url="{account}.blob.core.windows.net".format(account=account_name),
+            credential=sas_token or EnvironmentCredential(),
         )
+
         return blob_service_client
 
     @classmethod
@@ -30,7 +33,7 @@ def _split_url(self, url):
         see: https://docs.microsoft.com/en-us/azure/storage/common/storage-dotnet-shared-access-signature-part-1  # noqa: E501
         abs://myaccount.blob.core.windows.net/sascontainer/sasblob.txt?sastoken
         """
-        match = re.match(r"abs://(.*)\.blob\.core\.windows\.net\/(.*?)\/(.*)\?(.*)$", url)
+        match = re.match(r"abs://(.*)\.blob\.core\.windows\.net\/(.*?)\/([^\?]*)\??(.*)$", url)
         if not match:
             raise Exception("Invalid azure blob url '{0}'".format(url))
         else:

papermill/cli.py

@@ -201,6 +201,10 @@ def papermill(
     from stdin and write it out to stdout.
 
     """
+    # Jupyter deps use frozen modules, so we disable the python 3.11+ warning about debugger if running the CLI
+    if 'PYDEVD_DISABLE_FILE_VALIDATION' not in os.environ:
+        os.environ['PYDEVD_DISABLE_FILE_VALIDATION'] = '1'
+
     if not help_notebook:
         required_output_path = not (INPUT_PIPED or OUTPUT_PIPED)
         if required_output_path and not output_path:

papermill/tests/test_abs.py

@@ -1,7 +1,8 @@
+import os
 import unittest
 
 from unittest.mock import Mock, patch
-
+from azure.identity import EnvironmentCredential
 from ..abs import AzureBlobStore
 
 
@@ -33,6 +34,9 @@ def setUp(self):
         )
         self.abs = AzureBlobStore()
         self.abs._blob_service_client = Mock(return_value=self._blob_service_client)
+        os.environ["AZURE_TENANT_ID"] = "mytenantid"
+        os.environ["AZURE_CLIENT_ID"] = "myclientid"
+        os.environ["AZURE_CLIENT_SECRET"] = "myclientsecret"
 
     def test_split_url_raises_exception_on_invalid_url(self):
         with self.assertRaises(Exception) as context:
@@ -50,6 +54,15 @@ def test_split_url_splits_valid_url(self):
         self.assertEqual(params["blob"], "sasblob.txt")
         self.assertEqual(params["sas_token"], "sastoken")
 
+    def test_split_url_splits_valid_url_no_sas(self):
+        params = AzureBlobStore._split_url(
+            "abs://myaccount.blob.core.windows.net/container/blob.txt"
+        )
+        self.assertEqual(params["account"], "myaccount")
+        self.assertEqual(params["container"], "container")
+        self.assertEqual(params["blob"], "blob.txt")
+        self.assertEqual(params["sas_token"], "")
+
     def test_split_url_splits_valid_url_with_prefix(self):
         params = AzureBlobStore._split_url(
             "abs://myaccount.blob.core.windows.net/sascontainer/A/B/sasblob.txt?sastoken"
@@ -97,3 +110,12 @@ def test_blob_service_client(self):
         self.assertEqual(blob.account_name, "myaccount")
         # Credentials gets funky with v12.0.0, so I comment this out
         # self.assertEqual(blob.credential, "sastoken")
+
+    def test_blob_service_client_environment_credentials(self):
+        abs = AzureBlobStore()
+        blob = abs._blob_service_client(account_name="myaccount", sas_token="")
+        self.assertEqual(blob.account_name, "myaccount")
+        self.assertIsInstance(blob.credential, EnvironmentCredential)
+        self.assertEqual(blob.credential._credential._tenant_id, "mytenantid")
+        self.assertEqual(blob.credential._credential._client_id, "myclientid")
+        self.assertEqual(blob.credential._credential._client_credential, "myclientsecret")

papermill/tests/test_hdfs.py

@@ -1,6 +1,8 @@
 import unittest
 from unittest.mock import MagicMock, patch
 
+import pytest
+
 from ..iorw import HDFSHandler
 
 
@@ -38,6 +40,7 @@ def __init__(self, path):
         self.path = path
 
 
+@pytest.mark.skip(reason="No valid dep package for python 3.12 yet")
 @patch('papermill.iorw.HadoopFileSystem', side_effect=MockHadoopFileSystem())
 class HDFSTest(unittest.TestCase):
     def setUp(self):

requirements.txt

@@ -6,3 +6,4 @@ tqdm >= 4.32.2
 requests
 entrypoints
 tenacity >= 5.0.2
+aiohttp==3.9.0b0;python_version=="3.12"

requirements/azure.txt

@@ -1,3 +1,4 @@
 azure-datalake-store >= 0.0.30
 azure-storage-blob >= 12.1.0
 requests >= 2.21.0
+azure-identity>=1.3.1