diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7bd031d5..f50b4612 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,12 +12,12 @@ jobs: name: "Python ${{ matrix.python-version }}" runs-on: "ubuntu-latest" env: - USING_COVERAGE: '3.8,3.9,3.10' + USING_COVERAGE: '3.8,3.9,3.10,3.11,3.12' strategy: fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: "actions/checkout@v3" @@ -54,7 +54,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: 3.8 + python-version: 3.12 - name: "Install dependencies" run: | set -xe diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3684225c..e1c6e9a9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -69,7 +69,7 @@ For a full test suite of all envs and linting checks simply run tox without any tox ``` -This will require python3.7, python3.8, python3.9 and python3.10 to be installed. **Note** that python 3.7 has problems with the alpha build which is the available package version on many linux distros. Local build failures with 3.7 can happen as a result (you'll see a seg fault or exist code -11). +This will require python3.8, 3.9, 3.10, 3.11, and 3.12 to be installed. Alternavitely pytest can be used if you have an environment already setup which works or has custom packages not present in the tox build. diff --git a/MANIFEST.in b/MANIFEST.in index e9069731..48d3f063 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -8,7 +8,9 @@ recursive-include papermill *.txt include setup.py include requirements.txt +include tox_py_installer recursive-include requirements *.txt +include docs/requirements.txt include tox.ini include pytest.ini include README.md @@ -29,6 +31,9 @@ prune binder graft scripts # Test env prune .tox +# Exclude notebooks checkpoints generated by testing +recursive-exclude papermill/.ipynb_checkpoints *.ipynb +recursive-exclude papermill/tests/notebooks/.ipynb_checkpoints *.ipynb # Build files exclude .github diff --git a/README.md b/README.md index 9b0e4933..a52abccc 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ pip install papermill[all] ## Python Version Support -This library currently supports Python 3.7+ versions. As minor Python +This library currently supports Python 3.8+ versions. As minor Python versions are officially sunset by the Python org papermill will similarly drop support in the future. diff --git a/binder/cli-simple/simple_output.ipynb b/binder/cli-simple/simple_output.ipynb index b36708ed..88766e1a 100644 --- a/binder/cli-simple/simple_output.ipynb +++ b/binder/cli-simple/simple_output.ipynb @@ -2,12 +2,13 @@ "cells": [ { "cell_type": "markdown", + "id": "df2b0bde", "metadata": { "papermill": { - "duration": 0.00964, - "end_time": "2020-08-17T19:57:24.239226", + "duration": 0.003083, + "end_time": "2023-11-01T18:34:41.235066", "exception": false, - "start_time": "2020-08-17T19:57:24.229586", + "start_time": "2023-11-01T18:34:41.231983", "status": "completed" }, "tags": [] @@ -19,18 +20,19 @@ { "cell_type": "code", "execution_count": 1, + "id": "0d7af2f3", "metadata": { "execution": { - "iopub.execute_input": "2020-08-17T19:57:24.263404Z", - "iopub.status.busy": "2020-08-17T19:57:24.262659Z", - "iopub.status.idle": "2020-08-17T19:57:24.265395Z", - "shell.execute_reply": "2020-08-17T19:57:24.264689Z" + "iopub.execute_input": "2023-11-01T18:34:41.301099Z", + "iopub.status.busy": "2023-11-01T18:34:41.300307Z", + "iopub.status.idle": "2023-11-01T18:34:41.306171Z", + "shell.execute_reply": "2023-11-01T18:34:41.305038Z" }, "papermill": { - "duration": 0.012719, - "end_time": "2020-08-17T19:57:24.265567", + "duration": 0.012465, + "end_time": "2023-11-01T18:34:41.308519", "exception": false, - "start_time": "2020-08-17T19:57:24.252848", + "start_time": "2023-11-01T18:34:41.296054", "status": "completed" }, "tags": [ @@ -45,18 +47,19 @@ { "cell_type": "code", "execution_count": 2, + "id": "a5d4ad35", "metadata": { "execution": { - "iopub.execute_input": "2020-08-17T19:57:24.274405Z", - "iopub.status.busy": "2020-08-17T19:57:24.273934Z", - "iopub.status.idle": "2020-08-17T19:57:24.276194Z", - "shell.execute_reply": "2020-08-17T19:57:24.275695Z" + "iopub.execute_input": "2023-11-01T18:34:41.314681Z", + "iopub.status.busy": "2023-11-01T18:34:41.314338Z", + "iopub.status.idle": "2023-11-01T18:34:41.318662Z", + "shell.execute_reply": "2023-11-01T18:34:41.317802Z" }, "papermill": { - "duration": 0.007713, - "end_time": "2020-08-17T19:57:24.276300", + "duration": 0.010058, + "end_time": "2023-11-01T18:34:41.320883", "exception": false, - "start_time": "2020-08-17T19:57:24.268587", + "start_time": "2023-11-01T18:34:41.310825", "status": "completed" }, "tags": [ @@ -72,18 +75,19 @@ { "cell_type": "code", "execution_count": 3, + "id": "99512b79", "metadata": { "execution": { - "iopub.execute_input": "2020-08-17T19:57:24.283119Z", - "iopub.status.busy": "2020-08-17T19:57:24.282711Z", - "iopub.status.idle": "2020-08-17T19:57:24.284846Z", - "shell.execute_reply": "2020-08-17T19:57:24.284482Z" + "iopub.execute_input": "2023-11-01T18:34:41.327359Z", + "iopub.status.busy": "2023-11-01T18:34:41.326530Z", + "iopub.status.idle": "2023-11-01T18:34:41.333035Z", + "shell.execute_reply": "2023-11-01T18:34:41.331879Z" }, "papermill": { - "duration": 0.006366, - "end_time": "2020-08-17T19:57:24.284929", + "duration": 0.012117, + "end_time": "2023-11-01T18:34:41.335151", "exception": false, - "start_time": "2020-08-17T19:57:24.278563", + "start_time": "2023-11-01T18:34:41.323034", "status": "completed" }, "tags": [] @@ -120,11 +124,12 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.2" + "version": "3.12.0" }, "papermill": { - "duration": 0.727728, - "end_time": "2020-08-17T19:57:24.494284", + "default_parameters": {}, + "duration": 1.612177, + "end_time": "2023-11-01T18:34:41.659543", "environment_variables": {}, "exception": null, "input_path": "binder/cli-simple/simple_input.ipynb", @@ -132,10 +137,10 @@ "parameters": { "msg": "Hello" }, - "start_time": "2020-08-17T19:57:23.766556", - "version": "2.1.3" + "start_time": "2023-11-01T18:34:40.047366", + "version": "2.4.0" } }, "nbformat": 4, - "nbformat_minor": 1 -} + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/docs/changelog.md b/docs/changelog.md index b634af0c..21e1d9da 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,6 +2,8 @@ ## Unreleased +- Added support for python 3.11 and 3.12 + ## 2.4.0 - Add tracking cell executions with cell descriptions [PR #650](https://github.com/nteract/papermill/pull/650) diff --git a/docs/conf.py b/docs/conf.py index abed19f1..a4a58dd7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -73,7 +73,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line foexitr these cases. -language = None +language = 'python' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -163,4 +163,4 @@ ] # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'https://docs.python.org/': None} +intersphinx_mapping = {'python': ('https://docs.python.org/', None)} diff --git a/docs/index.rst b/docs/index.rst index 71074f93..d05f7eb4 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -34,7 +34,7 @@ This opens up new opportunities for how notebooks can be used. For example: Python Version Support ---------------------- -This library currently supports python 3.7+ versions. As minor python +This library currently supports python 3.8+ versions. As minor python versions are officially sunset by the python org papermill will similarly drop support in the future. diff --git a/docs/requirements.txt b/docs/requirements.txt index cb450c5e..8f5c903e 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,8 +1,8 @@ # Pin packages for RTD builds # https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html#pinning-dependencies --r ../requirements.txt -Sphinx==3.5.4 -furo==2021.4.11b34 -myst-parser==0.13.7 -moto==2.0.5 -sphinx-copybutton==0.3.1 +#-r ../requirements.txt +Sphinx>=3.5.4 +furo>=2021.4.11b34 +myst-parser>=0.13.7 +moto>=2.0.5 +sphinx-copybutton>=0.3.1 diff --git a/docs/usage-inspect.rst b/docs/usage-inspect.rst index e81875e9..48978476 100644 --- a/docs/usage-inspect.rst +++ b/docs/usage-inspect.rst @@ -22,7 +22,7 @@ The `inspect_notebook` function can be called to inspect a notebook: pm.inspect_notebook('path/to/input.ipynb') .. note:: - If your path is parametrized, you can pass those parameters in a dictionary + If your path is parameterized, you can pass those parameters in a dictionary as second parameter: ``inspect_notebook('path/to/input_{month}.ipynb', parameters={month='Feb'})`` diff --git a/papermill/abs.py b/papermill/abs.py index 700164f5..2e86a0fa 100644 --- a/papermill/abs.py +++ b/papermill/abs.py @@ -3,6 +3,7 @@ import io from azure.storage.blob import BlobServiceClient +from azure.identity import EnvironmentCredential class AzureBlobStore(object): @@ -17,11 +18,13 @@ class AzureBlobStore(object): - write """ - def _blob_service_client(self, account_name, sas_token): + def _blob_service_client(self, account_name, sas_token=None): blob_service_client = BlobServiceClient( - "{account}.blob.core.windows.net".format(account=account_name), sas_token + account_url="{account}.blob.core.windows.net".format(account=account_name), + credential=sas_token or EnvironmentCredential(), ) + return blob_service_client @classmethod @@ -30,7 +33,7 @@ def _split_url(self, url): see: https://docs.microsoft.com/en-us/azure/storage/common/storage-dotnet-shared-access-signature-part-1 # noqa: E501 abs://myaccount.blob.core.windows.net/sascontainer/sasblob.txt?sastoken """ - match = re.match(r"abs://(.*)\.blob\.core\.windows\.net\/(.*?)\/(.*)\?(.*)$", url) + match = re.match(r"abs://(.*)\.blob\.core\.windows\.net\/(.*?)\/([^\?]*)\??(.*)$", url) if not match: raise Exception("Invalid azure blob url '{0}'".format(url)) else: diff --git a/papermill/cli.py b/papermill/cli.py index daa8b520..07be8d44 100755 --- a/papermill/cli.py +++ b/papermill/cli.py @@ -201,6 +201,10 @@ def papermill( from stdin and write it out to stdout. """ + # Jupyter deps use frozen modules, so we disable the python 3.11+ warning about debugger if running the CLI + if 'PYDEVD_DISABLE_FILE_VALIDATION' not in os.environ: + os.environ['PYDEVD_DISABLE_FILE_VALIDATION'] = '1' + if not help_notebook: required_output_path = not (INPUT_PIPED or OUTPUT_PIPED) if required_output_path and not output_path: diff --git a/papermill/tests/test_abs.py b/papermill/tests/test_abs.py index d5b2124c..1307e8fc 100644 --- a/papermill/tests/test_abs.py +++ b/papermill/tests/test_abs.py @@ -1,7 +1,8 @@ +import os import unittest from unittest.mock import Mock, patch - +from azure.identity import EnvironmentCredential from ..abs import AzureBlobStore @@ -33,6 +34,9 @@ def setUp(self): ) self.abs = AzureBlobStore() self.abs._blob_service_client = Mock(return_value=self._blob_service_client) + os.environ["AZURE_TENANT_ID"] = "mytenantid" + os.environ["AZURE_CLIENT_ID"] = "myclientid" + os.environ["AZURE_CLIENT_SECRET"] = "myclientsecret" def test_split_url_raises_exception_on_invalid_url(self): with self.assertRaises(Exception) as context: @@ -50,6 +54,15 @@ def test_split_url_splits_valid_url(self): self.assertEqual(params["blob"], "sasblob.txt") self.assertEqual(params["sas_token"], "sastoken") + def test_split_url_splits_valid_url_no_sas(self): + params = AzureBlobStore._split_url( + "abs://myaccount.blob.core.windows.net/container/blob.txt" + ) + self.assertEqual(params["account"], "myaccount") + self.assertEqual(params["container"], "container") + self.assertEqual(params["blob"], "blob.txt") + self.assertEqual(params["sas_token"], "") + def test_split_url_splits_valid_url_with_prefix(self): params = AzureBlobStore._split_url( "abs://myaccount.blob.core.windows.net/sascontainer/A/B/sasblob.txt?sastoken" @@ -97,3 +110,12 @@ def test_blob_service_client(self): self.assertEqual(blob.account_name, "myaccount") # Credentials gets funky with v12.0.0, so I comment this out # self.assertEqual(blob.credential, "sastoken") + + def test_blob_service_client_environment_credentials(self): + abs = AzureBlobStore() + blob = abs._blob_service_client(account_name="myaccount", sas_token="") + self.assertEqual(blob.account_name, "myaccount") + self.assertIsInstance(blob.credential, EnvironmentCredential) + self.assertEqual(blob.credential._credential._tenant_id, "mytenantid") + self.assertEqual(blob.credential._credential._client_id, "myclientid") + self.assertEqual(blob.credential._credential._client_credential, "myclientsecret") diff --git a/papermill/tests/test_hdfs.py b/papermill/tests/test_hdfs.py index 9d4e7f3f..de73ca03 100644 --- a/papermill/tests/test_hdfs.py +++ b/papermill/tests/test_hdfs.py @@ -1,6 +1,8 @@ import unittest from unittest.mock import MagicMock, patch +import pytest + from ..iorw import HDFSHandler @@ -38,6 +40,7 @@ def __init__(self, path): self.path = path +@pytest.mark.skip(reason="No valid dep package for python 3.12 yet") @patch('papermill.iorw.HadoopFileSystem', side_effect=MockHadoopFileSystem()) class HDFSTest(unittest.TestCase): def setUp(self): diff --git a/requirements.txt b/requirements.txt index 246004dc..559744f4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,4 @@ tqdm >= 4.32.2 requests entrypoints tenacity >= 5.0.2 +aiohttp==3.9.0b0;python_version=="3.12" \ No newline at end of file diff --git a/requirements/azure.txt b/requirements/azure.txt index c2242ad1..82f60816 100644 --- a/requirements/azure.txt +++ b/requirements/azure.txt @@ -1,3 +1,4 @@ azure-datalake-store >= 0.0.30 azure-storage-blob >= 12.1.0 requests >= 2.21.0 +azure-identity>=1.3.1 \ No newline at end of file diff --git a/setup.py b/setup.py index 4ec42722..cdf04abc 100644 --- a/setup.py +++ b/setup.py @@ -44,9 +44,12 @@ def read_reqs(fname, folder=None): gcs_reqs = read_reqs('gcs.txt', folder='requirements') hdfs_reqs = read_reqs('hdfs.txt', folder='requirements') github_reqs = read_reqs('github.txt', folder='requirements') +docs_only_reqs = read_reqs('requirements.txt', folder='docs') black_reqs = ['black >= 19.3b0'] all_reqs = s3_reqs + azure_reqs + gcs_reqs + hdfs_reqs + black_reqs -dev_reqs = read_reqs('dev.txt', folder='requirements') + all_reqs +docs_reqs = all_reqs + docs_only_reqs +# Temporarily remove hdfs_reqs from dev deps until the pyarrow package is available for Python 3.12 +dev_reqs = read_reqs('dev.txt', folder='requirements') + s3_reqs + azure_reqs + gcs_reqs + black_reqs # all_reqs extras_require = { "test": dev_reqs, "dev": dev_reqs, @@ -57,6 +60,7 @@ def read_reqs(fname, folder=None): "hdfs": hdfs_reqs, "github": github_reqs, "black": black_reqs, + "docs": docs_reqs, } # Get the long description from the README file @@ -66,7 +70,7 @@ def read_reqs(fname, folder=None): setup( name='papermill', version=version(), - description='Parametrize and run Jupyter and nteract Notebooks', + description='Parameterize and run Jupyter and nteract Notebooks', author='nteract contributors', author_email='nteract@googlegroups.com', license='BSD', @@ -76,7 +80,7 @@ def read_reqs(fname, folder=None): long_description_content_type='text/markdown', url='https://github.com/nteract/papermill', packages=['papermill'], - python_requires='>=3.7', + python_requires='>=3.8', install_requires=read_reqs('requirements.txt'), extras_require=extras_require, entry_points={'console_scripts': ['papermill = papermill.__main__:papermill']}, @@ -93,9 +97,10 @@ def read_reqs(fname, folder=None): 'License :: OSI Approved :: BSD License', 'Programming Language :: Python', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', ], ) diff --git a/tox.ini b/tox.ini index aef640a0..e7993169 100644 --- a/tox.ini +++ b/tox.ini @@ -1,13 +1,14 @@ [tox] skipsdist = true -envlist = py{37,38,39,310}, flake8, dist, manifest, docs, binder +envlist = py{38,39,310,311,312}, flake8, dist, manifest, docs, binder [gh-actions] python = - 3.7: py37 - 3.8: py38, dist + 3.8: py38 3.9: py39 3.10: py310 + 3.11: py311 + 3.12: py312, dist # Linters [testenv:flake8] @@ -27,9 +28,7 @@ ignore = [testenv:docs] description = invoke sphinx-build to build the HTML docs deps = - .[dev] - .[azure] - -r docs/requirements.txt + .[docs] extras = docs commands = sphinx-build -d "{toxworkdir}/docs_doctree" docs "{toxworkdir}/docs_out" --color -W -bhtml {posargs} @@ -45,16 +44,14 @@ commands = python -c "import glob; import papermill as pm; [pm.execute_notebook( # Distro [testenv:dist] skip_install = true -# Have to use /bin/bash or the `*` will cause that argument to get quoted by the tox command line... commands = python setup.py sdist --dist-dir={distdir} bdist_wheel --dist-dir={distdir} /bin/bash -c 'python -m pip install -U --force-reinstall {distdir}/papermill*.whl' - /bin/bash -c 'python -m pip install -U --force-reinstall --no-deps {distdir}/papermill*.tar.gz' # Black [testenv:black] description = apply black linter with desired rules -basepython = python3.8 +basepython = python3.12 deps = black commands = black -S -l 100 . @@ -67,15 +64,20 @@ setenv = AWS_SECRET_ACCESS_KEY=foobar_secret passenv = * basepython = - py37: python3.7 py38: python3.8 py39: python3.9 py310: python3.10 - flake8: python3.8 - manifest: python3.8 - dist: python3.8 - docs: python3.8 - binder: python3.8 - black: python3.8 + py311: python3.11 + py312: python3.12 + flake8: python3.12 + manifest: python3.12 + dist: python3.12 + docs: python3.12 + binder: python3.12 + black: python3.12 deps = .[dev] +# Have to use /bin/bash or the `*` will cause that argument to get quoted by the tox command line... +allowlist_externals = /bin/bash +# Python 3.12 breaks default pip/setuptools versions ... force an upgrade of these before anything else +install_command = /bin/bash ./tox_py_installer {opts} {packages} commands = pytest -v --maxfail=2 --cov=papermill -W always {posargs} diff --git a/tox_py_installer b/tox_py_installer new file mode 100755 index 00000000..4bcf405d --- /dev/null +++ b/tox_py_installer @@ -0,0 +1,5 @@ +#!/bin/bash +python -m ensurepip --upgrade +python -m pip install --upgrade setuptools +# python -m pip install {opts} {packages} +python -m pip install $1 $2