From ca27fd5aba27d248dc99756885033170cd6d1fb1 Mon Sep 17 00:00:00 2001 From: Jamiel Carter <88984480+jamielxcarter@users.noreply.github.com> Date: Thu, 17 Nov 2022 11:20:01 -0500 Subject: [PATCH] feat: S3 batch storage (#1087) * Added PyFilesystem/s3fs as an optional dependency * Test s3 protocol * Add dummy credentials to storage test * Add fs-s3fs to support nox tests * Shortening s3fs extras name to s3 * Include s3 extra instead of fs-s3fs directly * importlib-metadata 5.0.0 deprecated endpoint. No longer works with python 3.7. Going back down to 4.13.0 * Add s3 extra to cookiecutter toml, update docs * Mypy python 3.8 fix From ambiguous Any return type to explicity Traversable return type. * Sync versions * Add S3 extra to target cookiecutter * Update SDK minor version in cookiecutter * re-lock dependencies * Make explicit extra for s3 dependencies --- .../{{cookiecutter.tap_id}}/pyproject.toml | 6 +- .../{{cookiecutter.target_id}}/pyproject.toml | 6 +- docs/batch.md | 21 +- noxfile.py | 3 +- poetry.lock | 259 ++++++++++++------ pyproject.toml | 4 + singer_sdk/helpers/_resources.py | 3 +- tests/core/test_batch.py | 15 + 8 files changed, 224 insertions(+), 93 deletions(-) diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml index b850c97ee..9bce76242 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml @@ -12,7 +12,8 @@ license = "Apache 2.0" [tool.poetry.dependencies] python = "<3.11,>=3.7.1" requests = "^2.25.1" -singer-sdk = "^0.14.0" +singer-sdk = { version="^0.14.0"} +fs-s3fs = { version = "^1.1.1", optional = true} [tool.poetry.dev-dependencies] pytest = "^6.2.5" @@ -24,6 +25,9 @@ mypy = "^0.910" types-requests = "^2.26.1" isort = "^5.10.1" +[tool.poetry.extras] +s3 = ["fs-s3fs"] + [tool.isort] profile = "black" multi_line_output = 3 # Vertical Hanging Indent diff --git a/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml b/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml index a4e478bd1..9f6771603 100644 --- a/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml +++ b/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml @@ -12,7 +12,8 @@ license = "Apache 2.0" [tool.poetry.dependencies] python = "<3.11,>=3.7.1" requests = "^2.25.1" -singer-sdk = "^0.14.0" +singer-sdk = { version="^0.14.0"} +fs-s3fs = { version = "^1.1.1", optional = true} [tool.poetry.dev-dependencies] pytest = "^6.2.5" @@ -24,6 +25,9 @@ mypy = "^0.910" types-requests = "^2.26.1" isort = "^5.10.1" +[tool.poetry.extras] +s3 = ["fs-s3fs"] + [tool.isort] profile = "black" multi_line_output = 3 # Vertical Hanging Indent diff --git a/docs/batch.md b/docs/batch.md index 592a3dc85..df6ef1178 100644 --- a/docs/batch.md +++ b/docs/batch.md @@ -14,10 +14,11 @@ This library's implementation of the `BATCH` message is used to send records in - when the tap outputs records at a much higher rate than the target can consume them, creating backpressure - when the source system can directly export data in bulk (e.g. a database dump) -Currently only a local filesystem is supported, but other filesystems like AWS S3, FTP, etc. could be supported in the future. +Currently only a local filesystem or AWS S3 are supported, but other filesystems like FTP, etc. could be supported in the future. ## The `BATCH` Message +Local ```json { "type": "BATCH", @@ -33,6 +34,22 @@ Currently only a local filesystem is supported, but other filesystems like AWS S } ``` +AWS S3 +```json +{ + "type": "BATCH", + "stream": "users", + "encoding": { + "format": "jsonl", + "compression": "gzip" + }, + "manifest": [ + "s3://path/to/batch/file/1", + "s3://path/to/batch/file/2" + ] +} +``` + ### `encoding` The `encoding` field is used to specify the format and compression of the batch files. Currently only `jsonl` and `gzip` are supported, respectively. @@ -43,7 +60,7 @@ The `manifest` field is used to specify the paths to the batch files. The paths ## Batch configuration -When local storage is used, targets do no require special configuration to process `BATCH` messages. +When local storage is used, targets do no require special configuration to process `BATCH` messages. Use of AWS S3 assumes S3/AWS credentials are already discoverable via the underlying S3 libraries (`AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` / `AWS_DEFAULT_REGION`) Taps may be configured to specify a root storage `root` directory, file path `prefix`, and `encoding` for batch files using a configuration like the below: diff --git a/noxfile.py b/noxfile.py index e91f95846..ca4e05c50 100644 --- a/noxfile.py +++ b/noxfile.py @@ -69,7 +69,8 @@ def mypy(session: Session) -> None: @session(python=python_versions) def tests(session: Session) -> None: """Execute pytest tests and compute coverage.""" - session.install(".") + + session.install(".[s3]") session.install(*test_dependencies) # temp fix until pyarrow is supported on python 3.11 diff --git a/poetry.lock b/poetry.lock index 9289ae076..ff5f40fad 100644 --- a/poetry.lock +++ b/poetry.lock @@ -93,6 +93,38 @@ d = ["aiohttp (>=3.7.4)"] jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] uvloop = ["uvloop (>=0.15.2)"] +[[package]] +name = "boto3" +version = "1.26.11" +description = "The AWS SDK for Python" +category = "main" +optional = true +python-versions = ">= 3.7" + +[package.dependencies] +botocore = ">=1.29.11,<1.30.0" +jmespath = ">=0.7.1,<2.0.0" +s3transfer = ">=0.6.0,<0.7.0" + +[package.extras] +crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] + +[[package]] +name = "botocore" +version = "1.29.11" +description = "Low-level, data-driven core of boto 3." +category = "main" +optional = true +python-versions = ">= 3.7" + +[package.dependencies] +jmespath = ">=0.7.1,<2.0.0" +python-dateutil = ">=2.1,<3.0.0" +urllib3 = ">=1.25.4,<1.27" + +[package.extras] +crt = ["awscrt (==0.14.0)"] + [[package]] name = "certifi" version = "2022.9.24" @@ -259,7 +291,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "exceptiongroup" -version = "1.0.1" +version = "1.0.4" description = "Backport of PEP 654 (exception groups)" category = "dev" optional = false @@ -334,9 +366,22 @@ six = ">=1.10,<2.0" [package.extras] scandir = ["scandir (>=1.5,<2.0)"] +[[package]] +name = "fs-s3fs" +version = "1.1.1" +description = "Amazon S3 filesystem for PyFilesystem2" +category = "main" +optional = true +python-versions = "*" + +[package.dependencies] +boto3 = ">=1.9,<2.0" +fs = ">=2.4,<3.0" +six = ">=1.10,<2.0" + [[package]] name = "greenlet" -version = "2.0.0.post0" +version = "2.0.1" description = "Lightweight in-process concurrent programming" category = "main" optional = false @@ -344,7 +389,7 @@ python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*" [package.extras] docs = ["Sphinx", "docutils (<0.18)"] -test = ["faulthandler", "objgraph"] +test = ["faulthandler", "objgraph", "psutil"] [[package]] name = "idna" @@ -436,6 +481,14 @@ python-versions = "*" arrow = "*" jinja2 = "*" +[[package]] +name = "jmespath" +version = "1.0.1" +description = "JSON Matching Expressions" +category = "main" +optional = true +python-versions = ">=3.7" + [[package]] name = "joblib" version = "1.2.0" @@ -639,7 +692,7 @@ pyparsing = ">=2.0.2,<3.0.5 || >3.0.5" [[package]] name = "pathspec" -version = "0.10.1" +version = "0.10.2" description = "Utility library for gitignore style pattern matching of file paths." category = "dev" optional = false @@ -667,15 +720,15 @@ python-versions = ">=3.6" [[package]] name = "platformdirs" -version = "2.5.2" -description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +version = "2.5.4" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." category = "dev" optional = false python-versions = ">=3.7" [package.extras] -docs = ["furo (>=2021.7.5b38)", "proselint (>=0.10.2)", "sphinx (>=4)", "sphinx-autodoc-typehints (>=1.12)"] -test = ["appdirs (==1.4.4)", "pytest (>=6)", "pytest-cov (>=2.7)", "pytest-mock (>=3.6)"] +docs = ["furo (>=2022.9.29)", "proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.4)"] +test = ["appdirs (==1.4.4)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] [[package]] name = "pluggy" @@ -751,15 +804,15 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "pygithub" -version = "1.56" +version = "1.57" description = "Use the full Github API v3" category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [package.dependencies] deprecated = "*" -pyjwt = ">=2.0" +pyjwt = ">=2.4.0" pynacl = ">=1.4.0" requests = ">=2.14.0" @@ -951,9 +1004,23 @@ six = "*" fixture = ["fixtures"] test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testrepository (>=0.0.18)", "testtools"] +[[package]] +name = "s3transfer" +version = "0.6.0" +description = "An Amazon S3 Transfer Manager" +category = "main" +optional = true +python-versions = ">= 3.7" + +[package.dependencies] +botocore = ">=1.12.36,<2.0a.0" + +[package.extras] +crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] + [[package]] name = "setuptools" -version = "65.5.0" +version = "65.5.1" description = "Easily download, build, install, upgrade, and uninstall Python packages" category = "main" optional = false @@ -961,7 +1028,7 @@ python-versions = ">=3.7" [package.extras] docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mock", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] [[package]] @@ -1258,7 +1325,7 @@ python-versions = "*" [[package]] name = "types-urllib3" -version = "1.26.25.1" +version = "1.26.25.4" description = "Typing stubs for urllib3" category = "dev" optional = false @@ -1344,12 +1411,13 @@ testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools" [extras] debugging = [] docs = ["sphinx", "sphinx-rtd-theme", "sphinx-copybutton", "myst-parser", "sphinx-autobuild"] +s3 = ["fs-s3fs"] samples = [] [metadata] lock-version = "1.1" python-versions = "<3.12,>=3.7.1" -content-hash = "45875f4451fa3f478d4943fa72081a068027dc1498109515137d2ff1a11071e4" +content-hash = "e1e852cdbb1830058eec4cf07b4e626ef0537e0e5bbd2930e693a72c4f6bb1a6" [metadata.files] alabaster = [ @@ -1403,6 +1471,14 @@ black = [ {file = "black-22.10.0-py3-none-any.whl", hash = "sha256:c957b2b4ea88587b46cf49d1dc17681c1e672864fd7af32fc1e9664d572b3458"}, {file = "black-22.10.0.tar.gz", hash = "sha256:f513588da599943e0cde4e32cc9879e825d58720d6557062d1098c5ad80080e1"}, ] +boto3 = [ + {file = "boto3-1.26.11-py3-none-any.whl", hash = "sha256:3309d04e47490a1b46aa8ae7bcf4845a8dae2ed419a505c770895773ba7455dc"}, + {file = "boto3-1.26.11.tar.gz", hash = "sha256:f8f7268afe62e3b34b7d0e1b4d8a804148e713f35b0e6dbc736346b28b3c6549"}, +] +botocore = [ + {file = "botocore-1.29.11-py3-none-any.whl", hash = "sha256:a86d7e558074e849933983bc2c4cc6ae6bbed3aef3f5fb4d4689cd19331e4399"}, + {file = "botocore-1.29.11.tar.gz", hash = "sha256:97532c8a571017891ad4afee7951bea95c9f0bfe96a6f38fdca9062cd6067ec1"}, +] certifi = [ {file = "certifi-2022.9.24-py3-none-any.whl", hash = "sha256:90c1a32f1d68f940488354e36370f6cca89f0f106db09518524c88d6ed83f382"}, {file = "certifi-2022.9.24.tar.gz", hash = "sha256:0d9c601124e5a6ba9712dbc60d9c53c21e34f5f641fe83002317394311bdce14"}, @@ -1591,8 +1667,8 @@ docutils = [ {file = "docutils-0.17.1.tar.gz", hash = "sha256:686577d2e4c32380bb50cbb22f575ed742d58168cee37e99117a854bcd88f125"}, ] exceptiongroup = [ - {file = "exceptiongroup-1.0.1-py3-none-any.whl", hash = "sha256:4d6c0aa6dd825810941c792f53d7b8d71da26f5e5f84f20f9508e8f2d33b140a"}, - {file = "exceptiongroup-1.0.1.tar.gz", hash = "sha256:73866f7f842ede6cb1daa42c4af078e2035e5f7607f0e2c762cc51bb31bbe7b2"}, + {file = "exceptiongroup-1.0.4-py3-none-any.whl", hash = "sha256:542adf9dea4055530d6e1279602fa5cb11dab2395fa650b8674eaec35fc4a828"}, + {file = "exceptiongroup-1.0.4.tar.gz", hash = "sha256:bd14967b79cd9bdb54d97323216f8fdf533e278df937aa2a90089e7d6e06e5ec"}, ] flake8 = [ {file = "flake8-3.9.2-py2.py3-none-any.whl", hash = "sha256:bf8fd333346d844f616e8d47905ef3a3384edae6b4e9beb0c5101e25e3110907"}, @@ -1614,67 +1690,68 @@ fs = [ {file = "fs-2.4.16-py2.py3-none-any.whl", hash = "sha256:660064febbccda264ae0b6bace80a8d1be9e089e0a5eb2427b7d517f9a91545c"}, {file = "fs-2.4.16.tar.gz", hash = "sha256:ae97c7d51213f4b70b6a958292530289090de3a7e15841e108fbe144f069d313"}, ] +fs-s3fs = [ + {file = "fs-s3fs-1.1.1.tar.gz", hash = "sha256:b57f8c7664460ff7b451b4b44ca2ea9623a374d74e1284c2d5e6df499dc7976c"}, + {file = "fs_s3fs-1.1.1-py2.py3-none-any.whl", hash = "sha256:9ba160eaa93390cc5992a857675666cb2fbb3721b872474dfdc659a715c39280"}, +] greenlet = [ - {file = "greenlet-2.0.0.post0-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:b75e5644cc353328cd57ec8dafaaf5f81b2c3ecf7c4b278b907e99ad53ba7839"}, - {file = "greenlet-2.0.0.post0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:35827f98fd0d768862b8f15777e6dbb03fe6ac6e7bd1bee3f3ded4536f350347"}, - {file = "greenlet-2.0.0.post0-cp27-cp27m-win32.whl", hash = "sha256:b31de27313abbb567c528ed123380fcf18a5dfd03134570dfd12227e21ac1184"}, - {file = "greenlet-2.0.0.post0-cp27-cp27m-win_amd64.whl", hash = "sha256:b8cfc8fc944bd7b704691bc28225a2635e377e92dc413459845868d3f7724982"}, - {file = "greenlet-2.0.0.post0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:029ca674b3a7e8427db8f5c65d5ed4e24a7417af2a415a5958598aefd71980c4"}, - {file = "greenlet-2.0.0.post0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:13d492a807a5c7334b5931e9b6d9b181991ccc6a40555a7b177f189feff59b4b"}, - {file = "greenlet-2.0.0.post0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:30ce47525f9a1515566429ac7de6b1ae76d32c3ccede256e3517a1a6419cf659"}, - {file = "greenlet-2.0.0.post0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d9453135e48cd631e3e9f06d9da9100d17c9f662e4a6d8b552c29be6c834a6b9"}, - {file = "greenlet-2.0.0.post0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c2055c52260808d87622293b57df1c68aeb12ddd8a0cfc0223fb57a5f629e202"}, - {file = "greenlet-2.0.0.post0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:118e708dd7bc88beaeeaa5a8601a7743b8835b7bbaf7c8f23ffa78f8bc8faf28"}, - {file = "greenlet-2.0.0.post0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0fee3240093b745efc857392f09379514ad84db4ca324514594bbdf6380016c8"}, - {file = "greenlet-2.0.0.post0-cp310-cp310-win_amd64.whl", hash = "sha256:3407b843b05da71fef0f1dd666059c08ad0e0f4afc3b9c93c998a2e53fac95e5"}, - {file = "greenlet-2.0.0.post0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0a5c03e2a68ec2ff1cba74ceaed899ec8cd353285f4f985c30c8cfbef9d3a3be"}, - {file = "greenlet-2.0.0.post0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5e22485256bb1c60bbcc6f8509b1a11042358a2462d5ecdb9a82dc472d2fdd60"}, - {file = "greenlet-2.0.0.post0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7e9e0d4c5c618b0442396715ffe6c2f84a60d593dad7e0184388aed36d568a65"}, - {file = "greenlet-2.0.0.post0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:697cfbfc19815c40213badcfe5f076418e0f9100cd25a66f513f32c1026b8bf4"}, - {file = "greenlet-2.0.0.post0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4e144ab0de56b4d2a2cf0d2fb9d568b59fce49aab3e129badf17c12b0252047d"}, - {file = "greenlet-2.0.0.post0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4c4310f0e42154995d92810f27b44ab7116a4a696feb0ff141ae2de59196efd7"}, - {file = "greenlet-2.0.0.post0-cp311-cp311-win_amd64.whl", hash = "sha256:8b7e5191b974fb66fcbac1818ba494d3512da9cf6eaef7acd952f9862eaaa20c"}, - {file = "greenlet-2.0.0.post0-cp35-cp35m-macosx_10_14_x86_64.whl", hash = "sha256:3c3327da2bdab61078e42e695307465c425671a5a9251e6c29ee130d51943f28"}, - {file = "greenlet-2.0.0.post0-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:b043782c8f6cccc8fae3a16db397eca1d36a41b0706cbf6f514aea1e1a260bab"}, - {file = "greenlet-2.0.0.post0-cp35-cp35m-win32.whl", hash = "sha256:6fc73fc8dd81d9efa842a55033b6b4cb233b134a0270e127c6874d053ef2049b"}, - {file = "greenlet-2.0.0.post0-cp35-cp35m-win_amd64.whl", hash = "sha256:4cfa629de5b2dea27c81b334c4536463e9a49ac0877e2008a276d58d4c72868a"}, - {file = "greenlet-2.0.0.post0-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:08dc04f49ed1ea5e6772bb5e8cf2a77d1b1744566f4eca471a55b35af1278b31"}, - {file = "greenlet-2.0.0.post0-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:3a22e5988f9d66b3e9ae9583bf9d8ef792b09f23afeb78707e6a4f47ab57cc5e"}, - {file = "greenlet-2.0.0.post0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6393ec3cecda53b20241e88bc33d87cbd8126cc10870fc69fa16ca2e20a5ac1b"}, - {file = "greenlet-2.0.0.post0-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dd0198006278291d9469309d655093df1f5e5107c0261e242b5f390baee32199"}, - {file = "greenlet-2.0.0.post0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ca723dfc2789c1fb991809822811896b198ecf0909dbccea4a07170d18c3e1b"}, - {file = "greenlet-2.0.0.post0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:e56a5a9f303e3ac011ba445a6d84f05d08666bf8db094afafcec5228622c30f5"}, - {file = "greenlet-2.0.0.post0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:4bbe2d074292e3646704371eb640ee52c386d633ed72ff223dadcd3fe8ecd8f9"}, - {file = "greenlet-2.0.0.post0-cp36-cp36m-win32.whl", hash = "sha256:335dcf676d5e4122e4006c16ae11eda2467af5461b949c265ce120b6b959ffe2"}, - {file = "greenlet-2.0.0.post0-cp36-cp36m-win_amd64.whl", hash = "sha256:8c80e9c41a83d8c90399af8c7dcdeae0c03c48b40b9d0ab84457533d5d7882bf"}, - {file = "greenlet-2.0.0.post0-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e93ae35f0fd3caf75e58c76a1cab71e6ece169aaa1b281782ef9efde0a6b83f2"}, - {file = "greenlet-2.0.0.post0-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:1cac9e9895aeff26434325404558783ee54f4ff3aec8daa56b8706796f7b01a0"}, - {file = "greenlet-2.0.0.post0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2bab49783858cf724fff6868395cbeb81d1188cba23616b53e79de0beda29f42"}, - {file = "greenlet-2.0.0.post0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:56565ac9ab4ff3dd473bfe959e0bf2a5062aabb89b7c94cabb417beb162c9fff"}, - {file = "greenlet-2.0.0.post0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:02bdb1e373b275bd705c43b249426e776c4f8a8ff2afaf8ec5ea0dde487d8a14"}, - {file = "greenlet-2.0.0.post0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:eb36b6570646227a63eda03916f1cc6f3744ee96d28f7a0a5629c59267a8055f"}, - {file = "greenlet-2.0.0.post0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:22eca421e3f2f3c18f4f54c0ff525aa9d397c6f116fce9ebd37b420174dbc027"}, - {file = "greenlet-2.0.0.post0-cp37-cp37m-win32.whl", hash = "sha256:f8c425a130e04d5404edaf6f5906e5ab12f3aa1168a1828aba6dfadac5910469"}, - {file = "greenlet-2.0.0.post0-cp37-cp37m-win_amd64.whl", hash = "sha256:81fdcf7c0c2df46a99ca421a552c4370117851c5e4dbd6cb53d569b896d62322"}, - {file = "greenlet-2.0.0.post0-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:538c9e8f65a32413ace426f8117ef019021adf8175f7c491fed65f5fe2083e0c"}, - {file = "greenlet-2.0.0.post0-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:a339e510a079dc8372e39ce1c7629414db51966235c9670c58d529def79243a2"}, - {file = "greenlet-2.0.0.post0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9f2f110b9cc325f6543e0e3f4ab8008c272a59052f9464047c29d4be4511ce05"}, - {file = "greenlet-2.0.0.post0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2fbdec204ca40b3d0c0992a19c1ba627441c17983ac4ffc45baec7f5f53e20ca"}, - {file = "greenlet-2.0.0.post0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46156ae88ee71c37b6c4f7af63fff5d3ab8f45ef72e1a660bcf6386c1647f106"}, - {file = "greenlet-2.0.0.post0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a1a6745c5dce202aa3f29a1736c53cf2179e9c3b280dc62cea9cb8c69977c83"}, - {file = "greenlet-2.0.0.post0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:08f44e938d142271b954405afb6570e0be48a9f556b6bf4d42d2e3ae6a251fad"}, - {file = "greenlet-2.0.0.post0-cp38-cp38-win32.whl", hash = "sha256:d0e210e17a6181a3fd3f8dce957043a4e74177ffa9f295514984b2b633940dce"}, - {file = "greenlet-2.0.0.post0-cp38-cp38-win_amd64.whl", hash = "sha256:00ebdaf0fa51c284fd2172837d751731a15971e0c20d1a9163cfbdf620ce8b49"}, - {file = "greenlet-2.0.0.post0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:c416106b3b8e905b6ab0e84ec90047a6401021aa023f9aa93978e57cd8f8189f"}, - {file = "greenlet-2.0.0.post0-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:8b26932be686f3582df039d79fe96f7ca13d63b39468162f816f9ff29584b9a4"}, - {file = "greenlet-2.0.0.post0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:602a69c24f1a9755dd1760b3b31bdfc495c4613260c876a01b7e6d5eb9bcae1b"}, - {file = "greenlet-2.0.0.post0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:666d2a0b269a68cd4fe0976544ab97970c5334d35d0e47ae9be1723f734d8204"}, - {file = "greenlet-2.0.0.post0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4ea67f303cec384b148774667c7e3cf02311e7026fc02bdcdcd206dfe4ea4fc9"}, - {file = "greenlet-2.0.0.post0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:2146d15429b4eeb412428737594acb5660a5bc0fdd1488d8a2a74a5ee32391fa"}, - {file = "greenlet-2.0.0.post0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:21ee1ae26d072b195edea764218623f6c15eba4ae06816908f33c82e0af018d3"}, - {file = "greenlet-2.0.0.post0-cp39-cp39-win32.whl", hash = "sha256:e1781bda1e787d3ad33788cc3be47f6e47a9581676d02670c15ee36c9460adfe"}, - {file = "greenlet-2.0.0.post0-cp39-cp39-win_amd64.whl", hash = "sha256:6442bbfb047dc1e47658954b72e1589f2bc4e12e67d51bbad0059a626180daa1"}, - {file = "greenlet-2.0.0.post0.tar.gz", hash = "sha256:ad9abc3e4d2370cecb524421cc5c8a664006aa11d5c1cb3c9250e3bf65ab546e"}, + {file = "greenlet-2.0.1-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:9ed358312e63bf683b9ef22c8e442ef6c5c02973f0c2a939ec1d7b50c974015c"}, + {file = "greenlet-2.0.1-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:4f09b0010e55bec3239278f642a8a506b91034f03a4fb28289a7d448a67f1515"}, + {file = "greenlet-2.0.1-cp27-cp27m-win32.whl", hash = "sha256:1407fe45246632d0ffb7a3f4a520ba4e6051fc2cbd61ba1f806900c27f47706a"}, + {file = "greenlet-2.0.1-cp27-cp27m-win_amd64.whl", hash = "sha256:3001d00eba6bbf084ae60ec7f4bb8ed375748f53aeaefaf2a37d9f0370558524"}, + {file = "greenlet-2.0.1-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:d566b82e92ff2e09dd6342df7e0eb4ff6275a3f08db284888dcd98134dbd4243"}, + {file = "greenlet-2.0.1-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:0722c9be0797f544a3ed212569ca3fe3d9d1a1b13942d10dd6f0e8601e484d26"}, + {file = "greenlet-2.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4d37990425b4687ade27810e3b1a1c37825d242ebc275066cfee8cb6b8829ccd"}, + {file = "greenlet-2.0.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:be35822f35f99dcc48152c9839d0171a06186f2d71ef76dc57fa556cc9bf6b45"}, + {file = "greenlet-2.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c140e7eb5ce47249668056edf3b7e9900c6a2e22fb0eaf0513f18a1b2c14e1da"}, + {file = "greenlet-2.0.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d21681f09e297a5adaa73060737e3aa1279a13ecdcfcc6ef66c292cb25125b2d"}, + {file = "greenlet-2.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fb412b7db83fe56847df9c47b6fe3f13911b06339c2aa02dcc09dce8bbf582cd"}, + {file = "greenlet-2.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:c6a08799e9e88052221adca55741bf106ec7ea0710bca635c208b751f0d5b617"}, + {file = "greenlet-2.0.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9e112e03d37987d7b90c1e98ba5e1b59e1645226d78d73282f45b326f7bddcb9"}, + {file = "greenlet-2.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:56961cfca7da2fdd178f95ca407fa330c64f33289e1804b592a77d5593d9bd94"}, + {file = "greenlet-2.0.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:13ba6e8e326e2116c954074c994da14954982ba2795aebb881c07ac5d093a58a"}, + {file = "greenlet-2.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1bf633a50cc93ed17e494015897361010fc08700d92676c87931d3ea464123ce"}, + {file = "greenlet-2.0.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:9f2c221eecb7ead00b8e3ddb913c67f75cba078fd1d326053225a3f59d850d72"}, + {file = "greenlet-2.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:13ebf93c343dd8bd010cd98e617cb4c1c1f352a0cf2524c82d3814154116aa82"}, + {file = "greenlet-2.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:6f61d71bbc9b4a3de768371b210d906726535d6ca43506737682caa754b956cd"}, + {file = "greenlet-2.0.1-cp35-cp35m-macosx_10_14_x86_64.whl", hash = "sha256:2d0bac0385d2b43a7bd1d651621a4e0f1380abc63d6fb1012213a401cbd5bf8f"}, + {file = "greenlet-2.0.1-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:f6327b6907b4cb72f650a5b7b1be23a2aab395017aa6f1adb13069d66360eb3f"}, + {file = "greenlet-2.0.1-cp35-cp35m-win32.whl", hash = "sha256:81b0ea3715bf6a848d6f7149d25bf018fd24554a4be01fcbbe3fdc78e890b955"}, + {file = "greenlet-2.0.1-cp35-cp35m-win_amd64.whl", hash = "sha256:38255a3f1e8942573b067510f9611fc9e38196077b0c8eb7a8c795e105f9ce77"}, + {file = "greenlet-2.0.1-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:04957dc96669be041e0c260964cfef4c77287f07c40452e61abe19d647505581"}, + {file = "greenlet-2.0.1-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:4aeaebcd91d9fee9aa768c1b39cb12214b30bf36d2b7370505a9f2165fedd8d9"}, + {file = "greenlet-2.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:974a39bdb8c90a85982cdb78a103a32e0b1be986d411303064b28a80611f6e51"}, + {file = "greenlet-2.0.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8dca09dedf1bd8684767bc736cc20c97c29bc0c04c413e3276e0962cd7aeb148"}, + {file = "greenlet-2.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a4c0757db9bd08470ff8277791795e70d0bf035a011a528ee9a5ce9454b6cba2"}, + {file = "greenlet-2.0.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:5067920de254f1a2dee8d3d9d7e4e03718e8fd2d2d9db962c8c9fa781ae82a39"}, + {file = "greenlet-2.0.1-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:5a8e05057fab2a365c81abc696cb753da7549d20266e8511eb6c9d9f72fe3e92"}, + {file = "greenlet-2.0.1-cp36-cp36m-win32.whl", hash = "sha256:3d75b8d013086b08e801fbbb896f7d5c9e6ccd44f13a9241d2bf7c0df9eda928"}, + {file = "greenlet-2.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:097e3dae69321e9100202fc62977f687454cd0ea147d0fd5a766e57450c569fd"}, + {file = "greenlet-2.0.1-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:cb242fc2cda5a307a7698c93173d3627a2a90d00507bccf5bc228851e8304963"}, + {file = "greenlet-2.0.1-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:72b00a8e7c25dcea5946692a2485b1a0c0661ed93ecfedfa9b6687bd89a24ef5"}, + {file = "greenlet-2.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5b0ff9878333823226d270417f24f4d06f235cb3e54d1103b71ea537a6a86ce"}, + {file = "greenlet-2.0.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:be9e0fb2ada7e5124f5282d6381903183ecc73ea019568d6d63d33f25b2a9000"}, + {file = "greenlet-2.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b493db84d124805865adc587532ebad30efa68f79ad68f11b336e0a51ec86c2"}, + {file = "greenlet-2.0.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a20d33124935d27b80e6fdacbd34205732660e0a1d35d8b10b3328179a2b51a1"}, + {file = "greenlet-2.0.1-cp37-cp37m-win32.whl", hash = "sha256:ea688d11707d30e212e0110a1aac7f7f3f542a259235d396f88be68b649e47d1"}, + {file = "greenlet-2.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:afe07421c969e259e9403c3bb658968702bc3b78ec0b6fde3ae1e73440529c23"}, + {file = "greenlet-2.0.1-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:cd4ccc364cf75d1422e66e247e52a93da6a9b73cefa8cad696f3cbbb75af179d"}, + {file = "greenlet-2.0.1-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:4c8b1c43e75c42a6cafcc71defa9e01ead39ae80bd733a2608b297412beede68"}, + {file = "greenlet-2.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:659f167f419a4609bc0516fb18ea69ed39dbb25594934bd2dd4d0401660e8a1e"}, + {file = "greenlet-2.0.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:356e4519d4dfa766d50ecc498544b44c0249b6de66426041d7f8b751de4d6b48"}, + {file = "greenlet-2.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:811e1d37d60b47cb8126e0a929b58c046251f28117cb16fcd371eed61f66b764"}, + {file = "greenlet-2.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:0109af1138afbfb8ae647e31a2b1ab030f58b21dd8528c27beaeb0093b7938a9"}, + {file = "greenlet-2.0.1-cp38-cp38-win32.whl", hash = "sha256:88c8d517e78acdf7df8a2134a3c4b964415b575d2840a2746ddb1cc6175f8608"}, + {file = "greenlet-2.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:d6ee1aa7ab36475035eb48c01efae87d37936a8173fc4d7b10bb02c2d75dd8f6"}, + {file = "greenlet-2.0.1-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:b1992ba9d4780d9af9726bbcef6a1db12d9ab1ccc35e5773685a24b7fb2758eb"}, + {file = "greenlet-2.0.1-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:b5e83e4de81dcc9425598d9469a624826a0b1211380ac444c7c791d4a2137c19"}, + {file = "greenlet-2.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:505138d4fa69462447a562a7c2ef723c6025ba12ac04478bc1ce2fcc279a2db5"}, + {file = "greenlet-2.0.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cce1e90dd302f45716a7715517c6aa0468af0bf38e814ad4eab58e88fc09f7f7"}, + {file = "greenlet-2.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e9744c657d896c7b580455e739899e492a4a452e2dd4d2b3e459f6b244a638d"}, + {file = "greenlet-2.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:41b825d65f31e394b523c84db84f9383a2f7eefc13d987f308f4663794d2687e"}, + {file = "greenlet-2.0.1-cp39-cp39-win32.whl", hash = "sha256:db38f80540083ea33bdab614a9d28bcec4b54daa5aff1668d7827a9fc769ae0a"}, + {file = "greenlet-2.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:b23d2a46d53210b498e5b701a1913697671988f4bf8e10f935433f6e7c332fb6"}, + {file = "greenlet-2.0.1.tar.gz", hash = "sha256:42e602564460da0e8ee67cb6d7236363ee5e131aa15943b6670e44e5c2ed0f67"}, ] idna = [ {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, @@ -1708,6 +1785,10 @@ jinja2-time = [ {file = "jinja2-time-0.2.0.tar.gz", hash = "sha256:d14eaa4d315e7688daa4969f616f226614350c48730bfa1692d2caebd8c90d40"}, {file = "jinja2_time-0.2.0-py2.py3-none-any.whl", hash = "sha256:d3eab6605e3ec8b7a0863df09cc1d23714908fa61aa6986a845c20ba488b4efa"}, ] +jmespath = [ + {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, + {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, +] joblib = [ {file = "joblib-1.2.0-py3-none-any.whl", hash = "sha256:091138ed78f800342968c523bdde947e7a305b8594b910a0fea2ab83c3c6d385"}, {file = "joblib-1.2.0.tar.gz", hash = "sha256:e1cee4a79e4af22881164f218d4311f60074197fb707e082e803b61f6d137018"}, @@ -1867,8 +1948,8 @@ packaging = [ {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, ] pathspec = [ - {file = "pathspec-0.10.1-py3-none-any.whl", hash = "sha256:46846318467efc4556ccfd27816e004270a9eeeeb4d062ce5e6fc7a87c573f93"}, - {file = "pathspec-0.10.1.tar.gz", hash = "sha256:7ace6161b621d31e7902eb6b5ae148d12cfd23f4a249b9ffb6b9fee12084323d"}, + {file = "pathspec-0.10.2-py3-none-any.whl", hash = "sha256:88c2606f2c1e818b978540f73ecc908e13999c6c3a383daf3705652ae79807a5"}, + {file = "pathspec-0.10.2.tar.gz", hash = "sha256:8f6bf73e5758fd365ef5d58ce09ac7c27d2833a8d7da51712eac6e27e35141b0"}, ] pendulum = [ {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, @@ -1898,8 +1979,8 @@ pkgutil-resolve-name = [ {file = "pkgutil_resolve_name-1.3.10.tar.gz", hash = "sha256:357d6c9e6a755653cfd78893817c0853af365dd51ec97f3d358a819373bbd174"}, ] platformdirs = [ - {file = "platformdirs-2.5.2-py3-none-any.whl", hash = "sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788"}, - {file = "platformdirs-2.5.2.tar.gz", hash = "sha256:58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19"}, + {file = "platformdirs-2.5.4-py3-none-any.whl", hash = "sha256:af0276409f9a02373d540bf8480021a048711d572745aef4b7842dad245eba10"}, + {file = "platformdirs-2.5.4.tar.gz", hash = "sha256:1006647646d80f16130f052404c6b901e80ee4ed6bef6792e1f238a8969106f7"}, ] pluggy = [ {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, @@ -1950,8 +2031,8 @@ pyflakes = [ {file = "pyflakes-2.3.1.tar.gz", hash = "sha256:f5bc8ecabc05bb9d291eb5203d6810b49040f6ff446a756326104746cc00c1db"}, ] pygithub = [ - {file = "PyGithub-1.56-py3-none-any.whl", hash = "sha256:d15f13d82165306da8a68aefc0f848a6f6432d5febbff13b60a94758ce3ef8b5"}, - {file = "PyGithub-1.56.tar.gz", hash = "sha256:80c6d85cf0f9418ffeb840fd105840af694c4f17e102970badbaf678251f2a01"}, + {file = "PyGithub-1.57-py3-none-any.whl", hash = "sha256:5822febeac2391f1306c55a99af2bc8f86c8bf82ded000030cd02c18f31b731f"}, + {file = "PyGithub-1.57.tar.gz", hash = "sha256:c273f252b278fb81f1769505cc6921bdb6791e1cebd6ac850cc97dad13c31ff3"}, ] pygments = [ {file = "Pygments-2.13.0-py3-none-any.whl", hash = "sha256:f643f331ab57ba3c9d89212ee4a2dabc6e94f117cf4eefde99a0574720d14c42"}, @@ -2079,9 +2160,13 @@ requests-mock = [ {file = "requests-mock-1.10.0.tar.gz", hash = "sha256:59c9c32419a9fb1ae83ec242d98e889c45bd7d7a65d48375cc243ec08441658b"}, {file = "requests_mock-1.10.0-py2.py3-none-any.whl", hash = "sha256:2fdbb637ad17ee15c06f33d31169e71bf9fe2bdb7bc9da26185be0dd8d842699"}, ] +s3transfer = [ + {file = "s3transfer-0.6.0-py3-none-any.whl", hash = "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd"}, + {file = "s3transfer-0.6.0.tar.gz", hash = "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947"}, +] setuptools = [ - {file = "setuptools-65.5.0-py3-none-any.whl", hash = "sha256:f62ea9da9ed6289bfe868cd6845968a2c854d1427f8548d52cae02a42b4f0356"}, - {file = "setuptools-65.5.0.tar.gz", hash = "sha256:512e5536220e38146176efb833d4a62aa726b7bbff82cfbc8ba9eaa3996e0b17"}, + {file = "setuptools-65.5.1-py3-none-any.whl", hash = "sha256:d0b9a8433464d5800cbe05094acf5c6d52a91bfac9b52bcfc4d41382be5d5d31"}, + {file = "setuptools-65.5.1.tar.gz", hash = "sha256:e197a19aa8ec9722928f2206f8de752def0e4c9fc6953527360d1c36d94ddb2f"}, ] simplejson = [ {file = "simplejson-3.18.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:17dbc7f71fa5b7e4a2acef38cf0be30461ae6659456a978ce7eeebeb5bdf9e1a"}, @@ -2309,8 +2394,8 @@ types-simplejson = [ {file = "types_simplejson-3.18.0.0-py3-none-any.whl", hash = "sha256:f8a8428f753574fa3b7eb290756776f0fb6b4ad9cae72bf8c0c9534eeda6398c"}, ] types-urllib3 = [ - {file = "types-urllib3-1.26.25.1.tar.gz", hash = "sha256:a948584944b2412c9a74b9cf64f6c48caf8652cb88b38361316f6d15d8a184cd"}, - {file = "types_urllib3-1.26.25.1-py3-none-any.whl", hash = "sha256:f6422596cc9ee5fdf68f9d547f541096a20c2dcfd587e37c804c9ea720bf5cb2"}, + {file = "types-urllib3-1.26.25.4.tar.gz", hash = "sha256:eec5556428eec862b1ac578fb69aab3877995a99ffec9e5a12cf7fbd0cc9daee"}, + {file = "types_urllib3-1.26.25.4-py3-none-any.whl", hash = "sha256:ed6b9e8a8be488796f72306889a06a3fc3cb1aa99af02ab8afb50144d7317e49"}, ] typing-extensions = [ {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, diff --git a/pyproject.toml b/pyproject.toml index a5f4f9f2d..f6adcf215 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,9 @@ sphinx-copybutton = {version = ">=0.3.1,<0.6.0", optional = true} myst-parser = {version = ">=0.17.2,<0.19.0", optional = true} sphinx-autobuild = {version = "^2021.3.14", optional = true} +# File storage dependencies installed as optional 'filesystem' extras +fs-s3fs = { version = "^1.1.1", optional = true} + [tool.poetry.extras] docs = [ "sphinx", @@ -76,6 +79,7 @@ docs = [ "myst-parser", "sphinx-autobuild", ] +s3 = ["fs-s3fs"] # Mark these as extras so that installs pass on python 3.11 samples = [ "pyarrow", diff --git a/singer_sdk/helpers/_resources.py b/singer_sdk/helpers/_resources.py index a458e0a55..5f624f8c1 100644 --- a/singer_sdk/helpers/_resources.py +++ b/singer_sdk/helpers/_resources.py @@ -2,6 +2,7 @@ import sys from types import ModuleType +from typing import cast if sys.version_info >= (3, 9): import importlib.resources as importlib_resources @@ -21,4 +22,4 @@ def get_package_files(package: str | ModuleType) -> Traversable: Returns: The file as a Traversable object. """ - return importlib_resources.files(package) + return cast(Traversable, importlib_resources.files(package)) diff --git a/tests/core/test_batch.py b/tests/core/test_batch.py index 813088bff..6bb5a3fc0 100644 --- a/tests/core/test_batch.py +++ b/tests/core/test_batch.py @@ -32,6 +32,16 @@ def test_storage_get_url(): assert url.replace("\\", "/").endswith("root_dir/prefix--file.jsonl.gz") +def test_storage_get_s3_url(): + storage = StorageTarget("s3://testing123:testing123@test_bucket") + + with storage.fs(create=True) as fs: + url = fs.geturl("prefix--file.jsonl.gz") + assert url.startswith( + "https://s3.amazonaws.com/test_bucket/prefix--file.jsonl.gz" + ) + + @pytest.mark.parametrize( "file_url,root", [ @@ -40,6 +50,11 @@ def test_storage_get_url(): "file:///Users/sdk/path/to", id="local", ), + pytest.param( + "s3://test_bucket/prefix--file.jsonl.gz", + "s3://test_bucket", + id="s3", + ), ], ) def test_storage_from_url(file_url: str, root: str):