From 07ae5b230db6cd31939b979668d9e116272caa29 Mon Sep 17 00:00:00 2001 From: Giorgio Basile Date: Thu, 29 Jun 2023 12:24:24 +0200 Subject: [PATCH 1/2] Cleanup template blocks, flows and tasks --- docs/gen_examples_catalog.py | 1 - prefect_eo/blocks.py | 35 ----------------------------------- prefect_eo/flows.py | 26 -------------------------- prefect_eo/tasks.py | 24 ------------------------ tests/__init__.py | 0 tests/conftest.py | 4 ++++ tests/test_flows.py | 6 ------ tests/test_tasks.py | 24 ------------------------ 8 files changed, 4 insertions(+), 116 deletions(-) delete mode 100644 prefect_eo/blocks.py delete mode 100644 prefect_eo/flows.py delete mode 100644 prefect_eo/tasks.py create mode 100644 tests/__init__.py delete mode 100644 tests/test_flows.py delete mode 100644 tests/test_tasks.py diff --git a/docs/gen_examples_catalog.py b/docs/gen_examples_catalog.py index 562ff93..a0380fd 100644 --- a/docs/gen_examples_catalog.py +++ b/docs/gen_examples_catalog.py @@ -72,7 +72,6 @@ def get_code_examples(obj: Union[ModuleType, Callable]) -> Set[str]: code_examples_grouping = defaultdict(set) for _, module_name, ispkg in iter_modules(prefect_eo.__path__): - module_nesting = f"{COLLECTION_SLUG}.{module_name}" module_obj = load_module(module_nesting) diff --git a/prefect_eo/blocks.py b/prefect_eo/blocks.py deleted file mode 100644 index 3f26c56..0000000 --- a/prefect_eo/blocks.py +++ /dev/null @@ -1,35 +0,0 @@ -"""This is an example blocks module""" - -from prefect.blocks.core import Block -from pydantic import Field - - -class EoBlock(Block): - """ - A sample block that holds a value. - - Attributes: - value (str): The value to store. - - Example: - Load a stored value: - ```python - from prefect_eo import EoBlock - block = EoBlock.load("BLOCK_NAME") - ``` - """ - - _block_type_name = "eo" - # replace this with a relevant logo; defaults to Prefect logo - _logo_url = "https://images.ctfassets.net/gm98wzqotmnx/08yCE6xpJMX9Kjl5VArDS/c2ede674c20f90b9b6edeab71feffac9/prefect-200x200.png?h=250" # noqa - _documentation_url = "https://giorgiobasile.github.io/prefect-eo/blocks/#prefect-eo.blocks.EoBlock" # noqa - - value: str = Field("The default value", description="The value to store.") - - @classmethod - def seed_value_for_example(cls): - """ - Seeds the field, value, so the block can be loaded. - """ - block = cls(value="A sample value") - block.save("sample-block", overwrite=True) diff --git a/prefect_eo/flows.py b/prefect_eo/flows.py deleted file mode 100644 index eff01cb..0000000 --- a/prefect_eo/flows.py +++ /dev/null @@ -1,26 +0,0 @@ -"""This is an example flows module""" -from prefect import flow - -from prefect_eo.blocks import EoBlock -from prefect_eo.tasks import ( - goodbye_prefect_eo, - hello_prefect_eo, -) - - -@flow -def hello_and_goodbye(): - """ - Sample flow that says hello and goodbye! - """ - EoBlock.seed_value_for_example() - block = EoBlock.load("sample-block") - - print(hello_prefect_eo()) - print(f"The block's value: {block.value}") - print(goodbye_prefect_eo()) - return "Done" - - -if __name__ == "__main__": - hello_and_goodbye() diff --git a/prefect_eo/tasks.py b/prefect_eo/tasks.py deleted file mode 100644 index 7af54be..0000000 --- a/prefect_eo/tasks.py +++ /dev/null @@ -1,24 +0,0 @@ -"""This is an example tasks module""" -from prefect import task - - -@task -def hello_prefect_eo() -> str: - """ - Sample task that says hello! - - Returns: - A greeting for your collection - """ - return "Hello, prefect-eo!" - - -@task -def goodbye_prefect_eo() -> str: - """ - Sample task that says goodbye! - - Returns: - A farewell for your collection - """ - return "Goodbye, prefect-eo!" diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py index ca7cae7..feb4011 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,6 +2,10 @@ from prefect.testing.utilities import prefect_test_harness +def pytest_configure(config): + config.addinivalue_line("markers", "flaky: mark test as flaky") + + @pytest.fixture(scope="session", autouse=True) def prefect_db(): """ diff --git a/tests/test_flows.py b/tests/test_flows.py deleted file mode 100644 index 35ddb4d..0000000 --- a/tests/test_flows.py +++ /dev/null @@ -1,6 +0,0 @@ -from prefect_eo.flows import hello_and_goodbye - - -def test_hello_and_goodbye_flow(): - result = hello_and_goodbye() - assert result == "Done" diff --git a/tests/test_tasks.py b/tests/test_tasks.py deleted file mode 100644 index acedf2c..0000000 --- a/tests/test_tasks.py +++ /dev/null @@ -1,24 +0,0 @@ -from prefect import flow - -from prefect_eo.tasks import ( - goodbye_prefect_eo, - hello_prefect_eo, -) - - -def test_hello_prefect_eo(): - @flow - def test_flow(): - return hello_prefect_eo() - - result = test_flow() - assert result == "Hello, prefect-eo!" - - -def goodbye_hello_prefect_eo(): - @flow - def test_flow(): - return goodbye_prefect_eo() - - result = test_flow() - assert result == "Goodbye, prefect-eo!" From 37d36323815c9e0f31aa42d8eade32464c7e90cb Mon Sep 17 00:00:00 2001 From: Giorgio Basile Date: Thu, 29 Jun 2023 12:24:47 +0200 Subject: [PATCH 2/2] Add EarthdataCredentials block --- .github/workflows/tests.yml | 1 - CHANGELOG.md | 2 ++ README.md | 2 +- docs/{flows.md => earthdata.md} | 2 +- docs/tasks.md | 6 ---- mkdocs.yml | 4 +-- prefect_eo/__init__.py | 2 +- prefect_eo/earthdata.py | 59 +++++++++++++++++++++++++++++++++ requirements-dev.txt | 2 +- requirements.txt | 1 + setup.py | 3 +- tests/test_earthdata.py | 50 ++++++++++++++++++++++++++++ 12 files changed, 119 insertions(+), 15 deletions(-) rename docs/{flows.md => earthdata.md} (80%) delete mode 100644 docs/tasks.md create mode 100644 prefect_eo/earthdata.py create mode 100644 tests/test_earthdata.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6e7a5d4..8540b79 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -9,7 +9,6 @@ jobs: strategy: matrix: python-version: - - "3.7" - "3.8" - "3.9" - "3.10" diff --git a/CHANGELOG.md b/CHANGELOG.md index 0998eb8..692f815 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added NASA Earthdata credentials block - [#4](https://github.com/giorgiobasile/prefect-eo/issues/4) + ### Changed ### Deprecated diff --git a/README.md b/README.md index 62758b5..2d12f10 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ Install `prefect-eo` with `pip`: pip install prefect-eo ``` -Requires an installation of Python 3.7+. +Requires an installation of Python 3.8+. We recommend using a Python virtual environment manager such as pipenv, conda or virtualenv. diff --git a/docs/flows.md b/docs/earthdata.md similarity index 80% rename from docs/flows.md rename to docs/earthdata.md index db2274c..e61b459 100644 --- a/docs/flows.md +++ b/docs/earthdata.md @@ -3,4 +3,4 @@ description: notes: This documentation page is generated from source file docstrings. --- -::: prefect_eo.flows \ No newline at end of file +::: prefect_eo.earthdata \ No newline at end of file diff --git a/docs/tasks.md b/docs/tasks.md deleted file mode 100644 index a9fdb4d..0000000 --- a/docs/tasks.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -description: -notes: This documentation page is generated from source file docstrings. ---- - -::: prefect_eo.tasks \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index 7a7f9fd..72a9a9f 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -76,7 +76,7 @@ nav: - Blocks Catalog: blocks_catalog.md - Examples Catalog: examples_catalog.md - API Reference: - - Tasks: tasks.md - - Flows: flows.md + - Earthdata: earthdata.md + diff --git a/prefect_eo/__init__.py b/prefect_eo/__init__.py index 00c9adf..a5f85f0 100644 --- a/prefect_eo/__init__.py +++ b/prefect_eo/__init__.py @@ -1,4 +1,4 @@ from . import _version -from .blocks import EoBlock # noqa +from .earthdata import EarthdataCredentials # noqa __version__ = _version.get_versions()["version"] diff --git a/prefect_eo/earthdata.py b/prefect_eo/earthdata.py new file mode 100644 index 0000000..2122ba2 --- /dev/null +++ b/prefect_eo/earthdata.py @@ -0,0 +1,59 @@ +"""Module handling NASA Earthdata credentials""" + +import os + +import earthaccess +from prefect.blocks.core import Block +from pydantic import Field, SecretStr + + +class EarthdataCredentials(Block): + """ + Block used to manage authentication with NASA Earthdata. + NASA Earthdata authentication is handled via the `earthaccess` module. + Refer to the [earthaccess docs](https://nsidc.github.io/earthaccess/) + for more info about the possible credential configurations. + + Example: + Load stored Earthdata credentials: + ```python + from prefect_eo import EarthdataCredentials + + ed_credentials_block = EarthdataCredentials.load("BLOCK_NAME") + ``` + """ # noqa E501 + + _logo_url = "https://yt3.googleusercontent.com/ytc/AGIKgqPjIUeAw3_hrkHWZgixdwD5jc-hTWweoCA6bJMhUg=s176-c-k-c0x00ffffff-no-rj" # noqa + _block_type_name = "NASA Earthdata Credentials" + _documentation_url = "https://nsidc.github.io/earthaccess/" # noqa + + earthdata_username: str = Field( + default=..., + description="The Earthdata username of a specific account.", + title="Earthdata username", + ) + earthdata_password: SecretStr = Field( + default=..., + description="The Earthdata password of a specific account.", + title="Earthdata password", + ) + + def login(self) -> earthaccess.Auth: + """ + Returns an authenticated session with NASA Earthdata + + Example: + ```python + earthdata_credentials_block = EarthdataCredentials( + earthdata_username = "username", + earthdata_password = "password" + ) + earthdata_auth = earthdata_credentials_block.login() + ``` + """ + + """""" + + os.environ["EARTHDATA_USERNAME"] = self.earthdata_username + os.environ["EARTHDATA_PASSWORD"] = self.earthdata_password.get_secret_value() + return earthaccess.login(strategy="environment") diff --git a/requirements-dev.txt b/requirements-dev.txt index dae3fd3..10e6c75 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,8 +8,8 @@ mkdocstrings[python] isort pre-commit pytest-asyncio -mock; python_version < '3.8' mkdocs-gen-files interrogate coverage pillow +requests_mock diff --git a/requirements.txt b/requirements.txt index 4ec3de6..40f8a7f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ prefect>=2.0.0 +earthaccess diff --git a/setup.py b/setup.py index 4d45d0a..ba619fd 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ version=versioneer.get_version(), cmdclass=versioneer.get_cmdclass(), packages=find_packages(exclude=("tests", "docs")), - python_requires=">=3.7", + python_requires=">=3.8", install_requires=install_requires, extras_require={"dev": dev_requires}, entry_points={ @@ -38,7 +38,6 @@ "Intended Audience :: System Administrators", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", diff --git a/tests/test_earthdata.py b/tests/test_earthdata.py new file mode 100644 index 0000000..081ef39 --- /dev/null +++ b/tests/test_earthdata.py @@ -0,0 +1,50 @@ +import pytest +import requests_mock +from earthaccess import Auth + +from prefect_eo.earthdata import EarthdataCredentials + + +@pytest.fixture +def mock_earthdata_responses(): + with requests_mock.Mocker() as m: + json_response = [ + {"access_token": "EDL-token-1", "expiration_date": "12/15/2023"}, + {"access_token": "EDL-token-2", "expiration_date": "12/16/2023"}, + ] + m.get( + "https://urs.earthdata.nasa.gov/api/users/tokens", + json=json_response, + status_code=200, + ) + m.get( + "https://urs.earthdata.nasa.gov/profile", + json={"uid": "test_username"}, + status_code=200, + ) + m.get( + "https://urs.earthdata.nasa.gov/api/users/user?client_id=ntD0YGC_SM3Bjs-Tnxd7bg", # noqa E501 + json={"uid": "test_username"}, + status_code=200, + ) + yield m + + +def test_earthdata_credentials_login(mock_earthdata_responses): # noqa + """ + Asserts that instantiated EarthdataCredentials block creates an + authenticated session. + """ + + # Set up mock user input + mock_username = "user" + mock_password = "password" + + # Instantiate EarthdataCredentials block + earthdata_credentials_block = EarthdataCredentials( + earthdata_username=mock_username, earthdata_password=mock_password + ) + earthdata_auth = earthdata_credentials_block.login() + + assert isinstance(earthdata_auth, Auth) + assert earthdata_auth.authenticated