diff --git a/MANIFEST.in b/MANIFEST.in index 2206d75..6322f63 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1 @@ -include maap/dps/*.xml \ No newline at end of file +include maap/dps/*.xml diff --git a/doc.txt b/doc.txt index 59fdb33..286490f 100644 --- a/doc.txt +++ b/doc.txt @@ -1,6 +1,6 @@ -****** -maapPy -****** +******* +maap-py +******* # Python MAAP Client Library diff --git a/maap/config_reader.py b/maap/config_reader.py index bafc159..2d256d6 100644 --- a/maap/config_reader.py +++ b/maap/config_reader.py @@ -17,6 +17,7 @@ def __init__(self, maap_host=None, config_file_path=''): self.__config = ConfigParser() configfile_present = False config_paths = list(map(self.__get_config_path, [os.path.dirname(config_file_path), os.curdir, os.path.expanduser("~"), os.environ.get("MAAP_CONF") or '.'])) + for loc in config_paths: try: with open(loc) as source: diff --git a/maap/dps/DpsHelper.py b/maap/dps/DpsHelper.py index 47e526b..e790939 100644 --- a/maap/dps/DpsHelper.py +++ b/maap/dps/DpsHelper.py @@ -2,10 +2,11 @@ import requests import xml.etree.ElementTree as ET import logging -import os import json from os.path import exists +import importlib_resources as resources + class DpsHelper: DPS_INTERNAL_FILE_JOB = "_job.json" @@ -16,7 +17,6 @@ class DpsHelper: """ def __init__(self, api_header, dps_token_endpoint): self._api_header = api_header - self._location = os.path.dirname(os.path.abspath(__file__)) self._logger = logging.getLogger(__name__) self.dps_token_endpoint = dps_token_endpoint self.running_in_dps = self._running_in_dps_mode() @@ -35,9 +35,9 @@ def _skit(self, lines, kwargs): return res def submit_job(self, request_url, **kwargs): - xml_file = os.path.join(self._location, 'execute.xml') - input_xml = os.path.join(self._location, 'execute_inputs.xml') - + xml_file = resources.files("maap.dps").joinpath("execute.xml") + input_xml = resources.files("maap.dps").joinpath("execute_inputs.xml") + # ================================== # Part 1: Parse Required Arguments # ================================== @@ -81,23 +81,19 @@ def submit_job(self, request_url, **kwargs): ins_xml = '' other = '' - with open(input_xml) as xml: - ins_xml = xml.read() + ins_xml = input_xml.read_text() # ------------------------------- # Insert XML for algorithm inputs # ------------------------------- for key in input_names: - other += ins_xml.format(name=key).format(value=input_names[key]) + other += ins_xml.format(name=key, value=input_names[key]) other += '\n' # print(other) params['other_inputs'] = other - with open(xml_file) as xml: - req_xml = xml.read() - - req_xml = req_xml.format(**params) + req_xml = xml_file.read_text().format(**params) # log request body logging.debug('request is') diff --git a/maap/dps/dps_job_model.py b/maap/dps/dps_job_model.py index 99f1541..8de7239 100644 --- a/maap/dps/dps_job_model.py +++ b/maap/dps/dps_job_model.py @@ -1,8 +1,8 @@ import logging -import os from datetime import datetime import xml.etree.ElementTree as ET +import importlib_resources as resources import requests from maap.config_reader import ConfigReader @@ -37,18 +37,16 @@ def __init__(self, not_self_signed=False): self.__algorithm_version = None self.__username = 'anonymous' self.__inputs = {} - current_location = os.path.dirname(os.path.abspath(__file__)) - self.__xml_file = os.path.join(current_location, 'execute.xml') - self.__input_xml = os.path.join(current_location, 'execute_inputs.xml') + self.__xml_file = resources.files("maap.dps").joinpath("execute.xml") + self.__input_xml = resources.files("maap.dps").joinpath("execute_inputs.xml") def with_param(self, key, val): self.__inputs[key] = val return self def __generate_xml_inputs(self): - with open(self.__input_xml) as xml: - input_xml = xml.read() - input_xmls = [input_xml.format(name=k).format(value=v) for k, v in self.__inputs.items()] + input_xml = self.__input_xml.read_text() + input_xmls = [input_xml.format(name=k, value=v) for k, v in self.__inputs.items()] return '\n'.join(input_xmls) def generate_request_xml(self): @@ -61,10 +59,7 @@ def generate_request_xml(self): 'inputs': '', # TODO this is needed? 'other_inputs': self.__generate_xml_inputs(), } - with open(self.__xml_file) as xml: - request_xml = xml.read() - request_xml = request_xml.format(**params) - return request_xml + return self.__xml_file.read_text().format(**params) def submit_job(self): """ diff --git a/maap/dps/execute_inputs.xml b/maap/dps/execute_inputs.xml index eedec18..a21b54c 100644 --- a/maap/dps/execute_inputs.xml +++ b/maap/dps/execute_inputs.xml @@ -1,5 +1,5 @@ - {{value}} + - \ No newline at end of file + diff --git a/maap/maap.py b/maap/maap.py index 3bdbdd5..f817c3c 100644 --- a/maap/maap.py +++ b/maap/maap.py @@ -4,8 +4,10 @@ import uuid import urllib.parse import os -from mapboxgl.utils import * -from mapboxgl.viz import * +import sys + +import importlib_resources as resources +import requests from .Result import Collection, Granule, Result from maap.config_reader import ConfigReader from maap.dps.dps_job import DPSJob @@ -386,6 +388,8 @@ def _get_capabilities(self, granule_ur): return response def show(self, granule, display_config={}): + from mapboxgl.viz import RasterTilesViz + granule_ur = granule['Granule']['GranuleUR'] browse_file = json.loads(self._get_browse(granule_ur).text)['browse'] capabilities = json.loads(self._get_capabilities(granule_ur).text)['body'] @@ -409,4 +413,3 @@ def show(self, granule, display_config={}): if __name__ == "__main__": print("initialized") - diff --git a/maap/utils/HTTPServerHandler.py b/maap/utils/HTTPServerHandler.py deleted file mode 100644 index 8f29419..0000000 --- a/maap/utils/HTTPServerHandler.py +++ /dev/null @@ -1,56 +0,0 @@ -from http.server import BaseHTTPRequestHandler -import requests -import json - -REDIRECT_URL = 'http://localhost:8080/' - - -class HTTPServerHandler(BaseHTTPRequestHandler): - - """ - HTTP Server callbacks to handle Earthdata OAuth redirects - """ - def __init__(self, request, address, server, a_id): - self.app_id = a_id - super().__init__(request, address, server) - - def do_GET(self): - - EARTHDATA_API_AUTH_URI = 'https://uat.urs.earthdata.nasa.gov/oauth/token' - - self.send_response(200) - self.send_header('Content-type', 'text/html') - self.end_headers() - if 'code' in self.path: - self.auth_code = self.path.split('=')[1] - self.wfile.write(bytes('

You may now close this window.

', 'utf-8')) - self.server.access_token = self.get_access_token_from_url( - EARTHDATA_API_AUTH_URI, self.auth_code) - - def get_access_token_from_url(self, url, code): - """ - Parse the access token from Earthdata's response - Args: - url: the Earthdata api oauth URI containing valid client_id - code: Earthdata auth_code argument - Returns: - a string containing the access key - """ - - body = { - 'grant_type': 'authorization_code', - 'code': code, - 'redirect_uri': REDIRECT_URL - } - - r = requests.post(url, data=body, auth=('edl_client_name', 'edl_client_password')) - - if r.status_code == 401: - return "unauthorized" - else: - j = json.loads(r.text) - return j['access_token'] - - # Disable logging from the HTTP Server - def log_message(self, format, *args): - return diff --git a/maap/utils/TokenHandler.py b/maap/utils/TokenHandler.py deleted file mode 100644 index 160187e..0000000 --- a/maap/utils/TokenHandler.py +++ /dev/null @@ -1,39 +0,0 @@ -from http.server import HTTPServer -from webbrowser import open_new -from maap.utils.HTTPServerHandler import HTTPServerHandler - -REDIRECT_URL = 'http://localhost:8080/' -PORT = 8080 - - -# Command-line SSO work in progress -# Known issues: -# 1) browser window is spawned during execution; investigating running chrome in headless mode -# 2) credentials are required as input on initial authentication; -# investigating CAS python libraries to avoid this concern. -class TokenHandler: - """ - Functions used to handle Earthdata oAuth - """ - def __init__(self, a_id): - self._id = a_id - - def get_access_token(self): - """ - Fetches the access key using an HTTP server to handle oAuth - requests - Args: - appId: The URS assigned App ID - """ - - access_uri = ('https://uat.urs.earthdata.nasa.gov/oauth/' - + 'authorize?client_id=' + self._id + '&redirect_uri=' - + REDIRECT_URL + "&response_type=code") - - open_new(access_uri) - http_server = HTTPServer( - ('localhost', PORT), - lambda request, address, server: HTTPServerHandler( - request, address, server, self._id)) - http_server.handle_request() - return http_server.access_token \ No newline at end of file diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 7f97434..0000000 --- a/requirements.txt +++ /dev/null @@ -1,17 +0,0 @@ -backoff==2.2.1 -black -boto3==1.26.138 -boto3-stubs[s3] -botocore-stubs -ConfigParser==5.3.0 -ipython==8.13.2 -isort -mapboxgl==0.10.2 -moto==4.1.10 -mypy_boto3_s3==1.26.127 -pytest==7.3.1 -PyYAML==6.0 -requests==2.28.1 -responses==0.23.1 -setuptools==65.5.0 -urllib3 \ No newline at end of file diff --git a/setup.py b/setup.py index 28a6820..d6fe40e 100644 --- a/setup.py +++ b/setup.py @@ -4,34 +4,59 @@ # Package data # ------------ -_author = 'Jet Propulsion Laboratory' -_author_email = 'bsatoriu@jpl.nasa.gov' +_author = "Jet Propulsion Laboratory" +_author_email = "bsatoriu@jpl.nasa.gov" _classifiers = [ - 'Environment :: Console', - 'Framework :: Pytest', - 'Intended Audience :: Developers', - 'Intended Audience :: Information Technology', - 'Intended Audience :: Science/Research', - 'Topic :: Scientific/Engineering', - 'Development Status :: 3 - Alpha', - 'Operating System :: OS Independent', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3.5', - 'Topic :: Internet :: WWW/HTTP', - 'Topic :: Software Development :: Libraries :: Python Modules', + "Environment :: Console", + "Framework :: Pytest", + "Intended Audience :: Developers", + "Intended Audience :: Information Technology", + "Intended Audience :: Science/Research", + "Topic :: Scientific/Engineering", + "Development Status :: 3 - Alpha", + "Operating System :: OS Independent", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3.5", + "Topic :: Internet :: WWW/HTTP", + "Topic :: Software Development :: Libraries :: Python Modules", ] -_description = 'maapPy Python API' -_download_url = '' -_requirements = ["backoff", "boto3==1.33.13", "ConfigParser", "ipython==8.12.0", "mapboxgl", "moto", "mypy_boto3_s3", "pytest", - "PyYAML", "requests", "responses", "setuptools"] -_keywords = ['dataset', 'granule', 'nasa', 'MAAP', 'CMR'] -_license = 'Apache License, Version 2.0' -_long_description = 'Python client API for interacting with the NASA MAAP API' -_name = 'maapPy' -_namespaces = [] -_test_suite = '' -_url = 'https://github.com/MAAP-Project/maap-py' -_version = '3.1.4' +_description = "maapPy Python API" +_download_url = "" +_boto3_version = "1.34.41" +_requirements = [ + "backoff~=2.2", + f"boto3~={_boto3_version}", + "ConfigParser~=6.0", + "importlib_resources~=6.0", + # We must explicitly specify ipython because mapboxgl requires it, but + # does not specify it in its own requirements. This is a bug in mapboxgl + # that has been fixed, but the fix has not been released even though it was + # fixed in 2019. See https://github.com/mapbox/mapboxgl-jupyter/pull/172. + "ipython==8.11.0", + "mapboxgl~=0.10", + "PyYAML~=6.0", + "requests~=2.31", + "setuptools~=69.0", +] +_extra_requirements = { + "dev": [ + f"boto3-stubs[s3]~={_boto3_version}", + "moto~=4.2", + "mypy~=1.8", + "pytest~=7.4", + "responses~=0.24", + "types-requests~=2.31", + "types-PyYAML~=6.0", + ] +} +_keywords = ["dataset", "granule", "nasa", "MAAP", "CMR"] +_license = "Apache License, Version 2.0" +_long_description = "Python client API for interacting with the NASA MAAP API" +_name = "maap-py" +_namespaces: list[str] = [] +_test_suite = "" +_url = "https://github.com/MAAP-Project/maap-py" +_version = "3.1.5" _zip_safe = False # Setup Metadata @@ -42,12 +67,11 @@ def _read(*rnames): return open(os.path.join(os.path.dirname(__file__), *rnames)).read() -_header = '*' * len(_name) + '\n' + _name + '\n' + '*' * len(_name) -_longDescription = '\n\n'.join([ - _header, - _read('README.md') -]) -open('doc.txt', 'w').write(_longDescription) +_header = "*" * len(_name) + "\n" + _name + "\n" + "*" * len(_name) +_longDescription = "\n\n".join([_header, _read("README.md")]) + +with open("doc.txt", "w") as doc: + doc.write(_longDescription) setup( author=_author, @@ -56,9 +80,9 @@ def _read(*rnames): description=_description, download_url=_download_url, include_package_data=True, + setup_requires=["pytest-runner"], install_requires=_requirements, - setup_requires=['pytest-runner'], - tests_require=['pytest', 'responses', 'moto'], + extras_require=_extra_requirements, keywords=_keywords, license=_license, long_description=_long_description, diff --git a/test/test_MAAP.py b/test/test_MAAP.py index b057260..5dcc082 100644 --- a/test/test_MAAP.py +++ b/test/test_MAAP.py @@ -1,6 +1,5 @@ from unittest import TestCase from maap.maap import MAAP -from maap.utils.TokenHandler import TokenHandler from unittest.mock import MagicMock import re @@ -64,11 +63,6 @@ def test_genFromEarthdata(self): 'data_center="MAAP Data Management Team", '\ 'bounding_box="-35.4375,-55.6875,-80.4375,37.6875")') - def test_TokenHandler(self): - th = TokenHandler("a-K9YbTr8h112zW5pLV8Fw") - token = th.get_access_token() - self.assertTrue(token != 'unauthorized' and len(token) > 0) - def test_uploadFiles(self): self.maap._upload_s3 = MagicMock(return_value=None) result = self.maap.uploadFiles(['test/s3-upload-testfile1.txt', 'test/s3-upload-testfile2.txt'])