From 0fdf2355fa529fd369cd624ca7e075f58c126783 Mon Sep 17 00:00:00 2001 From: Marjorie Lucas <47004511+marjo-luc@users.noreply.github.com> Date: Thu, 12 Sep 2024 15:35:48 -0700 Subject: [PATCH] Release/4.1.0 (#106) * Remove the need to track maap.cfg file (#93) * Remove the need to track maap.cfg * Fix merge issues * Allow overriding of MAAP_API_HOST * Split config url construction into functions * Add test for config reader * Add script to perform functional test of maap-py * Construct api root url from provided maap host * Apply suggestions from code review Co-authored-by: Chuck Daniels * Apply suggestions from code review to update url handling Co-authored-by: Chuck Daniels * Remove unused code and memoize get config function * Update request utils to drop class and use methods * Use new request utils and pass maap config object * Update imports and url handling * Simplify boolean self-signed for request utils * Remove usage of os.path.join for constructing urls * Fix variable name in exception message * Remove incorrect import * Update functional tests --------- Co-authored-by: Chuck Daniels * added pagination to listJobs endpoint * change request * Issues/95: Add CICD pipeline (#101) * Setup CICD * Setup CICD * Setup CICD * Setup CICD * Setup CICD * Setup CICD * Setup CICD * Setup CICD * Setup CICD * Setup CICD * Setup CICD * Setup CICD * Fetch full repo in checkout for sonar analysis * tweak pr template wording * /version 4.0.1a0 * /version 4.1.0a1 * added more query params (#98) * added more query params * added get_job_details flag * cleanup * Update maap/maap.py Co-authored-by: Chuck Daniels * Update maap/maap.py Co-authored-by: Chuck Daniels * Update maap/maap.py Co-authored-by: Chuck Daniels * added docstring | updated param handling * review updates * review updates * renamed file --------- Co-authored-by: Chuck Daniels * /version 4.1.0a2 * secret management (#104) * wip * cleanup * updated get_secret endpoint to return value * added tests for secrets management * updated error handling | added logging * review comments * /version 4.1.0a3 * /version 4.2.0a0 * added changelog entry * /version 4.1.0rc1 --------- Co-authored-by: Sujen Shah Co-authored-by: Chuck Daniels Co-authored-by: Frank Greguska <89428916+frankinspace@users.noreply.github.com> Co-authored-by: frankinspace Co-authored-by: marjo-luc --- CHANGELOG.md | 10 +++ maap/Secrets.py | 131 ++++++++++++++++++++++++++++++++++++++++ maap/maap.py | 88 +++++++++++++++++++++++++-- maap/utils/endpoints.py | 2 + maap/utils/job.py | 21 +++++++ pyproject.toml | 2 +- test/functional_test.py | 35 +++++++++++ 7 files changed, 283 insertions(+), 6 deletions(-) create mode 100644 maap/Secrets.py create mode 100644 maap/utils/job.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b39d4f6..2e05a83 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed ### Security +## [4.1.0] +### Added +- [feature/secrets](https://github.com/MAAP-Project/Community/issues/1048): Added support for user secrets management +- [feature/jobs-list-query](https://github.com/MAAP-Project/maap-api-nasa/issues/130): Added query filter parameters to listJobs endpoint +- [issues/95](https://github.com/MAAP-Project/maap-py/issues/95): Added CICD pipeline +- [feature/pagination](https://github.com/MAAP-Project/Community/issues/1027): Added pagination support for listJobs endpoint + +### Changed +- [community-909](https://github.com/MAAP-Project/Community/issues/909): Removed need to track maap.cfg + ## [4.0.1] ### Added - [issues/95](https://github.com/MAAP-Project/maap-py/issues/95): Added github action workflow for publishing and switched to poetry for builds diff --git a/maap/Secrets.py b/maap/Secrets.py new file mode 100644 index 0000000..51f75cf --- /dev/null +++ b/maap/Secrets.py @@ -0,0 +1,131 @@ +import requests +import logging +import json +from maap.utils import endpoints +from maap.utils import requests_utils +from maap.utils import endpoints + +logger = logging.getLogger(__name__) + +class Secrets: + """ + Functions used for member secrets API interfacing + """ + def __init__(self, member_endpoint, api_header): + self._api_header = api_header + self._members_endpoint = f"{member_endpoint}/{endpoints.MEMBERS_SECRETS}" + + + def get_secrets(self): + """ + Returns a list of secrets for a given user. + + Returns: + list: Returns a list of dicts containing secret names e.g. [{'secret_name': 'secret1'}, {'secret_name': 'secret2'}]. + """ + try: + response = requests.get( + url = self._members_endpoint, + headers=self._api_header + ) + logger.debug(f"Response from get_secrets request: {response.text}") + return json.loads(response.text) + except Exception as e: + raise(f"Error retrieving secrets: {e}") + + + def get_secret(self, secret_name): + """ + Returns secret value for provided secret name. + + Args: + secret_name (str, required): Secret name. + + Returns: + string: Secret value. + + Raises: + ValueError: If secret name is not provided. + """ + if secret_name is None: + raise ValueError("Secret name parameter cannot be None.") + + try: + response = requests.get( + url = f"{self._members_endpoint}/{secret_name}", + headers=self._api_header + ) + + # Return secret value directly for user ease-of-use + if response.ok: + response = response.json() + return response["secret_value"] + + logger.debug(f"Response from get_secret request: {response.text}") + return json.loads(response.text) + except Exception as e: + raise(f"Error retrieving secret: {e}") + + + def add_secret(self, secret_name=None, secret_value=None): + """ + Adds a secret. Secret name must be provided. Secret value may be null. + + Args: + secret_name (str, required): Secret name. + secret_value (str, optional): Secret value. + + Returns: + dict: Containing name and value of secret that was just added. + + Raises: + ValueError: If secret name or secret value is not provided. + """ + if secret_name is None or secret_value is None: + raise ValueError("Failed to add secret. Secret name and secret value must not be 'None'.") + + try: + response = requests.post( + url = self._members_endpoint, + headers=self._api_header, + data=json.dumps({"secret_name": secret_name, "secret_value": secret_value}) + ) + + logger.debug(f"Response from add_secret: {response.text}") + return json.loads(response.text) + except Exception as e: + raise(f"Error adding secret: {e}") + + + def delete_secret(self, secret_name=None): + """ + Deletes a secret. + + Args: + secret_name (str, required): Secret name. + + Returns: + dict: Containing response code and message indicating whether or not deletion was successful. + + Raises: + ValueError: If secret name is not provided. + """ + if secret_name is None: + raise ValueError("Failed to delete secret. Please provide secret name.") + + try: + response = requests.delete( + url = f"{self._members_endpoint}/{secret_name}", + headers=self._api_header + ) + + logger.debug(f"Response from delete_secret: {response.text}") + return json.loads(response.text) + except Exception as e: + raise(f"Error deleting secret: {e}") + + + + + + diff --git a/maap/maap.py b/maap/maap.py index 993fa1c..19514f1 100644 --- a/maap/maap.py +++ b/maap/maap.py @@ -17,8 +17,10 @@ from maap.utils import algorithm_utils from maap.Profile import Profile from maap.AWS import AWS +from maap.Secrets import Secrets from maap.dps.DpsHelper import DpsHelper from maap.utils import endpoints +from maap.utils import job logger = logging.getLogger(__name__) @@ -40,10 +42,11 @@ def __init__(self, maap_host=os.getenv('MAAP_API_HOST', 'api.maap-project.org')) self.config.workspace_bucket_credentials, self._get_api_header() ) + self.secrets = Secrets(self.config.member, self._get_api_header(content_type="application/json")) def _get_api_header(self, content_type=None): - api_header = {'Accept': content_type if content_type else self.config.content_type, 'token': self.config.maap_token} + api_header = {'Accept': content_type if content_type else self.config.content_type, 'token': self.config.maap_token, 'Content-Type': content_type if content_type else self.config.content_type} if os.environ.get("MAAP_PGT"): api_header['proxy-ticket'] = os.environ.get("MAAP_PGT") @@ -256,17 +259,92 @@ def cancelJob(self, jobid): job.id = jobid return job.cancel_job() - def listJobs(self, username=None): - if username==None and self.profile is not None and 'username' in self.profile.account_info().keys(): + def listJobs(self, username=None, *, + algo_id=None, + end_time=None, + get_job_details=True, + offset=0, + page_size=10, + queue=None, + start_time=None, + status=None, + tag=None, + version=None): + """ + Returns a list of jobs for a given user that matches query params provided. + + Args: + username (str, optional): Platform user. If no username is provided, the profile username will be used. + algo_id (str, optional): Algorithm type. + end_time (str, optional): Specifying this parameter will return all jobs that have completed from the provided end time to now. e.g. 2024-01-01 or 2024-01-01T00:00:00.000000Z. + get_job_details (bool, optional): Flag that determines whether to return a detailed job list or a compact list containing just the job ids and their associated job tags. Default is True. + offset (int, optional): Offset for pagination. Default is 0. + page_size (int, optional): Page size for pagination. Default is 10. + queue (str, optional): Job processing resource. + start_time (str, optional): Specifying this parameter will return all jobs that have started from the provided start time to now. e.g. 2024-01-01 or 2024-01-01T00:00:00.000000Z. + status (str, optional): Job status, e.g. job-completed, job-failed, job-started, job-queued. + tag (str, optional): User job tag/identifier. + version (str, optional): Algorithm version, e.g. GitHub branch or tag. + + Returns: + list: List of jobs for a given user that matches query params provided. + + Raises: + ValueError: If username is not provided and cannot be obtained from the user's profile. + ValueError: If either algo_id or version is provided, but not both. + """ + if username is None and self.profile is not None and 'username' in self.profile.account_info().keys(): username = self.profile.account_info()['username'] - url = os.path.join(self.config.dps_job, username, endpoints.DPS_JOB_LIST) + + if username is None: + raise ValueError("Unable to determine username from profile. Please provide a username.") + + url = "/".join( + segment.strip("/") + for segment in (self.config.dps_job, username, endpoints.DPS_JOB_LIST) + ) + + params = { + k: v + for k, v in ( + ("algo_id", algo_id), + ("end_time", end_time), + ("get_job_details", get_job_details), + ("offset", offset), + ("page_size", page_size), + ("queue", queue), + ("start_time", start_time), + ("status", status), + ("tag", tag), + ("username", username), + ("version", version), + ) + if v is not None + } + + if (not algo_id) != (not version): + # Either algo_id or version was supplied as a non-empty string, but not both. + # Either both must be non-empty strings or both must be None. + raise ValueError("Either supply non-empty strings for both algo_id and version, or supply neither.") + + # DPS requests use 'job_type', which is a concatenation of 'algo_id' and 'version' + if algo_id and version: + params['job_type'] = f"{algo_id}:{version}" + + algo_id = params.pop('algo_id', None) + version = params.pop('version', None) + + if status is not None: + params['status'] = job.validate_job_status(status) + headers = self._get_api_header() logger.debug('GET request sent to {}'.format(url)) logger.debug('headers:') logger.debug(headers) response = requests.get( url=url, - headers=headers + headers=headers, + params=params, ) return response diff --git a/maap/utils/endpoints.py b/maap/utils/endpoints.py index 33bc816..8937f8e 100644 --- a/maap/utils/endpoints.py +++ b/maap/utils/endpoints.py @@ -3,3 +3,5 @@ DPS_JOB_DISMISS = "cancel" DPS_JOB_LIST = "list" CMR_ALGORITHM_DATA = "data" + +MEMBERS_SECRETS = "secrets" diff --git a/maap/utils/job.py b/maap/utils/job.py new file mode 100644 index 0000000..549d5e1 --- /dev/null +++ b/maap/utils/job.py @@ -0,0 +1,21 @@ +# Valid job statuses (loosely based on OGC job status types) +JOB_STATUSES = {'Accepted', 'Running', 'Succeeded', 'Failed', 'Dismissed', 'Deduped', 'Offline'} + +def validate_job_status(status): + ''' + Validates job status + + Args: + status (str): Job status. Accepted values are: 'Accepted', 'Running', 'Succeeded', 'Failed, 'Dismissed', 'Deduped', and 'Offline'. + + Returns: + status (str): Returns unmodified job status if job status is valid. + + Raises: + ValueError: If invalid job status is provided. + ''' + if status not in JOB_STATUSES: + valid_statuses = ", ".join(str(status) for status in JOB_STATUSES) + raise ValueError("Invalid job status: '{}'. Job status must be one of the following: {}".format(status, valid_statuses)) + + return status \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 8339458..cf62767 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "maap-py" -version = "4.0.1" +version = "4.1.0rc1" description = "Python client API for interacting with the NASA MAAP API" repository = "https://github.com/MAAP-Project/maap-py" authors = ["Jet Propulsion Laboratory "] diff --git a/test/functional_test.py b/test/functional_test.py index 64a3580..de55154 100644 --- a/test/functional_test.py +++ b/test/functional_test.py @@ -108,6 +108,31 @@ def cancel_job(maap: MAAP, job_id): assert resp is not None assert 'Accepted' in str(resp) +@log_decorator +def add_secret(maap: MAAP, secret_name=None, secret_value=None): + resp = maap.secrets.add_secret(secret_name, secret_value) + print(resp) + assert resp is not None + +@log_decorator +def get_secrets(maap: MAAP): + resp = maap.secrets.get_secrets() + print(resp) + assert resp is not None + +@log_decorator +def get_secret(maap: MAAP, secret_name=None, secret_value=None): + resp = maap.secrets.get_secret(secret_name) + print(resp) + assert resp is not None + assert resp == secret_value + +@log_decorator +def delete_secret(maap: MAAP, secret_name=None): + resp = maap.secrets.delete_secret(secret_name) + print(resp) + assert resp is not None + def main(): if os.environ.get('MAAP_PGT') is None: @@ -118,9 +143,19 @@ def main(): # list_algorithms(maap) job = submit_job(maap, queue="maap-dps-sandbox") cancel_job(maap, job.id) + + # Test secrets management + secret_name = "test_secret" + secret_value = "test_value" + get_secrets(maap) + add_secret(maap, secret_name, secret_value) + get_secret(maap, secret_name, secret_value) + delete_secret(maap, secret_name) + # submit_job(maap, wait_for_completion=True) # delete_algorithm(maap, "maap_functional_test_algo:main") + if __name__ == '__main__': main()