Release/4.1.0 (#106)

* Remove the need to track maap.cfg file (#93) * Remove the need to track maap.cfg * Fix merge issues * Allow overriding of MAAP_API_HOST * Split config url construction into functions * Add test for config reader * Add script to perform functional test of maap-py * Construct api root url from provided maap host * Apply suggestions from code review Co-authored-by: Chuck Daniels <[email protected]> * Apply suggestions from code review to update url handling Co-authored-by: Chuck Daniels <[email protected]> * Remove unused code and memoize get config function * Update request utils to drop class and use methods * Use new request utils and pass maap config object * Update imports and url handling * Simplify boolean self-signed for request utils * Remove usage of os.path.join for constructing urls * Fix variable name in exception message * Remove incorrect import * Update functional tests --------- Co-authored-by: Chuck Daniels <[email protected]> * added pagination to listJobs endpoint * change request * Issues/95: Add CICD pipeline (#101) * Setup CICD * Setup CICD * Setup CICD * Setup CICD * Setup CICD * Setup CICD * Setup CICD * Setup CICD * Setup CICD * Setup CICD * Setup CICD * Setup CICD * Fetch full repo in checkout for sonar analysis * tweak pr template wording * /version 4.0.1a0 * /version 4.1.0a1 * added more query params (#98) * added more query params * added get_job_details flag * cleanup * Update maap/maap.py Co-authored-by: Chuck Daniels <[email protected]> * Update maap/maap.py Co-authored-by: Chuck Daniels <[email protected]> * Update maap/maap.py Co-authored-by: Chuck Daniels <[email protected]> * added docstring | updated param handling * review updates * review updates * renamed file --------- Co-authored-by: Chuck Daniels <[email protected]> * /version 4.1.0a2 * secret management (#104) * wip * cleanup * updated get_secret endpoint to return value * added tests for secrets management * updated error handling | added logging * review comments * /version 4.1.0a3 * /version 4.2.0a0 * added changelog entry * /version 4.1.0rc1 --------- Co-authored-by: Sujen Shah <[email protected]> Co-authored-by: Chuck Daniels <[email protected]> Co-authored-by: Frank Greguska <[email protected]> Co-authored-by: frankinspace <[email protected]> Co-authored-by: marjo-luc <[email protected]>
MAAP-Project · Sep 12, 2024 · 0fdf235 · 0fdf235
1 parent d3ffa47
commit 0fdf235
Show file tree

Hide file tree

Showing 7 changed files with 283 additions and 6 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -12,6 +12,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Fixed
 ### Security
 
+## [4.1.0]
+### Added
+- [feature/secrets](https://github.com/MAAP-Project/Community/issues/1048): Added support for user secrets management
+- [feature/jobs-list-query](https://github.com/MAAP-Project/maap-api-nasa/issues/130): Added query filter parameters to listJobs endpoint
+- [issues/95](https://github.com/MAAP-Project/maap-py/issues/95): Added CICD pipeline
+- [feature/pagination](https://github.com/MAAP-Project/Community/issues/1027): Added pagination support for listJobs endpoint
+
+### Changed
+- [community-909](https://github.com/MAAP-Project/Community/issues/909): Removed need to track maap.cfg
+
 ## [4.0.1]
 ### Added
 - [issues/95](https://github.com/MAAP-Project/maap-py/issues/95): Added github action workflow for publishing and switched to poetry for builds

diff --git a/maap/Secrets.py b/maap/Secrets.py
@@ -0,0 +1,131 @@
+import requests
+import logging
+import json
+from maap.utils import endpoints
+from maap.utils import requests_utils
+from maap.utils import endpoints
+
+logger = logging.getLogger(__name__)
+
+class Secrets:
+    """
+    Functions used for member secrets API interfacing
+    """
+    def __init__(self, member_endpoint, api_header):
+        self._api_header = api_header
+        self._members_endpoint = f"{member_endpoint}/{endpoints.MEMBERS_SECRETS}"
+
+
+    def get_secrets(self):
+        """
+        Returns a list of secrets for a given user.
+
+        Returns:
+            list: Returns a list of dicts containing secret names e.g. [{'secret_name': 'secret1'}, {'secret_name': 'secret2'}].
+        """
+        try:
+            response = requests.get(
+                url = self._members_endpoint,
+                headers=self._api_header
+            )
+            logger.debug(f"Response from get_secrets request: {response.text}")
+            return json.loads(response.text)
+        except Exception as e:
+            raise(f"Error retrieving secrets: {e}")
+
+
+    def get_secret(self, secret_name):
+        """
+        Returns secret value for provided secret name.
+
+        Args:
+            secret_name (str, required): Secret name.
+
+        Returns:
+            string: Secret value.
+
+        Raises:
+            ValueError: If secret name is not provided.
+        """
+        if secret_name is None:
+            raise ValueError("Secret name parameter cannot be None.")
+
+        try:
+            response = requests.get(
+                url = f"{self._members_endpoint}/{secret_name}",
+                headers=self._api_header
+            )
+
+            # Return secret value directly for user ease-of-use
+            if response.ok:
+                response = response.json()
+                return response["secret_value"]
+
+            logger.debug(f"Response from get_secret request: {response.text}")
+            return json.loads(response.text)
+        except Exception as e:
+            raise(f"Error retrieving secret: {e}")
+
+
+    def add_secret(self, secret_name=None, secret_value=None):
+        """
+        Adds a secret. Secret name must be provided. Secret value may be null.
+
+        Args:
+            secret_name (str, required): Secret name.
+            secret_value (str, optional): Secret value.
+
+        Returns:
+            dict: Containing name and value of secret that was just added.
+
+        Raises:
+            ValueError: If secret name or secret value is not provided.
+        """
+        if secret_name is None or secret_value is None:
+            raise ValueError("Failed to add secret. Secret name and secret value must not be 'None'.")
+
+        try:
+            response = requests.post(
+                url = self._members_endpoint,
+                headers=self._api_header,
+                data=json.dumps({"secret_name": secret_name, "secret_value": secret_value})
+            )
+
+            logger.debug(f"Response from add_secret: {response.text}")
+            return json.loads(response.text)
+        except Exception as e:
+            raise(f"Error adding secret: {e}")
+
+
+    def delete_secret(self, secret_name=None):
+        """
+        Deletes a secret.
+
+        Args:
+            secret_name (str, required): Secret name.
+
+        Returns:
+            dict: Containing response code and message indicating whether or not deletion was successful.
+
+        Raises:
+            ValueError: If secret name is not provided.
+        """
+        if secret_name is None:
+            raise ValueError("Failed to delete secret. Please provide secret name.")
+
+        try:
+            response = requests.delete(
+                url = f"{self._members_endpoint}/{secret_name}",
+                headers=self._api_header
+            )
+
+            logger.debug(f"Response from delete_secret: {response.text}")
+            return json.loads(response.text)
+        except Exception as e:
+            raise(f"Error deleting secret: {e}")
+
+
+
+
+
+
diff --git a/maap/maap.py b/maap/maap.py
@@ -17,8 +17,10 @@
 from maap.utils import algorithm_utils
 from maap.Profile import Profile
 from maap.AWS import AWS
+from maap.Secrets import Secrets
 from maap.dps.DpsHelper import DpsHelper
 from maap.utils import endpoints
+from maap.utils import job
 
 logger = logging.getLogger(__name__)
 
@@ -40,10 +42,11 @@ def __init__(self, maap_host=os.getenv('MAAP_API_HOST', 'api.maap-project.org'))
             self.config.workspace_bucket_credentials,
             self._get_api_header()
         )
+        self.secrets = Secrets(self.config.member, self._get_api_header(content_type="application/json"))
 
     def _get_api_header(self, content_type=None):
 
-        api_header = {'Accept': content_type if content_type else self.config.content_type, 'token': self.config.maap_token}
+        api_header = {'Accept': content_type if content_type else self.config.content_type, 'token': self.config.maap_token, 'Content-Type': content_type if content_type else self.config.content_type}
 
         if os.environ.get("MAAP_PGT"):
             api_header['proxy-ticket'] = os.environ.get("MAAP_PGT")
@@ -256,17 +259,92 @@ def cancelJob(self, jobid):
         job.id = jobid
         return job.cancel_job()
 
-    def listJobs(self, username=None):
-        if username==None and self.profile is not None and 'username' in self.profile.account_info().keys():
+    def listJobs(self, username=None, *,
+                       algo_id=None, 
+                       end_time=None, 
+                       get_job_details=True, 
+                       offset=0, 
+                       page_size=10, 
+                       queue=None,
+                       start_time=None,
+                       status=None,
+                       tag=None, 
+                       version=None):
+        """
+        Returns a list of jobs for a given user that matches query params provided.
+
+        Args:
+            username (str, optional): Platform user. If no username is provided, the profile username will be used.
+            algo_id (str, optional): Algorithm type.
+            end_time (str, optional): Specifying this parameter will return all jobs that have completed from the provided end time to now. e.g. 2024-01-01 or 2024-01-01T00:00:00.000000Z.
+            get_job_details (bool, optional): Flag that determines whether to return a detailed job list or a compact list containing just the job ids and their associated job tags. Default is True.
+            offset (int, optional): Offset for pagination. Default is 0.
+            page_size (int, optional): Page size for pagination. Default is 10.
+            queue (str, optional): Job processing resource.
+            start_time (str, optional): Specifying this parameter will return all jobs that have started from the provided start time to now. e.g. 2024-01-01 or 2024-01-01T00:00:00.000000Z.
+            status (str, optional): Job status, e.g. job-completed, job-failed, job-started, job-queued.
+            tag (str, optional): User job tag/identifier.
+            version (str, optional): Algorithm version, e.g. GitHub branch or tag.
+
+        Returns:
+            list: List of jobs for a given user that matches query params provided.
+
+        Raises:
+            ValueError: If username is not provided and cannot be obtained from the user's profile.
+            ValueError: If either algo_id or version is provided, but not both.
+        """
+        if username is None and self.profile is not None and 'username' in self.profile.account_info().keys():
             username = self.profile.account_info()['username']
-        url = os.path.join(self.config.dps_job, username, endpoints.DPS_JOB_LIST)
+
+        if username is None:
+            raise ValueError("Unable to determine username from profile. Please provide a username.")
+
+        url = "/".join(
+            segment.strip("/")
+            for segment in (self.config.dps_job, username, endpoints.DPS_JOB_LIST)
+        )
+
+        params = {
+            k: v
+            for k, v in (
+                ("algo_id", algo_id),
+                ("end_time", end_time),
+                ("get_job_details", get_job_details),
+                ("offset", offset),
+                ("page_size", page_size),
+                ("queue", queue),
+                ("start_time", start_time),
+                ("status", status),
+                ("tag", tag),
+                ("username", username),
+                ("version", version),
+            )
+            if v is not None
+        }
+
+        if (not algo_id) != (not version):
+            # Either algo_id or version was supplied as a non-empty string, but not both.
+            # Either both must be non-empty strings or both must be None.
+            raise ValueError("Either supply non-empty strings for both algo_id and version, or supply neither.")
+
+        # DPS requests use 'job_type', which is a concatenation of 'algo_id' and 'version'
+        if algo_id and version:
+            params['job_type'] = f"{algo_id}:{version}"
+
+        algo_id = params.pop('algo_id', None)
+        version = params.pop('version', None)
+
+        if status is not None:
+            params['status'] = job.validate_job_status(status)
+
         headers = self._get_api_header()
         logger.debug('GET request sent to {}'.format(url))
         logger.debug('headers:')
         logger.debug(headers)
         response = requests.get(
             url=url,
-            headers=headers
+            headers=headers,
+            params=params,
         )
         return response
 

diff --git a/maap/utils/endpoints.py b/maap/utils/endpoints.py
@@ -3,3 +3,5 @@
 DPS_JOB_DISMISS = "cancel"
 DPS_JOB_LIST = "list"
 CMR_ALGORITHM_DATA = "data"
+
+MEMBERS_SECRETS = "secrets"
diff --git a/maap/utils/job.py b/maap/utils/job.py
@@ -0,0 +1,21 @@
+# Valid job statuses (loosely based on OGC job status types)
+JOB_STATUSES = {'Accepted', 'Running', 'Succeeded', 'Failed', 'Dismissed', 'Deduped', 'Offline'}
+
+def validate_job_status(status):
+    '''
+    Validates job status
+
+    Args:
+        status (str): Job status. Accepted values are: 'Accepted', 'Running', 'Succeeded', 'Failed, 'Dismissed', 'Deduped', and 'Offline'.
+
+    Returns:
+        status (str): Returns unmodified job status if job status is valid.
+
+    Raises: 
+        ValueError: If invalid job status is provided.
+    '''
+    if status not in JOB_STATUSES:
+        valid_statuses = ", ".join(str(status) for status in JOB_STATUSES)
+        raise ValueError("Invalid job status: '{}'. Job status must be one of the following: {}".format(status, valid_statuses))
+
+    return status
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "maap-py"
-version = "4.0.1"
+version = "4.1.0rc1"
 description = "Python client API for interacting with the NASA MAAP API"
 repository = "https://github.com/MAAP-Project/maap-py"
 authors = ["Jet Propulsion Laboratory <[email protected]>"]

diff --git a/test/functional_test.py b/test/functional_test.py
@@ -108,6 +108,31 @@ def cancel_job(maap: MAAP, job_id):
     assert resp is not None
     assert 'Accepted' in str(resp)
 
+@log_decorator
+def add_secret(maap: MAAP, secret_name=None, secret_value=None):
+    resp = maap.secrets.add_secret(secret_name, secret_value)
+    print(resp)
+    assert resp is not None
+
+@log_decorator
+def get_secrets(maap: MAAP):
+    resp = maap.secrets.get_secrets()
+    print(resp)
+    assert resp is not None
+
+@log_decorator
+def get_secret(maap: MAAP, secret_name=None, secret_value=None):
+    resp = maap.secrets.get_secret(secret_name)
+    print(resp)
+    assert resp is not None
+    assert resp == secret_value
+
+@log_decorator
+def delete_secret(maap: MAAP, secret_name=None):
+    resp = maap.secrets.delete_secret(secret_name)
+    print(resp)
+    assert resp is not None
+
 
 def main():
     if os.environ.get('MAAP_PGT') is None:
@@ -118,9 +143,19 @@ def main():
     # list_algorithms(maap)
     job = submit_job(maap, queue="maap-dps-sandbox")
     cancel_job(maap, job.id)
+
+    # Test secrets management
+    secret_name = "test_secret"
+    secret_value = "test_value"
+    get_secrets(maap)
+    add_secret(maap, secret_name, secret_value)
+    get_secret(maap, secret_name, secret_value)
+    delete_secret(maap, secret_name)
+
     # submit_job(maap, wait_for_completion=True)
     # delete_algorithm(maap, "maap_functional_test_algo:main")
 
 
+
 if __name__ == '__main__':
     main()