diff --git a/maap.cfg b/maap.cfg index ace0cf1..47b8a9a 100755 --- a/maap.cfg +++ b/maap.cfg @@ -18,6 +18,8 @@ query_endpoint = https://%(maap_host)s/api/query/ [aws] aws_access_key_id = ${AWS_ACCESS_KEY_ID} aws_secret_access_key = ${AWS_SECRET_ACCESS_KEY} +user_upload_bucket = maap-landing-zone +user_upload_directory = user-added/uploaded_objects [search] indexed_attributes = [ diff --git a/maap/maap.py b/maap/maap.py index 5998018..9a2bba8 100644 --- a/maap/maap.py +++ b/maap/maap.py @@ -2,6 +2,8 @@ import os import requests import json +import boto3 +import uuid import urllib.parse import time from mapboxgl.utils import * @@ -17,6 +19,8 @@ logger = logging.getLogger(__name__) +s3_client = boto3.client('s3') + try: from configparser import ConfigParser except ImportError: @@ -57,6 +61,8 @@ def __init__(self): self._AWS_ACCESS_KEY = os.environ.get("AWS_ACCESS_KEY_ID") or self.config.get("aws", "aws_access_key_id") self._AWS_ACCESS_SECRET = os.environ.get("AWS_SECRET_ACCESS_KEY") or self.config.get("aws", "aws_secret_access_key") + self._S3_USER_UPLOAD_BUCKET = os.environ.get("S3_USER_UPLOAD_BUCKET") or self.config.get("aws", "user_upload_bucket") + self._S3_USER_UPLOAD_DIR = os.environ.get("S3_USER_UPLOAD_DIR") or self.config.get("aws", "user_upload_directory") self._MAPBOX_TOKEN = os.environ.get("MAPBOX_ACCESS_TOKEN") or '' self._INDEXED_ATTRIBUTES = json.loads(self.config.get("search", "indexed_attributes")) @@ -136,6 +142,16 @@ def _get_search_results(self, url, limit, **kwargs): page_num += 1 return results + def _upload_s3(self, filename, bucket, objectKey): + """ + Upload file to S3, utility function useful for mocking in tests. + :param filename (string) - local filename (and path) + :param bucket (string) - S3 bucket to upload to + :param objectKey (string) - S3 directory and filename to upload the local file to + :return: S3 upload_file response + """ + s3_client.upload_file(filename, bucket, objectKey) + def searchGranule(self, limit=20, **kwargs): """ Search the CMR granules @@ -200,6 +216,22 @@ def getJobStatus(self, jobid): ) return response + def uploadFiles(self, filenames): + """ + Uploads files to a user-added staging directory. + Enables users of maap-py to potentially share files generated on the MAAP. + :param filenames: List of one or more filenames to upload + :return: String message including UUID of subdirectory of files + """ + bucket = self._S3_USER_UPLOAD_BUCKET + prefix = self._S3_USER_UPLOAD_DIR + uuid_dir = uuid.uuid4() + # TODO(aimee): This should upload to a user-namespaced directory + for filename in filenames: + basename = os.path.basename(filename) + response = self._upload_s3(filename, bucket, f"{prefix}/{uuid_dir}/{basename}") + return f"Upload file subdirectory: {uuid_dir} (keep a record of this if you want to share these files with other users)" + def executeQuery(self, src, query={}, poll_results=True, timeout=180, wait_interval=.5, max_redirects=5): """ Helper to execute query and poll results URL until results are returned diff --git a/test/s3-upload-testfile1.txt b/test/s3-upload-testfile1.txt new file mode 100644 index 0000000..bc7774a --- /dev/null +++ b/test/s3-upload-testfile1.txt @@ -0,0 +1 @@ +hello world! \ No newline at end of file diff --git a/test/s3-upload-testfile2.txt b/test/s3-upload-testfile2.txt new file mode 100644 index 0000000..1cd2973 --- /dev/null +++ b/test/s3-upload-testfile2.txt @@ -0,0 +1 @@ +hi again! \ No newline at end of file diff --git a/test/test_MAAP.py b/test/test_MAAP.py index 3a938e9..35ca2b7 100644 --- a/test/test_MAAP.py +++ b/test/test_MAAP.py @@ -1,11 +1,13 @@ from unittest import TestCase from maap.maap import MAAP from maap.utils.TokenHandler import TokenHandler - +from unittest.mock import MagicMock +import re class TestMAAP(TestCase): @classmethod def setUpClass(cls): + config_file_path = "./maap.cfg" cls.maap = MAAP() @@ -117,6 +119,12 @@ def test_TokenHandler(self): token = th.get_access_token() self.assertTrue(token != 'unauthorized' and len(token) > 0) + def test_uploadFiles(self): + self.maap._upload_s3 = MagicMock(return_value=None) + result = self.maap.uploadFiles(['test/s3-upload-testfile1.txt', 'test/s3-upload-testfile2.txt']) + upload_msg_regex = re.compile('Upload file subdirectory: .+ \\(keep a record of this if you want to share these files with other users\\)') + self.assertTrue(re.match(upload_msg_regex, result)) + def test_executeQuery(self): response = self.maap.executeQuery( src={