Merge pull request #4 from MAAP-Project/abarciauskas-bgse_add-file-up…

…load

Add files upload to S3

maap.cfg

@@ -18,6 +18,8 @@ query_endpoint = https://%(maap_host)s/api/query/
 [aws]
 aws_access_key_id = ${AWS_ACCESS_KEY_ID}
 aws_secret_access_key = ${AWS_SECRET_ACCESS_KEY}
+user_upload_bucket = maap-landing-zone
+user_upload_directory = user-added/uploaded_objects
 
 [search]
 indexed_attributes = [

maap/maap.py

@@ -2,6 +2,8 @@
 import os
 import requests
 import json
+import boto3
+import uuid
 import urllib.parse
 import time
 from mapboxgl.utils import *
@@ -17,6 +19,8 @@
 
 logger = logging.getLogger(__name__)
 
+s3_client = boto3.client('s3')
+
 try:
     from configparser import ConfigParser
 except ImportError:
@@ -57,6 +61,8 @@ def __init__(self):
 
         self._AWS_ACCESS_KEY = os.environ.get("AWS_ACCESS_KEY_ID") or self.config.get("aws", "aws_access_key_id")
         self._AWS_ACCESS_SECRET = os.environ.get("AWS_SECRET_ACCESS_KEY") or self.config.get("aws", "aws_secret_access_key")
+        self._S3_USER_UPLOAD_BUCKET = os.environ.get("S3_USER_UPLOAD_BUCKET") or self.config.get("aws", "user_upload_bucket")
+        self._S3_USER_UPLOAD_DIR = os.environ.get("S3_USER_UPLOAD_DIR") or self.config.get("aws", "user_upload_directory")
         self._MAPBOX_TOKEN = os.environ.get("MAPBOX_ACCESS_TOKEN") or ''
         self._INDEXED_ATTRIBUTES = json.loads(self.config.get("search", "indexed_attributes"))
 
@@ -136,6 +142,16 @@ def _get_search_results(self, url, limit, **kwargs):
                 page_num += 1
         return results
 
+    def _upload_s3(self, filename, bucket, objectKey):
+        """
+        Upload file to S3, utility function useful for mocking in tests.
+        :param filename (string) - local filename (and path)
+        :param bucket (string) - S3 bucket to upload to
+        :param objectKey (string) - S3 directory and filename to upload the local file to
+        :return: S3 upload_file response
+        """
+        s3_client.upload_file(filename, bucket, objectKey)
+
     def searchGranule(self, limit=20, **kwargs):
         """
             Search the CMR granules
@@ -200,6 +216,22 @@ def getJobStatus(self, jobid):
         )
         return response
 
+    def uploadFiles(self, filenames):
+        """
+        Uploads files to a user-added staging directory.
+        Enables users of maap-py to potentially share files generated on the MAAP.
+        :param filenames: List of one or more filenames to upload
+        :return: String message including UUID of subdirectory of files
+        """
+        bucket = self._S3_USER_UPLOAD_BUCKET
+        prefix = self._S3_USER_UPLOAD_DIR
+        uuid_dir = uuid.uuid4()
+        # TODO(aimee): This should upload to a user-namespaced directory
+        for filename in filenames:
+            basename = os.path.basename(filename)
+            response = self._upload_s3(filename, bucket, f"{prefix}/{uuid_dir}/{basename}")
+        return f"Upload file subdirectory: {uuid_dir} (keep a record of this if you want to share these files with other users)"
+
     def executeQuery(self, src, query={}, poll_results=True, timeout=180, wait_interval=.5, max_redirects=5):
         """
         Helper to execute query and poll results URL until results are returned

test/s3-upload-testfile1.txt

@@ -0,0 +1 @@
+hello world!

test/s3-upload-testfile2.txt

@@ -0,0 +1 @@
+hi again!

test/test_MAAP.py

@@ -1,11 +1,13 @@
 from unittest import TestCase
 from maap.maap import MAAP
 from maap.utils.TokenHandler import TokenHandler
-
+from unittest.mock import MagicMock
+import re
 
 class TestMAAP(TestCase):
     @classmethod
     def setUpClass(cls):
+        config_file_path = "./maap.cfg"
 
         cls.maap = MAAP()
 
@@ -117,6 +119,12 @@ def test_TokenHandler(self):
         token = th.get_access_token()
         self.assertTrue(token != 'unauthorized' and len(token) > 0)
 
+    def test_uploadFiles(self):
+        self.maap._upload_s3 = MagicMock(return_value=None)
+        result = self.maap.uploadFiles(['test/s3-upload-testfile1.txt', 'test/s3-upload-testfile2.txt'])
+        upload_msg_regex = re.compile('Upload file subdirectory: .+ \\(keep a record of this if you want to share these files with other users\\)')
+        self.assertTrue(re.match(upload_msg_regex, result))
+
     def test_executeQuery(self):
         response = self.maap.executeQuery(
             src={