[Issue 2512] S3 Presign URL

api/local.env

@@ -109,3 +109,6 @@ IS_LOCAL_FOREIGN_TABLE=true
 
 # File path for the export_opportunity_data task
 EXPORT_OPP_DATA_FILE_PATH=/tmp
+
+# This env var is used to set local AWS credentials
+IS_LOCAL_AWS=1

api/openapi.generated.yml

@@ -2036,9 +2036,9 @@ components:
     OpportunityAttachmentV1:
       type: object
       properties:
-        file_location:
+        download_path:
           type: string
-          description: The URL to download the attachment
+          description: The pre-signed s3 URL to download the attachment
           example: https://...
         mime_type:
           type: string

api/src/adapters/aws/s3_adapter.py

@@ -8,6 +8,7 @@ class S3Config(PydanticBaseEnvConfig):
     # We should generally not need to set this except
     # locally to use localstack
     s3_endpoint_url: str | None = None
+    presigned_s3_duration: int = 1800
 
     ### S3 Buckets
     # note that we default these to None

api/src/api/opportunities_v1/opportunity_schemas.py

@@ -318,9 +318,9 @@ class OpportunityV1Schema(Schema):
 
 
 class OpportunityAttachmentV1Schema(Schema):
-    file_location = fields.String(
+    download_path = fields.String(
         metadata={
-            "description": "The URL to download the attachment",
+            "description": "The pre-signed s3 URL to download the attachment",
             "example": "https://...",
         }
     )

api/src/services/opportunities_v1/get_opportunity.py

@@ -1,13 +1,18 @@
+import os
 from datetime import date
+from typing import List
 
+import boto3
 from sqlalchemy import select
 from sqlalchemy.orm import noload, selectinload
 
 import src.adapters.db as db
 import src.util.datetime_util as datetime_util
+from src.adapters.aws import S3Config, get_s3_client
 from src.api.route_utils import raise_flask_error
 from src.db.models.agency_models import Agency
-from src.db.models.opportunity_models import Opportunity, OpportunitySummary
+from src.db.models.opportunity_models import Opportunity, OpportunityAttachment, OpportunitySummary
+from src.util.file_util import split_s3_url
 
 
 def _fetch_opportunity(
@@ -35,8 +40,41 @@ def _fetch_opportunity(
     return opportunity
 
 
+def get_boto_session() -> boto3.Session:
+    is_local = bool(os.getenv("IS_LOCAL_AWS", False))
+    if is_local:
+        return boto3.Session(aws_access_key_id="NO_CREDS", aws_secret_access_key="NO_CREDS")
+
+    return boto3.Session()
+
+
+def pre_sign_opportunity_file_location(
+    opp_atts: List,
+) -> List[OpportunityAttachment]:
+    s3_config = S3Config()
+
+    s3_client = get_s3_client(s3_config, get_boto_session())
+    for opp_att in opp_atts:
+        file_loc = opp_att.file_location
+        bucket, key = split_s3_url(file_loc)
+        pre_sign_file_loc = s3_client.generate_presigned_url(
+            "get_object",
+            Params={"Bucket": bucket, "Key": key},
+            ExpiresIn=s3_config.presigned_s3_duration,
+        )
+        opp_att.download_path = pre_sign_file_loc
+
+    return opp_atts
+
+
 def get_opportunity(db_session: db.Session, opportunity_id: int) -> Opportunity:
-    return _fetch_opportunity(db_session, opportunity_id, load_all_opportunity_summaries=False)
+    opportunity = _fetch_opportunity(
+        db_session, opportunity_id, load_all_opportunity_summaries=False
+    )
+
+    pre_sign_opportunity_file_location(opportunity.opportunity_attachments)
+
+    return opportunity
 
 
 def get_opportunity_versions(db_session: db.Session, opportunity_id: int) -> dict:

api/tests/conftest.py

@@ -1,4 +1,6 @@
 import logging
+import os
+import pathlib
 import uuid
 
 import _pytest.monkeypatch
@@ -50,6 +52,26 @@ def test_example(monkeypatch):
     load_local_env_vars()
 
 
+### Uploads test files
+@pytest.fixture
+def upload_opportunity_attachment_s3(reset_aws_env_vars, mock_s3_bucket):
+    s3_client = boto3.client("s3")
+    test_folder_path = (
+        pathlib.Path(__file__).parent.resolve() / "lib/opportunity_attachment_test_files"
+    )
+
+    for root, _, files in os.walk(test_folder_path):
+        for file in files:
+            file_path = os.path.join(root, file)
+            s3_client.upload_file(
+                file_path, Bucket=mock_s3_bucket, Key=os.path.relpath(file_path, test_folder_path)
+            )
+
+    # Check files were uploaded to mock s3
+    s3_files = s3_client.list_objects_v2(Bucket=mock_s3_bucket)
+    assert len(s3_files["Contents"]) == 5
+
+
 ####################
 # Test DB session
 ####################
@@ -244,6 +266,7 @@ def reset_aws_env_vars(monkeypatch):
     monkeypatch.setenv("AWS_SECURITY_TOKEN", "testing")
     monkeypatch.setenv("AWS_SESSION_TOKEN", "testing")
     monkeypatch.setenv("AWS_DEFAULT_REGION", "us-east-1")
+    monkeypatch.delenv("S3_ENDPOINT_URL")
 
 
 @pytest.fixture

api/tests/lib/opportunity_attachment_test_files/test_file_1.txt

@@ -0,0 +1 @@
+Hello, world

api/tests/lib/opportunity_attachment_test_files/test_file_2.txt

@@ -0,0 +1 @@
+Hello, world - again!

api/tests/lib/opportunity_attachment_test_files/test_file_3.txt

@@ -0,0 +1 @@
+Hello, motto

api/tests/lib/seed_local_db.py

@@ -1,13 +1,18 @@
 import logging
+import os
+import pathlib
 import random
 
+import boto3
 import click
+from botocore.exceptions import ClientError
 from sqlalchemy import func
 
 import src.adapters.db as db
 import src.logging
 import src.util.datetime_util as datetime_util
 import tests.src.db.models.factories as factories
+from src.adapters.aws import S3Config, get_s3_client
 from src.adapters.db import PostgresDBClient
 from src.db.models.agency_models import Agency
 from src.db.models.opportunity_models import Opportunity
@@ -16,6 +21,30 @@
 
 logger = logging.getLogger(__name__)
 
+TESTS_FOLDER = pathlib.Path(__file__).parent.resolve()
+
+
+def _upload_opportunity_attachments_s3():
+    s3_config = S3Config()
+    s3_client = get_s3_client(
+        s3_config, boto3.Session(aws_access_key_id="NO_CREDS", aws_secret_access_key="NO_CREDS")
+    )
+    test_folder_path = TESTS_FOLDER / "opportunity_attachment_test_files"
+
+    for root, _, files in os.walk(test_folder_path):
+        for file in files:
+            file_path = os.path.join(root, file)
+            object_name = os.path.relpath(file_path, test_folder_path)
+
+        try:
+            s3_client.upload_file(file_path, s3_config.s3_opportunity_bucket, object_name)
+            logger.info("Successfully uploaded files")
+        except ClientError as e:
+            logger.error(
+                "Error uploading to s3: %s",
+                extra={"object_name": object_name, "file_path": file_path, "error": e},
+            )
+
 
 def _add_history(
     opps: list[Opportunity],
@@ -173,6 +202,8 @@ def seed_local_db(iterations: int, include_history: bool) -> None:
         logger.info("Running seed script for local DB")
         error_if_not_local()
 
+        _upload_opportunity_attachments_s3()
+
         db_client = PostgresDBClient()
 
         with db_client.get_session() as db_session:

api/tests/src/api/opportunities_v1/test_opportunity_route_get.py

@@ -1,11 +1,13 @@
 import pytest
+import requests
 
 from tests.src.api.opportunities_v1.conftest import (
     validate_opportunity,
     validate_opportunity_with_attachments,
 )
 from tests.src.db.models.factories import (
     CurrentOpportunitySummaryFactory,
+    OpportunityAttachmentFactory,
     OpportunityFactory,
     OpportunitySummaryFactory,
 )
@@ -47,7 +49,6 @@ def test_get_opportunity_200(
     db_opportunity = OpportunityFactory.create(
         **opportunity_params, current_opportunity_summary=None
     )  # We'll set the current opportunity below
-
     if opportunity_summary_params is not None:
         db_opportunity_summary = OpportunitySummaryFactory.create(
             **opportunity_summary_params, opportunity=db_opportunity
@@ -70,8 +71,6 @@ def test_get_opportunity_with_attachment_200(
 ):
     # Create an opportunity with an attachment
     opportunity = OpportunityFactory.create()
-
-    # Ensure the opportunity is committed to the database
     db_session.commit()
 
     # Make the GET request
@@ -88,6 +87,32 @@ def test_get_opportunity_with_attachment_200(
     validate_opportunity_with_attachments(opportunity, response_data)
 
 
+def test_get_opportunity_s3_endpoint_url_200(
+    upload_opportunity_attachment_s3, client, api_auth_token, enable_factory_create, db_session
+):
+    # Create an opportunity with a specific attachment
+    opportunity = OpportunityFactory.create(opportunity_attachments=[])
+    bucket = "test_bucket"
+    object_name = "test_file_1.txt"
+    file_loc = f"s3://{bucket}/{object_name}"
+    OpportunityAttachmentFactory.create(file_location=file_loc, opportunity=opportunity)
+
+    # Make the GET request
+    resp = client.get(
+        f"/v1/opportunities/{opportunity.opportunity_id}", headers={"X-Auth": api_auth_token}
+    )
+
+    # Check the response
+    assert resp.status_code == 200
+    response_data = resp.get_json()["data"]
+    presigned_url = response_data["attachments"][0]["download_path"]
+
+    # Validate pre-signed url
+    response = requests.get(presigned_url)
+    assert response.status_code == 200
+    assert response.text == "Hello, world"
+
+
 def test_get_opportunity_404_not_found(client, api_auth_token, truncate_opportunities):
     resp = client.get("/v1/opportunities/1", headers={"X-Auth": api_auth_token})
     assert resp.status_code == 404

api/tests/src/db/models/factories.py

@@ -147,6 +147,14 @@ class CustomProvider(BaseProvider):
 
     YN_YESNO_BOOLEAN_VALUES = ["Y", "N", "Yes", "No"]
 
+    OPPORTUNITY_ATTACHMENT_S3_PATHS = [
+        "s3://local-opportunities/test_file_1.txt",
+        "s3://local-opportunities/test_file_2.txt",
+        "s3://local-opportunities/test_file_3.txt",
+        "s3://local-opportunities/test_file_4.pdf",
+        "s3://local-opportunities/test_file_5.pdf",
+    ]
+
     def agency(self) -> str:
         return self.random_element(self.AGENCIES)
 
@@ -188,6 +196,9 @@ def yn_boolean(self) -> str:
     def yn_yesno_boolean(self) -> str:
         return self.random_element(self.YN_YESNO_BOOLEAN_VALUES)
 
+    def s3_file_location(self) -> str:
+        return self.random_element(self.OPPORTUNITY_ATTACHMENT_S3_PATHS)
+
 
 fake = faker.Faker()
 fake.add_provider(CustomProvider)
@@ -239,7 +250,7 @@ class OpportunityAttachmentFactory(BaseFactory):
     class Meta:
         model = opportunity_models.OpportunityAttachment
 
-    file_location = factory.Faker("url")
+    file_location = factory.Faker("s3_file_location")
     mime_type = factory.Faker("mime_type")
     file_name = factory.Faker("file_name")
     file_description = factory.Faker("sentence")