diff --git a/api/local.env b/api/local.env index d6248be61..a93d17aec 100644 --- a/api/local.env +++ b/api/local.env @@ -84,6 +84,9 @@ S3_ENDPOINT_URL=http://localstack:4566 S3_OPPORTUNITY_BUCKET=local-opportunities +# This env var is used to set local AWS credentials +IS_LOCAL_AWS=1 + ############################ # Feature Flags ############################ diff --git a/api/openapi.generated.yml b/api/openapi.generated.yml index 9ccc945b0..91f2f8ccf 100644 --- a/api/openapi.generated.yml +++ b/api/openapi.generated.yml @@ -2036,7 +2036,7 @@ components: OpportunityAttachmentV1: type: object properties: - file_location: + download_path: type: string description: The URL to download the attachment example: https://... diff --git a/api/src/adapters/aws/__init__.py b/api/src/adapters/aws/__init__.py index 3f55ab312..f7484c740 100644 --- a/api/src/adapters/aws/__init__.py +++ b/api/src/adapters/aws/__init__.py @@ -1,3 +1,4 @@ +from .aws_session import get_boto_session from .s3_adapter import S3Config, get_s3_client -__all__ = ["get_s3_client", "S3Config"] +__all__ = ["get_s3_client", "S3Config", "get_boto_session"] diff --git a/api/src/adapters/aws/aws_session.py b/api/src/adapters/aws/aws_session.py new file mode 100644 index 000000000..de781999b --- /dev/null +++ b/api/src/adapters/aws/aws_session.py @@ -0,0 +1,11 @@ +import os + +import boto3 + + +def get_boto_session() -> boto3.Session: + is_local = bool(os.getenv("IS_LOCAL_AWS", False)) + if is_local: + return boto3.Session(aws_access_key_id="NO_CREDS", aws_secret_access_key="NO_CREDS") + + return boto3.Session() diff --git a/api/src/adapters/aws/s3_adapter.py b/api/src/adapters/aws/s3_adapter.py index 4a5f13fb8..6eae7f034 100644 --- a/api/src/adapters/aws/s3_adapter.py +++ b/api/src/adapters/aws/s3_adapter.py @@ -8,6 +8,7 @@ class S3Config(PydanticBaseEnvConfig): # We should generally not need to set this except # locally to use localstack s3_endpoint_url: str | None = None + presigned_s3_duration: int = 1800 ### S3 Buckets # note that we default these to None diff --git a/api/src/api/opportunities_v1/opportunity_schemas.py b/api/src/api/opportunities_v1/opportunity_schemas.py index a66ef8da7..a6b4347cb 100644 --- a/api/src/api/opportunities_v1/opportunity_schemas.py +++ b/api/src/api/opportunities_v1/opportunity_schemas.py @@ -318,7 +318,7 @@ class OpportunityV1Schema(Schema): class OpportunityAttachmentV1Schema(Schema): - file_location = fields.String( + download_path = fields.String( metadata={ "description": "The URL to download the attachment", "example": "https://...", diff --git a/api/src/services/opportunities_v1/get_opportunity.py b/api/src/services/opportunities_v1/get_opportunity.py index c21de45e8..fa1134bae 100644 --- a/api/src/services/opportunities_v1/get_opportunity.py +++ b/api/src/services/opportunities_v1/get_opportunity.py @@ -5,9 +5,11 @@ import src.adapters.db as db import src.util.datetime_util as datetime_util +from src.adapters.aws import S3Config, get_boto_session, get_s3_client from src.api.route_utils import raise_flask_error from src.db.models.agency_models import Agency -from src.db.models.opportunity_models import Opportunity, OpportunitySummary +from src.db.models.opportunity_models import Opportunity, OpportunityAttachment, OpportunitySummary +from src.util.file_util import split_s3_url def _fetch_opportunity( @@ -35,8 +37,39 @@ def _fetch_opportunity( return opportunity +def pre_sign_opportunity_file_location( + opp_atts: list, +) -> list[OpportunityAttachment]: + s3_config = S3Config() + + s3_client = get_s3_client(s3_config, get_boto_session()) + for opp_att in opp_atts: + file_loc = opp_att.file_location + bucket, key = split_s3_url(file_loc) + pre_sign_file_loc = s3_client.generate_presigned_url( + "get_object", + Params={"Bucket": bucket, "Key": key}, + ExpiresIn=s3_config.presigned_s3_duration, + ) + if s3_config.s3_endpoint_url: + # Only relevant when local, due to docker path issues + pre_sign_file_loc = pre_sign_file_loc.replace( + s3_config.s3_endpoint_url, "http://localhost:4566" + ) + + opp_att.download_path = pre_sign_file_loc + + return opp_atts + + def get_opportunity(db_session: db.Session, opportunity_id: int) -> Opportunity: - return _fetch_opportunity(db_session, opportunity_id, load_all_opportunity_summaries=False) + opportunity = _fetch_opportunity( + db_session, opportunity_id, load_all_opportunity_summaries=False + ) + + pre_sign_opportunity_file_location(opportunity.opportunity_attachments) + + return opportunity def get_opportunity_versions(db_session: db.Session, opportunity_id: int) -> dict: diff --git a/api/tests/conftest.py b/api/tests/conftest.py index ba8fde520..d751dad58 100644 --- a/api/tests/conftest.py +++ b/api/tests/conftest.py @@ -1,4 +1,6 @@ import logging +import os +import pathlib import uuid import _pytest.monkeypatch @@ -50,6 +52,26 @@ def test_example(monkeypatch): load_local_env_vars() +### Uploads test files +@pytest.fixture +def upload_opportunity_attachment_s3(reset_aws_env_vars, mock_s3_bucket): + s3_client = boto3.client("s3") + test_folder_path = ( + pathlib.Path(__file__).parent.resolve() / "lib/opportunity_attachment_test_files" + ) + + for root, _, files in os.walk(test_folder_path): + for file in files: + file_path = os.path.join(root, file) + s3_client.upload_file( + file_path, Bucket=mock_s3_bucket, Key=os.path.relpath(file_path, test_folder_path) + ) + + # Check files were uploaded to mock s3 + s3_files = s3_client.list_objects_v2(Bucket=mock_s3_bucket) + assert len(s3_files["Contents"]) == 5 + + #################### # Test DB session #################### @@ -244,6 +266,7 @@ def reset_aws_env_vars(monkeypatch): monkeypatch.setenv("AWS_SECURITY_TOKEN", "testing") monkeypatch.setenv("AWS_SESSION_TOKEN", "testing") monkeypatch.setenv("AWS_DEFAULT_REGION", "us-east-1") + monkeypatch.delenv("S3_ENDPOINT_URL") @pytest.fixture diff --git a/api/tests/lib/opportunity_attachment_test_files/test_file_1.txt b/api/tests/lib/opportunity_attachment_test_files/test_file_1.txt new file mode 100644 index 000000000..dbe9dba55 --- /dev/null +++ b/api/tests/lib/opportunity_attachment_test_files/test_file_1.txt @@ -0,0 +1 @@ +Hello, world \ No newline at end of file diff --git a/api/tests/lib/opportunity_attachment_test_files/test_file_2.txt b/api/tests/lib/opportunity_attachment_test_files/test_file_2.txt new file mode 100644 index 000000000..781090c1a --- /dev/null +++ b/api/tests/lib/opportunity_attachment_test_files/test_file_2.txt @@ -0,0 +1 @@ +Hello, world - again! \ No newline at end of file diff --git a/api/tests/lib/opportunity_attachment_test_files/test_file_3.txt b/api/tests/lib/opportunity_attachment_test_files/test_file_3.txt new file mode 100644 index 000000000..f94a7922b --- /dev/null +++ b/api/tests/lib/opportunity_attachment_test_files/test_file_3.txt @@ -0,0 +1 @@ +Hello, motto \ No newline at end of file diff --git a/api/tests/lib/opportunity_attachment_test_files/test_file_4.pdf b/api/tests/lib/opportunity_attachment_test_files/test_file_4.pdf new file mode 100644 index 000000000..4bb6fbe25 Binary files /dev/null and b/api/tests/lib/opportunity_attachment_test_files/test_file_4.pdf differ diff --git a/api/tests/lib/opportunity_attachment_test_files/test_file_5.pdf b/api/tests/lib/opportunity_attachment_test_files/test_file_5.pdf new file mode 100644 index 000000000..adf79e600 Binary files /dev/null and b/api/tests/lib/opportunity_attachment_test_files/test_file_5.pdf differ diff --git a/api/tests/lib/seed_local_db.py b/api/tests/lib/seed_local_db.py index 6a2ab9d0a..c634f280e 100644 --- a/api/tests/lib/seed_local_db.py +++ b/api/tests/lib/seed_local_db.py @@ -1,13 +1,18 @@ import logging +import os +import pathlib import random +import boto3 import click +from botocore.exceptions import ClientError from sqlalchemy import func import src.adapters.db as db import src.logging import src.util.datetime_util as datetime_util import tests.src.db.models.factories as factories +from src.adapters.aws import S3Config, get_s3_client from src.adapters.db import PostgresDBClient from src.db.models.agency_models import Agency from src.db.models.opportunity_models import Opportunity @@ -16,6 +21,30 @@ logger = logging.getLogger(__name__) +TESTS_FOLDER = pathlib.Path(__file__).parent.resolve() + + +def _upload_opportunity_attachments_s3(): + s3_config = S3Config() + s3_client = get_s3_client( + s3_config, boto3.Session(aws_access_key_id="NO_CREDS", aws_secret_access_key="NO_CREDS") + ) + test_folder_path = TESTS_FOLDER / "opportunity_attachment_test_files" + + for root, _, files in os.walk(test_folder_path): + for file in files: + file_path = os.path.join(root, file) + object_name = os.path.relpath(file_path, test_folder_path) + + try: + s3_client.upload_file(file_path, s3_config.s3_opportunity_bucket, object_name) + logger.info("Successfully uploaded files") + except ClientError as e: + logger.error( + "Error uploading to s3: %s", + extra={"object_name": object_name, "file_path": file_path, "error": e}, + ) + def _add_history( opps: list[Opportunity], @@ -173,6 +202,8 @@ def seed_local_db(iterations: int, include_history: bool) -> None: logger.info("Running seed script for local DB") error_if_not_local() + _upload_opportunity_attachments_s3() + db_client = PostgresDBClient() with db_client.get_session() as db_session: diff --git a/api/tests/src/api/opportunities_v1/conftest.py b/api/tests/src/api/opportunities_v1/conftest.py index 65273c62b..f08f3e3cd 100644 --- a/api/tests/src/api/opportunities_v1/conftest.py +++ b/api/tests/src/api/opportunities_v1/conftest.py @@ -153,7 +153,6 @@ def validate_opportunity_attachments( assert len(db_attachments) == len(resp_attachments) for db_attachment, resp_attachment in zip(db_attachments, resp_attachments, strict=True): - assert db_attachment.file_location == resp_attachment["file_location"] assert db_attachment.mime_type == resp_attachment["mime_type"] assert db_attachment.file_name == resp_attachment["file_name"] assert db_attachment.file_description == resp_attachment["file_description"] diff --git a/api/tests/src/api/opportunities_v1/test_opportunity_route_get.py b/api/tests/src/api/opportunities_v1/test_opportunity_route_get.py index d02950794..53fea4c85 100644 --- a/api/tests/src/api/opportunities_v1/test_opportunity_route_get.py +++ b/api/tests/src/api/opportunities_v1/test_opportunity_route_get.py @@ -1,4 +1,5 @@ import pytest +import requests from tests.src.api.opportunities_v1.conftest import ( validate_opportunity, @@ -6,6 +7,7 @@ ) from tests.src.db.models.factories import ( CurrentOpportunitySummaryFactory, + OpportunityAttachmentFactory, OpportunityFactory, OpportunitySummaryFactory, ) @@ -47,7 +49,6 @@ def test_get_opportunity_200( db_opportunity = OpportunityFactory.create( **opportunity_params, current_opportunity_summary=None ) # We'll set the current opportunity below - if opportunity_summary_params is not None: db_opportunity_summary = OpportunitySummaryFactory.create( **opportunity_summary_params, opportunity=db_opportunity @@ -70,8 +71,6 @@ def test_get_opportunity_with_attachment_200( ): # Create an opportunity with an attachment opportunity = OpportunityFactory.create() - - # Ensure the opportunity is committed to the database db_session.commit() # Make the GET request @@ -88,6 +87,33 @@ def test_get_opportunity_with_attachment_200( validate_opportunity_with_attachments(opportunity, response_data) +def test_get_opportunity_s3_endpoint_url_200( + upload_opportunity_attachment_s3, client, api_auth_token, enable_factory_create, db_session +): + # Create an opportunity with a specific attachment + opportunity = OpportunityFactory.create(opportunity_attachments=[]) + bucket = "test_bucket" + object_name = "test_file_1.txt" + file_loc = f"s3://{bucket}/{object_name}" + OpportunityAttachmentFactory.create(file_location=file_loc, opportunity=opportunity) + + # Make the GET request + resp = client.get( + f"/v1/opportunities/{opportunity.opportunity_id}", headers={"X-Auth": api_auth_token} + ) + + # Check the response + assert resp.status_code == 200 + response_data = resp.get_json()["data"] + print("response_data", response_data) + presigned_url = response_data["attachments"][0]["download_path"] + + # Validate pre-signed url + response = requests.get(presigned_url, timeout=5) + assert response.status_code == 200 + assert response.text == "Hello, world" + + def test_get_opportunity_404_not_found(client, api_auth_token, truncate_opportunities): resp = client.get("/v1/opportunities/1", headers={"X-Auth": api_auth_token}) assert resp.status_code == 404 diff --git a/api/tests/src/db/models/factories.py b/api/tests/src/db/models/factories.py index 4f64ae033..b0fed7d11 100644 --- a/api/tests/src/db/models/factories.py +++ b/api/tests/src/db/models/factories.py @@ -147,6 +147,14 @@ class CustomProvider(BaseProvider): YN_YESNO_BOOLEAN_VALUES = ["Y", "N", "Yes", "No"] + OPPORTUNITY_ATTACHMENT_S3_PATHS = [ + "s3://local-opportunities/test_file_1.txt", + "s3://local-opportunities/test_file_2.txt", + "s3://local-opportunities/test_file_3.txt", + "s3://local-opportunities/test_file_4.pdf", + "s3://local-opportunities/test_file_5.pdf", + ] + def agency(self) -> str: return self.random_element(self.AGENCIES) @@ -188,6 +196,9 @@ def yn_boolean(self) -> str: def yn_yesno_boolean(self) -> str: return self.random_element(self.YN_YESNO_BOOLEAN_VALUES) + def s3_file_location(self) -> str: + return self.random_element(self.OPPORTUNITY_ATTACHMENT_S3_PATHS) + fake = faker.Faker() fake.add_provider(CustomProvider) @@ -239,7 +250,7 @@ class OpportunityAttachmentFactory(BaseFactory): class Meta: model = opportunity_models.OpportunityAttachment - file_location = factory.Faker("url") + file_location = factory.Faker("s3_file_location") mime_type = factory.Faker("mime_type") file_name = factory.Faker("file_name") file_description = factory.Faker("sentence")