Skip to content

Commit

Permalink
Save local bulk import results to a dir that is ignored by git
Browse files Browse the repository at this point in the history
  • Loading branch information
annaCPR committed Oct 10, 2024
1 parent 0fc64c4 commit 137ea48
Show file tree
Hide file tree
Showing 3 changed files with 7 additions and 3 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ coverage.xml
*.py,cover
.hypothesis/
.pytest_cache/
bulk_import_results/

# Translations
*.mo
Expand Down
6 changes: 4 additions & 2 deletions app/clients/aws/s3bucket.py
Original file line number Diff line number Diff line change
Expand Up @@ -116,11 +116,13 @@ def upload_ingest_json_to_s3(
_LOGGER.info(os.getenv("BULK_IMPORT_BUCKET", "Not there"))
ingest_upload_bucket = os.environ["BULK_IMPORT_BUCKET"]
current_timestamp = datetime.now().strftime("%m-%d-%YT%H:%M:%S")

filename = f"{ingest_id}-{corpus_import_id}-{current_timestamp}.json"

if ingest_upload_bucket == "skip":
with open(filename, "w") as f:
json.dump(data, f, indent=4)
os.makedirs("bulk_import_results", exist_ok=True)
with open(f"bulk_import_results/{filename}", "w") as file:
json.dump(data, file, indent=4)
return

s3_client = get_s3_client()
Expand Down
3 changes: 2 additions & 1 deletion tests/unit_tests/clients/aws/test_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ def test_upload_json_to_s3_when_error(basic_s3_client):

@patch.dict(os.environ, {"BULK_IMPORT_BUCKET": "test_bucket"})
def test_upload_ingest_json_to_s3_success(basic_s3_client):
print(">>>>>>>>>>>>>>>>>>>", os.getenv("BULK_IMPORT_BUCKET", "Nothing"))
json_data = {"test": "test"}
upload_ingest_json_to_s3("1111-1111", "test_corpus_id", json_data)

Expand Down Expand Up @@ -69,4 +70,4 @@ def test_do_not_save_ingest_json_to_s3_when_in_local_development(basic_s3_client
)

assert "Contents" not in find_response
cleanup_local_files("1111-1111-test_corpus_id*")
cleanup_local_files("bulk_import_results/1111-1111-test_corpus_id*")

0 comments on commit 137ea48

Please sign in to comment.