bugfix/unique-temp-filenames

cdp_backend/pipeline/event_gather_pipeline.py

@@ -7,6 +7,7 @@
 from operator import attrgetter
 from pathlib import Path
 from typing import Any, Callable, Dict, List, NamedTuple, Optional, Set, Tuple, Union
+from uuid import uuid4
 
 from aiohttp.client_exceptions import ClientResponseError
 from fireo.fields.errors import FieldValidationFailed, InvalidFieldType, RequiredField
@@ -127,8 +128,11 @@ def create_event_gather_flow(
         for event in events:
             session_processing_results: List[SessionProcessingResult] = []
             for session in event.sessions:
-                # Download video to local copy
-                resource_copy_filepath = resource_copy_task(uri=session.video_uri)
+                # Download video to local copy making
+                # copy unique in case of shared session video
+                resource_copy_filepath = resource_copy_task(
+                    uri=session.video_uri, dst=f"{str(uuid4())}_temp.mp4"
+                )
 
                 # Handle video conversion or non-secure resource
                 # hosting
@@ -229,7 +233,7 @@ def create_event_gather_flow(
 
 
 @task(max_retries=3, retry_delay=timedelta(seconds=120))
-def resource_copy_task(uri: str) -> str:
+def resource_copy_task(uri: str, dst: str = None) -> str:
     """
     Copy a file to a temporary location for processing.
 
@@ -250,6 +254,7 @@ def resource_copy_task(uri: str) -> str:
     """
     return file_utils.resource_copy(
         uri=uri,
+        dst=dst,
         overwrite=True,
     )
 

cdp_backend/tests/pipeline/test_event_gather_pipeline.py

@@ -7,7 +7,7 @@
 from pathlib import Path
 from typing import List, Optional
 from unittest import mock
-from unittest.mock import MagicMock
+from unittest.mock import MagicMock, patch
 
 import pytest
 from prefect import Flow
@@ -577,6 +577,10 @@ def test_store_event_processing_results(
 EXISTING_REMOTE_M3U8_MINIMAL_EVENT.sessions[0].video_uri = EXAMPLE_M3U8_PLAYLIST_URI
 
 
+def path_rename(self: Path, newPath: Path) -> Path:
+    return newPath
+
+
 @mock.patch(f"{PIPELINE_PATH}.fs_functions.upload_file")
 @mock.patch(f"{PIPELINE_PATH}.fs_functions.get_open_url_for_gcs_file")
 @mock.patch(f"{PIPELINE_PATH}.fs_functions.remove_local_file")
@@ -633,20 +637,22 @@ def test_convert_video_and_handle_host(
     mock_convert_video_to_mp4.return_value = expected_filepath
     mock_hash_file_contents.return_value = "abc123"
 
-    (
-        mp4_filepath,
-        session_video_hosted_url,
-        session_content_hash,
-    ) = pipeline.convert_video_and_handle_host.run(
-        video_filepath=video_filepath,
-        session=session,
-        credentials_file="fake/credentials.json",
-        bucket="doesnt://matter",
-    )
+    with patch.object(Path, "rename", path_rename):
+
+        (
+            mp4_filepath,
+            session_video_hosted_url,
+            session_content_hash,
+        ) = pipeline.convert_video_and_handle_host.run(
+            video_filepath=video_filepath,
+            session=session,
+            credentials_file="fake/credentials.json",
+            bucket="doesnt://matter",
+        )
 
-    # Make sure mp4 files don't go through conversion
-    if Path(video_filepath).suffix == ".mp4":
-        assert not mock_convert_video_to_mp4.called
+        # Make sure mp4 files don't go through conversion
+        if Path(video_filepath).suffix == ".mp4":
+            assert not mock_convert_video_to_mp4.called
 
-    assert mp4_filepath == expected_filepath
-    assert session_video_hosted_url == expected_hosted_video_url
+        assert mp4_filepath == str(Path(video_filepath).with_suffix(".mp4"))
+        assert session_video_hosted_url == expected_hosted_video_url

cdp_backend/tests/utils/test_file_utils.py

@@ -35,6 +35,58 @@
 #############################################################################
 
 
+@pytest.mark.parametrize(
+    "path, stem, expected_result",
+    [
+        (Path("file.ext"), "new", "new.ext"),
+    ],
+)
+def test_with_stem(path: Path, stem: str, expected_result: str) -> None:
+    new_path = file_utils.with_stem(path, stem)
+    assert str(new_path) == expected_result
+
+
+@pytest.mark.parametrize(
+    "path, addition, expected_result",
+    [
+        (Path("file.ext"), "-new", "file-new.ext"),
+    ],
+)
+def test_append_to_stem(path: Path, addition: str, expected_result: str) -> None:
+    new_path = file_utils.append_to_stem(path, addition)
+    assert str(new_path) == expected_result
+
+
+@pytest.mark.parametrize(
+    "path, stem, expected_result",
+    [
+        (Path("file.ext"), "new", "new.ext"),
+    ],
+)
+def test_rename_with_stem(path: Path, stem: str, expected_result: str) -> None:
+    file = open(path, "w")
+    file.close()
+    new_path = file_utils.rename_with_stem(path, stem)
+    assert str(new_path) == expected_result
+    assert new_path.exists()
+    os.remove(new_path)
+
+
+@pytest.mark.parametrize(
+    "path, addition, expected_result",
+    [
+        (Path("file.ext"), "-new", "file-new.ext"),
+    ],
+)
+def test_rename_append_to_stem(path: Path, addition: str, expected_result: str) -> None:
+    file = open(path, "w")
+    file.close()
+    new_path = file_utils.rename_append_to_stem(path, addition)
+    assert str(new_path) == expected_result
+    assert new_path.exists()
+    os.remove(new_path)
+
+
 @pytest.mark.parametrize(
     "uri, expected_result",
     [

cdp_backend/utils/file_utils.py

@@ -28,6 +28,82 @@
 MAX_THUMBNAIL_WIDTH = 960
 
 
+def with_stem(path: Path, stem: str) -> Path:
+    """
+    Create a path with a new stem
+
+    Parameters
+    ----------
+    path: Path
+        The path to alter
+    stem: str
+        The string to be the new stem of the path
+
+    Returns
+    -------
+    path: Path
+        The new path with the replaced stem
+    """
+    return path.with_name(f"{stem}{path.suffix}")
+
+
+def append_to_stem(path: Path, addition: str) -> Path:
+    """
+    Rename a file with a string appended to the path stem
+
+    Parameters
+    ----------
+    path: Path
+        The path to alter
+    addition: str
+        The string to be appended to the path stem
+
+    Returns
+    -------
+    path: Path
+        The new path with the stem addition
+    """
+    return with_stem(path, f"{path.stem}{addition}")
+
+
+def rename_with_stem(path: Path, stem: str) -> Path:
+    """
+    Rename a file with a string appended to the path stem
+
+    Parameters
+    ----------
+    path: Path
+        The path to be renamed
+    stem: str
+        The string to become the new stem
+
+    Returns
+    -------
+    path: Path
+        The new path of the renamed file
+    """
+    return path.rename(with_stem(path, stem))
+
+
+def rename_append_to_stem(path: Path, addition: str) -> Path:
+    """
+    Rename a file with a string appended to the path stem
+
+    Parameters
+    ----------
+    path: Path
+        The path to be renamed
+    addition: str
+        The string to be appended to the path stem
+
+    Returns
+    -------
+    path: Path
+        The new path of the renamed file
+    """
+    return path.rename(append_to_stem(path, addition))
+
+
 def get_media_type(uri: str) -> Optional[str]:
     """
     Get the IANA media type for the provided URI.
@@ -520,7 +596,6 @@ def convert_video_to_mp4(
         The end time to trim the video in HH:MM:SS.
     output_path: Path
         The output path to place the clip at.
-        Must include a suffix to use for the reformatting.
 
     Returns
     -------
@@ -605,7 +680,7 @@ def clip_and_reformat_video(
     video_filepath: Path,
     start_time: Optional[str],
     end_time: Optional[str],
-    output_path: Path = Path("clipped.mp4"),
+    output_path: Path = None,
     output_format: str = "mp4",
 ) -> Path:
     """
@@ -621,8 +696,6 @@ def clip_and_reformat_video(
         The end time of the clip in HH:MM:SS.
     output_path: Path
         The output path to place the clip at.
-        Must include a suffix to use for the reformatting.
-        Default: "clipped.mp4"
     output_format: str
         The output format.
         Default: "mp4"
@@ -634,6 +707,8 @@ def clip_and_reformat_video(
     """
     import ffmpeg
 
+    output_path = output_path or rename_append_to_stem(video_filepath, "_clipped")
+
     try:
         ffmpeg_stdout, ffmpeg_stderr = (
             ffmpeg.input(