Add simple mock for Google Cloud Storage tests

giftless/storage/google_cloud.py

@@ -9,8 +9,8 @@
 from typing import Any, BinaryIO, cast
 
 import google.auth
+import google.cloud
 from google.auth import impersonated_credentials
-from google.cloud import storage
 from google.oauth2 import service_account
 
 from giftless.storage import ExternalStorage, StreamingStorage
@@ -40,7 +40,7 @@ def __init__(
             | impersonated_credentials.Credentials
             | None
         ) = self._load_credentials(account_key_file, account_key_base64)
-        self.storage_client = storage.Client(
+        self.storage_client = google.cloud.storage.Client(
             project=project_name, credentials=self.credentials
         )
         if not self.credentials:

tests/mocks/google_cloud_storage.py

@@ -0,0 +1,83 @@
+"""Mock for google_cloud_storage that just uses a temporary directory
+rather than talking to Google.  This effectively makes it a LocalStorage
+implementation, of course.
+"""
+
+import shutil
+from pathlib import Path
+from typing import Any, BinaryIO
+
+from giftless.storage.exc import ObjectNotFoundError
+from giftless.storage.google_cloud import GoogleCloudStorage
+
+
+class MockGoogleCloudStorage(GoogleCloudStorage):
+    """Mocks a GoogleCloudStorage object by simulating it with a local
+    directory.
+    """
+
+    def __init__(
+        self,
+        project_name: str,
+        bucket_name: str,
+        path: Path,
+        account_key_file: str | None = None,
+        account_key_base64: str | None = None,
+        path_prefix: str | None = None,
+        serviceaccount_email: str | None = None,
+        **_: Any,
+    ) -> None:
+        super().__init__(
+            project_name=project_name,
+            bucket_name=bucket_name,
+            account_key_file=account_key_file,
+            account_key_base64=account_key_base64,
+            serviceaccount_email=serviceaccount_email,
+        )
+        self._path = path
+
+    def _get_blob_path(self, prefix: str, oid: str) -> str:
+        return str(self._get_blob_pathlib_path(prefix, oid))
+
+    def _get_blob_pathlib_path(self, prefix: str, oid: str) -> Path:
+        return Path(self._path / Path(prefix) / oid)
+
+    @staticmethod
+    def _create_path(spath: str) -> None:
+        path = Path(spath)
+        if not path.is_dir():
+            path.mkdir(parents=True)
+
+    def _get_signed_url(
+        self,
+        prefix: str,
+        oid: str,
+        expires_in: int,
+        http_method: str = "GET",
+        filename: str | None = None,
+        disposition: str | None = None,
+    ) -> str:
+        return f"https://example.com/signed_blob/{prefix}/{oid}"
+
+    def get(self, prefix: str, oid: str) -> BinaryIO:
+        obj = self._get_blob_pathlib_path(prefix, oid)
+        if not obj.exists():
+            raise ObjectNotFoundError("Object does not exist")
+        return obj.open("rb")
+
+    def put(self, prefix: str, oid: str, data_stream: BinaryIO) -> int:
+        path = self._get_blob_pathlib_path(prefix, oid)
+        directory = path.parent
+        self._create_path(str(directory))
+        with path.open("bw") as dest:
+            shutil.copyfileobj(data_stream, dest)
+            return dest.tell()
+
+    def exists(self, prefix: str, oid: str) -> bool:
+        return self._get_blob_pathlib_path(prefix, oid).is_file()
+
+    def get_size(self, prefix: str, oid: str) -> int:
+        if not self.exists(prefix, oid):
+            raise ObjectNotFoundError("Object does not exist")
+        path = self._get_blob_pathlib_path(prefix, oid)
+        return path.stat().st_size

tests/storage/test_google_cloud.py

@@ -1,129 +1,78 @@
 """Tests for the Google Cloud Storage storage backend."""
-import os
-from collections.abc import Generator
-from typing import Any
+from pathlib import Path
 
+import google.cloud.storage  # noqa: F401 (used implicitly by storage backend)
 import pytest
-from google.api_core.exceptions import GoogleAPIError
 
-from giftless.storage.google_cloud import GoogleCloudStorage
+from ..mocks.google_cloud_storage import MockGoogleCloudStorage
+from . import ExternalStorageAbstractTests, StreamingStorageAbstractTests
 
 MOCK_GCP_PROJECT_NAME = "giftless-tests"
-MOCK_GCP_BUCKET_NAME = "giftless-tests-20200818"
+MOCK_GCP_BUCKET_NAME = "giftless-tests-20240115"
 
 # This is a valid but revoked key that we use in testing
 MOCK_GCP_KEY_B64 = (
-    "ewogICJ0eXBlIjogInNlcnZpY2VfYWNjb3VudCIsCiAgInByb2plY3RfaWQiOiAiZ2lmdGxlc3MtdGVz"
-    "dHMiLAogICJwcml2YXRlX2tleV9pZCI6ICI4MWRhNDcxNzhiYzhmYjE1MDU1NTg3OWRjZTczZThmZDlm"
-    "OWI4NmJkIiwKICAicHJpdmF0ZV9rZXkiOiAiLS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tXG5NSUlF"
-    "dkFJQkFEQU5CZ2txaGtpRzl3MEJBUUVGQUFTQ0JLWXdnZ1NpQWdFQUFvSUJBUUNsYXdDOUEvZHBnbVJW"
-    "XG5kYVg2UW5xY1N6YW5ueTdCVlgwVklwUHVjNzl2aFR2NWRwZXRaa29SQmV6Uzg2ZStHUHVyTmJIMU9r"
-    "WEZrL2tkXG5SNHFqMDV6SXlYeWxiQUVxSk1BV24zZFY0VUVRVFlmRitPY0ltZUxpcjR3cW9pTldDZDNJ"
-    "aHErNHVVeU1WRDMxXG5wc1FlcWVxcWV6bVoyNG1oTjBLK2NQczNuSXlIK0lzZXFsWjJob3U3bUU3U2Js"
-    "YXdjc04ramcyNmQ5YzFUZlpoXG42eFozVkpndGFtcUZvdlZmbEZwNFVvLy9tVGo0cXEwUWRUYk9SS1NE"
-    "eVkxTWhkQ24veSsyaForVm9IUitFM0Z4XG5XRmc2VGFwRGJhc29heEp5YjRoZEFFK0JhbW14bklTL09G"
-    "bElaMGVoL2tsRmlBTlJRMEpQb2dXRjFjVE9NcVFxXG4wMlVFV2V5ckFnTUJBQUVDZ2dFQUJNOE5odGVQ"
-    "NElhTEUxd2haclN0cEp5NWltMGgxenFXTVlCTU85WDR4KzJUXG5PZmRUYStLbWtpcUV1c1UyanNJdks1"
-    "VUJPakVBcncxVU1RYnBaaEtoTDhub2c3dGkyNjVoMG1Ba1pzWlZOWHU0XG5UKzQ4REZ4YzQ4THlzaktX"
-    "M1RCQVBSb2RRbkJLTVA3MnY4QThKMU5BYlMwZ3IvTW1TbEVidm1tT2FuTU9ONXAwXG43djlscm9GMzFO"
-    "akMzT05OY25pUlRYY01xT2tEbWt5LyszeVc2RldMMkJZV3RwcGN3L0s1TnYxdGNMTG5iajVhXG5Hc3dV"
-    "MENtQXgyTEVoWEo0bndJaWlFR3h6UGZYVXNLcVhLL2JEZENKbDUzMTgraU9aSHNrdXR1OFlqQVpsdktp"
-    "XG5yckNFUkFXZitLeTZ0WGhnKzJIRzJJbUc5cG8wRnUwTGlIU0ZVUURKY1FLQmdRRFQ5RDJEYm9SNWFG"
-    "WW0wQlVSXG5vNGd4OHZGc0NyTEx0a09EZGx3U2wrT20yblFvY0JXSTEyTmF5QXRlL2xhVFZNRlorVks1"
-    "bU9vYXl2WnljTU1YXG5SdXZJYmdCTFdHYkdwSXdXZnlDOGxRZEJYM09xZTZZSzZTMUU2VnNYUVN0aHQ0"
-    "YUx3ZGpGQ2J6VU1lc1ZzREV5XG5FYU85aXlTUVlFTmFTN2V3amFzNUFVU1F0d0tCZ1FESHl4WUp3bWxp"
-    "QzE4NEVyZ3lZSEFwYm9weXQzSVkzVGFKXG5yV2MrSGw5WDNzVEJzaFVQYy85SmhjanZKYWVzMlhrcEEw"
-    "YmY5cis1MEcxUndua3dMWHZsbDJSU0FBNE05TG4rXG45cVlsNEFXNU9QVTVJS0tKYVk1c0kzSHdXTXd6"
-    "elRya3FBV3hNallJME9OSnBaWUVnSTVKN09sek1jYnhLREZxXG51MmpjYkFubnJRS0JnRlUxaklGSkxm"
-    "TE5FazE2Tys0aWF6K0Jack5EdmN1TjA2aUhMYzYveDJLdDBpTHJwSXlsXG40cWg5WWF6bjNSQlA4NGRq"
-    "WjNGNzJ5bTRUTW1ITWJjcTZPRmo3N1JhcnI3UEtnNWxQMWp4Sk1DUVNpVFFudGttXG5FdS93VEpHVnZv"
-    "WURUUkRrZG13SVZTU05pTy9vTEc3dmpuOUY4QVltM1F6eEFjRDF3MDhnaGxzVEFvR0FidUthXG4vNTJq"
-    "eVdPUVhGbWZXMjVFc2VvRTh2ZzNYZTlnZG5jRUJ1anFkNlZPeEVYbkJHV1h1U0dFVEo0MGVtMVVubHVR"
-    "XG5PWHNFRzhlKzlKS2ZtZ3FVYWU5bElWR2dlclpVaUZveUNuRlVHK0d0MEIvNXRaUWRGSTF6ampacVZ4"
-    "Ry9idXFHXG5CanRjMi9XN1A4T2tDQ21sVHdncTVPRXFqZXVGeWJ2cnpmSTBhUjBDZ1lCdVlYWm5MMm1x"
-    "eVNma0FnaGswRVVmXG5XeElDb1FmRDdCQlJBV3lmL3VwRjQ2NlMvRmhONUVreG5vdkZ2RlZyQjU1SHVH"
-    "RTh2Qk4vTEZNVXlPU0xXQ0lIXG5RUG9ZcytNM0NLdGJWTXMxY1h2Tm5tZFRhMnRyYjQ0SlQ5ZlFLbkVw"
-    "a2VsbUdPdXJMNEVMdmFyUEFyR0x4VllTXG5jWFo1a1FBUy9GeGhFSDZSbnFSalFnPT1cbi0tLS0tRU5E"
-    "IFBSSVZBVEUgS0VZLS0tLS1cbiIsCiAgImNsaWVudF9lbWFpbCI6ICJzb21lLXNlcnZpY2UtYWNjb3Vu"
-    "dEBnaWZ0bGVzcy10ZXN0cy5pYW0uZ3NlcnZpY2VhY2NvdW50LmNvbSIsCiAgImNsaWVudF9pZCI6ICIx"
-    "MDk4NTYwMjgzNDI5MDI4ODI3MTUiLAogICJhdXRoX3VyaSI6ICJodHRwczovL2FjY291bnRzLmdvb2ds"
-    "ZS5jb20vby9vYXV0aDIvYXV0aCIsCiAgInRva2VuX3VyaSI6ICJodHRwczovL29hdXRoMi5nb29nbGVh"
-    "cGlzLmNvbS90b2tlbiIsCiAgImF1dGhfcHJvdmlkZXJfeDUwOV9jZXJ0X3VybCI6ICJodHRwczovL3d3"
-    "dy5nb29nbGVhcGlzLmNvbS9vYXV0aDIvdjEvY2VydHMiLAogICJjbGllbnRfeDUwOV9jZXJ0X3VybCI6"
-    "ICJodHRwczovL3d3dy5nb29nbGVhcGlzLmNvbS9yb2JvdC92MS9tZXRhZGF0YS94NTA5L3NvbWUtc2Vy"
-    "dmljZS1hY2NvdW50JTQwZ2lmdGxlc3MtdGVzdHMuaWFtLmdzZXJ2aWNlYWNjb3VudC5jb20iCn0K"
+    "ewogICJ0eXBlIjogInNlcnZpY2VfYWNjb3VudCIsCiAgInByb2plY3RfaWQiOiAiZ2lmdGxl"
+    "c3MtdGVzdHMiLAogICJwcml2YXRlX2tleV9pZCI6ICI4MWRhNDcxNzhiYzhmYjE1MDU1NTg3"
+    "OWRjZTczZThmZDlmOWI4NmJkIiwKICAicHJpdmF0ZV9rZXkiOiAiLS0tLS1CRUdJTiBQUklW"
+    "QVRFIEtFWS0tLS0tXG5NSUlFdkFJQkFEQU5CZ2txaGtpRzl3MEJBUUVGQUFTQ0JLWXdnZ1Np"
+    "QWdFQUFvSUJBUUNsYXdDOUEvZHBnbVJWXG5kYVg2UW5xY1N6YW5ueTdCVlgwVklwUHVjNzl2"
+    "aFR2NWRwZXRaa29SQmV6Uzg2ZStHUHVyTmJIMU9rWEZrL2tkXG5SNHFqMDV6SXlYeWxiQUVx"
+    "Sk1BV24zZFY0VUVRVFlmRitPY0ltZUxpcjR3cW9pTldDZDNJaHErNHVVeU1WRDMxXG5wc1Fl"
+    "cWVxcWV6bVoyNG1oTjBLK2NQczNuSXlIK0lzZXFsWjJob3U3bUU3U2JsYXdjc04ramcyNmQ5"
+    "YzFUZlpoXG42eFozVkpndGFtcUZvdlZmbEZwNFVvLy9tVGo0cXEwUWRUYk9SS1NEeVkxTWhk"
+    "Q24veSsyaForVm9IUitFM0Z4XG5XRmc2VGFwRGJhc29heEp5YjRoZEFFK0JhbW14bklTL09G"
+    "bElaMGVoL2tsRmlBTlJRMEpQb2dXRjFjVE9NcVFxXG4wMlVFV2V5ckFnTUJBQUVDZ2dFQUJN"
+    "OE5odGVQNElhTEUxd2haclN0cEp5NWltMGgxenFXTVlCTU85WDR4KzJUXG5PZmRUYStLbWtp"
+    "cUV1c1UyanNJdks1VUJPakVBcncxVU1RYnBaaEtoTDhub2c3dGkyNjVoMG1Ba1pzWlZOWHU0"
+    "XG5UKzQ4REZ4YzQ4THlzaktXM1RCQVBSb2RRbkJLTVA3MnY4QThKMU5BYlMwZ3IvTW1TbEVi"
+    "dm1tT2FuTU9ONXAwXG43djlscm9GMzFOakMzT05OY25pUlRYY01xT2tEbWt5LyszeVc2RldM"
+    "MkJZV3RwcGN3L0s1TnYxdGNMTG5iajVhXG5Hc3dVMENtQXgyTEVoWEo0bndJaWlFR3h6UGZY"
+    "VXNLcVhLL2JEZENKbDUzMTgraU9aSHNrdXR1OFlqQVpsdktpXG5yckNFUkFXZitLeTZ0WGhn"
+    "KzJIRzJJbUc5cG8wRnUwTGlIU0ZVUURKY1FLQmdRRFQ5RDJEYm9SNWFGWW0wQlVSXG5vNGd4"
+    "OHZGc0NyTEx0a09EZGx3U2wrT20yblFvY0JXSTEyTmF5QXRlL2xhVFZNRlorVks1bU9vYXl2"
+    "WnljTU1YXG5SdXZJYmdCTFdHYkdwSXdXZnlDOGxRZEJYM09xZTZZSzZTMUU2VnNYUVN0aHQ0"
+    "YUx3ZGpGQ2J6VU1lc1ZzREV5XG5FYU85aXlTUVlFTmFTN2V3amFzNUFVU1F0d0tCZ1FESHl4"
+    "WUp3bWxpQzE4NEVyZ3lZSEFwYm9weXQzSVkzVGFKXG5yV2MrSGw5WDNzVEJzaFVQYy85Smhj"
+    "anZKYWVzMlhrcEEwYmY5cis1MEcxUndua3dMWHZsbDJSU0FBNE05TG4rXG45cVlsNEFXNU9Q"
+    "VTVJS0tKYVk1c0kzSHdXTXd6elRya3FBV3hNallJME9OSnBaWUVnSTVKN09sek1jYnhLREZx"
+    "XG51MmpjYkFubnJRS0JnRlUxaklGSkxmTE5FazE2Tys0aWF6K0Jack5EdmN1TjA2aUhMYzYv"
+    "eDJLdDBpTHJwSXlsXG40cWg5WWF6bjNSQlA4NGRqWjNGNzJ5bTRUTW1ITWJjcTZPRmo3N1Jh"
+    "cnI3UEtnNWxQMWp4Sk1DUVNpVFFudGttXG5FdS93VEpHVnZvWURUUkRrZG13SVZTU05pTy9v"
+    "TEc3dmpuOUY4QVltM1F6eEFjRDF3MDhnaGxzVEFvR0FidUthXG4vNTJqeVdPUVhGbWZXMjVF"
+    "c2VvRTh2ZzNYZTlnZG5jRUJ1anFkNlZPeEVYbkJHV1h1U0dFVEo0MGVtMVVubHVRXG5PWHNF"
+    "RzhlKzlKS2ZtZ3FVYWU5bElWR2dlclpVaUZveUNuRlVHK0d0MEIvNXRaUWRGSTF6ampacVZ4"
+    "Ry9idXFHXG5CanRjMi9XN1A4T2tDQ21sVHdncTVPRXFqZXVGeWJ2cnpmSTBhUjBDZ1lCdVlY"
+    "Wm5MMm1xeVNma0FnaGswRVVmXG5XeElDb1FmRDdCQlJBV3lmL3VwRjQ2NlMvRmhONUVreG5v"
+    "dkZ2RlZyQjU1SHVHRTh2Qk4vTEZNVXlPU0xXQ0lIXG5RUG9ZcytNM0NLdGJWTXMxY1h2Tm5t"
+    "ZFRhMnRyYjQ0SlQ5ZlFLbkVwa2VsbUdPdXJMNEVMdmFyUEFyR0x4VllTXG5jWFo1a1FBUy9G"
+    "eGhFSDZSbnFSalFnPT1cbi0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS1cbiIsCiAgImNsaWVu"
+    "dF9lbWFpbCI6ICJzb21lLXNlcnZpY2UtYWNjb3VudEBnaWZ0bGVzcy10ZXN0cy5pYW0uZ3Nl"
+    "cnZpY2VhY2NvdW50LmNvbSIsCiAgImNsaWVudF9pZCI6ICIxMDk4NTYwMjgzNDI5MDI4ODI3"
+    "MTUiLAogICJhdXRoX3VyaSI6ICJodHRwczovL2FjY291bnRzLmdvb2dsZS5jb20vby9vYXV0"
+    "aDIvYXV0aCIsCiAgInRva2VuX3VyaSI6ICJodHRwczovL29hdXRoMi5nb29nbGVhcGlzLmNv"
+    "bS90b2tlbiIsCiAgImF1dGhfcHJvdmlkZXJfeDUwOV9jZXJ0X3VybCI6ICJodHRwczovL3d3"
+    "dy5nb29nbGVhcGlzLmNvbS9vYXV0aDIvdjEvY2VydHMiLAogICJjbGllbnRfeDUwOV9jZXJ0"
+    "X3VybCI6ICJodHRwczovL3d3dy5nb29nbGVhcGlzLmNvbS9yb2JvdC92MS9tZXRhZGF0YS94"
+    "NTA5L3NvbWUtc2VydmljZS1hY2NvdW50JTQwZ2lmdGxlc3MtdGVzdHMuaWFtLmdzZXJ2aWNl"
+    "YWNjb3VudC5jb20iCn0K"
 )
 
 
 @pytest.fixture
-def storage_backend() -> Generator[GoogleCloudStorage, None, None]:
-    """Provide a Google Cloud Storage backend for all GCS tests.
-
-    For this to work against production Google Cloud, you need to set
-    ``GCP_ACCOUNT_KEY_FILE``, ``GCP_PROJECT_NAME`` and ``GCP_BUCKET_NAME``
-    environment variables when running the tests.
-
-    If these variables are not set, and pytest-vcr is not in use, the
-    tests *will* fail.
-    """
-    account_key_file = os.environ.get("GCP_ACCOUNT_KEY_FILE")
-    project_name = os.environ.get("GCP_PROJECT_NAME")
-    bucket_name = os.environ.get("GCP_BUCKET_NAME")
-    prefix = "giftless-tests"
-
-    if account_key_file and project_name and bucket_name:
-        # We use a live GCS bucket to test
-        storage = GoogleCloudStorage(
-            project_name=project_name,
-            bucket_name=bucket_name,
-            account_key_file=account_key_file,
-            path_prefix=prefix,
-        )
-        try:
-            yield storage
-        finally:
-            bucket = storage.storage_client.bucket(bucket_name)
-            try:
-                blobs = bucket.list_blobs(prefix=prefix + "/")
-                bucket.delete_blobs(blobs)
-            except GoogleAPIError as e:
-                raise pytest.PytestWarning(
-                    f"Could not clean up after test: {e}"
-                ) from None
-    else:
-        yield GoogleCloudStorage(
-            project_name=MOCK_GCP_PROJECT_NAME,
-            bucket_name=MOCK_GCP_BUCKET_NAME,
-            account_key_base64=MOCK_GCP_KEY_B64,
-            path_prefix=prefix,
-        )
-
-
-@pytest.fixture(scope="module")
-def vcr_config() -> dict[str, Any]:
-    live_tests = bool(
-        os.environ.get("GCP_ACCOUNT_KEY_FILE")
-        and os.environ.get("GCP_PROJECT_NAME")
-        and os.environ.get("GCP_BUCKET_NAME")
+def storage_backend(
+    storage_path: Path,
+) -> MockGoogleCloudStorage:
+    """Provide a mock Google Cloud Storage backend for all GCS tests."""
+    return MockGoogleCloudStorage(
+        project_name=MOCK_GCP_PROJECT_NAME,
+        bucket_name=MOCK_GCP_BUCKET_NAME,
+        account_key_base64=MOCK_GCP_KEY_B64,
+        path=storage_path,
     )
-    mode = "once" if live_tests else "none"
-    return {
-        "filter_headers": [("authorization", "fake-authz-header")],
-        "record_mode": mode,
-    }
 
 
-# TODO @athornton: updating the storage backends has caused the VCR cassettes
-# to become invalid.  Datopian will need to rebuild those cassettes with data
-# from the current implementation, or (better) we should use something other
-# than pytest-vcr, which is opaque and unhelpful.
-#
-# I can confirm that the Google Cloud Storage Backend at least works in
-# conjunction with Workload Identity, since I'm using that for my own storage
-# in my Git LFS implementation.  -- AJT 20231220
-#
-# @pytest.mark.vcr()
-# class TestGoogleCloudStorageBackend(
-#           StreamingStorageAbstractTests, ExternalStorageAbstractTests
-#       ):
-#           pass
+class TestGoogleCloudStorageBackend(
+    StreamingStorageAbstractTests, ExternalStorageAbstractTests
+):
+    pass