diff --git a/storage/google/cloud/storage/_helpers.py b/storage/google/cloud/storage/_helpers.py index 11f2ad556ef1..fdcd80f4d4da 100644 --- a/storage/google/cloud/storage/_helpers.py +++ b/storage/google/cloud/storage/_helpers.py @@ -43,8 +43,9 @@ class _PropertyMixin(object): """Abstract mixin for cloud storage classes with associated properties. Non-abstract subclasses should implement: - - client - path + - client + - user_project :type name: str :param name: The name of the object. Bucket names must start and end with a @@ -71,6 +72,14 @@ def user_project(self): """Abstract getter for the object user_project.""" raise NotImplementedError + @property + def _query_params(self): + """Default query parameters.""" + params = {} + if self.user_project is not None: + params["userProject"] = self.user_project + return params + def _require_client(self, client): """Check client or verify over-ride. @@ -97,11 +106,10 @@ def reload(self, client=None): ``client`` stored on the current object. """ client = self._require_client(client) + query_params = self._query_params # Pass only '?projection=noAcl' here because 'acl' and related # are handled via custom endpoints. - query_params = {"projection": "noAcl"} - if self.user_project is not None: - query_params["userProject"] = self.user_project + query_params["projection"] = "noAcl" api_response = client._connection.api_request( method="GET", path=self.path, query_params=query_params, _target_object=self ) @@ -148,11 +156,10 @@ def patch(self, client=None): ``client`` stored on the current object. """ client = self._require_client(client) + query_params = self._query_params # Pass '?projection=full' here because 'PATCH' documented not # to work properly w/ 'noAcl'. - query_params = {"projection": "full"} - if self.user_project is not None: - query_params["userProject"] = self.user_project + query_params["projection"] = "full" update_properties = {key: self._properties[key] for key in self._changes} # Make the API call. @@ -178,9 +185,8 @@ def update(self, client=None): ``client`` stored on the current object. """ client = self._require_client(client) - query_params = {"projection": "full"} - if self.user_project is not None: - query_params["userProject"] = self.user_project + query_params = self._query_params + query_params["projection"] = "full" api_response = client._connection.api_request( method="PUT", path=self.path, diff --git a/storage/google/cloud/storage/blob.py b/storage/google/cloud/storage/blob.py index 58d25e5d9a02..825b55a03967 100644 --- a/storage/google/cloud/storage/blob.py +++ b/storage/google/cloud/storage/blob.py @@ -159,7 +159,13 @@ class Blob(_PropertyMixin): """ def __init__( - self, name, bucket, chunk_size=None, encryption_key=None, kms_key_name=None + self, + name, + bucket, + chunk_size=None, + encryption_key=None, + kms_key_name=None, + generation=None, ): name = _bytes_to_unicode(name) super(Blob, self).__init__(name=name) @@ -177,6 +183,9 @@ def __init__( if kms_key_name is not None: self._properties["kmsKeyName"] = kms_key_name + if generation is not None: + self._properties["generation"] = generation + @property def chunk_size(self): """Get the blob's default chunk size. @@ -257,6 +266,16 @@ def user_project(self): """ return self.bucket.user_project + @property + def _query_params(self): + """Default query parameters.""" + params = {} + if self.generation is not None: + params["generation"] = self.generation + if self.user_project is not None: + params["userProject"] = self.user_project + return params + @property def public_url(self): """The public URL for this blob. @@ -387,6 +406,9 @@ def exists(self, client=None): # minimize the returned payload. query_params = {"fields": "name"} + if self.generation: + query_params["generation"] = self.generation + if self.user_project is not None: query_params["userProject"] = self.user_project @@ -423,7 +445,9 @@ def delete(self, client=None): (propagated from :meth:`google.cloud.storage.bucket.Bucket.delete_blob`). """ - return self.bucket.delete_blob(self.name, client=client) + return self.bucket.delete_blob( + self.name, client=client, generation=self.generation + ) def _get_transport(self, client): """Return the client's transport. @@ -1485,6 +1509,9 @@ def rewrite(self, source, token=None, client=None): if token: query_params["rewriteToken"] = token + if source.generation: + query_params["sourceGeneration"] = source.generation + if self.user_project is not None: query_params["userProject"] = self.user_project @@ -1533,7 +1560,7 @@ def update_storage_class(self, new_class, client=None): raise ValueError("Invalid storage class: %s" % (new_class,)) # Update current blob's storage class prior to rewrite - self._patch_property('storageClass', new_class) + self._patch_property("storageClass", new_class) # Execute consecutive rewrite operations until operation is done token, _, _ = self.rewrite(self) diff --git a/storage/google/cloud/storage/bucket.py b/storage/google/cloud/storage/bucket.py index b20e0c39b04d..aca501a13faf 100644 --- a/storage/google/cloud/storage/bucket.py +++ b/storage/google/cloud/storage/bucket.py @@ -345,7 +345,22 @@ def user_project(self): """ return self._user_project - def blob(self, blob_name, chunk_size=None, encryption_key=None, kms_key_name=None): + @property + def _query_params(self): + """Default query parameters.""" + params = {} + if self.user_project is not None: + params["userProject"] = self.user_project + return params + + def blob( + self, + blob_name, + chunk_size=None, + encryption_key=None, + kms_key_name=None, + generation=None, + ): """Factory constructor for blob object. .. note:: @@ -368,6 +383,10 @@ def blob(self, blob_name, chunk_size=None, encryption_key=None, kms_key_name=Non :param kms_key_name: Optional resource name of KMS key used to encrypt blob's content. + :type generation: long + :param generation: Optional. If present, selects a specific revision of + this object. + :rtype: :class:`google.cloud.storage.blob.Blob` :returns: The blob object created. """ @@ -377,6 +396,7 @@ def blob(self, blob_name, chunk_size=None, encryption_key=None, kms_key_name=Non chunk_size=chunk_size, encryption_key=encryption_key, kms_key_name=kms_key_name, + generation=generation, ) def notification( @@ -549,7 +569,9 @@ def path(self): return self.path_helper(self.name) - def get_blob(self, blob_name, client=None, encryption_key=None, **kwargs): + def get_blob( + self, blob_name, client=None, encryption_key=None, generation=None, **kwargs + ): """Get a blob object by name. This will return None if the blob doesn't exist: @@ -574,6 +596,10 @@ def get_blob(self, blob_name, client=None, encryption_key=None, **kwargs): See https://cloud.google.com/storage/docs/encryption#customer-supplied. + :type generation: long + :param generation: Optional. If present, selects a specific revision of + this object. + :type kwargs: dict :param kwargs: Keyword arguments to pass to the :class:`~google.cloud.storage.blob.Blob` constructor. @@ -586,6 +612,10 @@ def get_blob(self, blob_name, client=None, encryption_key=None, **kwargs): if self.user_project is not None: query_params["userProject"] = self.user_project + + if generation is not None: + query_params["generation"] = generation + blob = Blob( bucket=self, name=blob_name, encryption_key=encryption_key, **kwargs ) @@ -791,7 +821,7 @@ def delete(self, force=False, client=None): _target_object=None, ) - def delete_blob(self, blob_name, client=None): + def delete_blob(self, blob_name, client=None, generation=None): """Deletes a blob from the current bucket. If the blob isn't found (backend 404), raises a @@ -813,6 +843,10 @@ def delete_blob(self, blob_name, client=None): :param client: Optional. The client to use. If not passed, falls back to the ``client`` stored on the current bucket. + :type generation: long + :param generation: Optional. If present, permanently deletes a specific + revision of this object. + :raises: :class:`google.cloud.exceptions.NotFound` (to suppress the exception, call ``delete_blobs``, passing a no-op ``on_error`` callback, e.g.: @@ -825,6 +859,9 @@ def delete_blob(self, blob_name, client=None): client = self._require_client(client) query_params = {} + if generation is not None: + query_params["generation"] = generation + if self.user_project is not None: query_params["userProject"] = self.user_project diff --git a/storage/tests/system.py b/storage/tests/system.py index 1854b2dcfbea..3cef76f15fbb 100644 --- a/storage/tests/system.py +++ b/storage/tests/system.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import io import os import tempfile import re @@ -78,6 +79,7 @@ def setUpModule(): # In the **very** rare case the bucket name is reserved, this # fails with a ConnectionError. Config.TEST_BUCKET = Config.CLIENT.bucket(bucket_name) + Config.TEST_BUCKET.versioning_enabled = True retry_429(Config.TEST_BUCKET.create)() @@ -414,6 +416,20 @@ def test_crud_blob_w_user_project(self): # Exercise 'objects.insert' w/ userProject. blob.upload_from_filename(file_data["path"]) + gen0 = blob.generation + + # Upload a second generation of the blob + blob.upload_from_file(io.StringIO(six.text_type("gen1"))) + gen1 = blob.generation + + blob0 = with_user_project.blob("SmallFile", generation=gen0) + blob1 = with_user_project.blob("SmallFile", generation=gen1) + + # Exercise 'objects.get' w/ generation + self.assertEqual(with_user_project.get_blob(blob.name).generation, gen1) + self.assertEqual( + with_user_project.get_blob(blob.name, generation=gen0).generation, gen0 + ) try: # Exercise 'objects.get' (metadata) w/ userProject. @@ -421,22 +437,31 @@ def test_crud_blob_w_user_project(self): blob.reload() # Exercise 'objects.get' (media) w/ userProject. - downloaded = blob.download_as_string() - self.assertEqual(downloaded, file_contents) + self.assertEqual(blob0.download_as_string(), file_contents) + self.assertEqual(blob1.download_as_string(), "gen1") # Exercise 'objects.patch' w/ userProject. - blob.content_language = "en" - blob.patch() - self.assertEqual(blob.content_language, "en") + blob0.content_language = "en" + blob0.patch() + self.assertEqual(blob0.content_language, "en") + self.assertIsNone(blob1.content_language) # Exercise 'objects.update' w/ userProject. metadata = {"foo": "Foo", "bar": "Bar"} - blob.metadata = metadata - blob.update() - self.assertEqual(blob.metadata, metadata) + blob0.metadata = metadata + blob0.update() + self.assertEqual(blob0.metadata, metadata) + self.assertIsNone(blob1.metadata) finally: # Exercise 'objects.delete' (metadata) w/ userProject. - blob.delete() + blobs = with_user_project.list_blobs(prefix=blob.name, versions=True) + self.assertEqual([each.generation for each in blobs], [gen0, gen1]) + + blob0.delete() + blobs = with_user_project.list_blobs(prefix=blob.name, versions=True) + self.assertEqual([each.generation for each in blobs], [gen1]) + + blob1.delete() @unittest.skipUnless(USER_PROJECT, "USER_PROJECT not set in environment.") def test_blob_acl_w_user_project(self): diff --git a/storage/tests/unit/test_blob.py b/storage/tests/unit/test_blob.py index 3dcf85ee43d0..f44a347e170c 100644 --- a/storage/tests/unit/test_blob.py +++ b/storage/tests/unit/test_blob.py @@ -101,6 +101,13 @@ def test_ctor_w_kms_key_name(self): self.assertEqual(blob._encryption_key, None) self.assertEqual(blob.kms_key_name, KMS_RESOURCE) + def test_ctor_with_generation(self): + BLOB_NAME = "blob-name" + GENERATION = 12345 + bucket = _Bucket() + blob = self._make_one(BLOB_NAME, bucket=bucket, generation=GENERATION) + self.assertEqual(blob.generation, GENERATION) + def _set_properties_helper(self, kms_key_name=None): import datetime from google.cloud._helpers import UTC @@ -576,7 +583,7 @@ def test_delete(self): bucket._blobs[BLOB_NAME] = 1 blob.delete() self.assertFalse(blob.exists()) - self.assertEqual(bucket._deleted, [(BLOB_NAME, None)]) + self.assertEqual(bucket._deleted, [(BLOB_NAME, None, blob.generation)]) def test__get_transport(self): client = mock.Mock(spec=[u"_credentials", "_http"]) @@ -2295,6 +2302,44 @@ def test_rewrite_response_without_resource(self): self.assertEqual(rewritten, 33) self.assertEqual(size, 42) + def test_rewrite_generation(self): + SOURCE_BLOB = "source" + SOURCE_GENERATION = 42 + DEST_BLOB = "dest" + DEST_BUCKET = "other-bucket" + TOKEN = "TOKEN" + RESPONSE = { + "totalBytesRewritten": 33, + "objectSize": 42, + "done": False, + "rewriteToken": TOKEN, + } + response = ({"status": http_client.OK}, RESPONSE) + connection = _Connection(response) + client = _Client(connection) + source_bucket = _Bucket(client=client) + source_blob = self._make_one(SOURCE_BLOB, bucket=source_bucket) + source_blob._set_properties({"generation": SOURCE_GENERATION}) + dest_bucket = _Bucket(client=client, name=DEST_BUCKET) + dest_blob = self._make_one(DEST_BLOB, bucket=dest_bucket) + + token, rewritten, size = dest_blob.rewrite(source_blob) + + self.assertEqual(token, TOKEN) + self.assertEqual(rewritten, 33) + self.assertEqual(size, 42) + + kw, = connection._requested + self.assertEqual(kw["method"], "POST") + self.assertEqual( + kw["path"], + "/b/%s/o/%s/rewriteTo/b/%s/o/%s" + % ( + (source_bucket.name, source_blob.name, dest_bucket.name, dest_blob.name) + ), + ) + self.assertEqual(kw["query_params"], {"sourceGeneration": SOURCE_GENERATION}) + def test_rewrite_other_bucket_other_name_no_encryption_partial(self): SOURCE_BLOB = "source" DEST_BLOB = "dest" @@ -2508,13 +2553,13 @@ def test_update_storage_class_large_file(self): "objectSize": 84, "done": False, "rewriteToken": TOKEN, - "resource": {"storageClass": STORAGE_CLASS} + "resource": {"storageClass": STORAGE_CLASS}, } COMPLETE_RESPONSE = { "totalBytesRewritten": 84, "objectSize": 84, "done": True, - "resource": {"storageClass": STORAGE_CLASS} + "resource": {"storageClass": STORAGE_CLASS}, } response_1 = ({"status": http_client.OK}, INCOMPLETE_RESPONSE) response_2 = ({"status": http_client.OK}, COMPLETE_RESPONSE) @@ -2534,7 +2579,7 @@ def test_update_storage_class_wo_encryption_key(self): "totalBytesRewritten": 42, "objectSize": 42, "done": True, - "resource": {"storageClass": STORAGE_CLASS} + "resource": {"storageClass": STORAGE_CLASS}, } response = ({"status": http_client.OK}, RESPONSE) connection = _Connection(response) @@ -2579,7 +2624,7 @@ def test_update_storage_class_w_encryption_key_w_user_project(self): "totalBytesRewritten": 42, "objectSize": 42, "done": True, - "resource": {"storageClass": STORAGE_CLASS} + "resource": {"storageClass": STORAGE_CLASS}, } response = ({"status": http_client.OK}, RESPONSE) connection = _Connection(response) @@ -3175,9 +3220,9 @@ def __init__(self, client=None, name="name", user_project=None): self.path = "/b/" + name self.user_project = user_project - def delete_blob(self, blob_name, client=None): + def delete_blob(self, blob_name, client=None, generation=None): del self._blobs[blob_name] - self._deleted.append((blob_name, client)) + self._deleted.append((blob_name, client, generation)) class _Signer(object): diff --git a/storage/tests/unit/test_bucket.py b/storage/tests/unit/test_bucket.py index cee84decfc42..2191b7750ed8 100644 --- a/storage/tests/unit/test_bucket.py +++ b/storage/tests/unit/test_bucket.py @@ -256,6 +256,21 @@ def test_blob_w_encryption_key(self): self.assertEqual(blob._encryption_key, KEY) self.assertIsNone(blob.kms_key_name) + def test_blob_w_generation(self): + from google.cloud.storage.blob import Blob + + BUCKET_NAME = "BUCKET_NAME" + BLOB_NAME = "BLOB_NAME" + GENERATION = 123 + + bucket = self._make_one(name=BUCKET_NAME) + blob = bucket.blob(BLOB_NAME, generation=GENERATION) + self.assertIsInstance(blob, Blob) + self.assertIs(blob.bucket, bucket) + self.assertIs(blob.client, bucket.client) + self.assertEqual(blob.name, BLOB_NAME) + self.assertEqual(blob.generation, GENERATION) + def test_blob_w_kms_key_name(self): from google.cloud.storage.blob import Blob @@ -569,6 +584,21 @@ def test_get_blob_hit_w_user_project(self): self.assertEqual(kw["path"], "/b/%s/o/%s" % (NAME, BLOB_NAME)) self.assertEqual(kw["query_params"], {"userProject": USER_PROJECT}) + def test_get_blob_hit_w_generation(self): + NAME = "name" + BLOB_NAME = "blob-name" + GENERATION = 1512565576797178 + connection = _Connection({"name": BLOB_NAME}) + client = _Client(connection) + bucket = self._make_one(name=NAME) + blob = bucket.get_blob(BLOB_NAME, client=client, generation=GENERATION) + self.assertIs(blob.bucket, bucket) + self.assertEqual(blob.name, BLOB_NAME) + kw, = connection._requested + self.assertEqual(kw["method"], "GET") + self.assertEqual(kw["path"], "/b/%s/o/%s" % (NAME, BLOB_NAME)) + self.assertEqual(kw["query_params"], {"generation": GENERATION}) + def test_get_blob_hit_with_kwargs(self): from google.cloud.storage.blob import _get_encryption_headers @@ -837,6 +867,20 @@ def test_delete_blob_hit_with_user_project(self): self.assertEqual(kw["path"], "/b/%s/o/%s" % (NAME, BLOB_NAME)) self.assertEqual(kw["query_params"], {"userProject": USER_PROJECT}) + def test_delete_blob_hit_with_generation(self): + NAME = "name" + BLOB_NAME = "blob-name" + GENERATION = 1512565576797178 + connection = _Connection({}) + client = _Client(connection) + bucket = self._make_one(client=client, name=NAME) + result = bucket.delete_blob(BLOB_NAME, generation=GENERATION) + self.assertIsNone(result) + kw, = connection._requested + self.assertEqual(kw["method"], "DELETE") + self.assertEqual(kw["path"], "/b/%s/o/%s" % (NAME, BLOB_NAME)) + self.assertEqual(kw["query_params"], {"generation": GENERATION}) + def test_delete_blobs_empty(self): NAME = "name" connection = _Connection()