From 6feffd54f35309f088abee8e1127990c2e39d0ec Mon Sep 17 00:00:00 2001 From: Stephen Date: Mon, 10 Sep 2018 13:44:41 -0400 Subject: [PATCH] Revert "Revert "Support Bucket/Object lock operations (#320)" (#373)" This reverts commit 702d7bdef598b7c49ad7b1c2d263603f209210a6. --- src/bucket.ts | 124 ++++++++++++++++++++++++- src/file.ts | 63 +++++++++++++ src/index.ts | 11 +++ system-test/storage.js | 206 +++++++++++++++++++++++++++++++++++++++++ test/bucket.ts | 60 ++++++++++++ test/file.ts | 60 ++++++++++++ 6 files changed, 523 insertions(+), 1 deletion(-) diff --git a/src/bucket.ts b/src/bucket.ts index 10defda2c..61769be45 100644 --- a/src/bucket.ts +++ b/src/bucket.ts @@ -1455,7 +1455,7 @@ class Bucket extends ServiceObject { /** * @callback GetBucketMetadataCallback * @param {?Error} err Request error, if any. - * @param {object} files The bucket metadata. + * @param {object} metadata The bucket metadata. * @param {object} apiResponse The full API response. */ /** @@ -1586,6 +1586,48 @@ class Bucket extends ServiceObject { }); } + /** + * Lock a previously-defined retention policy. This will prevent changes to + * the policy. + * + * @throws {Error} if a metageneration is not provided. + * + * @param {Number|String} metageneration The bucket's metageneration. This is + * accesssible from calling {@link File#getMetadata}. + * @param {SetBucketMetadataCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * const storage = require('@google-cloud/storage')(); + * const bucket = storage.bucket('albums'); + * + * const metageneration = 2; + * + * bucket.lock(metageneration, function(err, apiResponse) {}); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * bucket.lock(metageneration).then(function(data) { + * const apiResponse = data[0]; + * }); + */ + lock(metageneration, callback) { + if (!is.number(metageneration) && !is.string(metageneration)) { + throw new Error('A metageneration must be provided.'); + } + + this.request( + { + method: 'POST', + uri: '/lockRetentionPolicy', + qs: { + ifMetagenerationMatch: metageneration, + }, + }, + callback); + } + /** * @typedef {array} MakeBucketPrivateResponse * @property {File[]} 0 List of files made private. @@ -1858,6 +1900,34 @@ class Bucket extends ServiceObject { return new Notification(this, id); } + /** + * Remove an already-existing retention policy from this bucket, if it is not + * locked. + * + * @param {SetBucketMetadataCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * const storage = require('@google-cloud/storage')(); + * const bucket = storage.bucket('albums'); + * + * bucket.removeRetentionPeriod(function(err, apiResponse) {}); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * bucket.removeRetentionPeriod().then(function(data) { + * const apiResponse = data[0]; + * }); + */ + removeRetentionPeriod(callback) { + this.setMetadata( + { + retentionPolicy: null, + }, + callback); + } + /** * Makes request and applies userProject query parameter if necessary. * @@ -1989,6 +2059,13 @@ class Bucket extends ServiceObject { * }, function(err, apiResponse) {}); * * //- + * // Set the default event-based hold value for new objects in this bucket. + * //- + * bucket.setMetadata({ + * defaultEventBasedHold: true + * }, function(err, apiResponse) {}); + * + * //- * // If the callback is omitted, we'll return a Promise. * //- * bucket.setMetadata(metadata).then(function(data) { @@ -2022,6 +2099,51 @@ class Bucket extends ServiceObject { }); } + /** + * Lock all objects contained in the bucket, based on their creation time. Any + * attempt to overwrite or delete objects younger than the retention period + * will result in a `PERMISSION_DENIED` error. + * + * An unlocked retention policy can be modified or removed from the bucket via + * {@link File#removeRetentionPeriod} and {@link File#setRetentionPeriod}. A + * locked retention policy cannot be removed or shortened in duration for the + * lifetime of the bucket. Attempting to remove or decrease period of a locked + * retention policy will result in a `PERMISSION_DENIED` error. You can still + * increase the policy. + * + * @param {*} duration In seconds, the minimum retention time for all objects + * contained in this bucket. + * @param {SetBucketMetadataCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * const storage = require('@google-cloud/storage')(); + * const bucket = storage.bucket('albums'); + * + * const DURATION_SECONDS = 15780000; // 6 months. + * + * //- + * // Lock the objects in this bucket for 6 months. + * //- + * bucket.setRetentionPeriod(DURATION_SECONDS, function(err, apiResponse) {}); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * bucket.setRetentionPeriod(DURATION_SECONDS).then(function(data) { + * const apiResponse = data[0]; + * }); + */ + setRetentionPeriod(duration, callback) { + this.setMetadata( + { + retentionPolicy: { + retentionPeriod: duration, + }, + }, + callback); + } + /** * @callback SetStorageClassCallback * @param {?Error} err Request error, if any. diff --git a/src/file.ts b/src/file.ts index 82667916f..0169406e0 100644 --- a/src/file.ts +++ b/src/file.ts @@ -1500,6 +1500,51 @@ class File extends ServiceObject { (this.parent as any).get.call(this, options, callback); } + /** + * @typedef {array} GetExpirationDateResponse + * @property {date} 0 A Date object representing the earliest time this file's + * retention policy will expire. + */ + /** + * @callback GetExpirationDateCallback + * @param {?Error} err Request error, if any. + * @param {date} expirationDate A Date object representing the earliest time + * this file's retention policy will expire. + */ + /** + * If this bucket has a retention policy defined, use this method to get a + * Date object representing the earliest time this file will expire. + * + * @param {GetExpirationDateCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * const storage = require('@google-cloud/storage')(); + * const myBucket = storage.bucket('my-bucket'); + * + * const file = myBucket.file('my-file'); + * + * file.getExpirationDate(function(err, expirationDate) { + * // expirationDate is a Date object. + * }); + */ + getExpirationDate(callback) { + this.getMetadata((err, metadata, apiResponse) => { + if (err) { + callback(err, null, apiResponse); + return; + } + + if (!metadata.retentionExpirationTime) { + const error = new Error('An expiration time is not available.'); + callback(error, null, apiResponse); + return; + } + + callback(null, new Date(metadata.retentionExpirationTime), apiResponse); + }); + } + /** * @typedef {array} GetFileMetadataResponse * @property {object} 0 The {@link File} metadata. @@ -2377,6 +2422,24 @@ class File extends ServiceObject { * }); * * //- + * // Set a temporary hold on this file from its bucket's retention period + * // configuration. + * // + * file.setMetadata({ + * temporaryHold: true + * }, function(err, apiResponse) {}); + * + * //- + * // Alternatively, you may set a temporary hold. This will follow the same + * // behavior as an event-based hold, with the exception that the bucket's + * // retention policy will not renew for this file from the time the hold is + * // released. + * //- + * file.setMetadata({ + * eventBasedHold: true + * }, function(err, apiResponse) {}); + * + * //- * // If the callback is omitted, we'll return a Promise. * //- * file.setMetadata(metadata).then(function(data) { diff --git a/src/index.ts b/src/index.ts index c63213acc..6c73f0739 100644 --- a/src/index.ts +++ b/src/index.ts @@ -361,6 +361,17 @@ class Storage extends Service { * storage.createBucket('new-bucket', metadata, callback); * * //- + * // Create a bucket with a retention policy of 6 months. + * //- + * const metadata = { + * retentionPolicy: { + * retentionPeriod: 15780000 // 6 months in seconds. + * } + * }; + * + * storage.createBucket('new-bucket', metadata, callback); + * + * //- * // Enable versioning on a new bucket. * //- * const metadata = { diff --git a/system-test/storage.js b/system-test/storage.js index 181bc0a9d..6c957c802 100644 --- a/system-test/storage.js +++ b/system-test/storage.js @@ -928,6 +928,212 @@ describe('storage', function() { }); }); + describe('bucket retention policies', function() { + const RETENTION_DURATION_SECONDS = 10; + + describe('bucket', function() { + it('should create a bucket with a retention policy', function() { + const bucket = storage.bucket(generateName()); + + return bucket + .create({ + retentionPolicy: { + retentionPeriod: RETENTION_DURATION_SECONDS, + }, + }) + .then(() => bucket.getMetadata()) + .then(response => { + const metadata = response[0]; + + assert.strictEqual( + metadata.retentionPolicy.retentionPeriod, + `${RETENTION_DURATION_SECONDS}` + ); + }); + }); + + it('should set a retention policy', function() { + const bucket = storage.bucket(generateName()); + + return bucket + .create() + .then(() => bucket.setRetentionPeriod(RETENTION_DURATION_SECONDS)) + .then(() => bucket.getMetadata()) + .then(response => { + const metadata = response[0]; + + assert.strictEqual( + metadata.retentionPolicy.retentionPeriod, + `${RETENTION_DURATION_SECONDS}` + ); + }); + }); + + it('should lock the retention period', function(done) { + const bucket = storage.bucket(generateName()); + + bucket + .create() + .then(() => bucket.setRetentionPeriod(RETENTION_DURATION_SECONDS)) + .then(() => bucket.lock()) + .then(() => bucket.setRetentionPeriod(RETENTION_DURATION_SECONDS / 2)) + .catch(err => { + assert.strictEqual(err.code, 403); + done(); + }); + }); + + it('should remove a retention period', function() { + const bucket = storage.bucket(generateName()); + + return bucket + .create() + .then(() => bucket.setRetentionPeriod(RETENTION_DURATION_SECONDS)) + .then(() => bucket.removeRetentionPeriod()) + .then(() => bucket.getMetadata()) + .then(response => { + const metadata = response[0]; + + assert.strictEqual(metadata.retentionPolicy, undefined); + }); + }); + }); + + describe('file', function() { + const BUCKET = storage.bucket(generateName()); + const FILE = BUCKET.file(generateName()); + + before(function() { + return BUCKET.create({ + retentionPolicy: { + retentionPeriod: 1, + }, + }).then(() => FILE.save('data')); + }); + + afterEach(function() { + return FILE.setMetadata({temporaryHold: null, eventBasedHold: null}); + }); + + after(function() { + return FILE.delete(); + }); + + it('should set and release an event-based hold', function() { + return FILE.setMetadata({eventBasedHold: true}) + .then(response => { + const metadata = response[0]; + + assert.strictEqual(metadata.eventBasedHold, true); + }) + .then(() => FILE.setMetadata({eventBasedHold: false})) + .then(() => FILE.getMetadata()) + .then(response => { + const metadata = response[0]; + + assert.strictEqual(metadata.eventBasedHold, false); + }); + }); + + it('should set and release a temporary hold', function() { + return FILE.setMetadata({temporaryHold: true}) + .then(response => { + const metadata = response[0]; + + assert.strictEqual(metadata.temporaryHold, true); + }) + .then(() => FILE.setMetadata({temporaryHold: false})) + .then(() => FILE.getMetadata()) + .then(response => { + const metadata = response[0]; + + assert.strictEqual(metadata.temporaryHold, false); + }); + }); + + it('should get an expiration date', function() { + return FILE.getExpirationDate().then(response => { + const expirationDate = response[0]; + assert(expirationDate instanceof Date); + }); + }); + }); + + describe('operations on held objects', function() { + const BUCKET = storage.bucket(generateName()); + const FILES = []; + + const RETENTION_PERIOD_SECONDS = 5; // Each test has this much time! + + function createFile(callback) { + const file = BUCKET.file(generateName()); + FILES.push(file); + + file.save('data', function(err) { + if (err) { + callback(err); + return; + } + + callback(null, file); + }); + } + + function deleteFiles(callback) { + async.each( + FILES, + function(file, next) { + file.setMetadata({temporaryHold: null}, function(err) { + if (err) { + next(err); + return; + } + + file.delete(next); + }); + }, + callback + ); + } + + before(function() { + return BUCKET.create({ + retentionPolicy: { + retentionPeriod: RETENTION_PERIOD_SECONDS, + }, + }); + }); + + after(function(done) { + setTimeout(deleteFiles, RETENTION_PERIOD_SECONDS * 1000, done); + }); + + //verify how the client library behaves when holds are enabled + //and attempting to perform an overwrite and delete. + it('should block an overwrite request', function(done) { + createFile(function(err, file) { + assert.ifError(err); + + file.save('new data', function(err) { + assert.strictEqual(err.code, 403); + done(); + }); + }); + }); + + it('should block a delete request', function(done) { + createFile(function(err, file) { + assert.ifError(err); + + file.delete(function(err) { + assert.strictEqual(err.code, 403); + done(); + }); + }); + }); + }); + }); + describe('requester pays', function() { const HAS_2ND_PROJECT = is.defined(process.env.GCN_STORAGE_2ND_PROJECT_ID); let bucket; diff --git a/test/bucket.ts b/test/bucket.ts index 71de024af..140a974a5 100644 --- a/test/bucket.ts +++ b/test/bucket.ts @@ -1567,6 +1567,34 @@ describe('Bucket', () => { }); }); + describe('lock', () => { + it('should throw if a metageneration is not provided', () => { + const expectedError = new RegExp('A metageneration must be provided.'); + + assert.throws(() => { + bucket.lock(assert.ifError); + }, expectedError); + }); + + it('should make the correct request', done => { + const metageneration = 8; + + bucket.request = (reqOpts, callback) => { + assert.deepStrictEqual(reqOpts, { + method: 'POST', + uri: '/lockRetentionPolicy', + qs: { + ifMetagenerationMatch: metageneration, + }, + }); + + callback(); // done() + }; + + bucket.lock(metageneration, done); + }); + }); + describe('makePrivate', () => { it('should set predefinedAcl & privatize files', done => { let didSetPredefinedAcl = false; @@ -1728,6 +1756,20 @@ describe('Bucket', () => { }); }); + describe('removeRetentionPeriod', () => { + it('should call setMetadata correctly', done => { + bucket.setMetadata = (metadata, callback) => { + assert.deepStrictEqual(metadata, { + retentionPolicy: null, + }); + + callback(); // done() + }; + + bucket.removeRetentionPeriod(done); + }); + }); + describe('request', () => { const USER_PROJECT = 'grape-spaceship-123'; @@ -1901,6 +1943,24 @@ describe('Bucket', () => { }); }); + describe('setRetentionPeriod', () => { + it('should call setMetadata correctly', done => { + const duration = 90000; + + bucket.setMetadata = (metadata, callback) => { + assert.deepStrictEqual(metadata, { + retentionPolicy: { + retentionPeriod: duration, + }, + }); + + callback(); // done() + }; + + bucket.setRetentionPeriod(duration, done); + }); + }); + describe('setStorageClass', () => { const STORAGE_CLASS = 'NEW_STORAGE_CLASS'; const OPTIONS = {}; diff --git a/test/file.ts b/test/file.ts index 03f42780c..149ee424e 100644 --- a/test/file.ts +++ b/test/file.ts @@ -1957,6 +1957,66 @@ describe('File', () => { }); }); + describe('getExpirationDate', () => { + it('should refresh metadata', done => { + file.getMetadata = () => { + done(); + }; + + file.getExpirationDate(assert.ifError); + }); + + it('should return error from getMetadata', done => { + const error = new Error('Error.'); + const apiResponse = {}; + + file.getMetadata = callback => { + callback(error, null, apiResponse); + }; + + file.getExpirationDate((err, expirationDate, apiResponse_) => { + assert.strictEqual(err, error); + assert.strictEqual(expirationDate, null); + assert.strictEqual(apiResponse_, apiResponse); + done(); + }); + }); + + it('should return an error if there is no expiration time', done => { + const apiResponse = {}; + + file.getMetadata = callback => { + callback(null, {}, apiResponse); + }; + + file.getExpirationDate((err, expirationDate, apiResponse_) => { + assert.strictEqual(err.message, `An expiration time is not available.`); + assert.strictEqual(expirationDate, null); + assert.strictEqual(apiResponse_, apiResponse); + done(); + }); + }); + + it('should return the expiration time as a Date object', done => { + const expirationTime = new Date(); + + const apiResponse = { + retentionExpirationTime: expirationTime.toJSON(), + }; + + file.getMetadata = callback => { + callback(null, apiResponse, apiResponse); + }; + + file.getExpirationDate((err, expirationDate, apiResponse_) => { + assert.ifError(err); + assert.deepStrictEqual(expirationDate, expirationTime); + assert.strictEqual(apiResponse_, apiResponse); + done(); + }); + }); + }); + describe('getMetadata', () => { it('should make the correct request', done => { extend(file.parent, {