From d4c7c479dd671139a6e0f7b48cfcccd926e3ff71 Mon Sep 17 00:00:00 2001 From: Seth Hollyman Date: Thu, 17 Jan 2019 16:07:25 -0800 Subject: [PATCH 1/7] BigQuery: Add long term storage bytes for managed tables. --- .../cloud/bigquery/StandardTableDefinition.java | 16 ++++++++++++++++ .../bigquery/StandardTableDefinitionTest.java | 5 +++++ .../google/cloud/bigquery/it/ITBigQueryTest.java | 3 +++ 3 files changed, 24 insertions(+) diff --git a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/StandardTableDefinition.java b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/StandardTableDefinition.java index f12e6b608a91..986ddf131db4 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/StandardTableDefinition.java +++ b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/StandardTableDefinition.java @@ -130,6 +130,8 @@ public abstract static class Builder public abstract Builder setNumBytes(Long numBytes); + public abstract Builder setNumLongTermBytes(Long numLongTermBytes); + public abstract Builder setNumRows(Long numRows); public abstract Builder setLocation(String location); @@ -161,6 +163,16 @@ public abstract static class Builder @Nullable public abstract Long getNumBytes(); + /** Returns the number of bytes considered "long-term storage" for reduced + * billing purposes. + * + * @see + * Long Term Storage Pricing + */ + @Nullable + public abstract Long getNumLongTermBytes(); + /** Returns the number of rows in this table, excluding any data in the streaming buffer. */ @Nullable public abstract Long getNumRows(); @@ -221,6 +233,7 @@ Table toPb() { tablePb.setNumRows(BigInteger.valueOf(getNumRows())); } tablePb.setNumBytes(getNumBytes()); + tablePb.setNumLongTermBytes(getNumLongTermBytes()); tablePb.setLocation(getLocation()); if (getStreamingBuffer() != null) { tablePb.setStreamingBuffer(getStreamingBuffer().toPb()); @@ -249,6 +262,9 @@ static StandardTableDefinition fromPb(Table tablePb) { if (tablePb.getClustering() != null) { builder.setClustering(Clustering.fromPb(tablePb.getClustering())); } + if (tablePb.getNumLongTermBytes() != null) { + builder.setNumLongTermBytes(tablePb.getNumLongTermBytes()); + } return builder.setNumBytes(tablePb.getNumBytes()).setLocation(tablePb.getLocation()).build(); } } diff --git a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/StandardTableDefinitionTest.java b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/StandardTableDefinitionTest.java index d70d33fb119b..7bfbf62754b8 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/StandardTableDefinitionTest.java +++ b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/StandardTableDefinitionTest.java @@ -44,6 +44,7 @@ public class StandardTableDefinitionTest { .build(); private static final Schema TABLE_SCHEMA = Schema.of(FIELD_SCHEMA1, FIELD_SCHEMA2, FIELD_SCHEMA3); private static final Long NUM_BYTES = 42L; + private static final Long NUM_LONG_TERM_BYTES = 18L; private static final Long NUM_ROWS = 43L; private static final String LOCATION = "US"; private static final StreamingBuffer STREAMING_BUFFER = new StreamingBuffer(1L, 2L, 3L); @@ -56,6 +57,7 @@ public class StandardTableDefinitionTest { .setLocation(LOCATION) .setNumBytes(NUM_BYTES) .setNumRows(NUM_ROWS) + .setNumLongTermBytes(NUM_LONG_TERM_BYTES) .setStreamingBuffer(STREAMING_BUFFER) .setSchema(TABLE_SCHEMA) .setTimePartitioning(TIME_PARTITIONING) @@ -84,6 +86,7 @@ public void testBuilder() { assertEquals(TABLE_SCHEMA, TABLE_DEFINITION.getSchema()); assertEquals(LOCATION, TABLE_DEFINITION.getLocation()); assertEquals(NUM_BYTES, TABLE_DEFINITION.getNumBytes()); + assertEquals(NUM_LONG_TERM_BYTES, TABLE_DEFINITION.getNumLongTermBytes()); assertEquals(NUM_ROWS, TABLE_DEFINITION.getNumRows()); assertEquals(STREAMING_BUFFER, TABLE_DEFINITION.getStreamingBuffer()); assertEquals(TIME_PARTITIONING, TABLE_DEFINITION.getTimePartitioning()); @@ -97,6 +100,7 @@ public void testOf() { assertEquals(TABLE_SCHEMA, TABLE_DEFINITION.getSchema()); assertNull(definition.getLocation()); assertNull(definition.getNumBytes()); + assertNull(definition.getNumLongTermBytes()); assertNull(definition.getNumRows()); assertNull(definition.getStreamingBuffer()); assertNull(definition.getTimePartitioning()); @@ -131,6 +135,7 @@ private void compareStandardTableDefinition( assertEquals(expected.getSchema(), value.getSchema()); assertEquals(expected.getType(), value.getType()); assertEquals(expected.getNumBytes(), value.getNumBytes()); + assertEquals(expected.getNumLongTermBytes(), value.getNumLongTermBytes()); assertEquals(expected.getNumRows(), value.getNumRows()); assertEquals(expected.getLocation(), value.getLocation()); assertEquals(expected.getStreamingBuffer(), value.getStreamingBuffer()); diff --git a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index a814158016e5..195f36e7bd12 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -428,6 +428,7 @@ public void testCreateAndGetTable() { assertNotNull(remoteTable.getCreationTime()); assertNotNull(remoteTable.getLastModifiedTime()); assertNotNull(remoteTable.getDefinition().getNumBytes()); + assertNotNull(remoteTable.getDefinition().getNumLongTermBytes()); assertNotNull(remoteTable.getDefinition().getNumRows()); assertEquals( partitioning, remoteTable.getDefinition().getTimePartitioning()); @@ -460,6 +461,7 @@ public void testCreateAndGetTableWithSelectedField() { assertNull(remoteTable.getDefinition().getSchema()); assertNull(remoteTable.getLastModifiedTime()); assertNull(remoteTable.getDefinition().getNumBytes()); + assertNull(remoteTable.getDefinition().getNumLongTermBytes()); assertNull(remoteTable.getDefinition().getNumRows()); assertNull(remoteTable.getDefinition().getTimePartitioning()); assertNull(remoteTable.getDefinition().getClustering()); @@ -699,6 +701,7 @@ public void testUpdateTableWithSelectedFields() { assertNull(updatedTable.getDefinition().getSchema()); assertNull(updatedTable.getLastModifiedTime()); assertNull(updatedTable.getDefinition().getNumBytes()); + assertNull(updatedTable.getDefinition().getNumLongTermBytes()); assertNull(updatedTable.getDefinition().getNumRows()); assertTrue(createdTable.delete()); } From 8705e56de8accc78c63bfb6a513e5553e9fb840b Mon Sep 17 00:00:00 2001 From: Seth Hollyman Date: Thu, 17 Jan 2019 16:10:41 -0800 Subject: [PATCH 2/7] formatting --- .../com/google/cloud/bigquery/StandardTableDefinition.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/StandardTableDefinition.java b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/StandardTableDefinition.java index 986ddf131db4..b23475a1fa36 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/StandardTableDefinition.java +++ b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/StandardTableDefinition.java @@ -163,7 +163,8 @@ public abstract static class Builder @Nullable public abstract Long getNumBytes(); - /** Returns the number of bytes considered "long-term storage" for reduced + /** + * Returns the number of bytes considered "long-term storage" for reduced * billing purposes. * * @see Date: Thu, 17 Jan 2019 16:33:13 -0800 Subject: [PATCH 3/7] let maven format the things --- .../google/cloud/bigquery/StandardTableDefinition.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/StandardTableDefinition.java b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/StandardTableDefinition.java index b23475a1fa36..9b2ae343875f 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/StandardTableDefinition.java +++ b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/StandardTableDefinition.java @@ -163,13 +163,11 @@ public abstract static class Builder @Nullable public abstract Long getNumBytes(); - /** - * Returns the number of bytes considered "long-term storage" for reduced - * billing purposes. + /** + * Returns the number of bytes considered "long-term storage" for reduced billing purposes. * - * @see - * Long Term Storage Pricing + * @see Long Term Storage + * Pricing */ @Nullable public abstract Long getNumLongTermBytes(); From f0512e3521ab30850bb7aaa4b0076907025a95af Mon Sep 17 00:00:00 2001 From: Seth Hollyman Date: Fri, 18 Jan 2019 14:48:59 -0800 Subject: [PATCH 4/7] plumb this upwards into Table/TableInfo --- .../java/com/google/cloud/bigquery/Table.java | 6 +++++ .../com/google/cloud/bigquery/TableInfo.java | 23 +++++++++++++++++++ .../google/cloud/bigquery/TableInfoTest.java | 9 ++++++++ 3 files changed, 38 insertions(+) diff --git a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Table.java b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Table.java index d38957bd6186..ab341c62c4ff 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Table.java +++ b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Table.java @@ -108,6 +108,12 @@ Builder setNumBytes(Long numBytes) { return this; } + @Override + Builder setNumLongTermBytes(Long numLongTermBytes) { + infoBuilder.setNumLongTermBytes(numLongTermBytes); + return this; + } + @Override Builder setNumRows(BigInteger numRows) { infoBuilder.setNumRows(numRows); diff --git a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/TableInfo.java b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/TableInfo.java index 8fad66d20a7e..34ce90877626 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/TableInfo.java +++ b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/TableInfo.java @@ -66,6 +66,7 @@ public Table apply(TableInfo tableInfo) { private final Long expirationTime; private final Long lastModifiedTime; private final Long numBytes; + private final Long numLongTermBytes; private final BigInteger numRows; private final TableDefinition definition; private final EncryptionConfiguration encryptionConfiguration; @@ -96,6 +97,8 @@ public abstract static class Builder { abstract Builder setNumBytes(Long numBytes); + abstract Builder setNumLongTermBytes(Long numLongTermBytes); + abstract Builder setNumRows(BigInteger numRows); abstract Builder setSelfLink(String selfLink); @@ -141,6 +144,7 @@ static class BuilderImpl extends Builder { private Long expirationTime; private Long lastModifiedTime; private Long numBytes; + private Long numLongTermBytes; private BigInteger numRows; private TableDefinition definition; private EncryptionConfiguration encryptionConfiguration; @@ -159,6 +163,7 @@ static class BuilderImpl extends Builder { this.expirationTime = tableInfo.expirationTime; this.lastModifiedTime = tableInfo.lastModifiedTime; this.numBytes = tableInfo.numBytes; + this.numLongTermBytes = tableInfo.numLongTermBytes; this.numRows = tableInfo.numRows; this.definition = tableInfo.definition; this.encryptionConfiguration = tableInfo.encryptionConfiguration; @@ -178,6 +183,7 @@ static class BuilderImpl extends Builder { this.generatedId = tablePb.getId(); this.selfLink = tablePb.getSelfLink(); this.numBytes = tablePb.getNumBytes(); + this.numLongTermBytes = tablePb.getNumLongTermBytes(); this.numRows = tablePb.getNumRows(); this.definition = TableDefinition.fromPb(tablePb); if (tablePb.getEncryptionConfiguration() != null) { @@ -235,6 +241,11 @@ Builder setNumBytes(Long numBytes) { return this; } + @Override + Builder setNumLongTermBytes(Long numLongTermBytes) { + this.numLongTermBytes = numLongTermBytes; + } + @Override Builder setNumRows(BigInteger numRows) { this.numRows = numRows; @@ -288,6 +299,7 @@ public TableInfo build() { this.expirationTime = builder.expirationTime; this.lastModifiedTime = builder.lastModifiedTime; this.numBytes = builder.numBytes; + this.numLongTermBytes = builder.numLongTermBytes; this.numRows = builder.numRows; this.definition = builder.definition; this.encryptionConfiguration = builder.encryptionConfiguration; @@ -360,6 +372,16 @@ public Long getNumBytes() { return numBytes; } + /** + * Returns the number of bytes considered "long-term storage" for reduced billing purposes. + * + * @see Long Term Storage + * Pricing + */ + public Long getNumLongTermBytes() { + return numLongTermBytes; + } + /** Returns the number of rows of data in this table */ public BigInteger getNumRows() { return numRows; @@ -394,6 +416,7 @@ public String toString() { .add("creationTime", creationTime) .add("lastModifiedTime", lastModifiedTime) .add("numBytes", numBytes) + .add("numLongTermBytes", numLongTermBytes) .add("numRows", numRows) .add("definition", definition) .add("encryptionConfiguration", encryptionConfiguration) diff --git a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/TableInfoTest.java b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/TableInfoTest.java index 6e35ce6cb620..235d9e9961d5 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/TableInfoTest.java +++ b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/TableInfoTest.java @@ -54,6 +54,7 @@ public class TableInfoTest { .build(); private static final Schema TABLE_SCHEMA = Schema.of(FIELD_SCHEMA1, FIELD_SCHEMA2, FIELD_SCHEMA3); private static final Long NUM_BYTES = 42L; + private static final Long NUM_LONG_TERM_BYTES = 21L; private static final Long NUM_ROWS = 43L; private static final String LOCATION = "US"; private static final StandardTableDefinition.StreamingBuffer STREAMING_BUFFER = @@ -62,6 +63,7 @@ public class TableInfoTest { StandardTableDefinition.newBuilder() .setLocation(LOCATION) .setNumBytes(NUM_BYTES) + .setNumLongTermBytes(NUM_LONG_TERM_BYTES) .setNumRows(NUM_ROWS) .setStreamingBuffer(STREAMING_BUFFER) .setSchema(TABLE_SCHEMA) @@ -95,6 +97,7 @@ public class TableInfoTest { .setGeneratedId(GENERATED_ID) .setLastModifiedTime(LAST_MODIFIED_TIME) .setNumBytes(NUM_BYTES) + .setNumLongTermBytes(NUM_LONG_TERM_BYTES) .setNumRows(BigInteger.valueOf(NUM_ROWS)) .setSelfLink(SELF_LINK) .setLabels(Collections.singletonMap("a", "b")) @@ -155,6 +158,10 @@ public void testBuilder() { assertEquals(LAST_MODIFIED_TIME, TABLE_INFO.getLastModifiedTime()); assertEquals(TABLE_DEFINITION, TABLE_INFO.getDefinition()); assertEquals(SELF_LINK, TABLE_INFO.getSelfLink()); + assertEquals(NUM_BYTES, TABLE_INFO.getNumBytes()); + assertEquals(NUM_LONG_TERM_BYTES, TABLE_INFO.getNumLongTermBytes()); + assertEquals(NUM_ROWS, TABLE_INFO.getNumLongTermBytes()); + assertEquals(TABLE_ID, VIEW_INFO.getTableId()); assertEquals(VIEW_DEFINITION, VIEW_INFO.getDefinition()); assertEquals(CREATION_TIME, VIEW_INFO.getCreationTime()); @@ -166,6 +173,7 @@ public void testBuilder() { assertEquals(LAST_MODIFIED_TIME, VIEW_INFO.getLastModifiedTime()); assertEquals(VIEW_DEFINITION, VIEW_INFO.getDefinition()); assertEquals(SELF_LINK, VIEW_INFO.getSelfLink()); + assertEquals(TABLE_ID, EXTERNAL_TABLE_INFO.getTableId()); assertEquals(CREATION_TIME, EXTERNAL_TABLE_INFO.getCreationTime()); assertEquals(DESCRIPTION, EXTERNAL_TABLE_INFO.getDescription()); @@ -248,6 +256,7 @@ private void compareTableInfo(TableInfo expected, TableInfo value) { assertEquals(expected.getGeneratedId(), value.getGeneratedId()); assertEquals(expected.getLastModifiedTime(), value.getLastModifiedTime()); assertEquals(expected.getNumBytes(), value.getNumBytes()); + assertEquals(expected.getNumLongTermBytes(), value.getNumLongTermBytes()); assertEquals(expected.getNumRows(), value.getNumRows()); assertEquals(expected.getSelfLink(), value.getSelfLink()); assertEquals(expected.getLabels(), value.getLabels()); From 54e4284bd9ea496fe0d091e26e1472e9aa6daca5 Mon Sep 17 00:00:00 2001 From: Seth Hollyman Date: Fri, 18 Jan 2019 15:23:11 -0800 Subject: [PATCH 5/7] return --- .../src/main/java/com/google/cloud/bigquery/TableInfo.java | 1 + 1 file changed, 1 insertion(+) diff --git a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/TableInfo.java b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/TableInfo.java index 34ce90877626..e64e85ebae22 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/TableInfo.java +++ b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/TableInfo.java @@ -244,6 +244,7 @@ Builder setNumBytes(Long numBytes) { @Override Builder setNumLongTermBytes(Long numLongTermBytes) { this.numLongTermBytes = numLongTermBytes; + return this; } @Override From faff517697c3da57a64f77fde17ac510b9238903 Mon Sep 17 00:00:00 2001 From: Seth Hollyman Date: Fri, 18 Jan 2019 15:43:57 -0800 Subject: [PATCH 6/7] assertion mismatch --- .../src/test/java/com/google/cloud/bigquery/TableInfoTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/TableInfoTest.java b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/TableInfoTest.java index 235d9e9961d5..4c4c3c5d7be2 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/TableInfoTest.java +++ b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/TableInfoTest.java @@ -160,7 +160,7 @@ public void testBuilder() { assertEquals(SELF_LINK, TABLE_INFO.getSelfLink()); assertEquals(NUM_BYTES, TABLE_INFO.getNumBytes()); assertEquals(NUM_LONG_TERM_BYTES, TABLE_INFO.getNumLongTermBytes()); - assertEquals(NUM_ROWS, TABLE_INFO.getNumLongTermBytes()); + assertEquals(NUM_ROWS, TABLE_INFO.getNumRows()); assertEquals(TABLE_ID, VIEW_INFO.getTableId()); assertEquals(VIEW_DEFINITION, VIEW_INFO.getDefinition()); From b7e8f83760176db46bec707232516283e4c0f6cf Mon Sep 17 00:00:00 2001 From: Solomon Duskis Date: Thu, 7 Mar 2019 12:56:00 -0500 Subject: [PATCH 7/7] Update TableInfoTest.java --- .../src/test/java/com/google/cloud/bigquery/TableInfoTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/TableInfoTest.java b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/TableInfoTest.java index 4c4c3c5d7be2..324f562cdfdf 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/TableInfoTest.java +++ b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/TableInfoTest.java @@ -160,7 +160,7 @@ public void testBuilder() { assertEquals(SELF_LINK, TABLE_INFO.getSelfLink()); assertEquals(NUM_BYTES, TABLE_INFO.getNumBytes()); assertEquals(NUM_LONG_TERM_BYTES, TABLE_INFO.getNumLongTermBytes()); - assertEquals(NUM_ROWS, TABLE_INFO.getNumRows()); + assertEquals(BigInteger.valueOf(NUM_ROWS), TABLE_INFO.getNumRows()); assertEquals(TABLE_ID, VIEW_INFO.getTableId()); assertEquals(VIEW_DEFINITION, VIEW_INFO.getDefinition());