From e55787c850ce29e1140d982ace02ffda6a07c67b Mon Sep 17 00:00:00 2001 From: JacobStocklass Date: Fri, 19 Feb 2021 22:40:56 +0000 Subject: [PATCH 01/22] Place Holder Stress Test for Setup --- .../v1beta2/st/STBigQueryStressTest.java | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStressTest.java diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStressTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStressTest.java new file mode 100644 index 0000000000..365e0cb074 --- /dev/null +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStressTest.java @@ -0,0 +1,50 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package com.google.cloud.bigquery.storage.v1beta2.st; + +import static org.junit.Assert.assertEquals; + +public class STBigQueryStressTest { + + private static final Logger LOG = + Logger.getLogger(ITBigQueryStorageLongRunningTest.class.getName()); + + private static BigQueryReadClient client; + private static String parentProjectId; + + @BeforeClass + public static void beforeClass() throws IOException { + client = BigQueryReadClient.create(); + parentProjectId = String.format("projects/%s", ServiceOptions.getDefaultProjectId()); + LOG.info( + String.format( + "%s tests running with parent project: %s", + ITBigQueryStorageLongRunningTest.class.getSimpleName(), parentProjectId)); + } + @AfterClass + public static void afterClass() { + if (client != null) { + client.close(); + } + } + + @Test + public void testLongRunningReadSession() throws InterruptedException, ExecutionException { + assertEquals(1,1); + } +} From 01729f7d6adf930b0e52944edd2bf4a010f1b04b Mon Sep 17 00:00:00 2001 From: JacobStocklass Date: Wed, 24 Feb 2021 21:31:50 +0000 Subject: [PATCH 02/22] Making a basic long running test for the Write API --- .../it/ITBigQueryWriteLongRunningTest.java | 5 ++ .../v1beta2/st/STBigQueryStressTest.java | 50 ------------------- 2 files changed, 5 insertions(+), 50 deletions(-) create mode 100644 google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/it/ITBigQueryWriteLongRunningTest.java delete mode 100644 google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStressTest.java diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/it/ITBigQueryWriteLongRunningTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/it/ITBigQueryWriteLongRunningTest.java new file mode 100644 index 0000000000..9c4140bf0e --- /dev/null +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/it/ITBigQueryWriteLongRunningTest.java @@ -0,0 +1,5 @@ +package com.google.cloud.bigquery.storage.v1alpha2.it; + +public class ITBigQueryWriteLongRunningTest { + +} diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStressTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStressTest.java deleted file mode 100644 index 365e0cb074..0000000000 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStressTest.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright 2020 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package com.google.cloud.bigquery.storage.v1beta2.st; - -import static org.junit.Assert.assertEquals; - -public class STBigQueryStressTest { - - private static final Logger LOG = - Logger.getLogger(ITBigQueryStorageLongRunningTest.class.getName()); - - private static BigQueryReadClient client; - private static String parentProjectId; - - @BeforeClass - public static void beforeClass() throws IOException { - client = BigQueryReadClient.create(); - parentProjectId = String.format("projects/%s", ServiceOptions.getDefaultProjectId()); - LOG.info( - String.format( - "%s tests running with parent project: %s", - ITBigQueryStorageLongRunningTest.class.getSimpleName(), parentProjectId)); - } - @AfterClass - public static void afterClass() { - if (client != null) { - client.close(); - } - } - - @Test - public void testLongRunningReadSession() throws InterruptedException, ExecutionException { - assertEquals(1,1); - } -} From d1d591d7b9cbc2901a11c79cfa00efcb064ed4d1 Mon Sep 17 00:00:00 2001 From: JacobStocklass Date: Fri, 26 Feb 2021 20:56:39 +0000 Subject: [PATCH 03/22] Simple stress test placeholder in folder st. --- .../it/ITBigQueryWriteLongRunningTest.java | 5 - ...STBigQueryStorageLongRunningWriteTest.java | 301 ++++++++++++++++++ 2 files changed, 301 insertions(+), 5 deletions(-) delete mode 100644 google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/it/ITBigQueryWriteLongRunningTest.java create mode 100644 google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/it/ITBigQueryWriteLongRunningTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/it/ITBigQueryWriteLongRunningTest.java deleted file mode 100644 index 9c4140bf0e..0000000000 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/it/ITBigQueryWriteLongRunningTest.java +++ /dev/null @@ -1,5 +0,0 @@ -package com.google.cloud.bigquery.storage.v1alpha2.it; - -public class ITBigQueryWriteLongRunningTest { - -} diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java new file mode 100644 index 0000000000..a16c2b87a0 --- /dev/null +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java @@ -0,0 +1,301 @@ +package com.google.cloud.bigquery.storage.v1beta2.st; + +import static org.junit.Assert.assertEquals; + +import com.google.api.core.ApiFuture; +import com.google.api.gax.rpc.ServerStream; +import com.google.cloud.ServiceOptions; +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.DatasetInfo; +import com.google.cloud.bigquery.Field; +import com.google.cloud.bigquery.FieldValueList; +import com.google.cloud.bigquery.LegacySQLTypeName; +import com.google.cloud.bigquery.Schema; +import com.google.cloud.bigquery.StandardSQLTypeName; +import com.google.cloud.bigquery.StandardTableDefinition; +import com.google.cloud.bigquery.TableId; +import com.google.cloud.bigquery.TableInfo; +import com.google.cloud.bigquery.TableResult; +import com.google.cloud.bigquery.storage.v1beta2.AppendRowsResponse; +import com.google.cloud.bigquery.storage.v1beta2.BigQueryReadClient; +import com.google.cloud.bigquery.storage.v1beta2.BigQueryWriteClient; +import com.google.cloud.bigquery.storage.v1beta2.CreateWriteStreamRequest; +import com.google.cloud.bigquery.storage.v1beta2.DataFormat; +import com.google.cloud.bigquery.storage.v1beta2.JsonStreamWriter; +import com.google.cloud.bigquery.storage.v1beta2.ReadRowsRequest; +import com.google.cloud.bigquery.storage.v1beta2.ReadRowsResponse; +import com.google.cloud.bigquery.storage.v1beta2.ReadSession; +import com.google.cloud.bigquery.storage.v1beta2.ReadStream; +import com.google.cloud.bigquery.storage.v1beta2.StreamWriter; +import com.google.cloud.bigquery.storage.v1beta2.TableName; +import com.google.cloud.bigquery.storage.v1beta2.WriteStream; +import com.google.cloud.bigquery.storage.v1beta2.it.ITBigQueryStorageLongRunningTest; +import com.google.cloud.bigquery.testing.RemoteBigQueryHelper; +import com.google.protobuf.Descriptors; +import java.io.IOException; +import java.sql.Time; +import java.util.ArrayList; +import java.util.Date; +import java.util.Iterator; +import java.util.List; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; +import javax.xml.crypto.Data; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Assume; +import org.junit.BeforeClass; +import org.junit.Test; +import org.threeten.bp.Duration; +import org.threeten.bp.LocalDateTime; + +public class STBigQueryStorageLongRunningWriteTest { + private static final Logger LOG = + Logger.getLogger(ITBigQueryStorageLongRunningTest.class.getName()); + + private static final String LONG_TESTS_ENABLED_PROPERTY = + "bigquery.storage.enable_long_running_tests"; + + private static final String LONG_TESTS_DISABLED_MESSAGE = + String.format( + "BigQuery Storage long running tests are not enabled and will be skipped. " + + "To enable them, set system property '%s' to true.", + LONG_TESTS_ENABLED_PROPERTY); + private static BigQueryWriteClient client; + private static String parentProjectId; + private static final String DATASET = RemoteBigQueryHelper.generateDatasetName(); + private static final String TABLE = "testtable"; + private static final String TABLE2 = "testtable2"; + private static final String DESCRIPTION = "BigQuery Write Java long test dataset"; + + private static TableInfo tableInfo; + private static TableInfo tableInfo2; + private static String tableId; + private static String tableId2; + private static BigQuery bigquery; + + private static JSONObject MakeJsonObject(int size, int i) { + JSONObject object = new JSONObject(); + object.put("test_str", "a".concat(String.valueOf(i))); + JSONArray array = new JSONArray(); + //object.put("test_numerics", new JSONArray(new String[] {"123.4", "-9000000"})); + JSONArray jsonArray = new JSONArray(); + jsonArray.put("12345"); + for (int j = 0; j < size; j++){ + jsonArray.put("12345"); + } + object.put("test_numerics", jsonArray); + object.put("test_datetime", String.valueOf(LocalDateTime.now())); + return object; + } + + @BeforeClass + public static void beforeClass() throws IOException { + Assume.assumeTrue(LONG_TESTS_DISABLED_MESSAGE, Boolean.getBoolean(LONG_TESTS_ENABLED_PROPERTY)); + client = BigQueryWriteClient.create(); + parentProjectId = String.format("projects/%s", ServiceOptions.getDefaultProjectId()); + RemoteBigQueryHelper bigqueryHelper = RemoteBigQueryHelper.create(); + bigquery = bigqueryHelper.getOptions().getService(); + DatasetInfo datasetInfo = DatasetInfo.newBuilder(/* datasetId = */ DATASET).setDescription(DESCRIPTION).build(); + bigquery.create(datasetInfo); + LOG.info("Created test dataset: " + DATASET); + tableInfo = + TableInfo.newBuilder( + TableId.of(DATASET, TABLE), + StandardTableDefinition.of( + Schema.of( + com.google.cloud.bigquery.Field.newBuilder("foo", LegacySQLTypeName.STRING) + .setMode(Field.Mode.NULLABLE) + .build()))) + .build(); + com.google.cloud.bigquery.Field.Builder innerTypeFieldBuilder = + com.google.cloud.bigquery.Field.newBuilder( + "inner_type", + LegacySQLTypeName.RECORD, + com.google.cloud.bigquery.Field.newBuilder("value", LegacySQLTypeName.STRING) + .setMode(Field.Mode.REPEATED) + .build()); + + tableInfo2 = + TableInfo.newBuilder( + TableId.of(DATASET, TABLE2), + StandardTableDefinition.of( + Schema.of( + Field.newBuilder( + "nested_repeated_type", + LegacySQLTypeName.RECORD, + innerTypeFieldBuilder.setMode(Field.Mode.REPEATED).build()) + .setMode(Field.Mode.REPEATED) + .build(), + innerTypeFieldBuilder.setMode(Field.Mode.NULLABLE).build()))) + .build(); + bigquery.create(tableInfo); + bigquery.create(tableInfo2); + tableId = + String.format( + "projects/%s/datasets/%s/tables/%s", + ServiceOptions.getDefaultProjectId(), DATASET, TABLE); + tableId2 = + String.format( + "projects/%s/datasets/%s/tables/%s", + ServiceOptions.getDefaultProjectId(), DATASET, TABLE2); + LOG.info( + String.format( + "%s tests running with parent project: %s", + ITBigQueryStorageLongRunningTest.class.getSimpleName(), parentProjectId)); + } + + @AfterClass + public static void afterClass() { + if (client != null) { + client.close(); + } + if (bigquery != null) { + RemoteBigQueryHelper.forceDelete(bigquery, DATASET); + LOG.info("Deleted test dataset: " + DATASET); + } + } + + @Test + public void testDefaultStream() throws IOException, InterruptedException, ExecutionException, + Descriptors.DescriptorValidationException { + // Set up a default stream. Write to it for a long time, (a few minutes for now) and make + // sure that everything goes well. + String tableName = "JsonTableDefaultStream"; + TableInfo tableInfo = + TableInfo.newBuilder( + TableId.of(DATASET, tableName), + StandardTableDefinition.of( + Schema.of( + com.google.cloud.bigquery.Field.newBuilder( + "test_str", StandardSQLTypeName.STRING) + .build(), + com.google.cloud.bigquery.Field.newBuilder( + "test_numerics", StandardSQLTypeName.NUMERIC) + .setMode(Field.Mode.REPEATED) + .build(), + com.google.cloud.bigquery.Field.newBuilder( + "test_datetime", StandardSQLTypeName.DATETIME) + .build()))) + .build(); + bigquery.create(tableInfo); + TableName parent = TableName.of(ServiceOptions.getDefaultProjectId(), DATASET, tableName); + try (JsonStreamWriter jsonStreamWriter = + JsonStreamWriter.newBuilder(parent.toString(), tableInfo.getDefinition().getSchema()) + .createDefaultStream() + .setBatchingSettings( + StreamWriter.Builder.DEFAULT_BATCHING_SETTINGS + .toBuilder() + .setRequestByteThreshold(1024 * 1024L) // 1 Mb + .setElementCountThreshold(2L) + .setDelayThreshold(Duration.ofSeconds(2)) + .build()) + .build()) { + // TODO: Instead of just sending one message and then two messages, send generated messages + // for like a minute, we can scale this up later to as long as we want. + for (int i = 0; i < 10; i++){ + LOG.info("Sending a message"); + // Ramping up the size increases the latency + JSONObject row = MakeJsonObject(1, i); + JSONArray jsonArr = new JSONArray(new JSONObject[] {row}); + + LocalDateTime start = LocalDateTime.now(); + Date startTime = new Date(); + ApiFuture response = jsonStreamWriter.append(jsonArr, -1); + assertEquals(0, response.get().getAppendResult().getOffset().getValue()); + Assert.assertFalse(response.get().getAppendResult().hasOffset()); + Date finishTime = new Date(); + LOG.info("Latency: ".concat(String.valueOf(finishTime.getTime() - startTime.getTime())).concat(" ms")); + // seems like 2 or 3 seconds on average + //LOG.info(String.valueOf((Math.abs(finish.getSecond())-start.getSecond()))); + } + + TableResult result = + bigquery.listTableData( + tableInfo.getTableId(), BigQuery.TableDataListOption.startIndex(0L)); + Iterator iter = result.getValues().iterator(); + FieldValueList currentRow; + for (int i = 0; i < 10; i++) { + currentRow = iter.next(); + assertEquals("a".concat(String.valueOf(i)), currentRow.get(0).getStringValue()); + } + assertEquals(false, iter.hasNext()); + } + } + + @Test + public void testDedicatedStream() { + WriteStream writeStream = client.createWriteStream(CreateWriteStreamRequest.newBuilder() + .setParent(tableId) + .setWriteStream( + WriteStream.newBuilder().setType(WriteStream.Type.COMMITTED).build()) + .build()); + // set up a dedicated stream. Write to it for a long time, (a few minutes) and make + // sure that everything goes well. + } + + @Test + public void testBufferedStreamWithFlushOperation() { + // Set up a buffered stream with an ongoing flush operation and write to it for a long time, + // (a few minutes) and make sure that everything goes well. + + } +// public void testLongRunningReadSession() throws InterruptedException, ExecutionException { +// // This test writes a larger table with the goal of doing a simple validation of timeout settings +// // for a longer running session. +// +// String table = +// BigQueryResource.FormatTableResource( +// /* projectId = */ "bigquery-public-data", +// /* datasetId = */ "samples", +// /* tableId = */ "wikipedia"); +// +// WriteStream stream = client.createWriteStream(); +// +// ReadSession session = +// client.createReadSession( +// /* parent = */ parentProjectId, +// /* readSession = */ ReadSession.newBuilder() +// .setTable(table) +// .setDataFormat(DataFormat.AVRO) +// .build(), +// /* maxStreamCount = */ 5); +// +// assertEquals( +// String.format( +// "Did not receive expected number of streams for table '%s' CreateReadSession response:%n%s", +// table, session.toString()), +// 5, +// session.getStreamsCount()); +// +// List> tasks = new ArrayList<>(session.getStreamsCount()); +// for (final ReadStream stream : session.getStreamsList()) { +// tasks.add( +// new Callable() { +// @Override +// public Long call() throws Exception { +// return readAllRowsFromStream(stream); +// } +// }); +// } +// +// ExecutorService executor = Executors.newFixedThreadPool(tasks.size()); +// List> results = executor.invokeAll(tasks); +// +// long rowCount = 0; +// for (Future result : results) { +// rowCount += result.get(); +// } +// +// assertEquals(313_797_035, rowCount); +// } +} From 07c0525c09d33a020ac999180289c38466a93848 Mon Sep 17 00:00:00 2001 From: JacobStocklass Date: Mon, 1 Mar 2021 17:55:10 +0000 Subject: [PATCH 04/22] Add simple time caclulation --- ...STBigQueryStorageLongRunningWriteTest.java | 49 ++++++++++++------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java index a16c2b87a0..25c076b984 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java @@ -32,6 +32,8 @@ import com.google.cloud.bigquery.storage.v1beta2.it.ITBigQueryStorageLongRunningTest; import com.google.cloud.bigquery.testing.RemoteBigQueryHelper; import com.google.protobuf.Descriptors; +import java.io.File; +import java.io.FileWriter; import java.io.IOException; import java.sql.Time; import java.util.ArrayList; @@ -47,7 +49,6 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.logging.Logger; -import javax.xml.crypto.Data; import org.json.JSONArray; import org.json.JSONObject; import org.junit.AfterClass; @@ -83,18 +84,28 @@ public class STBigQueryStorageLongRunningWriteTest { private static String tableId2; private static BigQuery bigquery; - private static JSONObject MakeJsonObject(int size, int i) { + private static JSONObject MakeJsonObject(int size) throws IOException { JSONObject object = new JSONObject(); - object.put("test_str", "a".concat(String.valueOf(i))); - JSONArray array = new JSONArray(); - //object.put("test_numerics", new JSONArray(new String[] {"123.4", "-9000000"})); - JSONArray jsonArray = new JSONArray(); - jsonArray.put("12345"); - for (int j = 0; j < size; j++){ - jsonArray.put("12345"); + // size: (1, simple)(2,complex)() + if (size == 1) { + object.put("test_str", "aaa"); + object.put("test_numerics", new JSONArray(new String[]{"1234", "-900000"})); + object.put("test_datetime", String.valueOf(LocalDateTime.now())); + } + else if (size == 2) { // make it complicated and slow + // test_str + JSONObject test_str = new JSONObject(); + JSONObject test_str_nest = new JSONObject(); + test_str.put("testing", "lorem"); + test_str_nest.put("testing_testing", new JSONArray(new String[]{"ipsum","I","don't","know","this whole thing"})); + test_str.put("testing2", test_str_nest); + object.put("test_str", "testing testing"); + + // test_numerics + object.put("test_numerics", test_str); + + object.put("test_datetime", String.valueOf(LocalDateTime.now())); } - object.put("test_numerics", jsonArray); - object.put("test_datetime", String.valueOf(LocalDateTime.now())); return object; } @@ -202,10 +213,10 @@ public void testDefaultStream() throws IOException, InterruptedException, Execut .build()) { // TODO: Instead of just sending one message and then two messages, send generated messages // for like a minute, we can scale this up later to as long as we want. - for (int i = 0; i < 10; i++){ + for (int i = 0; i < 5; i++){ LOG.info("Sending a message"); // Ramping up the size increases the latency - JSONObject row = MakeJsonObject(1, i); + JSONObject row = MakeJsonObject(2); JSONArray jsonArr = new JSONArray(new JSONObject[] {row}); LocalDateTime start = LocalDateTime.now(); @@ -223,12 +234,12 @@ public void testDefaultStream() throws IOException, InterruptedException, Execut bigquery.listTableData( tableInfo.getTableId(), BigQuery.TableDataListOption.startIndex(0L)); Iterator iter = result.getValues().iterator(); - FieldValueList currentRow; - for (int i = 0; i < 10; i++) { - currentRow = iter.next(); - assertEquals("a".concat(String.valueOf(i)), currentRow.get(0).getStringValue()); - } - assertEquals(false, iter.hasNext()); +// FieldValueList currentRow; +// for (int i = 0; i < 5; i++) { +// currentRow = iter.next(); +// assertEquals("aaa", currentRow.get(0).getStringValue()); +// } +// assertEquals(false, iter.hasNext()); } } From 7ca9d6d3094b5aca255966b3470a230704b05168 Mon Sep 17 00:00:00 2001 From: JacobStocklass Date: Mon, 1 Mar 2021 18:10:21 +0000 Subject: [PATCH 05/22] Cleaning up before submitting pull --- ...STBigQueryStorageLongRunningWriteTest.java | 85 ++----------------- 1 file changed, 9 insertions(+), 76 deletions(-) diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java index 25c076b984..6876460f46 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java @@ -92,19 +92,8 @@ private static JSONObject MakeJsonObject(int size) throws IOException { object.put("test_numerics", new JSONArray(new String[]{"1234", "-900000"})); object.put("test_datetime", String.valueOf(LocalDateTime.now())); } - else if (size == 2) { // make it complicated and slow - // test_str - JSONObject test_str = new JSONObject(); - JSONObject test_str_nest = new JSONObject(); - test_str.put("testing", "lorem"); - test_str_nest.put("testing_testing", new JSONArray(new String[]{"ipsum","I","don't","know","this whole thing"})); - test_str.put("testing2", test_str_nest); - object.put("test_str", "testing testing"); - - // test_numerics - object.put("test_numerics", test_str); - - object.put("test_datetime", String.valueOf(LocalDateTime.now())); + else if (size == 2) { // make it complicated and large + // TODO(jstocklass): Make a better json object that doesn't break the format rules. } return object; } @@ -211,12 +200,10 @@ public void testDefaultStream() throws IOException, InterruptedException, Execut .setDelayThreshold(Duration.ofSeconds(2)) .build()) .build()) { - // TODO: Instead of just sending one message and then two messages, send generated messages - // for like a minute, we can scale this up later to as long as we want. for (int i = 0; i < 5; i++){ LOG.info("Sending a message"); // Ramping up the size increases the latency - JSONObject row = MakeJsonObject(2); + JSONObject row = MakeJsonObject(1); JSONArray jsonArr = new JSONArray(new JSONObject[] {row}); LocalDateTime start = LocalDateTime.now(); @@ -234,22 +221,17 @@ public void testDefaultStream() throws IOException, InterruptedException, Execut bigquery.listTableData( tableInfo.getTableId(), BigQuery.TableDataListOption.startIndex(0L)); Iterator iter = result.getValues().iterator(); -// FieldValueList currentRow; -// for (int i = 0; i < 5; i++) { -// currentRow = iter.next(); -// assertEquals("aaa", currentRow.get(0).getStringValue()); -// } -// assertEquals(false, iter.hasNext()); + FieldValueList currentRow; + for (int i = 0; i < 5; i++) { + currentRow = iter.next(); + assertEquals("aaa", currentRow.get(0).getStringValue()); + } + assertEquals(false, iter.hasNext()); } } @Test public void testDedicatedStream() { - WriteStream writeStream = client.createWriteStream(CreateWriteStreamRequest.newBuilder() - .setParent(tableId) - .setWriteStream( - WriteStream.newBuilder().setType(WriteStream.Type.COMMITTED).build()) - .build()); // set up a dedicated stream. Write to it for a long time, (a few minutes) and make // sure that everything goes well. } @@ -260,53 +242,4 @@ public void testBufferedStreamWithFlushOperation() { // (a few minutes) and make sure that everything goes well. } -// public void testLongRunningReadSession() throws InterruptedException, ExecutionException { -// // This test writes a larger table with the goal of doing a simple validation of timeout settings -// // for a longer running session. -// -// String table = -// BigQueryResource.FormatTableResource( -// /* projectId = */ "bigquery-public-data", -// /* datasetId = */ "samples", -// /* tableId = */ "wikipedia"); -// -// WriteStream stream = client.createWriteStream(); -// -// ReadSession session = -// client.createReadSession( -// /* parent = */ parentProjectId, -// /* readSession = */ ReadSession.newBuilder() -// .setTable(table) -// .setDataFormat(DataFormat.AVRO) -// .build(), -// /* maxStreamCount = */ 5); -// -// assertEquals( -// String.format( -// "Did not receive expected number of streams for table '%s' CreateReadSession response:%n%s", -// table, session.toString()), -// 5, -// session.getStreamsCount()); -// -// List> tasks = new ArrayList<>(session.getStreamsCount()); -// for (final ReadStream stream : session.getStreamsList()) { -// tasks.add( -// new Callable() { -// @Override -// public Long call() throws Exception { -// return readAllRowsFromStream(stream); -// } -// }); -// } -// -// ExecutorService executor = Executors.newFixedThreadPool(tasks.size()); -// List> results = executor.invokeAll(tasks); -// -// long rowCount = 0; -// for (Future result : results) { -// rowCount += result.get(); -// } -// -// assertEquals(313_797_035, rowCount); -// } } From 4aee9cad6a5ab3881fab66ab6894e1a1b43dd7cc Mon Sep 17 00:00:00 2001 From: JacobStocklass Date: Mon, 1 Mar 2021 21:25:13 +0000 Subject: [PATCH 06/22] caching changes --- ...STBigQueryStorageLongRunningWriteTest.java | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java index 6876460f46..abcd6c183c 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java @@ -3,7 +3,6 @@ import static org.junit.Assert.assertEquals; import com.google.api.core.ApiFuture; -import com.google.api.gax.rpc.ServerStream; import com.google.cloud.ServiceOptions; import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.DatasetInfo; @@ -17,37 +16,17 @@ import com.google.cloud.bigquery.TableInfo; import com.google.cloud.bigquery.TableResult; import com.google.cloud.bigquery.storage.v1beta2.AppendRowsResponse; -import com.google.cloud.bigquery.storage.v1beta2.BigQueryReadClient; import com.google.cloud.bigquery.storage.v1beta2.BigQueryWriteClient; -import com.google.cloud.bigquery.storage.v1beta2.CreateWriteStreamRequest; -import com.google.cloud.bigquery.storage.v1beta2.DataFormat; import com.google.cloud.bigquery.storage.v1beta2.JsonStreamWriter; -import com.google.cloud.bigquery.storage.v1beta2.ReadRowsRequest; -import com.google.cloud.bigquery.storage.v1beta2.ReadRowsResponse; -import com.google.cloud.bigquery.storage.v1beta2.ReadSession; -import com.google.cloud.bigquery.storage.v1beta2.ReadStream; import com.google.cloud.bigquery.storage.v1beta2.StreamWriter; import com.google.cloud.bigquery.storage.v1beta2.TableName; -import com.google.cloud.bigquery.storage.v1beta2.WriteStream; import com.google.cloud.bigquery.storage.v1beta2.it.ITBigQueryStorageLongRunningTest; import com.google.cloud.bigquery.testing.RemoteBigQueryHelper; import com.google.protobuf.Descriptors; -import java.io.File; -import java.io.FileWriter; import java.io.IOException; -import java.sql.Time; -import java.util.ArrayList; import java.util.Date; import java.util.Iterator; -import java.util.List; -import java.util.Timer; -import java.util.TimerTask; -import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; import java.util.logging.Logger; import org.json.JSONArray; import org.json.JSONObject; From 1d0b11977ed2ff6c3583af81a9a983ff348e9398 Mon Sep 17 00:00:00 2001 From: JacobStocklass Date: Mon, 1 Mar 2021 21:58:26 +0000 Subject: [PATCH 07/22] Cleaning up and adding TODO --- ...STBigQueryStorageLongRunningWriteTest.java | 30 ++----------------- 1 file changed, 2 insertions(+), 28 deletions(-) diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java index abcd6c183c..55c6d9c132 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java @@ -41,26 +41,17 @@ public class STBigQueryStorageLongRunningWriteTest { private static final Logger LOG = Logger.getLogger(ITBigQueryStorageLongRunningTest.class.getName()); - private static final String LONG_TESTS_ENABLED_PROPERTY = "bigquery.storage.enable_long_running_tests"; - private static final String LONG_TESTS_DISABLED_MESSAGE = - String.format( - "BigQuery Storage long running tests are not enabled and will be skipped. " - + "To enable them, set system property '%s' to true.", - LONG_TESTS_ENABLED_PROPERTY); private static BigQueryWriteClient client; private static String parentProjectId; private static final String DATASET = RemoteBigQueryHelper.generateDatasetName(); private static final String TABLE = "testtable"; - private static final String TABLE2 = "testtable2"; private static final String DESCRIPTION = "BigQuery Write Java long test dataset"; private static TableInfo tableInfo; - private static TableInfo tableInfo2; private static String tableId; - private static String tableId2; private static BigQuery bigquery; private static JSONObject MakeJsonObject(int size) throws IOException { @@ -79,7 +70,7 @@ else if (size == 2) { // make it complicated and large @BeforeClass public static void beforeClass() throws IOException { - Assume.assumeTrue(LONG_TESTS_DISABLED_MESSAGE, Boolean.getBoolean(LONG_TESTS_ENABLED_PROPERTY)); + //Assume.assumeTrue(LONG_TESTS_DISABLED_MESSAGE, Boolean.getBoolean(LONG_TESTS_ENABLED_PROPERTY)); client = BigQueryWriteClient.create(); parentProjectId = String.format("projects/%s", ServiceOptions.getDefaultProjectId()); RemoteBigQueryHelper bigqueryHelper = RemoteBigQueryHelper.create(); @@ -104,29 +95,11 @@ public static void beforeClass() throws IOException { .setMode(Field.Mode.REPEATED) .build()); - tableInfo2 = - TableInfo.newBuilder( - TableId.of(DATASET, TABLE2), - StandardTableDefinition.of( - Schema.of( - Field.newBuilder( - "nested_repeated_type", - LegacySQLTypeName.RECORD, - innerTypeFieldBuilder.setMode(Field.Mode.REPEATED).build()) - .setMode(Field.Mode.REPEATED) - .build(), - innerTypeFieldBuilder.setMode(Field.Mode.NULLABLE).build()))) - .build(); bigquery.create(tableInfo); - bigquery.create(tableInfo2); tableId = String.format( "projects/%s/datasets/%s/tables/%s", ServiceOptions.getDefaultProjectId(), DATASET, TABLE); - tableId2 = - String.format( - "projects/%s/datasets/%s/tables/%s", - ServiceOptions.getDefaultProjectId(), DATASET, TABLE2); LOG.info( String.format( "%s tests running with parent project: %s", @@ -187,6 +160,7 @@ public void testDefaultStream() throws IOException, InterruptedException, Execut LocalDateTime start = LocalDateTime.now(); Date startTime = new Date(); + //TODO(jstocklass): Make asynchronized calls instead of synchronized calls ApiFuture response = jsonStreamWriter.append(jsonArr, -1); assertEquals(0, response.get().getAppendResult().getOffset().getValue()); Assert.assertFalse(response.get().getAppendResult().hasOffset()); From ac1a6a08556d7f69c4a10cb7e85881d20237b63e Mon Sep 17 00:00:00 2001 From: JacobStocklass Date: Tue, 2 Mar 2021 00:37:53 +0000 Subject: [PATCH 08/22] Added copywrite info at beginning of file --- ...STBigQueryStorageLongRunningWriteTest.java | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java index 55c6d9c132..c0178b73e1 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java @@ -1,3 +1,19 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.google.cloud.bigquery.storage.v1beta2.st; import static org.junit.Assert.assertEquals; @@ -32,7 +48,6 @@ import org.json.JSONObject; import org.junit.AfterClass; import org.junit.Assert; -import org.junit.Assume; import org.junit.BeforeClass; import org.junit.Test; import org.threeten.bp.Duration; @@ -41,8 +56,6 @@ public class STBigQueryStorageLongRunningWriteTest { private static final Logger LOG = Logger.getLogger(ITBigQueryStorageLongRunningTest.class.getName()); - private static final String LONG_TESTS_ENABLED_PROPERTY = - "bigquery.storage.enable_long_running_tests"; private static BigQueryWriteClient client; private static String parentProjectId; From 0f10b05b79d96ed4a33ed7b83ee53ebc96680039 Mon Sep 17 00:00:00 2001 From: JacobStocklass Date: Tue, 2 Mar 2021 00:52:03 +0000 Subject: [PATCH 09/22] Removing error causing lines --- .../v1beta2/st/STBigQueryStorageLongRunningWriteTest.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java index c0178b73e1..388ea18cbb 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java @@ -32,7 +32,6 @@ import com.google.cloud.bigquery.TableInfo; import com.google.cloud.bigquery.TableResult; import com.google.cloud.bigquery.storage.v1beta2.AppendRowsResponse; -import com.google.cloud.bigquery.storage.v1beta2.BigQueryWriteClient; import com.google.cloud.bigquery.storage.v1beta2.JsonStreamWriter; import com.google.cloud.bigquery.storage.v1beta2.StreamWriter; import com.google.cloud.bigquery.storage.v1beta2.TableName; @@ -57,7 +56,6 @@ public class STBigQueryStorageLongRunningWriteTest { private static final Logger LOG = Logger.getLogger(ITBigQueryStorageLongRunningTest.class.getName()); - private static BigQueryWriteClient client; private static String parentProjectId; private static final String DATASET = RemoteBigQueryHelper.generateDatasetName(); private static final String TABLE = "testtable"; @@ -84,7 +82,6 @@ else if (size == 2) { // make it complicated and large @BeforeClass public static void beforeClass() throws IOException { //Assume.assumeTrue(LONG_TESTS_DISABLED_MESSAGE, Boolean.getBoolean(LONG_TESTS_ENABLED_PROPERTY)); - client = BigQueryWriteClient.create(); parentProjectId = String.format("projects/%s", ServiceOptions.getDefaultProjectId()); RemoteBigQueryHelper bigqueryHelper = RemoteBigQueryHelper.create(); bigquery = bigqueryHelper.getOptions().getService(); @@ -121,9 +118,6 @@ public static void beforeClass() throws IOException { @AfterClass public static void afterClass() { - if (client != null) { - client.close(); - } if (bigquery != null) { RemoteBigQueryHelper.forceDelete(bigquery, DATASET); LOG.info("Deleted test dataset: " + DATASET); From 246955aae768e3d935cdf58bcef4e557891b5d90 Mon Sep 17 00:00:00 2001 From: JacobStocklass Date: Tue, 2 Mar 2021 17:55:49 +0000 Subject: [PATCH 10/22] Moving Before class into single test to fix permissions issues --- ...STBigQueryStorageLongRunningWriteTest.java | 89 ++++++++++--------- 1 file changed, 47 insertions(+), 42 deletions(-) diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java index 388ea18cbb..f60f5a1bb2 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java @@ -13,7 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package com.google.cloud.bigquery.storage.v1beta2.st; import static org.junit.Assert.assertEquals; @@ -32,6 +31,7 @@ import com.google.cloud.bigquery.TableInfo; import com.google.cloud.bigquery.TableResult; import com.google.cloud.bigquery.storage.v1beta2.AppendRowsResponse; +import com.google.cloud.bigquery.storage.v1beta2.BigQueryWriteClient; import com.google.cloud.bigquery.storage.v1beta2.JsonStreamWriter; import com.google.cloud.bigquery.storage.v1beta2.StreamWriter; import com.google.cloud.bigquery.storage.v1beta2.TableName; @@ -55,7 +55,10 @@ public class STBigQueryStorageLongRunningWriteTest { private static final Logger LOG = Logger.getLogger(ITBigQueryStorageLongRunningTest.class.getName()); + private static final String LONG_TESTS_ENABLED_PROPERTY = + "bigquery.storage.enable_long_running_tests"; + private static BigQueryWriteClient client; private static String parentProjectId; private static final String DATASET = RemoteBigQueryHelper.generateDatasetName(); private static final String TABLE = "testtable"; @@ -70,24 +73,38 @@ private static JSONObject MakeJsonObject(int size) throws IOException { // size: (1, simple)(2,complex)() if (size == 1) { object.put("test_str", "aaa"); - object.put("test_numerics", new JSONArray(new String[]{"1234", "-900000"})); + object.put("test_numerics", new JSONArray(new String[] {"1234", "-900000"})); object.put("test_datetime", String.valueOf(LocalDateTime.now())); - } - else if (size == 2) { // make it complicated and large + } else if (size == 2) { // make it complicated and large // TODO(jstocklass): Make a better json object that doesn't break the format rules. } return object; } - @BeforeClass - public static void beforeClass() throws IOException { - //Assume.assumeTrue(LONG_TESTS_DISABLED_MESSAGE, Boolean.getBoolean(LONG_TESTS_ENABLED_PROPERTY)); + @AfterClass + public static void afterClass() { + if (client != null) { + client.close(); + } + if (bigquery != null) { + RemoteBigQueryHelper.forceDelete(bigquery, DATASET); + LOG.info("Deleted test dataset: " + DATASET); + } + } + + @Test + public void testDefaultStream() + throws IOException, InterruptedException, ExecutionException, + Descriptors.DescriptorValidationException { + // Set up a default stream. Write to it for a long time, (a few minutes for now) and make + // sure that everything goes well. parentProjectId = String.format("projects/%s", ServiceOptions.getDefaultProjectId()); RemoteBigQueryHelper bigqueryHelper = RemoteBigQueryHelper.create(); bigquery = bigqueryHelper.getOptions().getService(); - DatasetInfo datasetInfo = DatasetInfo.newBuilder(/* datasetId = */ DATASET).setDescription(DESCRIPTION).build(); + client = BigQueryWriteClient.create(); + DatasetInfo datasetInfo = + DatasetInfo.newBuilder(/* datasetId = */ DATASET).setDescription(DESCRIPTION).build(); bigquery.create(datasetInfo); - LOG.info("Created test dataset: " + DATASET); tableInfo = TableInfo.newBuilder( TableId.of(DATASET, TABLE), @@ -114,37 +131,22 @@ public static void beforeClass() throws IOException { String.format( "%s tests running with parent project: %s", ITBigQueryStorageLongRunningTest.class.getSimpleName(), parentProjectId)); - } - - @AfterClass - public static void afterClass() { - if (bigquery != null) { - RemoteBigQueryHelper.forceDelete(bigquery, DATASET); - LOG.info("Deleted test dataset: " + DATASET); - } - } - - @Test - public void testDefaultStream() throws IOException, InterruptedException, ExecutionException, - Descriptors.DescriptorValidationException { - // Set up a default stream. Write to it for a long time, (a few minutes for now) and make - // sure that everything goes well. String tableName = "JsonTableDefaultStream"; TableInfo tableInfo = TableInfo.newBuilder( - TableId.of(DATASET, tableName), - StandardTableDefinition.of( - Schema.of( - com.google.cloud.bigquery.Field.newBuilder( - "test_str", StandardSQLTypeName.STRING) - .build(), - com.google.cloud.bigquery.Field.newBuilder( - "test_numerics", StandardSQLTypeName.NUMERIC) - .setMode(Field.Mode.REPEATED) - .build(), - com.google.cloud.bigquery.Field.newBuilder( - "test_datetime", StandardSQLTypeName.DATETIME) - .build()))) + TableId.of(DATASET, tableName), + StandardTableDefinition.of( + Schema.of( + com.google.cloud.bigquery.Field.newBuilder( + "test_str", StandardSQLTypeName.STRING) + .build(), + com.google.cloud.bigquery.Field.newBuilder( + "test_numerics", StandardSQLTypeName.NUMERIC) + .setMode(Field.Mode.REPEATED) + .build(), + com.google.cloud.bigquery.Field.newBuilder( + "test_datetime", StandardSQLTypeName.DATETIME) + .build()))) .build(); bigquery.create(tableInfo); TableName parent = TableName.of(ServiceOptions.getDefaultProjectId(), DATASET, tableName); @@ -159,7 +161,7 @@ public void testDefaultStream() throws IOException, InterruptedException, Execut .setDelayThreshold(Duration.ofSeconds(2)) .build()) .build()) { - for (int i = 0; i < 5; i++){ + for (int i = 0; i < 5; i++) { LOG.info("Sending a message"); // Ramping up the size increases the latency JSONObject row = MakeJsonObject(1); @@ -167,14 +169,17 @@ public void testDefaultStream() throws IOException, InterruptedException, Execut LocalDateTime start = LocalDateTime.now(); Date startTime = new Date(); - //TODO(jstocklass): Make asynchronized calls instead of synchronized calls + // TODO(jstocklass): Make asynchronized calls instead of synchronized calls ApiFuture response = jsonStreamWriter.append(jsonArr, -1); assertEquals(0, response.get().getAppendResult().getOffset().getValue()); Assert.assertFalse(response.get().getAppendResult().hasOffset()); Date finishTime = new Date(); - LOG.info("Latency: ".concat(String.valueOf(finishTime.getTime() - startTime.getTime())).concat(" ms")); - // seems like 2 or 3 seconds on average - //LOG.info(String.valueOf((Math.abs(finish.getSecond())-start.getSecond()))); + LOG.info( + "Latency: " + .concat(String.valueOf(finishTime.getTime() - startTime.getTime())) + .concat(" ms")); + // seems like 2 or 3 seconds on average + // LOG.info(String.valueOf((Math.abs(finish.getSecond())-start.getSecond()))); } TableResult result = From 4eddd79781fc7fd1113881f44db1d0564aefdbab Mon Sep 17 00:00:00 2001 From: JacobStocklass Date: Tue, 2 Mar 2021 18:11:42 +0000 Subject: [PATCH 11/22] Ran mvn format, should fix lint errors --- .../st/STBigQueryStorageLongRunningWriteTest.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java index f60f5a1bb2..0ded646254 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java @@ -47,7 +47,6 @@ import org.json.JSONObject; import org.junit.AfterClass; import org.junit.Assert; -import org.junit.BeforeClass; import org.junit.Test; import org.threeten.bp.Duration; import org.threeten.bp.LocalDateTime; @@ -107,12 +106,12 @@ public void testDefaultStream() bigquery.create(datasetInfo); tableInfo = TableInfo.newBuilder( - TableId.of(DATASET, TABLE), - StandardTableDefinition.of( - Schema.of( - com.google.cloud.bigquery.Field.newBuilder("foo", LegacySQLTypeName.STRING) - .setMode(Field.Mode.NULLABLE) - .build()))) + TableId.of(DATASET, TABLE), + StandardTableDefinition.of( + Schema.of( + com.google.cloud.bigquery.Field.newBuilder("foo", LegacySQLTypeName.STRING) + .setMode(Field.Mode.NULLABLE) + .build()))) .build(); com.google.cloud.bigquery.Field.Builder innerTypeFieldBuilder = com.google.cloud.bigquery.Field.newBuilder( From 717de70a8ac6b48eef3496104a36322285e7ab60 Mon Sep 17 00:00:00 2001 From: JacobStocklass Date: Tue, 2 Mar 2021 22:14:37 +0000 Subject: [PATCH 12/22] Resolving comments, removing unneccsary code --- ...STBigQueryStorageLongRunningWriteTest.java | 93 ++++++------------- 1 file changed, 27 insertions(+), 66 deletions(-) diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java index 0ded646254..41fbe14496 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java @@ -16,6 +16,7 @@ package com.google.cloud.bigquery.storage.v1beta2.st; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import com.google.api.core.ApiFuture; import com.google.cloud.ServiceOptions; @@ -23,7 +24,6 @@ import com.google.cloud.bigquery.DatasetInfo; import com.google.cloud.bigquery.Field; import com.google.cloud.bigquery.FieldValueList; -import com.google.cloud.bigquery.LegacySQLTypeName; import com.google.cloud.bigquery.Schema; import com.google.cloud.bigquery.StandardSQLTypeName; import com.google.cloud.bigquery.StandardTableDefinition; @@ -33,7 +33,6 @@ import com.google.cloud.bigquery.storage.v1beta2.AppendRowsResponse; import com.google.cloud.bigquery.storage.v1beta2.BigQueryWriteClient; import com.google.cloud.bigquery.storage.v1beta2.JsonStreamWriter; -import com.google.cloud.bigquery.storage.v1beta2.StreamWriter; import com.google.cloud.bigquery.storage.v1beta2.TableName; import com.google.cloud.bigquery.storage.v1beta2.it.ITBigQueryStorageLongRunningTest; import com.google.cloud.bigquery.testing.RemoteBigQueryHelper; @@ -48,10 +47,14 @@ import org.junit.AfterClass; import org.junit.Assert; import org.junit.Test; -import org.threeten.bp.Duration; import org.threeten.bp.LocalDateTime; public class STBigQueryStorageLongRunningWriteTest { + public enum Complexity { + SIMPLE, + COMPLEX + } + private static final Logger LOG = Logger.getLogger(ITBigQueryStorageLongRunningTest.class.getName()); private static final String LONG_TESTS_ENABLED_PROPERTY = @@ -67,15 +70,20 @@ public class STBigQueryStorageLongRunningWriteTest { private static String tableId; private static BigQuery bigquery; - private static JSONObject MakeJsonObject(int size) throws IOException { + private static JSONObject MakeJsonObject(Complexity complexity) throws IOException { JSONObject object = new JSONObject(); // size: (1, simple)(2,complex)() - if (size == 1) { - object.put("test_str", "aaa"); - object.put("test_numerics", new JSONArray(new String[] {"1234", "-900000"})); - object.put("test_datetime", String.valueOf(LocalDateTime.now())); - } else if (size == 2) { // make it complicated and large - // TODO(jstocklass): Make a better json object that doesn't break the format rules. + switch (complexity) { + case SIMPLE: + object.put("test_str", "aaa"); + object.put("test_numerics", new JSONArray(new String[] {"1234", "-900000"})); + object.put("test_datetime", String.valueOf(LocalDateTime.now())); + break; + case COMPLEX: + // TODO(jstocklass): Make a better json object that doesn't break the format rules. + break; + default: + break; } return object; } @@ -95,8 +103,8 @@ public static void afterClass() { public void testDefaultStream() throws IOException, InterruptedException, ExecutionException, Descriptors.DescriptorValidationException { - // Set up a default stream. Write to it for a long time, (a few minutes for now) and make - // sure that everything goes well. + // TODO(jstocklass): Set up a default stream. Write to it for a long time, + // (a few minutes for now) and make sure that everything goes well, report stats. parentProjectId = String.format("projects/%s", ServiceOptions.getDefaultProjectId()); RemoteBigQueryHelper bigqueryHelper = RemoteBigQueryHelper.create(); bigquery = bigqueryHelper.getOptions().getService(); @@ -104,32 +112,11 @@ public void testDefaultStream() DatasetInfo datasetInfo = DatasetInfo.newBuilder(/* datasetId = */ DATASET).setDescription(DESCRIPTION).build(); bigquery.create(datasetInfo); - tableInfo = - TableInfo.newBuilder( - TableId.of(DATASET, TABLE), - StandardTableDefinition.of( - Schema.of( - com.google.cloud.bigquery.Field.newBuilder("foo", LegacySQLTypeName.STRING) - .setMode(Field.Mode.NULLABLE) - .build()))) - .build(); - com.google.cloud.bigquery.Field.Builder innerTypeFieldBuilder = - com.google.cloud.bigquery.Field.newBuilder( - "inner_type", - LegacySQLTypeName.RECORD, - com.google.cloud.bigquery.Field.newBuilder("value", LegacySQLTypeName.STRING) - .setMode(Field.Mode.REPEATED) - .build()); - - bigquery.create(tableInfo); - tableId = - String.format( - "projects/%s/datasets/%s/tables/%s", - ServiceOptions.getDefaultProjectId(), DATASET, TABLE); LOG.info( String.format( "%s tests running with parent project: %s", ITBigQueryStorageLongRunningTest.class.getSimpleName(), parentProjectId)); + String tableName = "JsonTableDefaultStream"; TableInfo tableInfo = TableInfo.newBuilder( @@ -152,33 +139,19 @@ public void testDefaultStream() try (JsonStreamWriter jsonStreamWriter = JsonStreamWriter.newBuilder(parent.toString(), tableInfo.getDefinition().getSchema()) .createDefaultStream() - .setBatchingSettings( - StreamWriter.Builder.DEFAULT_BATCHING_SETTINGS - .toBuilder() - .setRequestByteThreshold(1024 * 1024L) // 1 Mb - .setElementCountThreshold(2L) - .setDelayThreshold(Duration.ofSeconds(2)) - .build()) .build()) { for (int i = 0; i < 5; i++) { - LOG.info("Sending a message"); - // Ramping up the size increases the latency - JSONObject row = MakeJsonObject(1); + JSONObject row = MakeJsonObject(Complexity.SIMPLE); JSONArray jsonArr = new JSONArray(new JSONObject[] {row}); - LocalDateTime start = LocalDateTime.now(); Date startTime = new Date(); // TODO(jstocklass): Make asynchronized calls instead of synchronized calls ApiFuture response = jsonStreamWriter.append(jsonArr, -1); - assertEquals(0, response.get().getAppendResult().getOffset().getValue()); - Assert.assertFalse(response.get().getAppendResult().hasOffset()); Date finishTime = new Date(); - LOG.info( - "Latency: " - .concat(String.valueOf(finishTime.getTime() - startTime.getTime())) - .concat(" ms")); - // seems like 2 or 3 seconds on average - // LOG.info(String.valueOf((Math.abs(finish.getSecond())-start.getSecond()))); + Assert.assertFalse(response.get().getAppendResult().hasOffset()); + // TODO(jstocklass): Compute aggregate statistics instead of logging. Once we have a large + // number of requests, this won't be very usable. + LOG.info("Latency: " + String.valueOf(finishTime.getTime() - startTime.getTime()) + " ms"); } TableResult result = @@ -187,23 +160,11 @@ public void testDefaultStream() Iterator iter = result.getValues().iterator(); FieldValueList currentRow; for (int i = 0; i < 5; i++) { + assertTrue(iter.hasNext()); currentRow = iter.next(); assertEquals("aaa", currentRow.get(0).getStringValue()); } assertEquals(false, iter.hasNext()); } } - - @Test - public void testDedicatedStream() { - // set up a dedicated stream. Write to it for a long time, (a few minutes) and make - // sure that everything goes well. - } - - @Test - public void testBufferedStreamWithFlushOperation() { - // Set up a buffered stream with an ongoing flush operation and write to it for a long time, - // (a few minutes) and make sure that everything goes well. - - } } From 14a559b99494387594627d3a670bddafe1056f99 Mon Sep 17 00:00:00 2001 From: JacobStocklass Date: Wed, 3 Mar 2021 00:49:55 +0000 Subject: [PATCH 13/22] Fixing comments and cleaning up --- .../STBigQueryStorageLongRunningWriteTest.java | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java index 41fbe14496..31831c52d0 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java @@ -50,6 +50,9 @@ import org.threeten.bp.LocalDateTime; public class STBigQueryStorageLongRunningWriteTest { + + + public enum Complexity { SIMPLE, COMPLEX @@ -60,19 +63,17 @@ public enum Complexity { private static final String LONG_TESTS_ENABLED_PROPERTY = "bigquery.storage.enable_long_running_tests"; - private static BigQueryWriteClient client; - private static String parentProjectId; - private static final String DATASET = RemoteBigQueryHelper.generateDatasetName(); - private static final String TABLE = "testtable"; + private static String DATASET; private static final String DESCRIPTION = "BigQuery Write Java long test dataset"; - private static TableInfo tableInfo; - private static String tableId; + private static BigQueryWriteClient client; + private static String parentProjectId; private static BigQuery bigquery; private static JSONObject MakeJsonObject(Complexity complexity) throws IOException { JSONObject object = new JSONObject(); // size: (1, simple)(2,complex)() + // TODO(jstocklass): Add option for testing protobuf format using StreamWriter2 switch (complexity) { case SIMPLE: object.put("test_str", "aaa"); @@ -93,7 +94,7 @@ public static void afterClass() { if (client != null) { client.close(); } - if (bigquery != null) { + if (bigquery != null && DATASET != null) { RemoteBigQueryHelper.forceDelete(bigquery, DATASET); LOG.info("Deleted test dataset: " + DATASET); } @@ -106,9 +107,10 @@ public void testDefaultStream() // TODO(jstocklass): Set up a default stream. Write to it for a long time, // (a few minutes for now) and make sure that everything goes well, report stats. parentProjectId = String.format("projects/%s", ServiceOptions.getDefaultProjectId()); + client = BigQueryWriteClient.create(); RemoteBigQueryHelper bigqueryHelper = RemoteBigQueryHelper.create(); bigquery = bigqueryHelper.getOptions().getService(); - client = BigQueryWriteClient.create(); + DATASET = RemoteBigQueryHelper.generateDatasetName(); DatasetInfo datasetInfo = DatasetInfo.newBuilder(/* datasetId = */ DATASET).setDescription(DESCRIPTION).build(); bigquery.create(datasetInfo); From 002602a678afbe5b2303263748899bbcea63d5b6 Mon Sep 17 00:00:00 2001 From: JacobStocklass Date: Wed, 3 Mar 2021 19:25:19 +0000 Subject: [PATCH 14/22] Moved creation of client back into BeforeClass and resolved some other comments --- ...STBigQueryStorageLongRunningWriteTest.java | 46 ++++++++++--------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java index 31831c52d0..b22188bc4c 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java @@ -46,14 +46,12 @@ import org.json.JSONObject; import org.junit.AfterClass; import org.junit.Assert; +import org.junit.BeforeClass; import org.junit.Test; import org.threeten.bp.LocalDateTime; public class STBigQueryStorageLongRunningWriteTest { - - - - public enum Complexity { + public enum RowComplexity { SIMPLE, COMPLEX } @@ -63,14 +61,14 @@ public enum Complexity { private static final String LONG_TESTS_ENABLED_PROPERTY = "bigquery.storage.enable_long_running_tests"; - private static String DATASET; + private static String dataset; private static final String DESCRIPTION = "BigQuery Write Java long test dataset"; private static BigQueryWriteClient client; private static String parentProjectId; private static BigQuery bigquery; - private static JSONObject MakeJsonObject(Complexity complexity) throws IOException { + private static JSONObject MakeJsonObject(RowComplexity complexity) throws IOException { JSONObject object = new JSONObject(); // size: (1, simple)(2,complex)() // TODO(jstocklass): Add option for testing protobuf format using StreamWriter2 @@ -89,14 +87,27 @@ private static JSONObject MakeJsonObject(Complexity complexity) throws IOExcepti return object; } + @BeforeClass + public static void beforeClass() throws IOException{ + parentProjectId = String.format("projects/%s", ServiceOptions.getDefaultProjectId()); + + client = BigQueryWriteClient.create(); + RemoteBigQueryHelper bigqueryHelper = RemoteBigQueryHelper.create(); + bigquery = bigqueryHelper.getOptions().getService(); + dataset = RemoteBigQueryHelper.generateDatasetName(); + DatasetInfo datasetInfo = + DatasetInfo.newBuilder(/* datasetId = */ dataset).setDescription(DESCRIPTION).build(); + bigquery.create(datasetInfo); + } + @AfterClass public static void afterClass() { if (client != null) { client.close(); } - if (bigquery != null && DATASET != null) { - RemoteBigQueryHelper.forceDelete(bigquery, DATASET); - LOG.info("Deleted test dataset: " + DATASET); + if (bigquery != null && dataset != null) { + RemoteBigQueryHelper.forceDelete(bigquery, dataset); + LOG.info("Deleted test dataset: " + dataset); } } @@ -106,23 +117,15 @@ public void testDefaultStream() Descriptors.DescriptorValidationException { // TODO(jstocklass): Set up a default stream. Write to it for a long time, // (a few minutes for now) and make sure that everything goes well, report stats. - parentProjectId = String.format("projects/%s", ServiceOptions.getDefaultProjectId()); - client = BigQueryWriteClient.create(); - RemoteBigQueryHelper bigqueryHelper = RemoteBigQueryHelper.create(); - bigquery = bigqueryHelper.getOptions().getService(); - DATASET = RemoteBigQueryHelper.generateDatasetName(); - DatasetInfo datasetInfo = - DatasetInfo.newBuilder(/* datasetId = */ DATASET).setDescription(DESCRIPTION).build(); - bigquery.create(datasetInfo); LOG.info( String.format( "%s tests running with parent project: %s", - ITBigQueryStorageLongRunningTest.class.getSimpleName(), parentProjectId)); + STBigQueryStorageLongRunningWriteTest.class.getSimpleName(), parentProjectId)); String tableName = "JsonTableDefaultStream"; TableInfo tableInfo = TableInfo.newBuilder( - TableId.of(DATASET, tableName), + TableId.of(dataset, tableName), StandardTableDefinition.of( Schema.of( com.google.cloud.bigquery.Field.newBuilder( @@ -137,13 +140,13 @@ public void testDefaultStream() .build()))) .build(); bigquery.create(tableInfo); - TableName parent = TableName.of(ServiceOptions.getDefaultProjectId(), DATASET, tableName); + TableName parent = TableName.of(ServiceOptions.getDefaultProjectId(), dataset, tableName); try (JsonStreamWriter jsonStreamWriter = JsonStreamWriter.newBuilder(parent.toString(), tableInfo.getDefinition().getSchema()) .createDefaultStream() .build()) { for (int i = 0; i < 5; i++) { - JSONObject row = MakeJsonObject(Complexity.SIMPLE); + JSONObject row = MakeJsonObject(RowComplexity.SIMPLE); JSONArray jsonArr = new JSONArray(new JSONObject[] {row}); LocalDateTime start = LocalDateTime.now(); Date startTime = new Date(); @@ -168,5 +171,6 @@ public void testDefaultStream() } assertEquals(false, iter.hasNext()); } + } } From b29998ca740aa6bb17d735e183b0e53a01b28bed Mon Sep 17 00:00:00 2001 From: JacobStocklass Date: Wed, 3 Mar 2021 20:45:45 +0000 Subject: [PATCH 15/22] Formating fix --- .../v1beta2/st/STBigQueryStorageLongRunningWriteTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java index b22188bc4c..015620f3ba 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java @@ -88,7 +88,7 @@ private static JSONObject MakeJsonObject(RowComplexity complexity) throws IOExce } @BeforeClass - public static void beforeClass() throws IOException{ + public static void beforeClass() throws IOException { parentProjectId = String.format("projects/%s", ServiceOptions.getDefaultProjectId()); client = BigQueryWriteClient.create(); @@ -171,6 +171,5 @@ public void testDefaultStream() } assertEquals(false, iter.hasNext()); } - } } From edcfa5474f9abedcc89d13642a91262f7eb7dd1e Mon Sep 17 00:00:00 2001 From: JacobStocklass Date: Wed, 3 Mar 2021 23:34:20 +0000 Subject: [PATCH 16/22] Changed name from ST to IT to fix errors --- ...ITBigQueryStorageLongRunningWriteTest.java | 269 ++++++++++++++++++ ...STBigQueryStorageLongRunningWriteTest.java | 175 ------------ 2 files changed, 269 insertions(+), 175 deletions(-) create mode 100644 google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/ITBigQueryStorageLongRunningWriteTest.java delete mode 100644 google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/ITBigQueryStorageLongRunningWriteTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/ITBigQueryStorageLongRunningWriteTest.java new file mode 100644 index 0000000000..b19289f607 --- /dev/null +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/ITBigQueryStorageLongRunningWriteTest.java @@ -0,0 +1,269 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigquery.storage.v1beta2.st; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import com.google.api.core.ApiFuture; +import com.google.cloud.ServiceOptions; +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.DatasetInfo; +import com.google.cloud.bigquery.Field; +import com.google.cloud.bigquery.Field.Mode; +import com.google.cloud.bigquery.FieldValueList; +import com.google.cloud.bigquery.Schema; +import com.google.cloud.bigquery.StandardSQLTypeName; +import com.google.cloud.bigquery.StandardTableDefinition; +import com.google.cloud.bigquery.TableId; +import com.google.cloud.bigquery.TableInfo; +import com.google.cloud.bigquery.TableResult; +import com.google.cloud.bigquery.storage.v1beta2.AppendRowsResponse; +import com.google.cloud.bigquery.storage.v1beta2.BigQueryWriteClient; +import com.google.cloud.bigquery.storage.v1beta2.JsonStreamWriter; +import com.google.cloud.bigquery.storage.v1beta2.TableName; +import com.google.cloud.bigquery.storage.v1beta2.it.ITBigQueryStorageLongRunningTest; +import com.google.cloud.bigquery.testing.RemoteBigQueryHelper; +import com.google.protobuf.Descriptors; +import java.io.IOException; +import java.util.Date; +import java.util.Iterator; +import java.util.concurrent.ExecutionException; +import java.util.logging.Logger; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.threeten.bp.LocalDateTime; + +public class ITBigQueryStorageLongRunningWriteTest { + public enum RowComplexity { + SIMPLE, + COMPLEX + } + + private static final Logger LOG = + Logger.getLogger(ITBigQueryStorageLongRunningTest.class.getName()); + private static final String LONG_TESTS_ENABLED_PROPERTY = + "bigquery.storage.enable_long_running_tests"; + + private static String dataset; + private static final String DESCRIPTION = "BigQuery Write Java long test dataset"; + + private static BigQueryWriteClient client; + private static String parentProjectId; + private static BigQuery bigquery; + + private static JSONObject MakeJsonObject(RowComplexity complexity) throws IOException { + JSONObject object = new JSONObject(); + // size: (1, simple)(2,complex)() + // TODO(jstocklass): Add option for testing protobuf format using StreamWriter2 + switch (complexity) { + case SIMPLE: + object.put("test_str", "aaa"); + object.put("test_numerics", new JSONArray(new String[] {"1234", "-900000"})); + object.put("test_datetime", String.valueOf(LocalDateTime.now())); + break; + case COMPLEX: + // TODO(jstocklass): Make a better json object that doesn't break the format rules. + object.put("test_str", "aaa"); + object.put("test_numerics1", new JSONArray(new String[] {"1","2","3","4","5","6","7","8", + "9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25", + "26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42", + "43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59", + "60","61","62","63","64","65","66","67","68","69","70","71","72","73","74","75","76", + "77","78","79","80","81","82","83","84","85","86","87","88","89","90","91","92","93", + "94","95","96","97","98","99","100"})); + object.put("test_numerics2", new JSONArray(new String[] {"1","2","3","4","5","6","7","8", + "9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25", + "26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42", + "43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59", + "60","61","62","63","64","65","66","67","68","69","70","71","72","73","74","75","76", + "77","78","79","80","81","82","83","84","85","86","87","88","89","90","91","92","93", + "94","95","96","97","98","99","100"})); + object.put("test_numerics3", new JSONArray(new String[] {"1","2","3","4","5","6","7","8", + "9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25", + "26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42", + "43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59", + "60","61","62","63","64","65","66","67","68","69","70","71","72","73","74","75","76", + "77","78","79","80","81","82","83","84","85","86","87","88","89","90","91","92","93", + "94","95","96","97","98","99","100"})); + object.put("test_datetime", String.valueOf(LocalDateTime.now())); + object.put("test_bools", new JSONArray(new boolean[]{false, true, false, true, false, + true, false, true, false, true, false, true, false, true, false, true, false, true, + true, false, true, false, true, false, true, false, true, false, true, false, true, + true, false, true, false, true, false, true, false, true, false, true, false, true, + true, false, true, false, true, false, true, false, true, false, true, false, true, + true, false, true, false, true, false, true, false, true, false, true, false, true, + true, false, true, false, true, false, true, false, true, false, true, false, true, + true, false, true, false, true, false, true, false, true, false, true, false, true, + true, false, true, false, true, false, true, false, true, false, true, false, true, + true, false, true, false, true, false, true, false, true, false, true, false, true,})); + break; + default: + break; + } + return object; + } + + @BeforeClass + public static void beforeClass() throws IOException { + parentProjectId = String.format("projects/%s", ServiceOptions.getDefaultProjectId()); + + client = BigQueryWriteClient.create(); + RemoteBigQueryHelper bigqueryHelper = RemoteBigQueryHelper.create(); + bigquery = bigqueryHelper.getOptions().getService(); + dataset = RemoteBigQueryHelper.generateDatasetName(); + DatasetInfo datasetInfo = + DatasetInfo.newBuilder(/* datasetId = */ dataset).setDescription(DESCRIPTION).build(); + bigquery.create(datasetInfo); + } + + @AfterClass + public static void afterClass() { + if (client != null) { + client.close(); + } + if (bigquery != null && dataset != null) { + RemoteBigQueryHelper.forceDelete(bigquery, dataset); + LOG.info("Deleted test dataset: " + dataset); + } + } + + @Test + public void testDefaultStream() + throws IOException, InterruptedException, ExecutionException, + Descriptors.DescriptorValidationException { + // TODO(jstocklass): Set up a default stream. Write to it for a long time, + // (a few minutes for now) and make sure that everything goes well, report stats. + LOG.info( + String.format( + "%s tests running with parent project: %s", + ITBigQueryStorageLongRunningWriteTest.class.getSimpleName(), parentProjectId)); + + String tableName = "JsonSimpleTableDefaultStream"; + TableInfo tableInfo = + TableInfo.newBuilder( + TableId.of(dataset, tableName), + StandardTableDefinition.of( + Schema.of( + com.google.cloud.bigquery.Field.newBuilder( + "test_str", StandardSQLTypeName.STRING) + .build(), + com.google.cloud.bigquery.Field.newBuilder( + "test_numerics", StandardSQLTypeName.NUMERIC) + .setMode(Field.Mode.REPEATED) + .build(), + com.google.cloud.bigquery.Field.newBuilder( + "test_datetime", StandardSQLTypeName.DATETIME) + .build()))) + .build(); + bigquery.create(tableInfo); + + StandardSQLTypeName[] array = new StandardSQLTypeName[]{StandardSQLTypeName.INT64}; + String complexTableName = "JsonComplexTableDefaultStream"; + TableInfo tableInfo2 = + TableInfo.newBuilder( + TableId.of(dataset, complexTableName), + StandardTableDefinition.of( + Schema.of( + com.google.cloud.bigquery.Field.newBuilder( + "test_str", StandardSQLTypeName.STRING).build(), + com.google.cloud.bigquery.Field.newBuilder( + "test_numerics1", StandardSQLTypeName.NUMERIC).setMode(Mode.REPEATED) + .build(), + com.google.cloud.bigquery.Field.newBuilder( + "test_numerics2", StandardSQLTypeName.NUMERIC).setMode(Mode.REPEATED) + .build(), + com.google.cloud.bigquery.Field.newBuilder( + "test_numerics3", StandardSQLTypeName.NUMERIC).setMode(Mode.REPEATED) + .build(), + com.google.cloud.bigquery.Field.newBuilder( + "test_datetime", StandardSQLTypeName.DATETIME).build(), + com.google.cloud.bigquery.Field.newBuilder( + "test_bools", StandardSQLTypeName.BOOL).setMode( + Mode.REPEATED).build() + // How do i add sublevels here?? + ) + ) + ).build(); + bigquery.create(tableInfo2); + + TableName parent = TableName.of(ServiceOptions.getDefaultProjectId(), dataset, tableName); + try (JsonStreamWriter jsonStreamWriter = + JsonStreamWriter.newBuilder(parent.toString(), tableInfo.getDefinition().getSchema()) + .createDefaultStream() + .build()) { + for (int i = 0; i < 5; i++) { + JSONObject row = MakeJsonObject(RowComplexity.SIMPLE); + JSONArray jsonArr = new JSONArray(new JSONObject[]{row}); + LocalDateTime start = LocalDateTime.now(); + Date startTime = new Date(); + // TODO(jstocklass): Make asynchronized calls instead of synchronized calls + ApiFuture response = jsonStreamWriter.append(jsonArr, -1); + Date finishTime = new Date(); + Assert.assertFalse(response.get().getAppendResult().hasOffset()); + // TODO(jstocklass): Compute aggregate statistics instead of logging. Once we have a large + // number of requests, this won't be very usable. + LOG.info( + "Simple Latency: " + String.valueOf(finishTime.getTime() - startTime.getTime()) + " ms"); + } + + TableResult result = + bigquery.listTableData( + tableInfo.getTableId(), BigQuery.TableDataListOption.startIndex(0L)); + Iterator iter = result.getValues().iterator(); + FieldValueList currentRow; + for (int i = 0; i < 5; i++) { + assertTrue(iter.hasNext()); + currentRow = iter.next(); + assertEquals("aaa", currentRow.get(0).getStringValue()); + } + assertEquals(false, iter.hasNext()); + } + parent = TableName.of(ServiceOptions.getDefaultProjectId(), dataset, complexTableName); + try (JsonStreamWriter jsonStreamWriter = + JsonStreamWriter.newBuilder(parent.toString(), tableInfo2.getDefinition().getSchema()) + .createDefaultStream() + .build()) { + for (int i = 0; i < 5; i++) { + JSONObject row = MakeJsonObject(RowComplexity.COMPLEX); + JSONArray jsonArr = new JSONArray(new JSONObject[]{row}); + Date startTime = new Date(); + // TODO(jstocklass): Make asynchronized calls instead of synchronized calls + ApiFuture response = jsonStreamWriter.append(jsonArr, -1); + Date finishTime = new Date(); + Assert.assertFalse(response.get().getAppendResult().hasOffset()); + // TODO(jstocklass): Compute aggregate statistics instead of logging. Once we have a large + // number of requests, this won't be very usable. + LOG.info( + "Complex Latency: " + String.valueOf(finishTime.getTime() - startTime.getTime()) + " ms"); + } + + TableResult result2 = bigquery.listTableData(tableInfo2.getTableId(), BigQuery.TableDataListOption.startIndex(0L)); + Iterator iter = result2.getValues().iterator(); + FieldValueList currentRow2; + for (int i = 0; i < 5; i++) { + assertTrue(iter.hasNext()); + currentRow2 = iter.next(); + assertEquals("aaa", currentRow2.get(0).getStringValue()); + } + assertEquals(false, iter.hasNext()); + } + } +} diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java deleted file mode 100644 index 015620f3ba..0000000000 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/STBigQueryStorageLongRunningWriteTest.java +++ /dev/null @@ -1,175 +0,0 @@ -/* - * Copyright 2020 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.cloud.bigquery.storage.v1beta2.st; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import com.google.api.core.ApiFuture; -import com.google.cloud.ServiceOptions; -import com.google.cloud.bigquery.BigQuery; -import com.google.cloud.bigquery.DatasetInfo; -import com.google.cloud.bigquery.Field; -import com.google.cloud.bigquery.FieldValueList; -import com.google.cloud.bigquery.Schema; -import com.google.cloud.bigquery.StandardSQLTypeName; -import com.google.cloud.bigquery.StandardTableDefinition; -import com.google.cloud.bigquery.TableId; -import com.google.cloud.bigquery.TableInfo; -import com.google.cloud.bigquery.TableResult; -import com.google.cloud.bigquery.storage.v1beta2.AppendRowsResponse; -import com.google.cloud.bigquery.storage.v1beta2.BigQueryWriteClient; -import com.google.cloud.bigquery.storage.v1beta2.JsonStreamWriter; -import com.google.cloud.bigquery.storage.v1beta2.TableName; -import com.google.cloud.bigquery.storage.v1beta2.it.ITBigQueryStorageLongRunningTest; -import com.google.cloud.bigquery.testing.RemoteBigQueryHelper; -import com.google.protobuf.Descriptors; -import java.io.IOException; -import java.util.Date; -import java.util.Iterator; -import java.util.concurrent.ExecutionException; -import java.util.logging.Logger; -import org.json.JSONArray; -import org.json.JSONObject; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; -import org.threeten.bp.LocalDateTime; - -public class STBigQueryStorageLongRunningWriteTest { - public enum RowComplexity { - SIMPLE, - COMPLEX - } - - private static final Logger LOG = - Logger.getLogger(ITBigQueryStorageLongRunningTest.class.getName()); - private static final String LONG_TESTS_ENABLED_PROPERTY = - "bigquery.storage.enable_long_running_tests"; - - private static String dataset; - private static final String DESCRIPTION = "BigQuery Write Java long test dataset"; - - private static BigQueryWriteClient client; - private static String parentProjectId; - private static BigQuery bigquery; - - private static JSONObject MakeJsonObject(RowComplexity complexity) throws IOException { - JSONObject object = new JSONObject(); - // size: (1, simple)(2,complex)() - // TODO(jstocklass): Add option for testing protobuf format using StreamWriter2 - switch (complexity) { - case SIMPLE: - object.put("test_str", "aaa"); - object.put("test_numerics", new JSONArray(new String[] {"1234", "-900000"})); - object.put("test_datetime", String.valueOf(LocalDateTime.now())); - break; - case COMPLEX: - // TODO(jstocklass): Make a better json object that doesn't break the format rules. - break; - default: - break; - } - return object; - } - - @BeforeClass - public static void beforeClass() throws IOException { - parentProjectId = String.format("projects/%s", ServiceOptions.getDefaultProjectId()); - - client = BigQueryWriteClient.create(); - RemoteBigQueryHelper bigqueryHelper = RemoteBigQueryHelper.create(); - bigquery = bigqueryHelper.getOptions().getService(); - dataset = RemoteBigQueryHelper.generateDatasetName(); - DatasetInfo datasetInfo = - DatasetInfo.newBuilder(/* datasetId = */ dataset).setDescription(DESCRIPTION).build(); - bigquery.create(datasetInfo); - } - - @AfterClass - public static void afterClass() { - if (client != null) { - client.close(); - } - if (bigquery != null && dataset != null) { - RemoteBigQueryHelper.forceDelete(bigquery, dataset); - LOG.info("Deleted test dataset: " + dataset); - } - } - - @Test - public void testDefaultStream() - throws IOException, InterruptedException, ExecutionException, - Descriptors.DescriptorValidationException { - // TODO(jstocklass): Set up a default stream. Write to it for a long time, - // (a few minutes for now) and make sure that everything goes well, report stats. - LOG.info( - String.format( - "%s tests running with parent project: %s", - STBigQueryStorageLongRunningWriteTest.class.getSimpleName(), parentProjectId)); - - String tableName = "JsonTableDefaultStream"; - TableInfo tableInfo = - TableInfo.newBuilder( - TableId.of(dataset, tableName), - StandardTableDefinition.of( - Schema.of( - com.google.cloud.bigquery.Field.newBuilder( - "test_str", StandardSQLTypeName.STRING) - .build(), - com.google.cloud.bigquery.Field.newBuilder( - "test_numerics", StandardSQLTypeName.NUMERIC) - .setMode(Field.Mode.REPEATED) - .build(), - com.google.cloud.bigquery.Field.newBuilder( - "test_datetime", StandardSQLTypeName.DATETIME) - .build()))) - .build(); - bigquery.create(tableInfo); - TableName parent = TableName.of(ServiceOptions.getDefaultProjectId(), dataset, tableName); - try (JsonStreamWriter jsonStreamWriter = - JsonStreamWriter.newBuilder(parent.toString(), tableInfo.getDefinition().getSchema()) - .createDefaultStream() - .build()) { - for (int i = 0; i < 5; i++) { - JSONObject row = MakeJsonObject(RowComplexity.SIMPLE); - JSONArray jsonArr = new JSONArray(new JSONObject[] {row}); - LocalDateTime start = LocalDateTime.now(); - Date startTime = new Date(); - // TODO(jstocklass): Make asynchronized calls instead of synchronized calls - ApiFuture response = jsonStreamWriter.append(jsonArr, -1); - Date finishTime = new Date(); - Assert.assertFalse(response.get().getAppendResult().hasOffset()); - // TODO(jstocklass): Compute aggregate statistics instead of logging. Once we have a large - // number of requests, this won't be very usable. - LOG.info("Latency: " + String.valueOf(finishTime.getTime() - startTime.getTime()) + " ms"); - } - - TableResult result = - bigquery.listTableData( - tableInfo.getTableId(), BigQuery.TableDataListOption.startIndex(0L)); - Iterator iter = result.getValues().iterator(); - FieldValueList currentRow; - for (int i = 0; i < 5; i++) { - assertTrue(iter.hasNext()); - currentRow = iter.next(); - assertEquals("aaa", currentRow.get(0).getStringValue()); - } - assertEquals(false, iter.hasNext()); - } - } -} From ee5db70f01c4833af06d66605c54b1f983fd1df4 Mon Sep 17 00:00:00 2001 From: JacobStocklass Date: Wed, 3 Mar 2021 23:49:24 +0000 Subject: [PATCH 17/22] Formatting --- ...ITBigQueryStorageLongRunningWriteTest.java | 207 ++++++++++-------- 1 file changed, 121 insertions(+), 86 deletions(-) diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/ITBigQueryStorageLongRunningWriteTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/ITBigQueryStorageLongRunningWriteTest.java index b19289f607..2be092ff62 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/ITBigQueryStorageLongRunningWriteTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/ITBigQueryStorageLongRunningWriteTest.java @@ -82,38 +82,62 @@ private static JSONObject MakeJsonObject(RowComplexity complexity) throws IOExce case COMPLEX: // TODO(jstocklass): Make a better json object that doesn't break the format rules. object.put("test_str", "aaa"); - object.put("test_numerics1", new JSONArray(new String[] {"1","2","3","4","5","6","7","8", - "9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25", - "26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42", - "43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59", - "60","61","62","63","64","65","66","67","68","69","70","71","72","73","74","75","76", - "77","78","79","80","81","82","83","84","85","86","87","88","89","90","91","92","93", - "94","95","96","97","98","99","100"})); - object.put("test_numerics2", new JSONArray(new String[] {"1","2","3","4","5","6","7","8", - "9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25", - "26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42", - "43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59", - "60","61","62","63","64","65","66","67","68","69","70","71","72","73","74","75","76", - "77","78","79","80","81","82","83","84","85","86","87","88","89","90","91","92","93", - "94","95","96","97","98","99","100"})); - object.put("test_numerics3", new JSONArray(new String[] {"1","2","3","4","5","6","7","8", - "9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25", - "26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42", - "43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59", - "60","61","62","63","64","65","66","67","68","69","70","71","72","73","74","75","76", - "77","78","79","80","81","82","83","84","85","86","87","88","89","90","91","92","93", - "94","95","96","97","98","99","100"})); + object.put( + "test_numerics1", + new JSONArray( + new String[] { + "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", + "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", + "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", + "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", + "55", "56", "57", "58", "59", "60", "61", "62", "63", "64", "65", "66", "67", + "68", "69", "70", "71", "72", "73", "74", "75", "76", "77", "78", "79", "80", + "81", "82", "83", "84", "85", "86", "87", "88", "89", "90", "91", "92", "93", + "94", "95", "96", "97", "98", "99", "100" + })); + object.put( + "test_numerics2", + new JSONArray( + new String[] { + "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", + "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", + "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", + "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", + "55", "56", "57", "58", "59", "60", "61", "62", "63", "64", "65", "66", "67", + "68", "69", "70", "71", "72", "73", "74", "75", "76", "77", "78", "79", "80", + "81", "82", "83", "84", "85", "86", "87", "88", "89", "90", "91", "92", "93", + "94", "95", "96", "97", "98", "99", "100" + })); + object.put( + "test_numerics3", + new JSONArray( + new String[] { + "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", + "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", + "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", + "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", + "55", "56", "57", "58", "59", "60", "61", "62", "63", "64", "65", "66", "67", + "68", "69", "70", "71", "72", "73", "74", "75", "76", "77", "78", "79", "80", + "81", "82", "83", "84", "85", "86", "87", "88", "89", "90", "91", "92", "93", + "94", "95", "96", "97", "98", "99", "100" + })); object.put("test_datetime", String.valueOf(LocalDateTime.now())); - object.put("test_bools", new JSONArray(new boolean[]{false, true, false, true, false, - true, false, true, false, true, false, true, false, true, false, true, false, true, - true, false, true, false, true, false, true, false, true, false, true, false, true, - true, false, true, false, true, false, true, false, true, false, true, false, true, - true, false, true, false, true, false, true, false, true, false, true, false, true, - true, false, true, false, true, false, true, false, true, false, true, false, true, - true, false, true, false, true, false, true, false, true, false, true, false, true, - true, false, true, false, true, false, true, false, true, false, true, false, true, - true, false, true, false, true, false, true, false, true, false, true, false, true, - true, false, true, false, true, false, true, false, true, false, true, false, true,})); + object.put( + "test_bools", + new JSONArray( + new boolean[] { + false, true, false, true, false, true, false, true, false, true, false, true, + false, true, false, true, false, true, true, false, true, false, true, false, + true, false, true, false, true, false, true, true, false, true, false, true, + false, true, false, true, false, true, false, true, true, false, true, false, + true, false, true, false, true, false, true, false, true, true, false, true, + false, true, false, true, false, true, false, true, false, true, true, false, + true, false, true, false, true, false, true, false, true, false, true, true, + false, true, false, true, false, true, false, true, false, true, false, true, + true, false, true, false, true, false, true, false, true, false, true, false, + true, true, false, true, false, true, false, true, false, true, false, true, + false, true, + })); break; default: break; @@ -175,33 +199,38 @@ public void testDefaultStream() .build(); bigquery.create(tableInfo); - StandardSQLTypeName[] array = new StandardSQLTypeName[]{StandardSQLTypeName.INT64}; + StandardSQLTypeName[] array = new StandardSQLTypeName[] {StandardSQLTypeName.INT64}; String complexTableName = "JsonComplexTableDefaultStream"; TableInfo tableInfo2 = TableInfo.newBuilder( - TableId.of(dataset, complexTableName), - StandardTableDefinition.of( - Schema.of( - com.google.cloud.bigquery.Field.newBuilder( - "test_str", StandardSQLTypeName.STRING).build(), - com.google.cloud.bigquery.Field.newBuilder( - "test_numerics1", StandardSQLTypeName.NUMERIC).setMode(Mode.REPEATED) - .build(), - com.google.cloud.bigquery.Field.newBuilder( - "test_numerics2", StandardSQLTypeName.NUMERIC).setMode(Mode.REPEATED) - .build(), - com.google.cloud.bigquery.Field.newBuilder( - "test_numerics3", StandardSQLTypeName.NUMERIC).setMode(Mode.REPEATED) - .build(), - com.google.cloud.bigquery.Field.newBuilder( - "test_datetime", StandardSQLTypeName.DATETIME).build(), - com.google.cloud.bigquery.Field.newBuilder( - "test_bools", StandardSQLTypeName.BOOL).setMode( - Mode.REPEATED).build() - // How do i add sublevels here?? - ) - ) - ).build(); + TableId.of(dataset, complexTableName), + StandardTableDefinition.of( + Schema.of( + com.google.cloud.bigquery.Field.newBuilder( + "test_str", StandardSQLTypeName.STRING) + .build(), + com.google.cloud.bigquery.Field.newBuilder( + "test_numerics1", StandardSQLTypeName.NUMERIC) + .setMode(Mode.REPEATED) + .build(), + com.google.cloud.bigquery.Field.newBuilder( + "test_numerics2", StandardSQLTypeName.NUMERIC) + .setMode(Mode.REPEATED) + .build(), + com.google.cloud.bigquery.Field.newBuilder( + "test_numerics3", StandardSQLTypeName.NUMERIC) + .setMode(Mode.REPEATED) + .build(), + com.google.cloud.bigquery.Field.newBuilder( + "test_datetime", StandardSQLTypeName.DATETIME) + .build(), + com.google.cloud.bigquery.Field.newBuilder( + "test_bools", StandardSQLTypeName.BOOL) + .setMode(Mode.REPEATED) + .build() + // How do i add sublevels here?? + ))) + .build(); bigquery.create(tableInfo2); TableName parent = TableName.of(ServiceOptions.getDefaultProjectId(), dataset, tableName); @@ -209,41 +238,43 @@ public void testDefaultStream() JsonStreamWriter.newBuilder(parent.toString(), tableInfo.getDefinition().getSchema()) .createDefaultStream() .build()) { - for (int i = 0; i < 5; i++) { - JSONObject row = MakeJsonObject(RowComplexity.SIMPLE); - JSONArray jsonArr = new JSONArray(new JSONObject[]{row}); - LocalDateTime start = LocalDateTime.now(); - Date startTime = new Date(); - // TODO(jstocklass): Make asynchronized calls instead of synchronized calls - ApiFuture response = jsonStreamWriter.append(jsonArr, -1); - Date finishTime = new Date(); - Assert.assertFalse(response.get().getAppendResult().hasOffset()); - // TODO(jstocklass): Compute aggregate statistics instead of logging. Once we have a large - // number of requests, this won't be very usable. - LOG.info( - "Simple Latency: " + String.valueOf(finishTime.getTime() - startTime.getTime()) + " ms"); - } + for (int i = 0; i < 5; i++) { + JSONObject row = MakeJsonObject(RowComplexity.SIMPLE); + JSONArray jsonArr = new JSONArray(new JSONObject[] {row}); + LocalDateTime start = LocalDateTime.now(); + Date startTime = new Date(); + // TODO(jstocklass): Make asynchronized calls instead of synchronized calls + ApiFuture response = jsonStreamWriter.append(jsonArr, -1); + Date finishTime = new Date(); + Assert.assertFalse(response.get().getAppendResult().hasOffset()); + // TODO(jstocklass): Compute aggregate statistics instead of logging. Once we have a large + // number of requests, this won't be very usable. + LOG.info( + "Simple Latency: " + + String.valueOf(finishTime.getTime() - startTime.getTime()) + + " ms"); + } - TableResult result = - bigquery.listTableData( - tableInfo.getTableId(), BigQuery.TableDataListOption.startIndex(0L)); - Iterator iter = result.getValues().iterator(); - FieldValueList currentRow; - for (int i = 0; i < 5; i++) { - assertTrue(iter.hasNext()); - currentRow = iter.next(); - assertEquals("aaa", currentRow.get(0).getStringValue()); - } - assertEquals(false, iter.hasNext()); + TableResult result = + bigquery.listTableData( + tableInfo.getTableId(), BigQuery.TableDataListOption.startIndex(0L)); + Iterator iter = result.getValues().iterator(); + FieldValueList currentRow; + for (int i = 0; i < 5; i++) { + assertTrue(iter.hasNext()); + currentRow = iter.next(); + assertEquals("aaa", currentRow.get(0).getStringValue()); + } + assertEquals(false, iter.hasNext()); } parent = TableName.of(ServiceOptions.getDefaultProjectId(), dataset, complexTableName); try (JsonStreamWriter jsonStreamWriter = - JsonStreamWriter.newBuilder(parent.toString(), tableInfo2.getDefinition().getSchema()) - .createDefaultStream() - .build()) { + JsonStreamWriter.newBuilder(parent.toString(), tableInfo2.getDefinition().getSchema()) + .createDefaultStream() + .build()) { for (int i = 0; i < 5; i++) { JSONObject row = MakeJsonObject(RowComplexity.COMPLEX); - JSONArray jsonArr = new JSONArray(new JSONObject[]{row}); + JSONArray jsonArr = new JSONArray(new JSONObject[] {row}); Date startTime = new Date(); // TODO(jstocklass): Make asynchronized calls instead of synchronized calls ApiFuture response = jsonStreamWriter.append(jsonArr, -1); @@ -252,10 +283,14 @@ public void testDefaultStream() // TODO(jstocklass): Compute aggregate statistics instead of logging. Once we have a large // number of requests, this won't be very usable. LOG.info( - "Complex Latency: " + String.valueOf(finishTime.getTime() - startTime.getTime()) + " ms"); + "Complex Latency: " + + String.valueOf(finishTime.getTime() - startTime.getTime()) + + " ms"); } - TableResult result2 = bigquery.listTableData(tableInfo2.getTableId(), BigQuery.TableDataListOption.startIndex(0L)); + TableResult result2 = + bigquery.listTableData( + tableInfo2.getTableId(), BigQuery.TableDataListOption.startIndex(0L)); Iterator iter = result2.getValues().iterator(); FieldValueList currentRow2; for (int i = 0; i < 5; i++) { From 064c8d689339ba004127fec8a17d53841a92bc86 Mon Sep 17 00:00:00 2001 From: JacobStocklass Date: Thu, 4 Mar 2021 20:05:41 +0000 Subject: [PATCH 18/22] Aggregating data and logging once --- ...ITBigQueryStorageLongRunningWriteTest.java | 83 ++++++++++--------- 1 file changed, 43 insertions(+), 40 deletions(-) diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/ITBigQueryStorageLongRunningWriteTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/ITBigQueryStorageLongRunningWriteTest.java index 2be092ff62..7001cbd7fa 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/ITBigQueryStorageLongRunningWriteTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/ITBigQueryStorageLongRunningWriteTest.java @@ -39,7 +39,6 @@ import com.google.cloud.bigquery.testing.RemoteBigQueryHelper; import com.google.protobuf.Descriptors; import java.io.IOException; -import java.util.Date; import java.util.Iterator; import java.util.concurrent.ExecutionException; import java.util.logging.Logger; @@ -80,7 +79,6 @@ private static JSONObject MakeJsonObject(RowComplexity complexity) throws IOExce object.put("test_datetime", String.valueOf(LocalDateTime.now())); break; case COMPLEX: - // TODO(jstocklass): Make a better json object that doesn't break the format rules. object.put("test_str", "aaa"); object.put( "test_numerics1", @@ -138,6 +136,11 @@ private static JSONObject MakeJsonObject(RowComplexity complexity) throws IOExce true, true, false, true, false, true, false, true, false, true, false, true, false, true, })); + JSONObject sub = new JSONObject(); + sub.put("sub_bool", true); + sub.put("sub_int", 12); + sub.put("sub_string", "Test Test Test"); + object.put("test_subs", new JSONArray(new JSONObject[] {sub, sub, sub, sub, sub, sub})); break; default: break; @@ -206,61 +209,63 @@ public void testDefaultStream() TableId.of(dataset, complexTableName), StandardTableDefinition.of( Schema.of( - com.google.cloud.bigquery.Field.newBuilder( - "test_str", StandardSQLTypeName.STRING) - .build(), - com.google.cloud.bigquery.Field.newBuilder( - "test_numerics1", StandardSQLTypeName.NUMERIC) + Field.newBuilder("test_str", StandardSQLTypeName.STRING).build(), + Field.newBuilder("test_numerics1", StandardSQLTypeName.NUMERIC) .setMode(Mode.REPEATED) .build(), - com.google.cloud.bigquery.Field.newBuilder( - "test_numerics2", StandardSQLTypeName.NUMERIC) + Field.newBuilder("test_numerics2", StandardSQLTypeName.NUMERIC) .setMode(Mode.REPEATED) .build(), - com.google.cloud.bigquery.Field.newBuilder( - "test_numerics3", StandardSQLTypeName.NUMERIC) + Field.newBuilder("test_numerics3", StandardSQLTypeName.NUMERIC) .setMode(Mode.REPEATED) .build(), - com.google.cloud.bigquery.Field.newBuilder( - "test_datetime", StandardSQLTypeName.DATETIME) + Field.newBuilder("test_datetime", StandardSQLTypeName.DATETIME).build(), + Field.newBuilder("test_bools", StandardSQLTypeName.BOOL) + .setMode(Mode.REPEATED) .build(), - com.google.cloud.bigquery.Field.newBuilder( - "test_bools", StandardSQLTypeName.BOOL) + Field.newBuilder( + "test_subs", + StandardSQLTypeName.STRUCT, + Field.of("sub_bool", StandardSQLTypeName.BOOL), + Field.of("sub_int", StandardSQLTypeName.INT64), + Field.of("sub_string", StandardSQLTypeName.STRING)) .setMode(Mode.REPEATED) - .build() - // How do i add sublevels here?? - ))) + .build()))) .build(); bigquery.create(tableInfo2); + int requestLimit = 1000; + long averageLatency = 0; + long totalLatency = 0; TableName parent = TableName.of(ServiceOptions.getDefaultProjectId(), dataset, tableName); try (JsonStreamWriter jsonStreamWriter = JsonStreamWriter.newBuilder(parent.toString(), tableInfo.getDefinition().getSchema()) .createDefaultStream() .build()) { - for (int i = 0; i < 5; i++) { + for (int i = 0; i < requestLimit; i++) { JSONObject row = MakeJsonObject(RowComplexity.SIMPLE); JSONArray jsonArr = new JSONArray(new JSONObject[] {row}); - LocalDateTime start = LocalDateTime.now(); - Date startTime = new Date(); + long startTime = System.nanoTime(); // TODO(jstocklass): Make asynchronized calls instead of synchronized calls ApiFuture response = jsonStreamWriter.append(jsonArr, -1); - Date finishTime = new Date(); + long finishTime = System.nanoTime(); Assert.assertFalse(response.get().getAppendResult().hasOffset()); - // TODO(jstocklass): Compute aggregate statistics instead of logging. Once we have a large - // number of requests, this won't be very usable. - LOG.info( - "Simple Latency: " - + String.valueOf(finishTime.getTime() - startTime.getTime()) - + " ms"); + // Ignore first entry, it is way slower than the others and ruins expected behavior + if (i != 0) { + totalLatency += (finishTime - startTime); + } } + averageLatency = totalLatency / requestLimit; + // TODO(jstocklass): Is there a better way to get this than to log it? + LOG.info("Simple average Latency: " + String.valueOf(averageLatency) + " ns"); + averageLatency = totalLatency = 0; TableResult result = bigquery.listTableData( tableInfo.getTableId(), BigQuery.TableDataListOption.startIndex(0L)); Iterator iter = result.getValues().iterator(); FieldValueList currentRow; - for (int i = 0; i < 5; i++) { + for (int i = 0; i < requestLimit; i++) { assertTrue(iter.hasNext()); currentRow = iter.next(); assertEquals("aaa", currentRow.get(0).getStringValue()); @@ -272,28 +277,26 @@ public void testDefaultStream() JsonStreamWriter.newBuilder(parent.toString(), tableInfo2.getDefinition().getSchema()) .createDefaultStream() .build()) { - for (int i = 0; i < 5; i++) { + for (int i = 0; i < requestLimit; i++) { JSONObject row = MakeJsonObject(RowComplexity.COMPLEX); JSONArray jsonArr = new JSONArray(new JSONObject[] {row}); - Date startTime = new Date(); + long startTime = System.nanoTime(); // TODO(jstocklass): Make asynchronized calls instead of synchronized calls ApiFuture response = jsonStreamWriter.append(jsonArr, -1); - Date finishTime = new Date(); + long finishTime = System.nanoTime(); Assert.assertFalse(response.get().getAppendResult().hasOffset()); - // TODO(jstocklass): Compute aggregate statistics instead of logging. Once we have a large - // number of requests, this won't be very usable. - LOG.info( - "Complex Latency: " - + String.valueOf(finishTime.getTime() - startTime.getTime()) - + " ms"); + if (i != 0) { + totalLatency += (finishTime - startTime); + } } - + averageLatency = totalLatency / requestLimit; + LOG.info("Complex average Latency: " + String.valueOf(averageLatency) + " ns"); TableResult result2 = bigquery.listTableData( tableInfo2.getTableId(), BigQuery.TableDataListOption.startIndex(0L)); Iterator iter = result2.getValues().iterator(); FieldValueList currentRow2; - for (int i = 0; i < 5; i++) { + for (int i = 0; i < requestLimit; i++) { assertTrue(iter.hasNext()); currentRow2 = iter.next(); assertEquals("aaa", currentRow2.get(0).getStringValue()); From d38599502bc4cffc0048d031b9224222375fb735 Mon Sep 17 00:00:00 2001 From: JacobStocklass Date: Thu, 4 Mar 2021 23:00:20 +0000 Subject: [PATCH 19/22] Refactoring down to only the simple case. Complex case will be handled in a different PR --- ...ITBigQueryStorageLongRunningWriteTest.java | 138 +----------------- 1 file changed, 6 insertions(+), 132 deletions(-) diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/ITBigQueryStorageLongRunningWriteTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/ITBigQueryStorageLongRunningWriteTest.java index 7001cbd7fa..4bc04d3d9c 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/ITBigQueryStorageLongRunningWriteTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/ITBigQueryStorageLongRunningWriteTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2020 Google LLC + * Copyright 2021 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,7 +23,6 @@ import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.DatasetInfo; import com.google.cloud.bigquery.Field; -import com.google.cloud.bigquery.Field.Mode; import com.google.cloud.bigquery.FieldValueList; import com.google.cloud.bigquery.Schema; import com.google.cloud.bigquery.StandardSQLTypeName; @@ -60,10 +59,9 @@ public enum RowComplexity { Logger.getLogger(ITBigQueryStorageLongRunningTest.class.getName()); private static final String LONG_TESTS_ENABLED_PROPERTY = "bigquery.storage.enable_long_running_tests"; - - private static String dataset; private static final String DESCRIPTION = "BigQuery Write Java long test dataset"; + private static String dataset; private static BigQueryWriteClient client; private static String parentProjectId; private static BigQuery bigquery; @@ -79,69 +77,7 @@ private static JSONObject MakeJsonObject(RowComplexity complexity) throws IOExce object.put("test_datetime", String.valueOf(LocalDateTime.now())); break; case COMPLEX: - object.put("test_str", "aaa"); - object.put( - "test_numerics1", - new JSONArray( - new String[] { - "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", - "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", - "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", - "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", - "55", "56", "57", "58", "59", "60", "61", "62", "63", "64", "65", "66", "67", - "68", "69", "70", "71", "72", "73", "74", "75", "76", "77", "78", "79", "80", - "81", "82", "83", "84", "85", "86", "87", "88", "89", "90", "91", "92", "93", - "94", "95", "96", "97", "98", "99", "100" - })); - object.put( - "test_numerics2", - new JSONArray( - new String[] { - "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", - "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", - "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", - "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", - "55", "56", "57", "58", "59", "60", "61", "62", "63", "64", "65", "66", "67", - "68", "69", "70", "71", "72", "73", "74", "75", "76", "77", "78", "79", "80", - "81", "82", "83", "84", "85", "86", "87", "88", "89", "90", "91", "92", "93", - "94", "95", "96", "97", "98", "99", "100" - })); - object.put( - "test_numerics3", - new JSONArray( - new String[] { - "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", - "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", - "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", - "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", - "55", "56", "57", "58", "59", "60", "61", "62", "63", "64", "65", "66", "67", - "68", "69", "70", "71", "72", "73", "74", "75", "76", "77", "78", "79", "80", - "81", "82", "83", "84", "85", "86", "87", "88", "89", "90", "91", "92", "93", - "94", "95", "96", "97", "98", "99", "100" - })); - object.put("test_datetime", String.valueOf(LocalDateTime.now())); - object.put( - "test_bools", - new JSONArray( - new boolean[] { - false, true, false, true, false, true, false, true, false, true, false, true, - false, true, false, true, false, true, true, false, true, false, true, false, - true, false, true, false, true, false, true, true, false, true, false, true, - false, true, false, true, false, true, false, true, true, false, true, false, - true, false, true, false, true, false, true, false, true, true, false, true, - false, true, false, true, false, true, false, true, false, true, true, false, - true, false, true, false, true, false, true, false, true, false, true, true, - false, true, false, true, false, true, false, true, false, true, false, true, - true, false, true, false, true, false, true, false, true, false, true, false, - true, true, false, true, false, true, false, true, false, true, false, true, - false, true, - })); - JSONObject sub = new JSONObject(); - sub.put("sub_bool", true); - sub.put("sub_int", 12); - sub.put("sub_string", "Test Test Test"); - object.put("test_subs", new JSONArray(new JSONObject[] {sub, sub, sub, sub, sub, sub})); - break; + // TODO(jstocklass): make a complex object default: break; } @@ -158,6 +94,7 @@ public static void beforeClass() throws IOException { dataset = RemoteBigQueryHelper.generateDatasetName(); DatasetInfo datasetInfo = DatasetInfo.newBuilder(/* datasetId = */ dataset).setDescription(DESCRIPTION).build(); + LOG.info("Creating dataset: " + dataset); bigquery.create(datasetInfo); } @@ -173,7 +110,7 @@ public static void afterClass() { } @Test - public void testDefaultStream() + public void testSimpleDefaultStream() throws IOException, InterruptedException, ExecutionException, Descriptors.DescriptorValidationException { // TODO(jstocklass): Set up a default stream. Write to it for a long time, @@ -202,39 +139,7 @@ public void testDefaultStream() .build(); bigquery.create(tableInfo); - StandardSQLTypeName[] array = new StandardSQLTypeName[] {StandardSQLTypeName.INT64}; - String complexTableName = "JsonComplexTableDefaultStream"; - TableInfo tableInfo2 = - TableInfo.newBuilder( - TableId.of(dataset, complexTableName), - StandardTableDefinition.of( - Schema.of( - Field.newBuilder("test_str", StandardSQLTypeName.STRING).build(), - Field.newBuilder("test_numerics1", StandardSQLTypeName.NUMERIC) - .setMode(Mode.REPEATED) - .build(), - Field.newBuilder("test_numerics2", StandardSQLTypeName.NUMERIC) - .setMode(Mode.REPEATED) - .build(), - Field.newBuilder("test_numerics3", StandardSQLTypeName.NUMERIC) - .setMode(Mode.REPEATED) - .build(), - Field.newBuilder("test_datetime", StandardSQLTypeName.DATETIME).build(), - Field.newBuilder("test_bools", StandardSQLTypeName.BOOL) - .setMode(Mode.REPEATED) - .build(), - Field.newBuilder( - "test_subs", - StandardSQLTypeName.STRUCT, - Field.of("sub_bool", StandardSQLTypeName.BOOL), - Field.of("sub_int", StandardSQLTypeName.INT64), - Field.of("sub_string", StandardSQLTypeName.STRING)) - .setMode(Mode.REPEATED) - .build()))) - .build(); - bigquery.create(tableInfo2); - - int requestLimit = 1000; + int requestLimit = 10; long averageLatency = 0; long totalLatency = 0; TableName parent = TableName.of(ServiceOptions.getDefaultProjectId(), dataset, tableName); @@ -272,36 +177,5 @@ public void testDefaultStream() } assertEquals(false, iter.hasNext()); } - parent = TableName.of(ServiceOptions.getDefaultProjectId(), dataset, complexTableName); - try (JsonStreamWriter jsonStreamWriter = - JsonStreamWriter.newBuilder(parent.toString(), tableInfo2.getDefinition().getSchema()) - .createDefaultStream() - .build()) { - for (int i = 0; i < requestLimit; i++) { - JSONObject row = MakeJsonObject(RowComplexity.COMPLEX); - JSONArray jsonArr = new JSONArray(new JSONObject[] {row}); - long startTime = System.nanoTime(); - // TODO(jstocklass): Make asynchronized calls instead of synchronized calls - ApiFuture response = jsonStreamWriter.append(jsonArr, -1); - long finishTime = System.nanoTime(); - Assert.assertFalse(response.get().getAppendResult().hasOffset()); - if (i != 0) { - totalLatency += (finishTime - startTime); - } - } - averageLatency = totalLatency / requestLimit; - LOG.info("Complex average Latency: " + String.valueOf(averageLatency) + " ns"); - TableResult result2 = - bigquery.listTableData( - tableInfo2.getTableId(), BigQuery.TableDataListOption.startIndex(0L)); - Iterator iter = result2.getValues().iterator(); - FieldValueList currentRow2; - for (int i = 0; i < requestLimit; i++) { - assertTrue(iter.hasNext()); - currentRow2 = iter.next(); - assertEquals("aaa", currentRow2.get(0).getStringValue()); - } - assertEquals(false, iter.hasNext()); - } } } From 24c2154c5343a3ac2e4849007cabd5017658ed3b Mon Sep 17 00:00:00 2001 From: JacobStocklass Date: Thu, 4 Mar 2021 23:01:45 +0000 Subject: [PATCH 20/22] Quick rename --- .../v1beta2/st/ITBigQueryStorageLongRunningWriteTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/ITBigQueryStorageLongRunningWriteTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/ITBigQueryStorageLongRunningWriteTest.java index 4bc04d3d9c..7e978e7f36 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/ITBigQueryStorageLongRunningWriteTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/ITBigQueryStorageLongRunningWriteTest.java @@ -110,7 +110,7 @@ public static void afterClass() { } @Test - public void testSimpleDefaultStream() + public void testDefaultStreamSimpleSchema() throws IOException, InterruptedException, ExecutionException, Descriptors.DescriptorValidationException { // TODO(jstocklass): Set up a default stream. Write to it for a long time, From 081d25f2bc36b789aa7bb39bf0add23bd499cd0e Mon Sep 17 00:00:00 2001 From: JacobStocklass Date: Thu, 4 Mar 2021 23:39:04 +0000 Subject: [PATCH 21/22] Adding complex schema default stream test --- ...ITBigQueryStorageLongRunningWriteTest.java | 140 +++++++++++++++++- 1 file changed, 138 insertions(+), 2 deletions(-) diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/ITBigQueryStorageLongRunningWriteTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/ITBigQueryStorageLongRunningWriteTest.java index 7e978e7f36..dbe54039be 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/ITBigQueryStorageLongRunningWriteTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/ITBigQueryStorageLongRunningWriteTest.java @@ -23,6 +23,7 @@ import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.DatasetInfo; import com.google.cloud.bigquery.Field; +import com.google.cloud.bigquery.Field.Mode; import com.google.cloud.bigquery.FieldValueList; import com.google.cloud.bigquery.Schema; import com.google.cloud.bigquery.StandardSQLTypeName; @@ -65,6 +66,7 @@ public enum RowComplexity { private static BigQueryWriteClient client; private static String parentProjectId; private static BigQuery bigquery; + private static int requestLimit = 10; private static JSONObject MakeJsonObject(RowComplexity complexity) throws IOException { JSONObject object = new JSONObject(); @@ -77,7 +79,69 @@ private static JSONObject MakeJsonObject(RowComplexity complexity) throws IOExce object.put("test_datetime", String.valueOf(LocalDateTime.now())); break; case COMPLEX: - // TODO(jstocklass): make a complex object + object.put("test_str", "aaa"); + object.put( + "test_numerics1", + new JSONArray( + new String[] { + "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", + "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", + "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", + "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", + "55", "56", "57", "58", "59", "60", "61", "62", "63", "64", "65", "66", "67", + "68", "69", "70", "71", "72", "73", "74", "75", "76", "77", "78", "79", "80", + "81", "82", "83", "84", "85", "86", "87", "88", "89", "90", "91", "92", "93", + "94", "95", "96", "97", "98", "99", "100" + })); + object.put( + "test_numerics2", + new JSONArray( + new String[] { + "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", + "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", + "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", + "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", + "55", "56", "57", "58", "59", "60", "61", "62", "63", "64", "65", "66", "67", + "68", "69", "70", "71", "72", "73", "74", "75", "76", "77", "78", "79", "80", + "81", "82", "83", "84", "85", "86", "87", "88", "89", "90", "91", "92", "93", + "94", "95", "96", "97", "98", "99", "100" + })); + object.put( + "test_numerics3", + new JSONArray( + new String[] { + "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", + "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", + "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", + "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", + "55", "56", "57", "58", "59", "60", "61", "62", "63", "64", "65", "66", "67", + "68", "69", "70", "71", "72", "73", "74", "75", "76", "77", "78", "79", "80", + "81", "82", "83", "84", "85", "86", "87", "88", "89", "90", "91", "92", "93", + "94", "95", "96", "97", "98", "99", "100" + })); + object.put("test_datetime", String.valueOf(LocalDateTime.now())); + object.put( + "test_bools", + new JSONArray( + new boolean[] { + false, true, false, true, false, true, false, true, false, true, false, true, + false, true, false, true, false, true, true, false, true, false, true, false, + true, false, true, false, true, false, true, true, false, true, false, true, + false, true, false, true, false, true, false, true, true, false, true, false, + true, false, true, false, true, false, true, false, true, true, false, true, + false, true, false, true, false, true, false, true, false, true, true, false, + true, false, true, false, true, false, true, false, true, false, true, true, + false, true, false, true, false, true, false, true, false, true, false, true, + true, false, true, false, true, false, true, false, true, false, true, false, + true, true, false, true, false, true, false, true, false, true, false, true, + false, true, + })); + JSONObject sub = new JSONObject(); + sub.put("sub_bool", true); + sub.put("sub_int", 12); + sub.put("sub_string", "Test Test Test"); + object.put("test_subs", new JSONArray(new JSONObject[] {sub, sub, sub, sub, sub, sub})); + break; default: break; } @@ -139,7 +203,6 @@ public void testDefaultStreamSimpleSchema() .build(); bigquery.create(tableInfo); - int requestLimit = 10; long averageLatency = 0; long totalLatency = 0; TableName parent = TableName.of(ServiceOptions.getDefaultProjectId(), dataset, tableName); @@ -178,4 +241,77 @@ public void testDefaultStreamSimpleSchema() assertEquals(false, iter.hasNext()); } } + + @Test + public void testDefaultStreamComplexSchema() + throws IOException, InterruptedException, ExecutionException, + Descriptors.DescriptorValidationException { + StandardSQLTypeName[] array = new StandardSQLTypeName[] {StandardSQLTypeName.INT64}; + String complexTableName = "JsonComplexTableDefaultStream"; + TableInfo tableInfo2 = + TableInfo.newBuilder( + TableId.of(dataset, complexTableName), + StandardTableDefinition.of( + Schema.of( + Field.newBuilder("test_str", StandardSQLTypeName.STRING).build(), + Field.newBuilder("test_numerics1", StandardSQLTypeName.NUMERIC) + .setMode(Mode.REPEATED) + .build(), + Field.newBuilder("test_numerics2", StandardSQLTypeName.NUMERIC) + .setMode(Mode.REPEATED) + .build(), + Field.newBuilder("test_numerics3", StandardSQLTypeName.NUMERIC) + .setMode(Mode.REPEATED) + .build(), + Field.newBuilder("test_datetime", StandardSQLTypeName.DATETIME).build(), + Field.newBuilder("test_bools", StandardSQLTypeName.BOOL) + .setMode(Mode.REPEATED) + .build(), + Field.newBuilder( + "test_subs", + StandardSQLTypeName.STRUCT, + Field.of("sub_bool", StandardSQLTypeName.BOOL), + Field.of("sub_int", StandardSQLTypeName.INT64), + Field.of("sub_string", StandardSQLTypeName.STRING)) + .setMode(Mode.REPEATED) + .build()))) + .build(); + bigquery.create(tableInfo2); + + long totalLatency = 0; + long averageLatency = 0; + TableName parent = + TableName.of(ServiceOptions.getDefaultProjectId(), dataset, complexTableName); + try (JsonStreamWriter jsonStreamWriter = + JsonStreamWriter.newBuilder(parent.toString(), tableInfo2.getDefinition().getSchema()) + .createDefaultStream() + .build()) { + for (int i = 0; i < requestLimit; i++) { + JSONObject row = MakeJsonObject(RowComplexity.COMPLEX); + JSONArray jsonArr = new JSONArray(new JSONObject[] {row}); + long startTime = System.nanoTime(); + // TODO(jstocklass): Make asynchronized calls instead of synchronized calls + ApiFuture response = jsonStreamWriter.append(jsonArr, -1); + long finishTime = System.nanoTime(); + Assert.assertFalse(response.get().getAppendResult().hasOffset()); + if (i != 0) { + totalLatency += (finishTime - startTime); + } + } + averageLatency = totalLatency / requestLimit; + LOG.info("Complex average Latency: " + String.valueOf(averageLatency) + " ns"); + TableResult result2 = + bigquery.listTableData( + tableInfo2.getTableId(), BigQuery.TableDataListOption.startIndex(0L)); + Iterator iter = result2.getValues().iterator(); + FieldValueList currentRow2; + for (int i = 0; i < requestLimit; i++) { + assertTrue(iter.hasNext()); + currentRow2 = iter.next(); + assertEquals("aaa", currentRow2.get(0).getStringValue()); + } + assertEquals(false, iter.hasNext()); + } + } + } From 80375497fc77f288d2d84420da58de640d6b2ff8 Mon Sep 17 00:00:00 2001 From: JacobStocklass Date: Thu, 4 Mar 2021 23:40:04 +0000 Subject: [PATCH 22/22] Formatting --- ...ITBigQueryStorageLongRunningWriteTest.java | 123 +++++++++--------- 1 file changed, 61 insertions(+), 62 deletions(-) diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/ITBigQueryStorageLongRunningWriteTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/ITBigQueryStorageLongRunningWriteTest.java index dbe54039be..01c15bc00e 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/ITBigQueryStorageLongRunningWriteTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/st/ITBigQueryStorageLongRunningWriteTest.java @@ -84,57 +84,57 @@ private static JSONObject MakeJsonObject(RowComplexity complexity) throws IOExce "test_numerics1", new JSONArray( new String[] { - "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", - "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", - "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", - "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", - "55", "56", "57", "58", "59", "60", "61", "62", "63", "64", "65", "66", "67", - "68", "69", "70", "71", "72", "73", "74", "75", "76", "77", "78", "79", "80", - "81", "82", "83", "84", "85", "86", "87", "88", "89", "90", "91", "92", "93", - "94", "95", "96", "97", "98", "99", "100" + "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", + "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", + "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", + "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", + "55", "56", "57", "58", "59", "60", "61", "62", "63", "64", "65", "66", "67", + "68", "69", "70", "71", "72", "73", "74", "75", "76", "77", "78", "79", "80", + "81", "82", "83", "84", "85", "86", "87", "88", "89", "90", "91", "92", "93", + "94", "95", "96", "97", "98", "99", "100" })); object.put( "test_numerics2", new JSONArray( new String[] { - "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", - "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", - "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", - "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", - "55", "56", "57", "58", "59", "60", "61", "62", "63", "64", "65", "66", "67", - "68", "69", "70", "71", "72", "73", "74", "75", "76", "77", "78", "79", "80", - "81", "82", "83", "84", "85", "86", "87", "88", "89", "90", "91", "92", "93", - "94", "95", "96", "97", "98", "99", "100" + "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", + "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", + "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", + "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", + "55", "56", "57", "58", "59", "60", "61", "62", "63", "64", "65", "66", "67", + "68", "69", "70", "71", "72", "73", "74", "75", "76", "77", "78", "79", "80", + "81", "82", "83", "84", "85", "86", "87", "88", "89", "90", "91", "92", "93", + "94", "95", "96", "97", "98", "99", "100" })); object.put( "test_numerics3", new JSONArray( new String[] { - "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", - "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", - "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", - "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", - "55", "56", "57", "58", "59", "60", "61", "62", "63", "64", "65", "66", "67", - "68", "69", "70", "71", "72", "73", "74", "75", "76", "77", "78", "79", "80", - "81", "82", "83", "84", "85", "86", "87", "88", "89", "90", "91", "92", "93", - "94", "95", "96", "97", "98", "99", "100" + "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", + "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", + "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", + "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", + "55", "56", "57", "58", "59", "60", "61", "62", "63", "64", "65", "66", "67", + "68", "69", "70", "71", "72", "73", "74", "75", "76", "77", "78", "79", "80", + "81", "82", "83", "84", "85", "86", "87", "88", "89", "90", "91", "92", "93", + "94", "95", "96", "97", "98", "99", "100" })); object.put("test_datetime", String.valueOf(LocalDateTime.now())); object.put( "test_bools", new JSONArray( new boolean[] { - false, true, false, true, false, true, false, true, false, true, false, true, - false, true, false, true, false, true, true, false, true, false, true, false, - true, false, true, false, true, false, true, true, false, true, false, true, - false, true, false, true, false, true, false, true, true, false, true, false, - true, false, true, false, true, false, true, false, true, true, false, true, - false, true, false, true, false, true, false, true, false, true, true, false, - true, false, true, false, true, false, true, false, true, false, true, true, - false, true, false, true, false, true, false, true, false, true, false, true, - true, false, true, false, true, false, true, false, true, false, true, false, - true, true, false, true, false, true, false, true, false, true, false, true, - false, true, + false, true, false, true, false, true, false, true, false, true, false, true, + false, true, false, true, false, true, true, false, true, false, true, false, + true, false, true, false, true, false, true, true, false, true, false, true, + false, true, false, true, false, true, false, true, true, false, true, false, + true, false, true, false, true, false, true, false, true, true, false, true, + false, true, false, true, false, true, false, true, false, true, true, false, + true, false, true, false, true, false, true, false, true, false, true, true, + false, true, false, true, false, true, false, true, false, true, false, true, + true, false, true, false, true, false, true, false, true, false, true, false, + true, true, false, true, false, true, false, true, false, true, false, true, + false, true, })); JSONObject sub = new JSONObject(); sub.put("sub_bool", true); @@ -245,36 +245,36 @@ public void testDefaultStreamSimpleSchema() @Test public void testDefaultStreamComplexSchema() throws IOException, InterruptedException, ExecutionException, - Descriptors.DescriptorValidationException { + Descriptors.DescriptorValidationException { StandardSQLTypeName[] array = new StandardSQLTypeName[] {StandardSQLTypeName.INT64}; String complexTableName = "JsonComplexTableDefaultStream"; TableInfo tableInfo2 = TableInfo.newBuilder( - TableId.of(dataset, complexTableName), - StandardTableDefinition.of( - Schema.of( - Field.newBuilder("test_str", StandardSQLTypeName.STRING).build(), - Field.newBuilder("test_numerics1", StandardSQLTypeName.NUMERIC) - .setMode(Mode.REPEATED) - .build(), - Field.newBuilder("test_numerics2", StandardSQLTypeName.NUMERIC) - .setMode(Mode.REPEATED) - .build(), - Field.newBuilder("test_numerics3", StandardSQLTypeName.NUMERIC) - .setMode(Mode.REPEATED) - .build(), - Field.newBuilder("test_datetime", StandardSQLTypeName.DATETIME).build(), - Field.newBuilder("test_bools", StandardSQLTypeName.BOOL) - .setMode(Mode.REPEATED) - .build(), - Field.newBuilder( - "test_subs", - StandardSQLTypeName.STRUCT, - Field.of("sub_bool", StandardSQLTypeName.BOOL), - Field.of("sub_int", StandardSQLTypeName.INT64), - Field.of("sub_string", StandardSQLTypeName.STRING)) - .setMode(Mode.REPEATED) - .build()))) + TableId.of(dataset, complexTableName), + StandardTableDefinition.of( + Schema.of( + Field.newBuilder("test_str", StandardSQLTypeName.STRING).build(), + Field.newBuilder("test_numerics1", StandardSQLTypeName.NUMERIC) + .setMode(Mode.REPEATED) + .build(), + Field.newBuilder("test_numerics2", StandardSQLTypeName.NUMERIC) + .setMode(Mode.REPEATED) + .build(), + Field.newBuilder("test_numerics3", StandardSQLTypeName.NUMERIC) + .setMode(Mode.REPEATED) + .build(), + Field.newBuilder("test_datetime", StandardSQLTypeName.DATETIME).build(), + Field.newBuilder("test_bools", StandardSQLTypeName.BOOL) + .setMode(Mode.REPEATED) + .build(), + Field.newBuilder( + "test_subs", + StandardSQLTypeName.STRUCT, + Field.of("sub_bool", StandardSQLTypeName.BOOL), + Field.of("sub_int", StandardSQLTypeName.INT64), + Field.of("sub_string", StandardSQLTypeName.STRING)) + .setMode(Mode.REPEATED) + .build()))) .build(); bigquery.create(tableInfo2); @@ -313,5 +313,4 @@ public void testDefaultStreamComplexSchema() assertEquals(false, iter.hasNext()); } } - }