From 73778a795d71204e43b99d7fe5e8d07266d5b0e9 Mon Sep 17 00:00:00 2001 From: Mike Wasson <3992422+MikeWasson@users.noreply.github.com> Date: Tue, 8 Feb 2022 17:58:33 +0000 Subject: [PATCH 1/2] docs(tutorials): Add IT to JsonWriteDefaultStream tutorial Includes two small code fixes: - Change a field name in the schema. - Detect schema updates while streaming. --- pom.xml | 1 + tutorials/JsonWriterDefaultStream/pom.xml | 14 ++++- .../com/example/JsonWriterDefaultStream.java | 8 ++- .../example/JsonWriterDefaultStreamIT.java | 63 ++++++++++++++++++- .../src/test/resources/TestData.json | 5 ++ 5 files changed, 88 insertions(+), 3 deletions(-) create mode 100644 tutorials/JsonWriterDefaultStream/src/test/resources/TestData.json diff --git a/pom.xml b/pom.xml index e2a204f388..4777411072 100644 --- a/pom.xml +++ b/pom.xml @@ -253,6 +253,7 @@ include-samples samples + tutorials diff --git a/tutorials/JsonWriterDefaultStream/pom.xml b/tutorials/JsonWriterDefaultStream/pom.xml index b6ce7f9a94..23225bc2a9 100644 --- a/tutorials/JsonWriterDefaultStream/pom.xml +++ b/tutorials/JsonWriterDefaultStream/pom.xml @@ -41,6 +41,19 @@ arrow-memory-netty 6.0.1 + + + junit + junit + 4.13.2 + test + + + com.google.truth + truth + 1.1.3 + test + @@ -57,7 +70,6 @@ - . ../../samples/snippets/src/main/java/com/example/bigquerystorage diff --git a/tutorials/JsonWriterDefaultStream/src/main/java/com/example/JsonWriterDefaultStream.java b/tutorials/JsonWriterDefaultStream/src/main/java/com/example/JsonWriterDefaultStream.java index 70d5da08c8..63c223cf3b 100644 --- a/tutorials/JsonWriterDefaultStream/src/main/java/com/example/JsonWriterDefaultStream.java +++ b/tutorials/JsonWriterDefaultStream/src/main/java/com/example/JsonWriterDefaultStream.java @@ -67,7 +67,7 @@ public static void createDestinationTable( .build(), Field.of("author", StandardSQLTypeName.STRING), Field.of("committer", StandardSQLTypeName.STRING), - Field.of("ts", StandardSQLTypeName.DATETIME), + Field.of("commit_date", StandardSQLTypeName.DATETIME), Field.of("subject", StandardSQLTypeName.STRING), Field.of("message", StandardSQLTypeName.STRING), Field.of("repo_name", StandardSQLTypeName.STRING)); @@ -114,6 +114,12 @@ public static void writeToDefaultStream( } // batch ApiFuture future = writer.append(jsonArr); AppendRowsResponse response = future.get(); + if (response.hasUpdatedSchema()) { + // The destination table schema has changed. The client library automatically + // reestablishes a connection to the backend using the new schema, so we can continue + // to send data without interruption. + System.out.println("Table schema changed."); + } } System.out.println("Appended records successfully."); } catch (ExecutionException e) { diff --git a/tutorials/JsonWriterDefaultStream/src/test/java/com/example/JsonWriterDefaultStreamIT.java b/tutorials/JsonWriterDefaultStream/src/test/java/com/example/JsonWriterDefaultStreamIT.java index cb716c17bb..b55090eaa4 100644 --- a/tutorials/JsonWriterDefaultStream/src/test/java/com/example/JsonWriterDefaultStreamIT.java +++ b/tutorials/JsonWriterDefaultStream/src/test/java/com/example/JsonWriterDefaultStreamIT.java @@ -16,7 +16,68 @@ package com.example.bigquerystorage; +import static com.google.common.truth.Truth.assertThat; + +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.BigQuery.DatasetDeleteOption; +import com.google.cloud.bigquery.BigQueryOptions; +import com.google.cloud.bigquery.DatasetId; +import com.google.cloud.bigquery.DatasetInfo; +import java.io.ByteArrayOutputStream; +import java.nio.file.FileSystems; +import java.nio.file.Path; +import java.io.PrintStream; +import java.util.UUID; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + @RunWith(JUnit4.class) public class JsonWriterDefaultStreamIT { - // TODO(mwasson): ADD Integration Test + + private static final String GOOGLE_CLOUD_PROJECT = System.getenv("GOOGLE_CLOUD_PROJECT"); + + private ByteArrayOutputStream bout; + private PrintStream out; + private BigQuery bigquery; + private String datasetName; + + @BeforeClass + public static void beforeClass() { + + } + + @Before + public void setUp() { + bout = new ByteArrayOutputStream(); + out = new PrintStream(bout); + System.setOut(out); + + bigquery = BigQueryOptions.getDefaultInstance().getService(); + + // Create a new dataset for each test. + datasetName = "JAVA_WRITER_DEFAULT_STREAM_TEST" + UUID.randomUUID().toString().substring(0, 8); + bigquery.create(DatasetInfo.newBuilder(datasetName).build()); + } + + @Test + public void testJsonWriterDefaultStream() throws Exception { + Path dataFilePath = FileSystems.getDefault().getPath("src/test/resources", "TestData.json"); + + System.out.println(dataFilePath.toString()); + String[] args = {GOOGLE_CLOUD_PROJECT, datasetName, "github", dataFilePath.toString()}; + JsonWriterDefaultStream.main(args); + assertThat(bout.toString()).contains("Appended records successfully."); + } + + @After + public void tearDown() { + bigquery.delete( + DatasetId.of(GOOGLE_CLOUD_PROJECT, datasetName), DatasetDeleteOption.deleteContents()); + System.setOut(null); + } + } diff --git a/tutorials/JsonWriterDefaultStream/src/test/resources/TestData.json b/tutorials/JsonWriterDefaultStream/src/test/resources/TestData.json new file mode 100644 index 0000000000..a060639297 --- /dev/null +++ b/tutorials/JsonWriterDefaultStream/src/test/resources/TestData.json @@ -0,0 +1,5 @@ +{"commit":"0001","parent":["00001"],"author":"user1","committer":"GitHub","subject":"Commit 1","message":"This is a commit.\n\n","repo_name":"googleapis/java-bigquerystorage","commit_date":"2019-07-23T20:28:01"} +{"commit":"0002","parent":["00002"],"author":"user1","committer":"GitHub","subject":"Commit 2","message":"This is a commit.\n\n","repo_name":"googleapis/java-bigquerystorage","commit_date":"2019-12-05T16:05:16"} +{"commit":"0003","parent":["00003"],"author":"user1","committer":"GitHub","subject":"Commit 3","message":"This is a commit.\n\n","repo_name":"googleapis/java-bigquerystorage","commit_date":"2019-03-21T16:59:23"} +{"commit":"0004","parent":["00004"],"author":"user1","committer":"GitHub","subject":"Commit 4","message":"This is a commit.\n\n","repo_name":"googleapis/java-bigquerystorage","commit_date":"2019-01-11T01:31:39"} +{"commit":"0005","parent":["00005"],"author":"user1","committer":"GitHub","subject":"Commit 5","message":"This is a commit.\n\n","repo_name":"googleapis/java-bigquerystorage","commit_date":"2019-07-31T19:09:09"} From 56982c61706a83f0806cd1cedc118f135b7a7394 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Tue, 8 Feb 2022 19:58:28 +0000 Subject: [PATCH 2/2] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- README.md | 2 +- .../test/java/com/example/JsonWriterDefaultStreamIT.java | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 7c1c7f1ac0..1fab747ef2 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ If you are using Maven without BOM, add this to your dependencies: If you are using Gradle 5.x or later, add this to your dependencies ```Groovy -implementation platform('com.google.cloud:libraries-bom:24.2.0') +implementation platform('com.google.cloud:libraries-bom:24.3.0') implementation 'com.google.cloud:google-cloud-bigquerystorage' ``` diff --git a/tutorials/JsonWriterDefaultStream/src/test/java/com/example/JsonWriterDefaultStreamIT.java b/tutorials/JsonWriterDefaultStream/src/test/java/com/example/JsonWriterDefaultStreamIT.java index b55090eaa4..3fe67b885d 100644 --- a/tutorials/JsonWriterDefaultStream/src/test/java/com/example/JsonWriterDefaultStreamIT.java +++ b/tutorials/JsonWriterDefaultStream/src/test/java/com/example/JsonWriterDefaultStreamIT.java @@ -24,9 +24,9 @@ import com.google.cloud.bigquery.DatasetId; import com.google.cloud.bigquery.DatasetInfo; import java.io.ByteArrayOutputStream; +import java.io.PrintStream; import java.nio.file.FileSystems; import java.nio.file.Path; -import java.io.PrintStream; import java.util.UUID; import org.junit.After; import org.junit.Before; @@ -46,9 +46,7 @@ public class JsonWriterDefaultStreamIT { private String datasetName; @BeforeClass - public static void beforeClass() { - - } + public static void beforeClass() {} @Before public void setUp() { @@ -79,5 +77,4 @@ public void tearDown() { DatasetId.of(GOOGLE_CLOUD_PROJECT, datasetName), DatasetDeleteOption.deleteContents()); System.setOut(null); } - }