From 9fd7aca8b6032a2c02d0ea91dd063c9ac9e151aa Mon Sep 17 00:00:00 2001 From: Veronica Wasson <3992422+VeronicaWasson@users.noreply.github.com> Date: Tue, 8 Feb 2022 13:46:53 -0800 Subject: [PATCH] docs(tutorials): Add IT to JsonWriteDefaultStream tutorial (#1522) Includes two small code fixes: - Change a field name in the schema. - Detect schema updates while streaming. --- pom.xml | 1 + tutorials/JsonWriterDefaultStream/pom.xml | 14 ++++- .../com/example/JsonWriterDefaultStream.java | 8 ++- .../example/JsonWriterDefaultStreamIT.java | 60 ++++++++++++++++++- .../src/test/resources/TestData.json | 5 ++ 5 files changed, 85 insertions(+), 3 deletions(-) create mode 100644 tutorials/JsonWriterDefaultStream/src/test/resources/TestData.json diff --git a/pom.xml b/pom.xml index 0e27e7833a..831d939520 100644 --- a/pom.xml +++ b/pom.xml @@ -253,6 +253,7 @@ include-samples samples + tutorials diff --git a/tutorials/JsonWriterDefaultStream/pom.xml b/tutorials/JsonWriterDefaultStream/pom.xml index b6ce7f9a94..23225bc2a9 100644 --- a/tutorials/JsonWriterDefaultStream/pom.xml +++ b/tutorials/JsonWriterDefaultStream/pom.xml @@ -41,6 +41,19 @@ arrow-memory-netty 6.0.1 + + + junit + junit + 4.13.2 + test + + + com.google.truth + truth + 1.1.3 + test + @@ -57,7 +70,6 @@ - . ../../samples/snippets/src/main/java/com/example/bigquerystorage diff --git a/tutorials/JsonWriterDefaultStream/src/main/java/com/example/JsonWriterDefaultStream.java b/tutorials/JsonWriterDefaultStream/src/main/java/com/example/JsonWriterDefaultStream.java index 70d5da08c8..63c223cf3b 100644 --- a/tutorials/JsonWriterDefaultStream/src/main/java/com/example/JsonWriterDefaultStream.java +++ b/tutorials/JsonWriterDefaultStream/src/main/java/com/example/JsonWriterDefaultStream.java @@ -67,7 +67,7 @@ public static void createDestinationTable( .build(), Field.of("author", StandardSQLTypeName.STRING), Field.of("committer", StandardSQLTypeName.STRING), - Field.of("ts", StandardSQLTypeName.DATETIME), + Field.of("commit_date", StandardSQLTypeName.DATETIME), Field.of("subject", StandardSQLTypeName.STRING), Field.of("message", StandardSQLTypeName.STRING), Field.of("repo_name", StandardSQLTypeName.STRING)); @@ -114,6 +114,12 @@ public static void writeToDefaultStream( } // batch ApiFuture future = writer.append(jsonArr); AppendRowsResponse response = future.get(); + if (response.hasUpdatedSchema()) { + // The destination table schema has changed. The client library automatically + // reestablishes a connection to the backend using the new schema, so we can continue + // to send data without interruption. + System.out.println("Table schema changed."); + } } System.out.println("Appended records successfully."); } catch (ExecutionException e) { diff --git a/tutorials/JsonWriterDefaultStream/src/test/java/com/example/JsonWriterDefaultStreamIT.java b/tutorials/JsonWriterDefaultStream/src/test/java/com/example/JsonWriterDefaultStreamIT.java index cb716c17bb..3fe67b885d 100644 --- a/tutorials/JsonWriterDefaultStream/src/test/java/com/example/JsonWriterDefaultStreamIT.java +++ b/tutorials/JsonWriterDefaultStream/src/test/java/com/example/JsonWriterDefaultStreamIT.java @@ -16,7 +16,65 @@ package com.example.bigquerystorage; +import static com.google.common.truth.Truth.assertThat; + +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.BigQuery.DatasetDeleteOption; +import com.google.cloud.bigquery.BigQueryOptions; +import com.google.cloud.bigquery.DatasetId; +import com.google.cloud.bigquery.DatasetInfo; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.file.FileSystems; +import java.nio.file.Path; +import java.util.UUID; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + @RunWith(JUnit4.class) public class JsonWriterDefaultStreamIT { - // TODO(mwasson): ADD Integration Test + + private static final String GOOGLE_CLOUD_PROJECT = System.getenv("GOOGLE_CLOUD_PROJECT"); + + private ByteArrayOutputStream bout; + private PrintStream out; + private BigQuery bigquery; + private String datasetName; + + @BeforeClass + public static void beforeClass() {} + + @Before + public void setUp() { + bout = new ByteArrayOutputStream(); + out = new PrintStream(bout); + System.setOut(out); + + bigquery = BigQueryOptions.getDefaultInstance().getService(); + + // Create a new dataset for each test. + datasetName = "JAVA_WRITER_DEFAULT_STREAM_TEST" + UUID.randomUUID().toString().substring(0, 8); + bigquery.create(DatasetInfo.newBuilder(datasetName).build()); + } + + @Test + public void testJsonWriterDefaultStream() throws Exception { + Path dataFilePath = FileSystems.getDefault().getPath("src/test/resources", "TestData.json"); + + System.out.println(dataFilePath.toString()); + String[] args = {GOOGLE_CLOUD_PROJECT, datasetName, "github", dataFilePath.toString()}; + JsonWriterDefaultStream.main(args); + assertThat(bout.toString()).contains("Appended records successfully."); + } + + @After + public void tearDown() { + bigquery.delete( + DatasetId.of(GOOGLE_CLOUD_PROJECT, datasetName), DatasetDeleteOption.deleteContents()); + System.setOut(null); + } } diff --git a/tutorials/JsonWriterDefaultStream/src/test/resources/TestData.json b/tutorials/JsonWriterDefaultStream/src/test/resources/TestData.json new file mode 100644 index 0000000000..a060639297 --- /dev/null +++ b/tutorials/JsonWriterDefaultStream/src/test/resources/TestData.json @@ -0,0 +1,5 @@ +{"commit":"0001","parent":["00001"],"author":"user1","committer":"GitHub","subject":"Commit 1","message":"This is a commit.\n\n","repo_name":"googleapis/java-bigquerystorage","commit_date":"2019-07-23T20:28:01"} +{"commit":"0002","parent":["00002"],"author":"user1","committer":"GitHub","subject":"Commit 2","message":"This is a commit.\n\n","repo_name":"googleapis/java-bigquerystorage","commit_date":"2019-12-05T16:05:16"} +{"commit":"0003","parent":["00003"],"author":"user1","committer":"GitHub","subject":"Commit 3","message":"This is a commit.\n\n","repo_name":"googleapis/java-bigquerystorage","commit_date":"2019-03-21T16:59:23"} +{"commit":"0004","parent":["00004"],"author":"user1","committer":"GitHub","subject":"Commit 4","message":"This is a commit.\n\n","repo_name":"googleapis/java-bigquerystorage","commit_date":"2019-01-11T01:31:39"} +{"commit":"0005","parent":["00005"],"author":"user1","committer":"GitHub","subject":"Commit 5","message":"This is a commit.\n\n","repo_name":"googleapis/java-bigquerystorage","commit_date":"2019-07-31T19:09:09"}