Merge branch 'main' into feature/dateStrings

.github/.OwlBot.lock.yaml

@@ -1,4 +1,4 @@
-# Copyright 2022 Google LLC
+# Copyright 2023 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,4 +13,5 @@
 # limitations under the License.
 docker:
   image: gcr.io/cloud-devrel-public-resources/owlbot-java:latest
-  digest: sha256:27b1b1884dce60460d7521b23c2a73376cba90c0ef3d9f0d32e4bdb786959cfd
+  digest: sha256:81f75d962cd28b7ad10740a643b8069b8fa0357cb495b782eef8560bb7a8fd65
+# created: 2023-12-05T19:16:19.735195992Z

.github/.OwlBot.yaml

@@ -22,6 +22,8 @@ deep-remove-regex:
 
 deep-preserve-regex:
 - "/google-.*/src/test/java/com/google/cloud/.*/v.*/it/IT.*Test.java"
+- "/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/SchemaAwareStreamWriter.java"
+- "/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/ToProtoConverter.java"
 - "/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/BigQuerySchemaUtil.java"
 - "/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/BigQuerySchemaUtilTest.java"
 - "/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/ConnectionWorkerPoolTest.java"

.github/dependabot.yml

@@ -0,0 +1,19 @@
+version: 2
+updates:
+  - package-ecosystem: "maven"
+    directory: "/"
+    schedule:
+      interval: "daily"
+    # Disable version updates for Maven dependencies
+    # we use renovate-bot as well as shared-dependencies BOM to update maven dependencies.
+    ignore:
+      - dependency-name: "*"
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "daily"
+    # Disable version updates for pip dependencies
+    # If a security vulnerability comes in, we will be notified about 
+    # it via template in the synthtool repository.
+    ignore:
+      - dependency-name: "*"

.github/release-please.yml

@@ -18,3 +18,7 @@ branches:
     handleGHRelease: true
     releaseType: java-backport
     branch: 2.25.x
+  - bumpMinorPreMajor: true
+    handleGHRelease: true
+    releaseType: java-backport
+    branch: 2.38.x

.github/release-trigger.yml

@@ -1 +1,2 @@
 enabled: true
+multiScmName: java-bigquerystorage

.github/sync-repo-settings.yaml

@@ -8,17 +8,15 @@ branchProtectionRules:
     requiresCodeOwnerReviews: true
     requiresStrictStatusChecks: false
     requiredStatusCheckContexts:
-      - dependencies (8)
-      - dependencies (11)
+      - dependencies (17)
       - lint
       - clirr
       - units (8)
       - units (11)
       - 'Kokoro - Test: Integration'
       - cla/google
       - OwlBot Post Processor
-      - 'Kokoro - Test: Java GraalVM Native Image'
-      - 'Kokoro - Test: Java 17 GraalVM Native Image'
+      - javadoc
   - pattern: java7
     isAdminEnforced: true
     requiredApprovingReviewCount: 1
@@ -79,8 +77,20 @@ branchProtectionRules:
       - 'Kokoro - Test: Integration'
       - cla/google
       - OwlBot Post Processor
-      - 'Kokoro - Test: Java GraalVM Native Image'
-      - 'Kokoro - Test: Java 17 GraalVM Native Image'
+  - pattern: 2.38.x
+    isAdminEnforced: true
+    requiredApprovingReviewCount: 1
+    requiresCodeOwnerReviews: true
+    requiresStrictStatusChecks: false
+    requiredStatusCheckContexts:
+      - dependencies (17)
+      - lint
+      - clirr
+      - units (8)
+      - units (11)
+      - 'Kokoro - Test: Integration'
+      - cla/google
+      - OwlBot Post Processor
 permissionRules:
   - team: yoshi-admins
     permission: admin

.github/workflows/approve-readme.yaml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     if: github.repository_owner == 'googleapis' && github.head_ref == 'autosynth-readme'
     steps:
-      - uses: actions/github-script@v6
+      - uses: actions/github-script@v7
         with:
           github-token: ${{secrets.YOSHI_APPROVER_TOKEN}}
           script: |

.github/workflows/auto-release.yaml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     if: contains(github.head_ref, 'release-please')
     steps:
-    - uses: actions/github-script@v6
+    - uses: actions/github-script@v7
       with:
         github-token: ${{secrets.YOSHI_APPROVER_TOKEN}}
         debug: true

.github/workflows/ci.yaml

@@ -12,24 +12,46 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        java: [8, 11, 17]
+        java: [11, 17]
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
     - uses: actions/setup-java@v2
       with:
-        distribution: zulu
+        distribution: temurin
         java-version: ${{matrix.java}}
     - run: java -version
     - run: .kokoro/build.sh
       env:
         JOB_TYPE: test
+  units-java8:
+    # Building using Java 17 and run the tests with Java 8 runtime
+    name: "units (8)"
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
+    - uses: actions/setup-java@v3
+      with:
+        java-version: 8
+        distribution: zulu
+    - name: "Set jvm system property environment variable for surefire plugin (unit tests)"
+      # Maven surefire plugin (unit tests) allows us to specify JVM to run the tests.
+      # https://maven.apache.org/surefire/maven-surefire-plugin/test-mojo.html#jvm
+      run: echo "SUREFIRE_JVM_OPT=-Djvm=${JAVA_HOME}/bin/java" >> $GITHUB_ENV
+      shell: bash
+    - uses: actions/setup-java@v3
+      with:
+        java-version: 17
+        distribution: zulu
+    - run: .kokoro/build.sh
+      env:
+        JOB_TYPE: test
   windows:
     runs-on: windows-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
     - uses: actions/setup-java@v2
       with:
-        distribution: zulu
+        distribution: temurin
         java-version: 8
     - run: java -version
     - run: .kokoro/build.bat
@@ -39,22 +61,34 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        java: [8, 11, 17]
+        java: [17]
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
     - uses: actions/setup-java@v2
       with:
-        distribution: zulu
+        distribution: temurin
         java-version: ${{matrix.java}}
     - run: java -version
     - run: .kokoro/dependencies.sh
+  javadoc:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
+      - uses: actions/setup-java@v3
+        with:
+          distribution: temurin
+          java-version: 17
+      - run: java -version
+      - run: .kokoro/build.sh
+        env:
+          JOB_TYPE: javadoc
   lint:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
     - uses: actions/setup-java@v2
       with:
-        distribution: zulu
+        distribution: temurin
         java-version: 11
     - run: java -version
     - run: .kokoro/build.sh
@@ -63,10 +97,10 @@ jobs:
   clirr:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
     - uses: actions/setup-java@v2
       with:
-        distribution: zulu
+        distribution: temurin
         java-version: 8
     - run: java -version
     - run: .kokoro/build.sh

.github/workflows/samples.yaml

@@ -7,7 +7,7 @@ jobs:
   checkstyle:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
       - uses: actions/setup-java@v1
         with:
           java-version: 8

.gitignore

@@ -9,6 +9,7 @@ target/
 # Intellij
 *.iml
 .idea/
+.vscode/
 
 # python utilities
 *.pyc

.kokoro/build.sh

@@ -29,7 +29,7 @@ echo ${JOB_TYPE}
 
 # attempt to install 3 times with exponential backoff (starting with 10 seconds)
 retry_with_backoff 3 10 \
-  mvn install -B -V \
+  mvn install -B -V -U \
     -DskipTests=true \
     -Dclirr.skip=true \
     -Denforcer.skip=true \
@@ -47,7 +47,8 @@ set +e
 
 case ${JOB_TYPE} in
 test)
-    mvn test -B -Dclirr.skip=true -Denforcer.skip=true
+    echo "SUREFIRE_JVM_OPT: ${SUREFIRE_JVM_OPT}"
+    mvn test -B -ntp -Dclirr.skip=true -Denforcer.skip=true ${SUREFIRE_JVM_OPT}
     RETURN_CODE=$?
     ;;
 lint)
@@ -64,18 +65,21 @@ integration)
       -DtrimStackTrace=false \
       -Dclirr.skip=true \
       -Denforcer.skip=true \
+      -Dit.test=!ITBigQueryWrite*RetryTest \
+      -Dsurefire.failIfNoSpecifiedTests=false \
+      -Dfailsafe.failIfNoSpecifiedTests=false \
       -fae \
       verify
     RETURN_CODE=$?
     ;;
 graalvm)
     # Run Unit and Integration Tests with Native Image
-    mvn -B ${INTEGRATION_TEST_ARGS} -ntp -Pnative -Penable-integration-tests test
+    mvn -B ${INTEGRATION_TEST_ARGS} -ntp -PcustomNative -Penable-integration-tests  test
     RETURN_CODE=$?
     ;;
 graalvm17)
     # Run Unit and Integration Tests with Native Image
-    mvn -B ${INTEGRATION_TEST_ARGS} -ntp -Pnative -Penable-integration-tests test
+    mvn -B ${INTEGRATION_TEST_ARGS} -ntp -PcustomNative -Penable-integration-tests test
     RETURN_CODE=$?
     ;;
 samples)
@@ -111,6 +115,22 @@ clirr)
     mvn -B -Denforcer.skip=true clirr:check
     RETURN_CODE=$?
     ;;
+retry_quota)
+    mvn -B ${INTEGRATION_TEST_ARGS} \
+      -Dit.test=ITBigQueryWriteQuotaRetryTest \
+      -Dsurefire.failIfNoSpecifiedTests=false \
+      -Dfailsafe.failIfNoSpecifiedTests=false \
+      test
+    RETURN_CODE=$?
+    ;;
+retry_non_quota)
+    mvn -B ${INTEGRATION_TEST_ARGS} \
+      -Dit.test=ITBigQueryWriteNonQuotaRetryTest \
+      -Dsurefire.failIfNoSpecifiedTests=false \
+      -Dfailsafe.failIfNoSpecifiedTests=false \
+      test
+    RETURN_CODE=$?
+    ;;
 *)
     ;;
 esac

.kokoro/presubmit/graalvm-native-17.cfg → .kokoro/continuous/graalvm-native-17.cfg

@@ -3,7 +3,7 @@
 # Configure the docker image for kokoro-trampoline.
 env_vars: {
   key: "TRAMPOLINE_IMAGE"
-  value: "gcr.io/cloud-devrel-kokoro-resources/graalvm17:22.3.0"
+  value: "gcr.io/cloud-devrel-kokoro-resources/graalvm17:22.3.3"
 }
 
 env_vars: {
@@ -30,4 +30,9 @@ env_vars: {
 env_vars: {
   key: "SECRET_MANAGER_KEYS"
   value: "java-it-service-account"
+}
+
+env_vars: {
+  key: "ENABLE_FLAKYBOT"
+  value: "true"
 }

.kokoro/presubmit/graalvm-native.cfg → .kokoro/continuous/graalvm-native.cfg

@@ -3,7 +3,7 @@
 # Configure the docker image for kokoro-trampoline.
 env_vars: {
   key: "TRAMPOLINE_IMAGE"
-  value: "gcr.io/cloud-devrel-kokoro-resources/graalvm:22.3.0"
+  value: "gcr.io/cloud-devrel-kokoro-resources/graalvm:22.3.3"
 }
 
 env_vars: {
@@ -31,3 +31,8 @@ env_vars: {
   key: "SECRET_MANAGER_KEYS"
   value: "java-it-service-account"
 }
+
+env_vars: {
+  key: "ENABLE_FLAKYBOT"
+  value: "true"
+}

.kokoro/nightly/integration.cfg

@@ -35,3 +35,4 @@ env_vars: {
   key: "SECRET_MANAGER_KEYS"
   value: "java-it-service-account"
 }
+

.kokoro/nightly/java11-integration.cfg

@@ -35,3 +35,4 @@ env_vars: {
   key: "SECRET_MANAGER_KEYS"
   value: "java-it-service-account"
 }
+

.kokoro/nightly/retry_non_quota.cfg

@@ -0,0 +1,37 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+env_vars: {
+  key: "TRAMPOLINE_IMAGE"
+  value: "gcr.io/cloud-devrel-kokoro-resources/java8"
+}
+
+# TODO: remove this after we've migrated all tests and scripts
+env_vars: {
+  key: "GCLOUD_PROJECT"
+  value: "bq-write-api-java-retry-test"
+}
+
+env_vars: {
+  key: "GOOGLE_CLOUD_PROJECT"
+  value: "bq-write-api-java-retry-test"
+}
+
+env_vars: {
+  key: "JOB_TYPE"
+  value: "retry_non_quota"
+}
+
+env_vars: {
+  key: "ENABLE_FLAKYBOT"
+  value: "true"
+}
+
+env_vars: {
+  key: "GOOGLE_APPLICATION_CREDENTIALS"
+  value: "secret_manager/java-it-service-account"
+}
+
+env_vars: {
+  key: "SECRET_MANAGER_KEYS"
+  value: "java-it-service-account"
+}