From 0a1b8d92f49e826979de6b9f4e2e2956bd838ad7 Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Mon, 23 Jul 2018 15:24:20 -0400 Subject: [PATCH] Bigtable: remove regen scripts in favor of utilities/generate_api.py (#3495) --- .../google-cloud-bigtable/DEVELOPING.md | 8 +- .../google-cloud-bigtable/autogen_files.lst | 26 ---- .../handwritten_files.lst | 84 ---------- .../google-cloud-bigtable/scripts/README.md | 14 -- .../scripts/regenerate.sh | 144 ------------------ .../scripts/update-deps.sh | 95 ------------ 6 files changed, 3 insertions(+), 368 deletions(-) delete mode 100644 google-cloud-clients/google-cloud-bigtable/autogen_files.lst delete mode 100644 google-cloud-clients/google-cloud-bigtable/handwritten_files.lst delete mode 100644 google-cloud-clients/google-cloud-bigtable/scripts/README.md delete mode 100755 google-cloud-clients/google-cloud-bigtable/scripts/regenerate.sh delete mode 100755 google-cloud-clients/google-cloud-bigtable/scripts/update-deps.sh diff --git a/google-cloud-clients/google-cloud-bigtable/DEVELOPING.md b/google-cloud-clients/google-cloud-bigtable/DEVELOPING.md index 3c59561ac646..1f1113785328 100644 --- a/google-cloud-clients/google-cloud-bigtable/DEVELOPING.md +++ b/google-cloud-clients/google-cloud-bigtable/DEVELOPING.md @@ -11,11 +11,9 @@ The Cloud Bigtable client is composed of two layers: ### Generated GAPIC base layer The GAPIC layer is generated from a combination of protobuf and GAPIC yaml definitions found in -[googleapis](https://github.com/googleapis/googleapis/tree/master/google/bigtable). This layer is -split into 2 parts: admin and data apis. The admin part is straightforward enough to be consumed -directly by the consumer. However the data client is too protocol focused and is too low level to -be consumed by an application directly. Instead, it's intended to be an implementation detail of the -handwritten layer. +[googleapis](https://github.com/googleapis/googleapis/tree/master/google/bigtable). However the +API is too protocol focused and is too low level to be consumed by an application directly. +Instead, it's intended to be an implementation detail of the handwritten layer. ### Handwritten layer diff --git a/google-cloud-clients/google-cloud-bigtable/autogen_files.lst b/google-cloud-clients/google-cloud-bigtable/autogen_files.lst deleted file mode 100644 index ef763544fed7..000000000000 --- a/google-cloud-clients/google-cloud-bigtable/autogen_files.lst +++ /dev/null @@ -1,26 +0,0 @@ -src/test/java/com/google/cloud/bigtable/data/v2/MockBigtableImpl.java -src/test/java/com/google/cloud/bigtable/data/v2/BaseBigtableDataClientTest.java -src/test/java/com/google/cloud/bigtable/data/v2/MockBigtable.java -src/main/java/com/google/cloud/bigtable/data/v2/package-info.java -src/main/java/com/google/cloud/bigtable/data/v2/BaseBigtableDataSettings.java -src/main/java/com/google/cloud/bigtable/data/v2/stub/BigtableStub.java -src/main/java/com/google/cloud/bigtable/data/v2/stub/GrpcBigtableStub.java -src/main/java/com/google/cloud/bigtable/data/v2/stub/BigtableStubSettings.java -src/main/java/com/google/cloud/bigtable/data/v2/BaseBigtableDataClient.java -src/test/java/com/google/cloud/bigtable/admin/v2/MockBigtableTableAdmin.java -src/test/java/com/google/cloud/bigtable/admin/v2/BigtableInstanceAdminClientTest.java -src/test/java/com/google/cloud/bigtable/admin/v2/BigtableTableAdminClientTest.java -src/test/java/com/google/cloud/bigtable/admin/v2/MockBigtableTableAdminImpl.java -src/test/java/com/google/cloud/bigtable/admin/v2/MockBigtableInstanceAdmin.java -src/test/java/com/google/cloud/bigtable/admin/v2/MockBigtableInstanceAdminImpl.java -src/main/java/com/google/cloud/bigtable/admin/v2/package-info.java -src/main/java/com/google/cloud/bigtable/admin/v2/BigtableInstanceAdminSettings.java -src/main/java/com/google/cloud/bigtable/admin/v2/stub/BigtableInstanceAdminStub.java -src/main/java/com/google/cloud/bigtable/admin/v2/stub/GrpcBigtableInstanceAdminStub.java -src/main/java/com/google/cloud/bigtable/admin/v2/stub/GrpcBigtableTableAdminStub.java -src/main/java/com/google/cloud/bigtable/admin/v2/stub/BigtableTableAdminStub.java -src/main/java/com/google/cloud/bigtable/admin/v2/stub/BigtableTableAdminStubSettings.java -src/main/java/com/google/cloud/bigtable/admin/v2/stub/BigtableInstanceAdminStubSettings.java -src/main/java/com/google/cloud/bigtable/admin/v2/BigtableTableAdminClient.java -src/main/java/com/google/cloud/bigtable/admin/v2/BigtableTableAdminSettings.java -src/main/java/com/google/cloud/bigtable/admin/v2/BigtableInstanceAdminClient.java diff --git a/google-cloud-clients/google-cloud-bigtable/handwritten_files.lst b/google-cloud-clients/google-cloud-bigtable/handwritten_files.lst deleted file mode 100644 index 7d385c98eb25..000000000000 --- a/google-cloud-clients/google-cloud-bigtable/handwritten_files.lst +++ /dev/null @@ -1,84 +0,0 @@ -src/test/resources/com/google/cloud/bigtable/data/v2/stub/readrows/read-rows-acceptance-test.json -src/test/java/com/google/cloud/bigtable/gaxx/reframing/ReframingResponseObserverTest.java -src/test/java/com/google/cloud/bigtable/gaxx/testing/FakeStatusCode.java -src/test/java/com/google/cloud/bigtable/gaxx/testing/package-info.java -src/test/java/com/google/cloud/bigtable/gaxx/testing/MockStreamingApi.java -src/test/java/com/google/cloud/bigtable/gaxx/testing/FakeStreamingApi.java -src/test/java/com/google/cloud/bigtable/data/v2/stub/CheckAndMutateRowCallableTest.java -src/test/java/com/google/cloud/bigtable/data/v2/stub/mutaterows/MutateRowsBatchingDescriptorTest.java -src/test/java/com/google/cloud/bigtable/data/v2/stub/mutaterows/BulkMutateRowsRetryTest.java -src/test/java/com/google/cloud/bigtable/data/v2/stub/SampleRowKeysCallableTest.java -src/test/java/com/google/cloud/bigtable/data/v2/stub/MutateRowCallableTest.java -src/test/java/com/google/cloud/bigtable/data/v2/stub/readrows/ReadRowsRetryTest.java -src/test/java/com/google/cloud/bigtable/data/v2/stub/readrows/RowMergingCallableTest.java -src/test/java/com/google/cloud/bigtable/data/v2/stub/readrows/ReadRowsMergingAcceptanceTest.java -src/test/java/com/google/cloud/bigtable/data/v2/stub/readrows/ReadRowsUserCallableTest.java -src/test/java/com/google/cloud/bigtable/data/v2/stub/readrows/FilterMarkerRowsCallableTest.java -src/test/java/com/google/cloud/bigtable/data/v2/stub/readrows/ReadRowsFirstCallableTest.java -src/test/java/com/google/cloud/bigtable/data/v2/stub/ReadModifyWriteRowCallableTest.java -src/test/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubSettingsTest.java -src/test/java/com/google/cloud/bigtable/data/v2/BigtableDataClientTest.java -src/test/java/com/google/cloud/bigtable/data/v2/models/FiltersTest.java -src/test/java/com/google/cloud/bigtable/data/v2/models/DefaultRowAdapterTest.java -src/test/java/com/google/cloud/bigtable/data/v2/models/QueryTest.java -src/test/java/com/google/cloud/bigtable/data/v2/models/ConditionalRowMutationTest.java -src/test/java/com/google/cloud/bigtable/data/v2/models/ReadModifyWriteRowTest.java -src/test/java/com/google/cloud/bigtable/data/v2/models/MutationTest.java -src/test/java/com/google/cloud/bigtable/data/v2/models/RowTest.java -src/test/java/com/google/cloud/bigtable/data/v2/models/RowMutationTest.java -src/test/java/com/google/cloud/bigtable/data/v2/models/RangeTest.java -src/test/java/com/google/cloud/bigtable/data/v2/internal/RegexUtilTest.java -src/test/java/com/google/cloud/bigtable/data/v2/internal/ByteStringComparatorTest.java -src/main/java/com/google/cloud/bigtable/gaxx/package-info.java -src/main/java/com/google/cloud/bigtable/gaxx/reframing/package-info.java -src/main/java/com/google/cloud/bigtable/gaxx/reframing/IncompleteStreamException.java -src/main/java/com/google/cloud/bigtable/gaxx/reframing/ReframingResponseObserver.java -src/main/java/com/google/cloud/bigtable/gaxx/reframing/Reframer.java -src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java -src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataClient.java -src/main/java/com/google/cloud/bigtable/data/v2/stub/CheckAndMutateRowCallable.java -src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java -src/main/java/com/google/cloud/bigtable/data/v2/stub/mutaterows/MutateRowsUserFacingCallable.java -src/main/java/com/google/cloud/bigtable/data/v2/stub/mutaterows/package-info.java -src/main/java/com/google/cloud/bigtable/data/v2/stub/mutaterows/MutateRowsBatchingDescriptor.java -src/main/java/com/google/cloud/bigtable/data/v2/stub/mutaterows/MutateRowsSpoolingCallable.java -src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubSettings.java -src/main/java/com/google/cloud/bigtable/data/v2/stub/readrows/RowMergingCallable.java -src/main/java/com/google/cloud/bigtable/data/v2/stub/readrows/package-info.java -src/main/java/com/google/cloud/bigtable/data/v2/stub/readrows/ReadRowsFirstCallable.java -src/main/java/com/google/cloud/bigtable/data/v2/stub/readrows/ReadRowsResumptionStrategy.java -src/main/java/com/google/cloud/bigtable/data/v2/stub/readrows/RowMerger.java -src/main/java/com/google/cloud/bigtable/data/v2/stub/readrows/StateMachine.java -src/main/java/com/google/cloud/bigtable/data/v2/stub/readrows/ReadRowsRetryCompletedCallable.java -src/main/java/com/google/cloud/bigtable/data/v2/stub/readrows/ReadRowsUserCallable.java -src/main/java/com/google/cloud/bigtable/data/v2/stub/readrows/FilterMarkerRowsCallable.java -src/main/java/com/google/cloud/bigtable/data/v2/stub/SampleRowKeysCallable.java -src/main/java/com/google/cloud/bigtable/data/v2/stub/MutateRowCallable.java -src/main/java/com/google/cloud/bigtable/data/v2/stub/ReadModifyWriteRowCallable.java -src/main/java/com/google/cloud/bigtable/data/v2/models/ReadModifyWriteRow.java -src/main/java/com/google/cloud/bigtable/data/v2/models/RowCell.java -src/main/java/com/google/cloud/bigtable/data/v2/models/Validations.java -src/main/java/com/google/cloud/bigtable/data/v2/models/RowMutation.java -src/main/java/com/google/cloud/bigtable/data/v2/models/BulkMutationBatcher.java -src/main/java/com/google/cloud/bigtable/data/v2/models/Mutation.java -src/main/java/com/google/cloud/bigtable/data/v2/models/DefaultRowAdapter.java -src/main/java/com/google/cloud/bigtable/data/v2/models/RowAdapter.java -src/main/java/com/google/cloud/bigtable/data/v2/models/KeyOffset.java -src/main/java/com/google/cloud/bigtable/data/v2/models/Range.java -src/main/java/com/google/cloud/bigtable/data/v2/models/ConditionalRowMutation.java -src/main/java/com/google/cloud/bigtable/data/v2/models/Filters.java -src/main/java/com/google/cloud/bigtable/data/v2/models/MutationApi.java -src/main/java/com/google/cloud/bigtable/data/v2/models/Row.java -src/main/java/com/google/cloud/bigtable/data/v2/models/Query.java -src/main/java/com/google/cloud/bigtable/data/v2/internal/ByteStringComparator.java -src/main/java/com/google/cloud/bigtable/data/v2/internal/DummyBatchingDescriptor.java -src/main/java/com/google/cloud/bigtable/data/v2/internal/RequestContext.java -src/main/java/com/google/cloud/bigtable/data/v2/internal/RegexUtil.java -src/main/java/com/google/cloud/bigtable/admin/v2/TableAdminClient.java -src/main/java/com/google/cloud/bigtable/admin/v2/models/GCRules.java -src/main/java/com/google/cloud/bigtable/admin/v2/models/TableAdminRequests.java -src/main/java/com/google/cloud/bigtable/admin/v2/models/TableAdminResponses.java -src/test/java/com/google/cloud/bigtable/admin/v2/TableAdminClientTest.java -src/test/java/com/google/cloud/bigtable/admin/v2/it/TableAdminClientIT.java -src/test/java/com/google/cloud/bigtable/admin/v2/models/TableAdminRequestsTest.java -src/test/java/com/google/cloud/bigtable/admin/v2/models/TableAdminResponsesTest.java diff --git a/google-cloud-clients/google-cloud-bigtable/scripts/README.md b/google-cloud-clients/google-cloud-bigtable/scripts/README.md deleted file mode 100644 index 0c215aae191a..000000000000 --- a/google-cloud-clients/google-cloud-bigtable/scripts/README.md +++ /dev/null @@ -1,14 +0,0 @@ -# Developer Scripts - -This directory contains scripts to help in development of this client. Specifically to test out -changes in the autogenerated protobufs and GAPIC client before publishing the changes to googleapis. - -- update-deps.sh downloads dependencies needed to regenerate the protobufs and the GAPIC client. - It will stage everything in `deps/`. It can be run again to refresh the dependencies. It will - try rebase any changes that are committed locally in the dependency. - -- regenerate.sh is used to regenerate the base GAPIC layer. It will regenerate the protobufs and - all of the GAPIC classes in this project. - -- autogen_files.lst and handwritten_files.lst track the origin of the files in `src/`, they will be - updated when `regenerate.sh` is run diff --git a/google-cloud-clients/google-cloud-bigtable/scripts/regenerate.sh b/google-cloud-clients/google-cloud-bigtable/scripts/regenerate.sh deleted file mode 100755 index 255fbeea531d..000000000000 --- a/google-cloud-clients/google-cloud-bigtable/scripts/regenerate.sh +++ /dev/null @@ -1,144 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script is used to regenerate protos & GAPIC stubs. -# It stages all of the generated code in deps/ and then installs it into the local maven repo. -# It is meant to be used to test changes to the gapic config before publishing -# To pull in upstream changes, please use update-deps.sh - -set -o errexit -o errtrace - -handle_error() { - echo "Failed, exiting. See deps/output.log for more details" - exit 1 -} -trap 'handle_error' ERR - - -PROJECT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )" -WORKDIR="$PROJECT_DIR/deps" -LOG="$WORKDIR/output.log" - -DOCKER_IMAGE=googleapis/artman:stable -TOOLKIT="$WORKDIR/toolkit" - -function run { - run_artman - copy_api_client_staging - copy_google_cloud_java - install_protos - - echo "Successfully completed!" -} - -function run_artman { - rm -rf artman-genfiles/* - - run_gapic "google/bigtable/artman_bigtable.yaml" - run_gapic "google/bigtable/admin/artman_bigtableadmin.yaml" -} - -function run_gapic { - CONFIG="$1" - - log "Running gapic for $CONFIG" - - INNER_CMD="artman --local \ - --config /googleapis/$CONFIG \ - --root-dir /googleapis \ - --output /tmp/artman-genfiles \ - generate java_gapic" - - DOCKER_CMD="docker run --name igor-artman --rm -it \ - -e RUNNING_IN_ARTMAN_DOCKER=True \ - -e HOST_USER_ID=$UID -e HOST_GROUP_ID=`id -g` \ - -v $WORKDIR/googleapis:/googleapis \ - -v $WORKDIR/artman-genfiles:/tmp/artman-genfiles" - - if [ -n "$TOOLKIT" ]; then - DOCKER_CMD="$DOCKER_CMD -v $WORKDIR/toolkit:/toolkit" - fi - - FULL_CMD="$DOCKER_CMD $DOCKER_IMAGE $INNER_CMD" - log "$FULL_CMD" - $FULL_CMD >>$LOG 2>&1 -} - -function copy_api_client_staging { - log "Copying files to api-client-staging" - - rm -rf \ - "$WORKDIR/api-client-staging/generated/java/proto-google-cloud-bigtable-v2" \ - "$WORKDIR/api-client-staging/generated/java/proto-google-cloud-bigtable-admin-v2" \ - "$WORKDIR/api-client-staging/generated/java/grpc-google-cloud-bigtable-v2" \ - "$WORKDIR/api-client-staging/generated/java/grpc-google-cloud-bigtable-admin-v2" \ - "$WORKDIR/api-client-staging/generated/java/gapic-google-cloud-bigtable-v2" \ - "$WORKDIR/api-client-staging/generated/java/gapic-google-cloud-bigtable-admin-v2" - - cp -r \ - "$WORKDIR/artman-genfiles/java/grpc-google-cloud-bigtable-v2" \ - "$WORKDIR/artman-genfiles/java/grpc-google-cloud-bigtable-admin-v2" \ - "$WORKDIR/artman-genfiles/java/proto-google-cloud-bigtable-v2" \ - "$WORKDIR/artman-genfiles/java/proto-google-cloud-bigtable-admin-v2" \ - "$WORKDIR/artman-genfiles/java/gapic-google-cloud-bigtable-v2" \ - "$WORKDIR/artman-genfiles/java/gapic-google-cloud-bigtable-admin-v2" \ - "$WORKDIR/api-client-staging/generated/java/" -} - -function copy_google_cloud_java { - log "Copying google-cloud-java" - - # Delete old files - xargs rm -f < autogen_files.lst - cat /dev/null > autogen_files.lst - - # Update manual files - find src -type f > $PROJECT_DIR/handwritten_files.lst - - # Generate a list of new files - # Data - pushd "$WORKDIR/artman-genfiles/java/gapic-google-cloud-bigtable-v2" - find src -type f >> $PROJECT_DIR/autogen_files.lst - cp -r "src" "$PROJECT_DIR" - popd - # Admin - pushd "$WORKDIR/artman-genfiles/java/gapic-google-cloud-bigtable-admin-v2" - find src -type f >> $PROJECT_DIR/autogen_files.lst - cp -r "src" "$PROJECT_DIR" - popd -} - -function install_protos { - log "Installing protos from api-client-staging" - - pushd "$WORKDIR/api-client-staging/generated/java" > /dev/null - ./gradlew \ - :proto-google-cloud-bigtable-v2:install \ - :grpc-google-cloud-bigtable-v2:install \ - :proto-google-cloud-bigtable-admin-v2:install \ - :grpc-google-cloud-bigtable-admin-v2:install \ - -x javadoc \ - >>$LOG 2>&1 - - popd > /dev/null -} - -function log { - echo ">>>>>>> $@" - echo ">>>>>>> $@" >> $LOG -} - -run diff --git a/google-cloud-clients/google-cloud-bigtable/scripts/update-deps.sh b/google-cloud-clients/google-cloud-bigtable/scripts/update-deps.sh deleted file mode 100755 index ea116bbbca73..000000000000 --- a/google-cloud-clients/google-cloud-bigtable/scripts/update-deps.sh +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script pulls in upstream updates from the veneer code generator and rebases your local -# changes on top. -# -# Specifically it: -# - fetches changes from googleapis for updated protos -# - fetches changes from toolkit for updates to the code generator -# - fetches api-client-staging to use as a staging area for the generated grpc stubs -# -# This is meant to be used conjunction with update-deps.sh to regenerate the underlying client with -# the latest upstream changes. - -set -o errexit -o errtrace - -handle_error() { - echo "Failed, exiting. See deps/output.log for more details" - exit 1 -} -trap 'handle_error' ERR - -PROJECT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )" -WORKDIR="$PROJECT_DIR/deps" -LOG="$WORKDIR/output.log" - -DIRS=( googleapis toolkit api-client-staging ) - -DOCKER_IMAGE=googleapis/artman:stable - -function run { - init - update -} - -# Make sure all of the dependencies are checked out -function init() { - mkdir -p "$WORKDIR" - echo "" > $WORKDIR/output.log - - log "Pulling docker image" - docker pull "$DOCKER_IMAGE" >>$LOG 2>&1 - - if [ ! -d "$WORKDIR/googleapis" ]; then - log "Fetching googleapis" - git clone "https://github.com/googleapis/googleapis.git" "$WORKDIR/googleapis" >>$LOG 2>&1 - fi - - if [ ! -d "$WORKDIR/toolkit" ]; then - log "Fetching toolkit" - git clone "https://github.com/googleapis/toolkit.git" "$WORKDIR/toolkit" >>$LOG 2>&1 - fi - - if [ ! -d "$WORKDIR/api-client-staging" ]; then - log "Fetching api-client-staging" - git clone "https://github.com/googleapis/api-client-staging.git" "$WORKDIR/api-client-staging" >>$LOG 2>&1 - fi -} - -function update() { - for DIR in "${DIRS[@]}"; do - log "Updating $DIR" - DIR="$WORKDIR/$DIR" - - log "Fetching upstream" - git -C "$DIR" fetch origin >>$LOG 2>&1 - - log "Dropping uncommitted local changes" - git -C "$DIR" reset --hard HEAD >>$LOG 2>&1 - git -C "$DIR" clean -f >>$LOG 2>&1 - - log "Rebasing local changes" - git -C "$DIR" rebase origin/master >>$LOG 2>&1 - done -} - -function log { - echo ">>>>>>> $@" - echo ">>>>>>> $@" >> $LOG -} - -run