Skip to content

Commit

Permalink
[ggj][parsing] feat: add Service.yaml parsing (#415)
Browse files Browse the repository at this point in the history
* fix: refactor requestBuilder into separate method in ServiceClientClassComposer

* feat: add varargs to AnonClass and ref setter methods

* feat: add HTTP annotation parsing/validation

* feat: Generate RequestParamsExtractor in GrpcServiceStub

* feat: add GrpcPublisherStub test to exercise HTTP subfields

* fix: add ByteString to DefaultValueComposer

* fix: Use repeated field name for paged RPC unit tests

* fix: refactor exception field, use paged repeated field name, add pubsub client test

* fix: ensure all testgen methods throw Exceptions

* fix: Fix resname helper method names for of* and format*

* fix: use only generated resnames in codegen

* fix: propagate of*Name changes to resname codegen

* fix: fix method arg resname mappings, add logging test

* fix: ensure paged tests use the right repeated resp. type

* feat: add PackageInfoDefinition AST node

* feat: add package-info.java codegen

* feat: add Service.yaml parsing

* Update BUILD.bazel
  • Loading branch information
miraleung authored Oct 25, 2020
1 parent 9923680 commit 8189648
Show file tree
Hide file tree
Showing 13 changed files with 359 additions and 5 deletions.
3 changes: 2 additions & 1 deletion dependencies.properties
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ maven.com_google_guava_guava=com.google.guava:guava:26.0-jre
maven.com_google_code_findbugs_jsr305=com.google.code.findbugs:jsr305:3.0.0
maven.com_google_auto_value_auto_value=com.google.auto.value:auto-value:1.7.2
maven.com_google_auto_value_auto_value_annotations=com.google.auto.value:auto-value-annotations:1.7.2
maven.com_google_code_gson=com.google.code.gson:gson:2.8.6
maven.com_google_protobuf_protobuf_java=com.google.protobuf:protobuf-java:3.12.2
maven.io_github_java_diff_utils=io.github.java-diff-utils:java-diff-utils:4.0
maven.javax_annotation_javax_annotation_api=javax.annotation:javax.annotation-api:1.3.2
Expand All @@ -33,7 +34,7 @@ maven.org_threeten_threetenbp=org.threeten:threetenbp:1.3.3

# Testing.
maven.junit_junit=junit:junit:4.13
# This hamcrest-core dependency is for running JUnit test manually, before JUnit 4.11 it's wrapped along with JUnit package.
# This hamcrest-core dependency is for running JUnit test manually, before JUnit 4.11 it's wrapped along with JUnit package.
# But now it has to be explicitly added.
maven.org_hamcrest_hamcrest_core=org.hamcrest:hamcrest-core:1.3
maven.org_mockito_mockito_core=org.mockito:mockito-core:2.21.0
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ java_library(
"@com_google_auto_value_auto_value//jar",
"@com_google_auto_value_auto_value_annotations//jar",
"@com_google_code_findbugs_jsr305//jar",
"@com_google_googleapis//google/api:api_java_proto",
"@com_google_googleapis//google/rpc:rpc_java_proto",
"@com_google_guava_guava//jar",
"@com_google_protobuf//:protobuf_java",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,13 @@ public abstract class GapicContext {
@Nullable
public abstract GapicServiceConfig serviceConfig();

@Nullable
public abstract com.google.api.Service serviceYamlProto();

public boolean hasServiceYamlProto() {
return serviceYamlProto() != null;
}

public static Builder builder() {
return new AutoValue_GapicContext.Builder();
}
Expand All @@ -54,6 +61,8 @@ public abstract static class Builder {

public abstract Builder setServiceConfig(GapicServiceConfig serviceConfig);

public abstract Builder setServiceYamlProto(com.google.api.Service serviceYamlProto);

public abstract GapicContext build();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ java_library(
"//src/main/java/com/google/api/generator/gapic/utils",
"@com_google_api_api_common//jar",
"@com_google_code_findbugs_jsr305//jar",
"@com_google_code_gson//jar",
"@com_google_googleapis//google/api:api_java_proto",
"@com_google_googleapis//google/longrunning:longrunning_java_proto",
"@com_google_guava_guava//jar",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,8 @@
public class Parser {
private static final String COMMA = ",";
private static final String COLON = ":";
private static final String DOT = ".";
private static final String DEFAULT_PORT = "443";
private static final String DOT = ".";

// Allow other parsers to access this.
protected static final SourceCodeInfoParser SOURCE_CODE_INFO_PARSER = new SourceCodeInfoParser();
Expand All @@ -88,6 +88,11 @@ public static GapicContext parse(CodeGeneratorRequest request) {
Optional<GapicServiceConfig> serviceConfigOpt =
ServiceConfigParser.parse(serviceConfigPath, batchingSettingsOpt);

Optional<String> serviceYamlConfigPathOpt =
PluginArgumentParser.parseServiceYamlConfigPath(request);
Optional<com.google.api.Service> serviceYamlProtoOpt =
ServiceYamlParser.parse(serviceYamlConfigPathOpt.get());

// Keep message and resource name parsing separate for cleaner logic.
// While this takes an extra pass through the protobufs, the extra time is relatively trivial
// and is worth the larger reduced maintenance cost.
Expand All @@ -96,22 +101,26 @@ public static GapicContext parse(CodeGeneratorRequest request) {
messages = updateResourceNamesInMessages(messages, resourceNames.values());
Set<ResourceName> outputArgResourceNames = new HashSet<>();
List<Service> services =
parseServices(request, messages, resourceNames, outputArgResourceNames);
parseServices(
request, messages, resourceNames, outputArgResourceNames, serviceYamlProtoOpt);
return GapicContext.builder()
.setServices(services)
.setMessages(messages)
.setResourceNames(resourceNames)
.setHelperResourceNames(outputArgResourceNames)
.setServiceConfig(serviceConfigOpt.isPresent() ? serviceConfigOpt.get() : null)
.setServiceYamlProto(serviceYamlProtoOpt.isPresent() ? serviceYamlProtoOpt.get() : null)
.build();
}

public static List<Service> parseServices(
CodeGeneratorRequest request,
Map<String, Message> messageTypes,
Map<String, ResourceName> resourceNames,
Set<ResourceName> outputArgResourceNames) {
Set<ResourceName> outputArgResourceNames,
Optional<com.google.api.Service> serviceYamlProtoOpt) {
Map<String, FileDescriptor> fileDescriptors = getFilesToGenerate(request);

List<Service> services = new ArrayList<>();
for (String fileToGenerate : request.getFileToGenerateList()) {
FileDescriptor fileDescriptor =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,11 @@ public class PluginArgumentParser {
// Synced to rules_java_gapic/java_gapic.bzl.
@VisibleForTesting static final String KEY_GRPC_SERVICE_CONFIG = "grpc-service-config";
@VisibleForTesting static final String KEY_GAPIC_CONFIG = "gapic-config";
@VisibleForTesting static final String KEY_SERVICE_YAML_CONFIG = "gapic-service-config";

private static final String JSON_FILE_ENDING = "grpc_service_config.json";
private static final String GAPIC_YAML_FILE_ENDING = "gapic.yaml";
private static final String SERVICE_YAML_FILE_ENDING = ".yaml";

static Optional<String> parseJsonConfigPath(CodeGeneratorRequest request) {
return parseJsonConfigPath(request.getParameter());
Expand All @@ -39,6 +41,10 @@ static Optional<String> parseGapicYamlConfigPath(CodeGeneratorRequest request) {
return parseGapicYamlConfigPath(request.getParameter());
}

static Optional<String> parseServiceYamlConfigPath(CodeGeneratorRequest request) {
return parseServiceYamlConfigPath(request.getParameter());
}

/** Expects a comma-separated list of file paths. */
@VisibleForTesting
static Optional<String> parseJsonConfigPath(String pluginProtocArgument) {
Expand All @@ -50,6 +56,11 @@ static Optional<String> parseGapicYamlConfigPath(String pluginProtocArgument) {
return parseArgument(pluginProtocArgument, KEY_GAPIC_CONFIG, GAPIC_YAML_FILE_ENDING);
}

@VisibleForTesting
static Optional<String> parseServiceYamlConfigPath(String pluginProtocArgument) {
return parseArgument(pluginProtocArgument, KEY_SERVICE_YAML_CONFIG, SERVICE_YAML_FILE_ENDING);
}

private static Optional<String> parseArgument(
String pluginProtocArgument, String key, String fileEnding) {
if (Strings.isNullOrEmpty(pluginProtocArgument)) {
Expand All @@ -62,7 +73,12 @@ private static Optional<String> parseArgument(
}
String keyVal = args[0];
String valueVal = args[1];
if (keyVal.equals(key) && valueVal.endsWith(fileEnding)) {
boolean valueMeetsCriteria = keyVal.equals(key) && valueVal.endsWith(fileEnding);
if (fileEnding.equals(SERVICE_YAML_FILE_ENDING)) {
valueMeetsCriteria &= !valueVal.endsWith(GAPIC_YAML_FILE_ENDING);
}

if (valueMeetsCriteria) {
return Optional.of(valueVal);
}
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.api.generator.gapic.protoparser;

import com.google.common.base.Strings;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.protobuf.InvalidProtocolBufferException;
import com.google.protobuf.util.JsonFormat;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Optional;
import org.yaml.snakeyaml.Yaml;
import org.yaml.snakeyaml.constructor.SafeConstructor;

public class ServiceYamlParser {
public static Optional<com.google.api.Service> parse(String serviceYamlFilePath) {
if (Strings.isNullOrEmpty(serviceYamlFilePath) || !(new File(serviceYamlFilePath)).exists()) {
return Optional.empty();
}

String fileContents = null;
try {
fileContents = new String(Files.readAllBytes(Paths.get(serviceYamlFilePath)));
} catch (IOException e) {
return Optional.empty();
}

Yaml yaml = new Yaml(new SafeConstructor());
Map<String, Object> yamlMap = yaml.load(fileContents);
Gson gson = new GsonBuilder().setPrettyPrinting().setLenient().create();
String jsonString = gson.toJson(yamlMap, LinkedHashMap.class);
// Use the full name instead of an import, to avoid reader confusion with this and
// model.Service.
com.google.api.Service.Builder serviceBuilder = com.google.api.Service.newBuilder();
try {
JsonFormat.parser().ignoringUnknownFields().merge(jsonString, serviceBuilder);
} catch (InvalidProtocolBufferException e) {
return Optional.empty();
}
return Optional.of(serviceBuilder.build());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ TESTS = [
"ResourceNameParserTest",
"ResourceReferenceParserTest",
"ServiceConfigParserTest",
"ServiceYamlParserTest",
"SourceCodeInfoParserTest",
]

Expand All @@ -26,6 +27,7 @@ filegroup(
"//src/test/java/com/google/api/generator/gapic/testdata:basic_proto_descriptor",
"//src/test/java/com/google/api/generator/gapic/testdata:gapic_config_files",
"//src/test/java/com/google/api/generator/gapic/testdata:service_config_files",
"//src/test/java/com/google/api/generator/gapic/testdata:service_yaml_files",
],
test_class = "com.google.api.generator.gapic.protoparser.{0}".format(test_name),
deps = [
Expand All @@ -39,6 +41,7 @@ filegroup(
"//src/test/java/com/google/api/generator/gapic/testdata:showcase_java_proto",
"//src/test/java/com/google/api/generator/gapic/testdata:testgapic_java_proto",
"@com_google_api_api_common//jar",
"@com_google_googleapis//google/api:api_java_proto",
"@com_google_googleapis//google/rpc:rpc_java_proto",
"@com_google_protobuf//:protobuf_java",
"@com_google_protobuf//:protobuf_java_util",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -148,11 +148,92 @@ public void parseGapicYamlPath_noneFound() {
assertFalse(PluginArgumentParser.parseGapicYamlConfigPath(rawArgument).isPresent());
}

@Test
public void parseServiceYamlPath_onlyOnePresent() {
String servicePath = "/tmp/something.yaml";
assertEquals(
servicePath,
PluginArgumentParser.parseServiceYamlConfigPath(createServiceConfig(servicePath)).get());
}

@Test
public void parseServiceYamlPath_returnsFirstOneFound() {
String servicePathOne = "/tmp/something.yaml";
String servicePathTwo = "/tmp/other.yaml";
assertEquals(
servicePathOne,
PluginArgumentParser.parseServiceYamlConfigPath(
String.join(
",",
Arrays.asList(
createServiceConfig(servicePathOne), createServiceConfig(servicePathTwo))))
.get());
}

@Test
public void parseServiceYamlPath_gapicFilePresent() {
String gapicPath = "/tmp/something_gapic.yaml";
String servicePath = "/tmp/something.yaml";
// Both passed under the service yaml flag.
String rawArgument =
String.join(
",", Arrays.asList(createServiceConfig(gapicPath), createServiceConfig(servicePath)));
assertEquals(servicePath, PluginArgumentParser.parseServiceYamlConfigPath(rawArgument).get());

// Passed under the right flags.
rawArgument =
String.join(
",", Arrays.asList(createGapicConfig(gapicPath), createServiceConfig(servicePath)));
assertEquals(servicePath, PluginArgumentParser.parseServiceYamlConfigPath(rawArgument).get());

// Swapped flags.
rawArgument =
String.join(
",", Arrays.asList(createGapicConfig(servicePath), createServiceConfig(gapicPath)));
assertFalse(PluginArgumentParser.parseServiceYamlConfigPath(rawArgument).isPresent());

// Both passed under the Gapic yaml flag.
rawArgument =
String.join(
",", Arrays.asList(createGapicConfig(gapicPath), createGapicConfig(servicePath)));
assertFalse(PluginArgumentParser.parseServiceYamlConfigPath(rawArgument).isPresent());
}

@Test
public void parseServiceYamlPath_similarFileAppearsFirst() {
String jsonPath = "/tmp/foo_grpc_service_config.json";
String gapicPath = "/tmp/something_gapic.yaml";
String servicePath = "/tmp/something.yaml";
String rawArgument =
String.join(
",",
Arrays.asList(
createGrpcServiceConfig(jsonPath),
createGapicConfig("/tmp/something.yaml"),
createGapicConfig("/tmp/some_gapicyaml"),
createServiceConfig(gapicPath),
createServiceConfig(servicePath)));
assertEquals(servicePath, PluginArgumentParser.parseServiceYamlConfigPath(rawArgument).get());
}

@Test
public void parseServiceYamlPath_noneFound() {
String jsonPath = "/tmp/foo_grpc_service_config.json";
String gapicPath = "";
String rawArgument =
String.join(",", Arrays.asList(createGrpcServiceConfig(jsonPath), gapicPath));
assertFalse(PluginArgumentParser.parseServiceYamlConfigPath(rawArgument).isPresent());
}

private static String createGrpcServiceConfig(String path) {
return String.format("%s=%s", PluginArgumentParser.KEY_GRPC_SERVICE_CONFIG, path);
}

private static String createGapicConfig(String path) {
return String.format("%s=%s", PluginArgumentParser.KEY_GAPIC_CONFIG, path);
}

private static String createServiceConfig(String path) {
return String.format("%s=%s", PluginArgumentParser.KEY_SERVICE_YAML_CONFIG, path);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.api.generator.gapic.protoparser;

import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertTrue;

import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Optional;
import org.junit.Test;

public class ServiceYamlParserTest {
private static final String YAML_DIRECTORY =
"src/test/java/com/google/api/generator/gapic/testdata/";

@Test
public void parseServiceYaml_basic() {
String yamlFilename = "logging.yaml";
Path yamlPath = Paths.get(YAML_DIRECTORY, yamlFilename);
Optional<com.google.api.Service> serviceYamlProtoOpt =
ServiceYamlParser.parse(yamlPath.toString());
assertTrue(serviceYamlProtoOpt.isPresent());

com.google.api.Service serviceYamlProto = serviceYamlProtoOpt.get();
assertEquals("logging.googleapis.com", serviceYamlProto.getName());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,11 @@ filegroup(
srcs = glob(["*_gapic.yaml"]),
)

filegroup(
name = "service_yaml_files",
srcs = ["logging.yaml"],
)

genrule(
name = "basic_proto_descriptor",
srcs = [
Expand Down
Loading

0 comments on commit 8189648

Please sign in to comment.