From 26611d752e54e7c0c20bb6d6588d507afa90de96 Mon Sep 17 00:00:00 2001 From: Xiaozhen Liu Date: Thu, 24 Sep 2020 10:40:36 -0700 Subject: [PATCH] [ggj][infra][3/5] feat: implement update golden bazel rules for dummy test (#314) * add fileDiffUtils * add new dummy unit test * format * add assertUtils * format * move dummy test to separate folder * add compare strings method in diffUtils * format * dummy test pass * reformat * add simple strings comparison * add java_diff_test bzl * group test framework helpers together * run testRunner * format * run single JUnit test and save output * add copy of golden * format * working pipeline * format * clean up * get codegen in local_tmp * format * add golden path to bazel rules * working pipeline * format * work! * clean up * fix * feedback * clean * comment * bazel rules feedback * java code feedback * move dependency * clean up * comment * move hamcrest dep back * add helpers in Utils --- BUILD.bazel | 22 +++++ dependencies.properties | 3 + rules_bazel/java/java_diff_test.bzl | 87 +++++++++++++++++++ .../api/generator/gapic/dummy/BUILD.bazel | 18 +++- .../gapic/dummy/FileDiffInfraDummyTest.java | 17 ++-- .../api/generator/test/framework/BUILD.bazel | 2 + .../test/framework/SingleJUnitTestRunner.java | 63 ++++++++++++++ .../api/generator/test/framework/Utils.java | 75 ++++++++++++++++ 8 files changed, 280 insertions(+), 7 deletions(-) create mode 100644 rules_bazel/java/java_diff_test.bzl create mode 100644 src/test/java/com/google/api/generator/test/framework/SingleJUnitTestRunner.java create mode 100644 src/test/java/com/google/api/generator/test/framework/Utils.java diff --git a/BUILD.bazel b/BUILD.bazel index a72c509528..ba95d15151 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -46,6 +46,28 @@ java_binary( ], ) +# JUnit runner binary, this is used to generate test output for updating goldens files. +# Run `bazel run testTarget_update` will trigger this runner. +java_binary( + name = "junit_runner", + srcs = [ + "//src/test/java/com/google/api/generator/gapic/dummy:dummy_files", + "//src/test/java/com/google/api/generator/test/framework:framework_files", + ], + data = ["//src/test/java/com/google/api/generator/gapic/dummy/goldens:goldens_files"], + jvm_flags = ["-Xmx512m"], + main_class = "com.google.api.generator.test.framework.SingleJUnitTestRunner", + visibility = ["//visibility:public"], + deps = [ + "//src/main/java/com/google/api/generator/engine/ast", + "//src/main/java/com/google/api/generator/engine/writer", + "//src/test/java/com/google/api/generator/test/framework", + "@io_github_java_diff_utils//jar", + "@junit_junit//jar", + "@org_hamcrest_hamcrest_core//jar", + ], +) + # google-java-format java_binary( name = "google_java_format_binary", diff --git a/dependencies.properties b/dependencies.properties index 761f5a1cba..007ec2bd56 100644 --- a/dependencies.properties +++ b/dependencies.properties @@ -33,6 +33,9 @@ maven.org_threeten_threetenbp=org.threeten:threetenbp:1.3.3 # Testing. maven.junit_junit=junit:junit:4.13 +# This hamcrest-core dependency is for running JUnit test manually, before JUnit 4.11 it's wrapped along with JUnit package. +# But now it has to be explicitly added. +maven.org_hamcrest_hamcrest_core=org.hamcrest:hamcrest-core:1.3 maven.org_mockito_mockito_core=org.mockito:mockito-core:2.21.0 # Keep in sync with gax-java. maven.com_google_truth_truth=com.google.truth:truth:1.0 diff --git a/rules_bazel/java/java_diff_test.bzl b/rules_bazel/java/java_diff_test.bzl new file mode 100644 index 0000000000..0b3ec96168 --- /dev/null +++ b/rules_bazel/java/java_diff_test.bzl @@ -0,0 +1,87 @@ +def _junit_output_impl(ctx): + test_class_name = ctx.attr.test_class_name + inputs = ctx.files.srcs + output = ctx.outputs.output + test_runner = ctx.executable.test_runner + + command = """ + mkdir local_tmp + TEST_OUTPUT_HOME="$(pwd)/local_tmp" \ + {test_runner_path} $@ + cd local_tmp + # Zip all files under local_tmp with all nested parent folders except for local_tmp itself. + # Zip files because there are cases that one Junit test can produce multiple goldens. + zip -r ../{output} . + """.format( + test_runner_path = test_runner.path, + output=output.path, + ) + + ctx.actions.run_shell( + inputs = inputs, + outputs = [output], + arguments = [test_class_name], + tools = [test_runner], + command = command, + ) + +junit_output_zip = rule( + attrs = { + "test_class_name": attr.string(mandatory=True), + "srcs": attr.label_list( + allow_files = True, + mandatory = True, + ), + "test_runner": attr.label( + mandatory = True, + executable = True, + cfg = "host", + ), + }, + outputs = { + "output": "%{name}%.zip", + }, + implementation = _junit_output_impl, +) + +def _overwritten_golden_impl(ctx): + script_content = """ + #!/bin/bash + cd ${{BUILD_WORKSPACE_DIRECTORY}} + unzip -ao {unit_test_results} -d src/test/java + """.format( + unit_test_results = ctx.file.unit_test_results.path, + ) + ctx.actions.write( + output = ctx.outputs.bin, + content = script_content, + is_executable = True, + ) + return [DefaultInfo(executable = ctx.outputs.bin)] + + +overwritten_golden = rule( + attrs = { + "unit_test_results": attr.label( + mandatory = True, + allow_single_file = True), + }, + outputs = { + "bin": "%{name}.sh", + }, + executable = True, + implementation = _overwritten_golden_impl, +) + +def updated_golden(name, test_class_name, srcs): + junit_output_name = "%s_output" % name + junit_output_zip( + name = junit_output_name, + test_class_name = test_class_name, + test_runner = "//:junit_runner", + srcs = srcs, + ) + overwritten_golden( + name = name, + unit_test_results = ":%s" % junit_output_name + ) diff --git a/src/test/java/com/google/api/generator/gapic/dummy/BUILD.bazel b/src/test/java/com/google/api/generator/gapic/dummy/BUILD.bazel index ace0cd26f4..7337d63159 100644 --- a/src/test/java/com/google/api/generator/gapic/dummy/BUILD.bazel +++ b/src/test/java/com/google/api/generator/gapic/dummy/BUILD.bazel @@ -1,3 +1,5 @@ +load("//:rules_bazel/java/java_diff_test.bzl", "updated_golden") + package(default_visibility = ["//visibility:public"]) TESTS = [ @@ -12,7 +14,7 @@ filegroup( [java_test( name = test_name, srcs = ["{0}.java".format(test_name)], - data = ["//src/test/java/com/google/api/generator/gapic/dummy/goldens:goldens_files"], + data = glob(["goldens/*.golden"]), test_class = "com.google.api.generator.gapic.dummy.{0}".format(test_name), deps = [ "//src/main/java/com/google/api/generator/engine/ast", @@ -21,3 +23,17 @@ filegroup( "@junit_junit//jar", ], ) for test_name in TESTS] + +TEST_CLASS_NAME = "com.google.api.generator.gapic.dummy.FileDiffInfraDummyTest" + +# Run `bazel run src/test/java/com/google/api/generator/gapic/dummy:FileDiffInfraDummyTest_update` +# to update goldens as expected generated code. +updated_golden( + name = "FileDiffInfraDummyTest_update", + srcs = [ + ":dummy_files", + "//src/test/java/com/google/api/generator/gapic/dummy/goldens:goldens_files", + "//src/test/java/com/google/api/generator/test/framework:framework_files", + ], + test_class_name = TEST_CLASS_NAME, +) diff --git a/src/test/java/com/google/api/generator/gapic/dummy/FileDiffInfraDummyTest.java b/src/test/java/com/google/api/generator/gapic/dummy/FileDiffInfraDummyTest.java index 98f766cab5..060a613dac 100644 --- a/src/test/java/com/google/api/generator/gapic/dummy/FileDiffInfraDummyTest.java +++ b/src/test/java/com/google/api/generator/gapic/dummy/FileDiffInfraDummyTest.java @@ -21,6 +21,7 @@ import com.google.api.generator.engine.ast.ScopeNode; import com.google.api.generator.engine.writer.JavaWriterVisitor; import com.google.api.generator.test.framework.Assert; +import com.google.api.generator.test.framework.Utils; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; @@ -34,8 +35,11 @@ public class FileDiffInfraDummyTest { // created. // // TODO(xiaozhenliu): remove this test class once the file-diff infra is in place and well-tested. - private static final String GOLDENFILES_DIRECTORY = - "src/test/java/com/google/api/generator/gapic/dummy/goldens/"; + private final String GOLDENFILES_DIRECTORY = Utils.getGoldenDir(this.getClass()); + private final String GOLDENFILES_SIMPLE_CLASS = + Utils.getClassName(this.getClass()) + "SimpleClass.golden"; + private final String GOLDENFILES_CLASS_WITH_HEADER = + Utils.getClassName(this.getClass()) + "ClassWithHeader.golden"; @Test public void simpleClass() { @@ -51,8 +55,8 @@ public void simpleClass() { .build(); JavaWriterVisitor visitor = new JavaWriterVisitor(); classDef.accept(visitor); - Path goldenFilePath = - Paths.get(GOLDENFILES_DIRECTORY, "FileDiffInfraDummyTestSimpleClass.golden"); + Utils.saveCodegenToFile(this.getClass(), GOLDENFILES_SIMPLE_CLASS, visitor.write()); + Path goldenFilePath = Paths.get(GOLDENFILES_DIRECTORY, GOLDENFILES_SIMPLE_CLASS); Assert.assertCodeEquals(goldenFilePath, visitor.write()); } @@ -69,8 +73,9 @@ public void classWithHeader() { .build(); JavaWriterVisitor visitor = new JavaWriterVisitor(); classDef.accept(visitor); - Path goldenFilePath = - Paths.get(GOLDENFILES_DIRECTORY, "FileDiffInfraDummyTestClassWithHeader.golden"); + // Save the generated code to a file for updating goldens if needed. + Utils.saveCodegenToFile(this.getClass(), GOLDENFILES_CLASS_WITH_HEADER, visitor.write()); + Path goldenFilePath = Paths.get(GOLDENFILES_DIRECTORY, GOLDENFILES_CLASS_WITH_HEADER); Assert.assertCodeEquals(goldenFilePath, visitor.write()); } diff --git a/src/test/java/com/google/api/generator/test/framework/BUILD.bazel b/src/test/java/com/google/api/generator/test/framework/BUILD.bazel index 8cdc676778..31b6d7d6f7 100644 --- a/src/test/java/com/google/api/generator/test/framework/BUILD.bazel +++ b/src/test/java/com/google/api/generator/test/framework/BUILD.bazel @@ -10,8 +10,10 @@ java_library( srcs = [ ":framework_files", ], + data = ["//src/test/java/com/google/api/generator/gapic/dummy/goldens:goldens_files"], deps = [ "@io_github_java_diff_utils//jar", "@junit_junit//jar", + "@org_hamcrest_hamcrest_core//jar", ], ) diff --git a/src/test/java/com/google/api/generator/test/framework/SingleJUnitTestRunner.java b/src/test/java/com/google/api/generator/test/framework/SingleJUnitTestRunner.java new file mode 100644 index 0000000000..a11ad6437a --- /dev/null +++ b/src/test/java/com/google/api/generator/test/framework/SingleJUnitTestRunner.java @@ -0,0 +1,63 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.api.generator.test.framework; + +import org.junit.runner.JUnitCore; +import org.junit.runner.Request; +import org.junit.runner.Result; + +public class SingleJUnitTestRunner { + // SingleJUnitTestRunner runs single JUnit test whose class name is passed through `args`. + // This is used to prepare codegen for updating goldens files. + public static void main(String... args) { + // Check whether the test class name is passed correctly e.g. + // `com.google.api.generator.gapic.composer.ComposerTest` + if (args.length < 1) { + throw new MissingRequiredArgException("Missing the JUnit class name argument."); + } + String className = args[0]; + Class clazz = null; + try { + clazz = Class.forName(className); + } catch (ClassNotFoundException e) { + throw new JUnitClassNotFoundException( + String.format("JUnit test class %s is not found.", className)); + } + Result result = new JUnitCore().run(Request.aClass(clazz)); + if (!result.wasSuccessful()) { + System.out.println("Tests have failures: " + result.getFailures()); + } + } + + public static class JUnitClassNotFoundException extends RuntimeException { + public JUnitClassNotFoundException(String errorMessage) { + super(errorMessage); + } + + public JUnitClassNotFoundException(String errorMessage, Throwable cause) { + super(errorMessage, cause); + } + } + + public static class MissingRequiredArgException extends RuntimeException { + public MissingRequiredArgException(String errorMessage) { + super(errorMessage); + } + + public MissingRequiredArgException(String errorMessage, Throwable cause) { + super(errorMessage, cause); + } + } +} diff --git a/src/test/java/com/google/api/generator/test/framework/Utils.java b/src/test/java/com/google/api/generator/test/framework/Utils.java new file mode 100644 index 0000000000..728be5f5b5 --- /dev/null +++ b/src/test/java/com/google/api/generator/test/framework/Utils.java @@ -0,0 +1,75 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.api.generator.test.framework; + +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; + +public class Utils { + /** + * Save the generated code from JUnit test to a file for updating goldens. These files will be + * saved as a zip file, then unzipped to overwrite goldens files. The relative path + * `com/google/..` which is identical with the location of goldens files which will help us easily + * replace the original goldens. For example: + * `src/test/java/com/google/api/generator/gapic/composer/ComposerTest.java` will save the + * generated code into a file called `ComposerTest.golden` at + * `$TEST_OUTPUT_HOME/com/google/api/generator/gapic/composer/goldens/ComposerTest.golden`. + * + * @param clazz the test class. + * @param fileName the name of saved file, usually it is test method name with suffix `.golden`. + * @param codegen the generated code from JUnit test. + */ + public static void saveCodegenToFile(Class clazz, String fileName, String codegen) { + // This system environment variable `TEST_OUTPUT_HOME` is used to specify a folder + // which contains generated output from JUnit test. + // It will be set when running `bazel run testTarget_update` command. + String testOutputHome = System.getenv("TEST_OUTPUT_HOME"); + String relativeGoldenDir = getTestoutGoldenDir(clazz); + Path testOutputDir = Paths.get(testOutputHome, relativeGoldenDir); + testOutputDir.toFile().mkdirs(); + try (FileWriter myWriter = + new FileWriter(Paths.get(testOutputHome, relativeGoldenDir, fileName).toFile())) { + myWriter.write(codegen); + } catch (IOException e) { + throw new SaveCodegenToFileException( + String.format( + "Error occured when saving codegen to file %s/%s", relativeGoldenDir, fileName)); + } + } + + private static String getTestoutGoldenDir(Class clazz) { + return clazz.getPackage().getName().replace(".", "/") + "/goldens/"; + } + + public static String getGoldenDir(Class clazz) { + return "src/test/java/" + clazz.getPackage().getName().replace(".", "/") + "/goldens/"; + } + + public static String getClassName(Class clazz) { + return clazz.getSimpleName(); + } + + public static class SaveCodegenToFileException extends RuntimeException { + public SaveCodegenToFileException(String errorMessage) { + super(errorMessage); + } + + public SaveCodegenToFileException(String errorMessage, Throwable cause) { + super(errorMessage, cause); + } + } +}