From 40271d7642fb6b76d68289d73b73a1eb4acb80bd Mon Sep 17 00:00:00 2001 From: Googler Date: Fri, 29 Dec 2023 11:14:05 -0800 Subject: [PATCH] Adds `GlobsValue` object which also wraps the `SkyKey` for the future `GlobsFunction` The `SkyKey` serialization test is also added within this change. PiperOrigin-RevId: 594478635 Change-Id: I310b77f13516dab2a38bd45b7e7649d5bfce307d --- .../google/devtools/build/lib/skyframe/BUILD | 26 ++ .../build/lib/skyframe/GlobsFunction.java | 28 +++ .../build/lib/skyframe/GlobsValue.java | 237 ++++++++++++++++++ .../build/lib/skyframe/SkyFunctions.java | 1 + .../google/devtools/build/lib/skyframe/BUILD | 1 + .../build/lib/skyframe/GlobsValueTest.java | 54 ++++ 6 files changed, 347 insertions(+) create mode 100644 src/main/java/com/google/devtools/build/lib/skyframe/GlobsFunction.java create mode 100644 src/main/java/com/google/devtools/build/lib/skyframe/GlobsValue.java create mode 100644 src/test/java/com/google/devtools/build/lib/skyframe/GlobsValueTest.java diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/BUILD b/src/main/java/com/google/devtools/build/lib/skyframe/BUILD index cadff4889cf96e..d45efb585f0e36 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/BUILD +++ b/src/main/java/com/google/devtools/build/lib/skyframe/BUILD @@ -1464,6 +1464,15 @@ java_library( ], ) +java_library( + name = "globs_function", + srcs = ["GlobsFunction.java"], + deps = [ + "//src/main/java/com/google/devtools/build/skyframe", + "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects", + ], +) + java_library( name = "glob_value", srcs = [ @@ -1485,6 +1494,23 @@ java_library( ], ) +java_library( + name = "globs_value", + srcs = ["GlobsValue.java"], + deps = [ + ":invalid_glob_pattern_exception", + ":sky_functions", + "//src/main/java/com/google/devtools/build/lib/cmdline", + "//src/main/java/com/google/devtools/build/lib/packages:globber", + "//src/main/java/com/google/devtools/build/lib/skyframe/serialization:visible-for-serialization", + "//src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec", + "//src/main/java/com/google/devtools/build/lib/vfs", + "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment", + "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects", + "//third_party:guava", + ], +) + java_library( name = "graph_backed_recursive_package_provider", srcs = ["GraphBackedRecursivePackageProvider.java"], diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/GlobsFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/GlobsFunction.java new file mode 100644 index 00000000000000..5551d410dacd3f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/GlobsFunction.java @@ -0,0 +1,28 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +/** A SkyFunction for {@link GlobsValue}s. */ +// TODO(b/290998109): Implement GlobsFunction with StateMachine mechanism. +public class GlobsFunction implements SkyFunction { + + @Override + public SkyValue compute(SkyKey skyKey, Environment env) { + throw new UnsupportedOperationException(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/GlobsValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/GlobsValue.java new file mode 100644 index 00000000000000..90ad17a9fe7c07 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/GlobsValue.java @@ -0,0 +1,237 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.cmdline.PackageIdentifier; +import com.google.devtools.build.lib.packages.Globber; +import com.google.devtools.build.lib.packages.Globber.Operation; +import com.google.devtools.build.lib.skyframe.serialization.VisibleForSerialization; +import com.google.devtools.build.lib.skyframe.serialization.autocodec.AutoCodec; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.Root; +import com.google.devtools.build.lib.vfs.UnixGlob; +import com.google.devtools.build.skyframe.SkyFunctionName; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; +import java.util.Objects; + +/** {@link SkyValue} corresponding to the computation result of the {@link GlobsFunction}. */ +public class GlobsValue implements SkyValue { + + private final ImmutableSet matches; + + public GlobsValue(ImmutableSet matches) { + this.matches = matches; + } + + public ImmutableSet getMatches() { + return matches; + } + + @Override + public boolean equals(Object other) { + if (other == this) { + return true; + } + if (!(other instanceof GlobsValue)) { + return false; + } + + return getMatches().equals(((GlobsValue) other).getMatches()); + } + + @Override + public int hashCode() { + return matches.hashCode(); + } + + /** + * Representation of individual glob inside a package, including its expression and Globber + * operation type. + */ + public static class GlobRequest { + + private final String pattern; + private final Globber.Operation globOperation; + + public String getPattern() { + return pattern; + } + + public Operation getGlobOeration() { + return globOperation; + } + + private GlobRequest(String pattern, Globber.Operation globOperation) { + this.pattern = pattern; + this.globOperation = globOperation; + } + + @Override + public String toString() { + return String.format("GlobRequest: %s %s", pattern, globOperation); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof GlobRequest)) { + return false; + } + + GlobRequest other = (GlobRequest) obj; + return pattern.equals(other.pattern) && globOperation.equals(other.globOperation); + } + + @Override + public int hashCode() { + return Objects.hash(pattern, globOperation); + } + + /** + * Creates {@link GlobRequest} object iff pattern is a valid glob expression. + * + *

@throws InvalidGlobPatternException if the pattern is not valid. + */ + public static GlobRequest create(String pattern, Globber.Operation globOeration) + throws InvalidGlobPatternException { + if (pattern.indexOf('?') != -1) { + throw new InvalidGlobPatternException(pattern, "wildcard ? forbidden"); + } + + String error = UnixGlob.checkPatternForError(pattern); + if (error != null) { + throw new InvalidGlobPatternException(pattern, error); + } + return new GlobRequest(pattern, globOeration); + } + } + + /** + * Returns the interned {@link GlobsValue.Key} object which contains all glob deps of a package. + * + * @param packageIdentifier packageId the name of the owner package (must be an existing package) + * @param packageRoot the package root of {@code packageId} + * @param globRequests container of all glob expressions and types of Globber operations, all + * input glob expressions are expected to be valid. + */ + public static Key key( + PackageIdentifier packageIdentifier, + Root packageRoot, + ImmutableSet globRequests) { + return Key.create(packageIdentifier, packageRoot, globRequests); + } + + /** + * {@link SkyKey} type for {@link GlobsValue}, serving as the input to {@link GlobsFunction}. + * + *

Expects all glob expressions inside {@link Key#globRequests} are valid, as indicated by + * {@code UnixGlob#checkPatternForError}. + */ + @VisibleForSerialization + @AutoCodec + public static class Key implements SkyKey { + private static final SkyKeyInterner interner = SkyKey.newInterner(); + + private final PackageIdentifier packageIdentifier; + private final Root packageRoot; + private final ImmutableSet globRequests; + + private static Key create( + PackageIdentifier packageIdentifier, + Root packageRoot, + ImmutableSet globRequests) { + return interner.intern(new Key(packageIdentifier, packageRoot, globRequests)); + } + + @VisibleForSerialization + @AutoCodec.Interner + static Key intern(Key key) { + return interner.intern(key); + } + + private Key( + PackageIdentifier packageIdentifier, + Root packageRoot, + ImmutableSet globRequests) { + this.packageIdentifier = packageIdentifier; + this.packageRoot = packageRoot; + this.globRequests = globRequests; + } + + /** + * Returns the package that "owns" all globs. + * + *

The globs evaluation code ensures that the boundaries of this package are not crossed. + */ + public PackageIdentifier getPackageIdentifier() { + return packageIdentifier; + } + + /** Returns the package root of {@link #packageIdentifier}. */ + public Root getPackageRoot() { + return packageRoot; + } + + /** + * Returns an {@link ImmutableSet} containing all globs inside the package, including each glob + * expression and operation. + */ + public ImmutableSet getGlobRequests() { + return globRequests; + } + + @Override + public SkyFunctionName functionName() { + return SkyFunctions.GLOBS; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof Key)) { + return false; + } + Key other = (Key) obj; + return packageIdentifier.equals(other.packageIdentifier) + && packageRoot.equals(other.packageRoot) + && globRequests.equals(other.globRequests); + } + + @Override + public int hashCode() { + return Objects.hash(packageIdentifier, packageRoot, globRequests); + } + + @Override + public String toString() { + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append( + String.format( + " getSkyKeyInterner() { + return interner; + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java index a6398ead1866ba..496382e0e2a6a6 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java @@ -41,6 +41,7 @@ public final class SkyFunctions { SkyFunctionName.createHermetic("STARLARK_BUILTINS"); public static final SkyFunctionName BZL_LOAD = SkyFunctionName.createHermetic("BZL_LOAD"); public static final SkyFunctionName GLOB = SkyFunctionName.createHermetic("GLOB"); + public static final SkyFunctionName GLOBS = SkyFunctionName.createHermetic("GLOBS"); public static final SkyFunctionName PACKAGE = SkyFunctionName.createHermetic("PACKAGE"); static final SkyFunctionName PACKAGE_ERROR = SkyFunctionName.createHermetic("PACKAGE_ERROR"); public static final SkyFunctionName PACKAGE_ERROR_MESSAGE = diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/BUILD b/src/test/java/com/google/devtools/build/lib/skyframe/BUILD index 67efc67ebd3885..093266564ef766 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/BUILD +++ b/src/test/java/com/google/devtools/build/lib/skyframe/BUILD @@ -229,6 +229,7 @@ java_test( "//src/main/java/com/google/devtools/build/lib/skyframe:glob_descriptor", "//src/main/java/com/google/devtools/build/lib/skyframe:glob_function", "//src/main/java/com/google/devtools/build/lib/skyframe:glob_value", + "//src/main/java/com/google/devtools/build/lib/skyframe:globs_value", "//src/main/java/com/google/devtools/build/lib/skyframe:ignored_package_prefixes_function", "//src/main/java/com/google/devtools/build/lib/skyframe:incompatible_view_exception", "//src/main/java/com/google/devtools/build/lib/skyframe:incremental_artifact_conflict_finder", diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/GlobsValueTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/GlobsValueTest.java new file mode 100644 index 00000000000000..9a24dbba18c099 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/skyframe/GlobsValueTest.java @@ -0,0 +1,54 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.cmdline.PackageIdentifier; +import com.google.devtools.build.lib.packages.Globber.Operation; +import com.google.devtools.build.lib.skyframe.GlobsValue.GlobRequest; +import com.google.devtools.build.lib.skyframe.serialization.testutils.FsUtils; +import com.google.devtools.build.lib.skyframe.serialization.testutils.SerializationTester; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.Root; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class GlobsValueTest { + + @Test + public void testSerialization() throws Exception { + PackageIdentifier packageId = PackageIdentifier.create("foo", PathFragment.create("//bar")); + Root packageRoot = Root.fromPath(FsUtils.TEST_FILESYSTEM.getPath("/packageRoot")); + + GlobRequest globRequest1 = GlobRequest.create("*", Operation.FILES_AND_DIRS); + GlobRequest globRequest2 = GlobRequest.create("foo/**", Operation.SUBPACKAGES); + GlobRequest globRequest3 = GlobRequest.create("**/*", Operation.FILES); + + SerializationTester serializationTester = + new SerializationTester( + GlobsValue.key(packageId, packageRoot, ImmutableSet.of(globRequest1, globRequest2)), + GlobsValue.key(packageId, packageRoot, ImmutableSet.of(globRequest2, globRequest3))) + .setVerificationFunction(GlobsValueTest::verifyEquivalent); + FsUtils.addDependencies(serializationTester); + serializationTester.runTests(); + } + + private static void verifyEquivalent(GlobsValue.Key orig, GlobsValue.Key deserialized) { + assertThat(deserialized).isSameInstanceAs(orig); + } +}