Open-source LocalDiffAwarenessIntegrationTest.

PiperOrigin-RevId: 420127324

src/test/java/com/google/devtools/build/lib/buildtool/util/BUILD

@@ -98,6 +98,7 @@ java_library(
         "//src/test/java/com/google/devtools/build/lib/testutil:TestUtils",
         "//src/test/java/com/google/devtools/build/lib/vfs/util",
         "//third_party:guava",
+        "//third_party:guava-testlib",
         "//third_party:jsr305",
         "//third_party:junit4",
         "//third_party:truth",

src/test/java/com/google/devtools/build/lib/buildtool/util/SkyframeIntegrationTestBase.java

@@ -0,0 +1,88 @@
+// Copyright 2022 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.buildtool.util;
+
+import static com.google.common.truth.Truth.assertThat;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.testing.GcFinalization;
+import com.google.devtools.build.lib.skyframe.SkyframeExecutor;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import java.lang.ref.WeakReference;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+/** Infrastructure to support Skyframe integration tests. */
+public abstract class SkyframeIntegrationTestBase extends BuildIntegrationTestCase {
+
+  protected SkyframeExecutor skyframeExecutor() {
+    return runtimeWrapper.getSkyframeExecutor();
+  }
+
+  protected static List<WeakReference<?>> weakRefs(Object... strongRefs) throws Exception {
+    List<WeakReference<?>> result = new ArrayList<>();
+    for (Object ref : strongRefs) {
+      result.add(new WeakReference<>(ref));
+    }
+    return result;
+  }
+
+  protected static void assertAllReleased(Iterable<WeakReference<?>> refs) {
+    for (WeakReference<?> ref : refs) {
+      GcFinalization.awaitClear(ref);
+    }
+  }
+
+  private String makeGenruleContents(String value) {
+    return String.format(
+        "genrule(name='target', outs=['out'], cmd='/bin/echo %s > $(location out)')", value);
+  }
+
+  protected void writeGenrule(String filename, String value) throws Exception {
+    write(filename, makeGenruleContents(value));
+  }
+
+  protected void writeGenruleAbsolute(Path file, String value) throws Exception {
+    writeAbsolute(file, makeGenruleContents(value));
+  }
+
+  protected void assertCharContentsIgnoringOrderAndWhitespace(
+      String expectedCharContents, String target) throws Exception {
+    Path path = Iterables.getOnlyElement(getArtifacts(target)).getPath();
+    char[] actualChars = FileSystemUtils.readContentAsLatin1(path);
+    char[] expectedChars = expectedCharContents.toCharArray();
+    Arrays.sort(actualChars);
+    Arrays.sort(expectedChars);
+    assertThat(new String(actualChars).trim()).isEqualTo(new String(expectedChars).trim());
+  }
+
+  protected void assertContents(String expectedContents, String target) throws Exception {
+    assertContents(expectedContents, Iterables.getOnlyElement(getArtifacts(target)).getPath());
+  }
+
+  protected void assertContents(String expectedContents, Path path) throws Exception {
+    String actualContents = new String(FileSystemUtils.readContentAsLatin1(path));
+    assertThat(actualContents.trim()).isEqualTo(expectedContents);
+  }
+
+  protected ImmutableList<String> getOnlyOutputContentAsLines(String target) throws Exception {
+    return FileSystemUtils.readLines(
+        Iterables.getOnlyElement(getArtifacts(target)).getPath(), UTF_8);
+  }
+}

src/test/java/com/google/devtools/build/lib/skyframe/BUILD

@@ -20,6 +20,7 @@ CROSS_PLATFORM_WINDOWS_TESTS = [
 
 # Tests that are broken out from the SkyframeTests target into separate targets.
 EXCLUDED_FROM_SKYFRAME_TESTS = [
+    "LocalDiffAwarenessIntegrationTest.java",
     "PrepareDepsOfTargetsUnderDirectoryFunctionTest.java",  # b/179148968
 ] + CROSS_PLATFORM_WINDOWS_TESTS
 
@@ -378,3 +379,28 @@ java_test(
         "//third_party:truth",
     ],
 )
+
+java_test(
+    name = "LocalDiffAwarenessIntegrationTest",
+    srcs = ["LocalDiffAwarenessIntegrationTest.java"],
+    # TODO(pcloudy): Even with --experimental_windows_watchfs, there's an extra
+    #  getValues() on the second build in
+    #  externalSymlink_doesNotTriggerFullGraphTraversal with Windows, and
+    #  non-deterministic failure to detect changes (watchfs bug?).
+    tags = ["no_windows"],
+    deps = [
+        "//src/main/java/com/google/devtools/build/lib:runtime",
+        "//src/main/java/com/google/devtools/build/lib/analysis:blaze_directories",
+        "//src/main/java/com/google/devtools/build/lib/skyframe:local_diff_awareness",
+        "//src/main/java/com/google/devtools/build/lib/util:abrupt_exit_exception",
+        "//src/main/java/com/google/devtools/build/lib/util:os",
+        "//src/main/java/com/google/devtools/build/lib/vfs",
+        "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
+        "//src/main/java/com/google/devtools/common/options",
+        "//src/test/java/com/google/devtools/build/lib/buildtool/util",
+        "//src/test/java/com/google/devtools/build/skyframe:testutil",
+        "//third_party:guava",
+        "//third_party:junit4",
+        "//third_party:truth",
+    ],
+)

src/test/java/com/google/devtools/build/lib/skyframe/LocalDiffAwarenessIntegrationTest.java

@@ -0,0 +1,180 @@
+// Copyright 2022 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.skyframe;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.truth.Truth.assertWithMessage;
+import static org.junit.Assert.assertThrows;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.analysis.BlazeDirectories;
+import com.google.devtools.build.lib.buildtool.util.SkyframeIntegrationTestBase;
+import com.google.devtools.build.lib.runtime.BlazeModule;
+import com.google.devtools.build.lib.runtime.BlazeRuntime;
+import com.google.devtools.build.lib.runtime.Command;
+import com.google.devtools.build.lib.runtime.WorkspaceBuilder;
+import com.google.devtools.build.lib.util.AbruptExitException;
+import com.google.devtools.build.lib.util.OS;
+import com.google.devtools.build.lib.vfs.DelegateFileSystem;
+import com.google.devtools.build.lib.vfs.FileStatus;
+import com.google.devtools.build.lib.vfs.FileSystem;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.skyframe.NotifyingHelper;
+import com.google.devtools.common.options.OptionsBase;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.junit.After;
+import org.junit.Assume;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * Tests for local diff awareness. A good place for general tests of Bazel's interactions with
+ * "smart" filesystems, so that open-source changes don't break Google-internal features around
+ * smart filesystems.
+ */
+@RunWith(JUnit4.class)
+public class LocalDiffAwarenessIntegrationTest extends SkyframeIntegrationTestBase {
+  private final Map<PathFragment, IOException> throwOnNextStatIfFound = new HashMap<>();
+
+  @Override
+  protected BlazeRuntime.Builder getRuntimeBuilder() throws Exception {
+    return super.getRuntimeBuilder()
+        .addBlazeModule(
+            new BlazeModule() {
+              @Override
+              public void workspaceInit(
+                  BlazeRuntime runtime, BlazeDirectories directories, WorkspaceBuilder builder) {
+                builder.addDiffAwarenessFactory(new LocalDiffAwareness.Factory(ImmutableList.of()));
+              }
+
+              @Override
+              public Iterable<Class<? extends OptionsBase>> getCommandOptions(Command command) {
+                return ImmutableList.of(LocalDiffAwareness.Options.class);
+              }
+            });
+  }
+
+  @Override
+  public FileSystem createFileSystem() throws Exception {
+    return new DelegateFileSystem(super.createFileSystem()) {
+      @Override
+      protected FileStatus statIfFound(PathFragment path, boolean followSymlinks)
+          throws IOException {
+        IOException e = throwOnNextStatIfFound.remove(path);
+        if (e != null) {
+          throw e;
+        }
+        return super.statIfFound(path, followSymlinks);
+      }
+    };
+  }
+
+  @Before
+  public void addOptions() {
+    addOptions("--watchfs", "--experimental_windows_watchfs");
+  }
+
+  @After
+  public void checkExceptionsThrown() {
+    assertWithMessage("Injected exception(s) not thrown").that(throwOnNextStatIfFound).isEmpty();
+  }
+
+  @Test
+  public void changedFile_detectsChange() throws Exception {
+    // TODO(bazel-team): Understand why these tests are flaky on Mac. Probably real watchfs bug?
+    Assume.assumeFalse(OS.DARWIN.equals(OS.getCurrent()));
+    write("foo/BUILD", "genrule(name='foo', outs=['out'], cmd='echo hello > $@')");
+    buildTarget("//foo");
+    assertContents("hello", "//foo");
+    write("foo/BUILD", "genrule(name='foo', outs=['out'], cmd='echo there > $@')");
+
+    buildTarget("//foo");
+
+    assertContents("there", "//foo");
+  }
+
+  @Test
+  public void changedFile_statFails_throwsError() throws Exception {
+    Assume.assumeFalse(OS.DARWIN.equals(OS.getCurrent()));
+    write("foo/BUILD", "genrule(name='foo', outs=['out'], cmd='echo hello > $@')");
+    buildTarget("//foo");
+    assertContents("hello", "//foo");
+    Path buildFile = write("foo/BUILD", "genrule(name='foo', outs=['out'], cmd='echo there > $@')");
+    IOException injectedException = new IOException("oh no!");
+    throwOnNextStatIfFound.put(buildFile.asFragment(), injectedException);
+
+    AbruptExitException e = assertThrows(AbruptExitException.class, () -> buildTarget("//foo"));
+
+    assertThat(e.getCause()).hasCauseThat().hasCauseThat().isSameInstanceAs(injectedException);
+  }
+
+  @Test
+  public void externalSymlink_doesNotTriggerFullGraphTraversal() throws Exception {
+    addOptions("--symlink_prefix=/");
+    AtomicInteger calledGetValues = new AtomicInteger(0);
+    skyframeExecutor()
+        .getEvaluator()
+        .injectGraphTransformerForTesting(
+            NotifyingHelper.makeNotifyingTransformer(
+                (key, type, order, context) -> {
+                  if (type == NotifyingHelper.EventType.GET_VALUES) {
+                    calledGetValues.incrementAndGet();
+                  }
+                }));
+    write(
+        "hello/BUILD",
+        "genrule(name='target', srcs = ['external'], outs=['out'], cmd='/bin/cat $(SRCS) > $@')");
+    String externalLink = System.getenv("TEST_TMPDIR") + "/target";
+    write(externalLink, "one");
+    createSymlink(externalLink, "hello/external");
+
+    // Trivial build: external symlink is not seen, so normal diff awareness is in play.
+    buildTarget("//hello:BUILD");
+    // New package path on first build triggers full-graph work.
+    calledGetValues.set(0);
+    // getValues() called during output file checking (although if an output service is able to
+    // report modified files in practice there is no iteration).
+    // If external repositories are being used, getValues called because of that too.
+    // TODO(bazel-team): get rid of this when we can disable checks for external repositories.
+    int numGetValuesInFullDiffAwarenessBuild =
+        1 + ("bazel".equals(this.getRuntime().getProductName()) ? 1 : 0);
+
+    buildTarget("//hello:BUILD");
+    assertThat(calledGetValues.getAndSet(0)).isEqualTo(numGetValuesInFullDiffAwarenessBuild);
+
+    // Now bring the external symlink into Bazel's awareness.
+    buildTarget("//hello:target");
+    assertContents("one", "//hello:target");
+    assertThat(calledGetValues.getAndSet(0)).isEqualTo(numGetValuesInFullDiffAwarenessBuild);
+
+    // Builds that follow a build containing an external file don't trigger a traversal.
+    buildTarget("//hello:target");
+    assertContents("one", "//hello:target");
+    assertThat(calledGetValues.getAndSet(0)).isEqualTo(numGetValuesInFullDiffAwarenessBuild);
+
+    write(externalLink, "two");
+
+    buildTarget("//hello:target");
+    // External file changes are tracked.
+    assertContents("two", "//hello:target");
+    assertThat(calledGetValues.getAndSet(0)).isEqualTo(numGetValuesInFullDiffAwarenessBuild);
+  }
+}