From 4ee7f114387187c8b578b7f41674b9e68593d20e Mon Sep 17 00:00:00 2001 From: David Ostrovsky Date: Wed, 2 May 2018 03:30:11 -0700 Subject: [PATCH] remote: introduce --disk_cache_flag Consolidate the --experimental_local_disk_cache and --experimental_local_disk_cache_path flags into a single --disk_cache= flag. Also, create the cache directory if it doesn't exist. RELNOTES: We replaced the --experimental_local_disk_cache and --experimental_local_disk_cache_path flags into a single --disk_cache flag. Additionally, Bazel now tries to create the disk cache directory if it doesn't exist. Closes #5119. PiperOrigin-RevId: 195070550 --- site/docs/remote-caching.md | 29 +++++++++++++++++-- .../remote/RemoteActionContextProvider.java | 2 +- .../build/lib/remote/RemoteOptions.java | 18 ++++-------- .../lib/remote/SimpleBlobStoreFactory.java | 23 ++++++++------- src/test/shell/bazel/BUILD | 4 +-- ...ction_cache_test.sh => disk_cache_test.sh} | 4 +-- 6 files changed, 49 insertions(+), 31 deletions(-) rename src/test/shell/bazel/{local_action_cache_test.sh => disk_cache_test.sh} (94%) diff --git a/site/docs/remote-caching.md b/site/docs/remote-caching.md index 163f1122f2ad69..939edce6c47f35 100644 --- a/site/docs/remote-caching.md +++ b/site/docs/remote-caching.md @@ -28,6 +28,7 @@ make builds significantly faster. * [Delete content from the remote cache](#delete-content-from-the-remote-cache) * [Known Issues](#known-issues) * [External Links](#external-links) +* [Disk cache](#disk-cache) * [Bazel remote execution (in development)](#bazel-remote-execution-in-development) ## Remote caching overview @@ -306,14 +307,36 @@ You may want to delete content from the cache to: * Create a clean cache after a cache was poisoned * Reduce the amount of storage used by deleting old outputs +## Disk cache + +Bazel can use a directory on the file system as a remote cache. This is +useful for sharing build artifacts when switching branches and/or working +on multiple workspaces of the same project, such as multiple checkouts. Since +Bazel does not garbage-collect the directory, so you might want to automate a +periodic cleanup of this directory. Enable disk cache as follows: + +``` +build --disk_cache=/path/to/build/cache +``` + +You can pass a user-specific path to the `--disk_cache` flag using the `~` alias +(Bazel will substitute the current user's home directory). This comes in handy +when enabling disk cache for all developers of a project via the project's +checked in `.bazelrc` file. + +To enable cache hits across different workspaces, use the following flag: + +``` +build --experimental_strict_action_env +``` + ## Known issues -**Input file modification during a Build** +**Input file modification during a build** When an input file is modified during a build, Bazel might upload invalid results to the remote cache. We are working on a solution for this problem. -See [issue #3360] for updates. Avoid this problem by not editing source -files during a build. +See [issue #3360] for updates. Avoid modifying source files during a build. **Environment variables leaking into an action** diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java index 8877f98572fb90..df8c312ef54a8f 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java @@ -64,7 +64,7 @@ public Iterable getActionContexts() { String buildRequestId = env.getBuildRequestId().toString(); String commandId = env.getCommandId().toString(); - if (remoteOptions.experimentalRemoteSpawnCache || remoteOptions.experimentalLocalDiskCache) { + if (remoteOptions.experimentalRemoteSpawnCache || remoteOptions.diskCache != null) { RemoteSpawnCache spawnCache = new RemoteSpawnCache( env.getExecRoot(), diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteOptions.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteOptions.java index dd4f5a0d04878a..e3b303db425668 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteOptions.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteOptions.java @@ -175,25 +175,17 @@ public final class RemoteOptions extends OptionsBase { ) public boolean experimentalRemoteSpawnCache; - // TODO(davido): Find a better place for this and the next option. @Option( - name = "experimental_local_disk_cache", - defaultValue = "false", - documentationCategory = OptionDocumentationCategory.UNCATEGORIZED, - effectTags = {OptionEffectTag.UNKNOWN}, - help = "Whether to use the experimental local disk cache." - ) - public boolean experimentalLocalDiskCache; - - @Option( - name = "experimental_local_disk_cache_path", + name = "disk_cache", defaultValue = "null", documentationCategory = OptionDocumentationCategory.UNCATEGORIZED, effectTags = {OptionEffectTag.UNKNOWN}, converter = OptionsUtils.PathFragmentConverter.class, - help = "A file path to a local disk cache." + help = + "A path to a directory where Bazel can read and write actions and action outputs. " + + "If the directory does not exist, it will be created." ) - public PathFragment experimentalLocalDiskCachePath; + public PathFragment diskCache; @Option( name = "experimental_guard_against_concurrent_changes", diff --git a/src/main/java/com/google/devtools/build/lib/remote/SimpleBlobStoreFactory.java b/src/main/java/com/google/devtools/build/lib/remote/SimpleBlobStoreFactory.java index 70a323ffeb8ba7..424a6886f41797 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/SimpleBlobStoreFactory.java +++ b/src/main/java/com/google/devtools/build/lib/remote/SimpleBlobStoreFactory.java @@ -21,6 +21,7 @@ import com.google.devtools.build.lib.remote.blobstore.SimpleBlobStore; import com.google.devtools.build.lib.remote.blobstore.http.HttpBlobStore; import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; import java.io.IOException; import java.net.URI; import java.util.concurrent.TimeUnit; @@ -34,8 +35,7 @@ public final class SimpleBlobStoreFactory { private SimpleBlobStoreFactory() {} - public static SimpleBlobStore createRest(RemoteOptions options, Credentials creds) - throws IOException { + public static SimpleBlobStore createRest(RemoteOptions options, Credentials creds) { try { return new HttpBlobStore( URI.create(options.remoteHttpCache), @@ -46,10 +46,13 @@ public static SimpleBlobStore createRest(RemoteOptions options, Credentials cred } } - public static SimpleBlobStore createLocalDisk(RemoteOptions options, Path workingDirectory) + public static SimpleBlobStore createDiskCache(Path workingDirectory, PathFragment diskCachePath) throws IOException { - return new OnDiskBlobStore( - workingDirectory.getRelative(checkNotNull(options.experimentalLocalDiskCachePath))); + Path cacheDir = workingDirectory.getRelative(checkNotNull(diskCachePath)); + if (!cacheDir.exists()) { + cacheDir.createDirectoryAndParents(); + } + return new OnDiskBlobStore(cacheDir); } public static SimpleBlobStore create( @@ -58,8 +61,8 @@ public static SimpleBlobStore create( if (isRestUrlOptions(options)) { return createRest(options, creds); } - if (workingDirectory != null && isLocalDiskCache(options)) { - return createLocalDisk(options, workingDirectory); + if (workingDirectory != null && isDiskCache(options)) { + return createDiskCache(workingDirectory, options.diskCache); } throw new IllegalArgumentException( "Unrecognized concurrent map RemoteOptions: must specify " @@ -67,11 +70,11 @@ public static SimpleBlobStore create( } public static boolean isRemoteCacheOptions(RemoteOptions options) { - return isRestUrlOptions(options) || isLocalDiskCache(options); + return isRestUrlOptions(options) || isDiskCache(options); } - public static boolean isLocalDiskCache(RemoteOptions options) { - return options.experimentalLocalDiskCache; + public static boolean isDiskCache(RemoteOptions options) { + return options.diskCache != null; } private static boolean isRestUrlOptions(RemoteOptions options) { diff --git a/src/test/shell/bazel/BUILD b/src/test/shell/bazel/BUILD index f9c51e0e888afb..bfa6d6cd935614 100644 --- a/src/test/shell/bazel/BUILD +++ b/src/test/shell/bazel/BUILD @@ -332,9 +332,9 @@ sh_test( ) sh_test( - name = "local_action_cache_test", + name = "disk_cache_test", size = "small", - srcs = ["local_action_cache_test.sh"], + srcs = ["disk_cache_test.sh"], data = [":test-deps"], ) diff --git a/src/test/shell/bazel/local_action_cache_test.sh b/src/test/shell/bazel/disk_cache_test.sh similarity index 94% rename from src/test/shell/bazel/local_action_cache_test.sh rename to src/test/shell/bazel/disk_cache_test.sh index 60c9c4ecc1abf4..ef224cafefdba7 100755 --- a/src/test/shell/bazel/local_action_cache_test.sh +++ b/src/test/shell/bazel/disk_cache_test.sh @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# Test the local action cache +# Test the local disk cache # # Load the test setup defined in the parent directory @@ -27,7 +27,7 @@ function test_local_action_cache() { local execution_file="${TEST_TMPDIR}/run.log" local input_file="foo.in" local output_file="bazel-genfiles/foo.txt" - local flags="--experimental_local_disk_cache_path=$cache --experimental_local_disk_cache" + local flags="--disk_cache=$cache" rm -rf $cache mkdir $cache