diff --git a/src/main/java/com/google/devtools/build/lib/authandtls/AuthAndTLSOptions.java b/src/main/java/com/google/devtools/build/lib/authandtls/AuthAndTLSOptions.java index 8641d1c1f8c23e..f648038182e026 100644 --- a/src/main/java/com/google/devtools/build/lib/authandtls/AuthAndTLSOptions.java +++ b/src/main/java/com/google/devtools/build/lib/authandtls/AuthAndTLSOptions.java @@ -139,6 +139,44 @@ public class AuthAndTLSOptions extends OptionsBase { + "pings are disabled, then this setting is ignored.") public Duration grpcKeepaliveTimeout; + @Option( + name = "experimental_credential_helper", + defaultValue = "null", + allowMultiple = true, + converter = UnresolvedScopedCredentialHelperConverter.class, + documentationCategory = OptionDocumentationCategory.UNCATEGORIZED, + effectTags = {OptionEffectTag.UNKNOWN}, + help = + "Configures Credential Helpers to use for retrieving credentials for the provided scope" + + " (domain).\n\n" + + "Credentials from Credential Helpers take precedence over credentials from" + + " --google_default_credentials, `--google_credentials`, or" + + " .netrc.\n\n" + + "See https://github.com/bazelbuild/proposals/blob/main/designs/2022-06-07-bazel-credential-helpers.md" + + " for details.") + public List credentialHelpers; + + @Option( + name = "experimental_credential_helper_timeout", + defaultValue = "5s", + converter = DurationConverter.class, + documentationCategory = OptionDocumentationCategory.UNCATEGORIZED, + effectTags = {OptionEffectTag.UNKNOWN}, + help = + "Configures the timeout for the Credential Helper.\n\n" + + "Credential Helpers failing to respond within this timeout will fail the" + + " invocation.") + public Duration credentialHelperTimeout; + + @Option( + name = "experimental_credential_helper_cache_duration", + defaultValue = "30m", + converter = DurationConverter.class, + documentationCategory = OptionDocumentationCategory.UNCATEGORIZED, + effectTags = {OptionEffectTag.UNKNOWN}, + help = "Configures the duration for which credentials from Credential Helpers are cached.") + public Duration credentialHelperCacheTimeout; + /** One of the values of the `--credential_helper` flag. */ @AutoValue public abstract static class UnresolvedScopedCredentialHelper { diff --git a/src/main/java/com/google/devtools/build/lib/authandtls/GoogleAuthUtils.java b/src/main/java/com/google/devtools/build/lib/authandtls/GoogleAuthUtils.java index 3d3ecac5bac70c..93c80f5396576c 100644 --- a/src/main/java/com/google/devtools/build/lib/authandtls/GoogleAuthUtils.java +++ b/src/main/java/com/google/devtools/build/lib/authandtls/GoogleAuthUtils.java @@ -19,10 +19,10 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.base.Strings; +import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperCredentials; import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperEnvironment; import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperProvider; import com.google.devtools.build.lib.events.Event; -import com.google.devtools.build.lib.events.Reporter; import com.google.devtools.build.lib.runtime.CommandLinePathFactory; import com.google.devtools.build.lib.vfs.FileSystem; import com.google.devtools.build.lib.vfs.Path; @@ -222,36 +222,49 @@ public static CallCredentialsProvider newCallCredentialsProvider(@Nullable Crede } /** - * Create a new {@link Credentials} with following order: + * Create a new {@link Credentials} retrieving call credentials in the following order: * *
    - *
  1. If authentication enabled by flags, use it to create credentials - *
  2. Use .netrc to provide credentials if exists - *
  3. Otherwise, return {@code null} + *
  4. If a Credential Helper is configured for the scope, use the credentials provided by the + * helper. + *
  5. If (Google) authentication is enabled by flags, use it to create credentials. + *
  6. Use {@code .netrc} to provide credentials if exists. *
* * @throws IOException in case the credentials can't be constructed. */ - @Nullable public static Credentials newCredentials( - Reporter reporter, - Map clientEnv, + CredentialHelperEnvironment credentialHelperEnvironment, + CommandLinePathFactory commandLinePathFactory, FileSystem fileSystem, AuthAndTLSOptions authAndTlsOptions) throws IOException { + Preconditions.checkNotNull(credentialHelperEnvironment); + Preconditions.checkNotNull(commandLinePathFactory); + Preconditions.checkNotNull(fileSystem); + Preconditions.checkNotNull(authAndTlsOptions); + Optional credentials = newGoogleCredentials(authAndTlsOptions); if (credentials.isEmpty()) { // Fallback to .netrc if it exists. try { - credentials = newCredentialsFromNetrc(clientEnv, fileSystem); + credentials = + newCredentialsFromNetrc(credentialHelperEnvironment.getClientEnvironment(), fileSystem); } catch (IOException e) { // TODO(yannic): Make this fail the build. - reporter.handle(Event.warn(e.getMessage())); + credentialHelperEnvironment.getEventReporter().handle(Event.warn(e.getMessage())); } } - return credentials.orElse(null); + return new CredentialHelperCredentials( + newCredentialHelperProvider( + credentialHelperEnvironment, + commandLinePathFactory, + authAndTlsOptions.credentialHelpers), + credentialHelperEnvironment, + credentials, + authAndTlsOptions.credentialHelperCacheTimeout); } /** diff --git a/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/BUILD b/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/BUILD index 8b11bce467a160..20500030d42356 100644 --- a/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/BUILD +++ b/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/BUILD @@ -17,9 +17,12 @@ java_library( "//src/main/java/com/google/devtools/build/lib/events", "//src/main/java/com/google/devtools/build/lib/shell", "//src/main/java/com/google/devtools/build/lib/vfs", + "//third_party:auth", "//third_party:auto_value", + "//third_party:caffeine", "//third_party:error_prone_annotations", "//third_party:gson", "//third_party:guava", + "//third_party:jsr305", ], ) diff --git a/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialHelperCredentials.java b/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialHelperCredentials.java new file mode 100644 index 00000000000000..ecc40e15707d8d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialHelperCredentials.java @@ -0,0 +1,141 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.authandtls.credentialhelper; + +import com.github.benmanes.caffeine.cache.CacheLoader; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.LoadingCache; +import com.google.auth.Credentials; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import java.io.IOException; +import java.net.URI; +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import javax.annotation.Nullable; + +/** + * Implementation of {@link Credentials} which fetches credentials by invoking a {@code credential + * helper} as subprocess, falling back to another {@link Credentials} if no suitable helper exists. + */ +public class CredentialHelperCredentials extends Credentials { + private final Optional fallbackCredentials; + + private final LoadingCache credentialCache; + + public CredentialHelperCredentials( + CredentialHelperProvider credentialHelperProvider, + CredentialHelperEnvironment credentialHelperEnvironment, + Optional fallbackCredentials, + Duration cacheTimeout) { + Preconditions.checkNotNull(credentialHelperProvider); + Preconditions.checkNotNull(credentialHelperEnvironment); + this.fallbackCredentials = Preconditions.checkNotNull(fallbackCredentials); + Preconditions.checkNotNull(cacheTimeout); + Preconditions.checkArgument( + !cacheTimeout.isNegative() && !cacheTimeout.isZero(), + "Cache timeout must be greater than 0"); + + credentialCache = + Caffeine.newBuilder() + .expireAfterWrite(cacheTimeout) + .build( + new CredentialHelperCacheLoader( + credentialHelperProvider, credentialHelperEnvironment)); + } + + @Override + public String getAuthenticationType() { + if (fallbackCredentials.isPresent()) { + return "credential-helper-with-fallback-" + fallbackCredentials.get().getAuthenticationType(); + } + + return "credential-helper"; + } + + @Override + public Map> getRequestMetadata(URI uri) throws IOException { + Preconditions.checkNotNull(uri); + + Optional>> credentials = getRequestMetadataFromCredentialHelper(uri); + if (credentials.isPresent()) { + return credentials.get(); + } + + if (fallbackCredentials.isPresent()) { + return fallbackCredentials.get().getRequestMetadata(uri); + } + + return ImmutableMap.of(); + } + + @SuppressWarnings("unchecked") // Map> to Map> + private Optional>> getRequestMetadataFromCredentialHelper(URI uri) { + Preconditions.checkNotNull(uri); + + GetCredentialsResponse response = credentialCache.get(uri); + + return Optional.ofNullable(response).map(value -> (Map) value.getHeaders()); + } + + @Override + public boolean hasRequestMetadata() { + return true; + } + + @Override + public boolean hasRequestMetadataOnly() { + return false; + } + + @Override + public void refresh() throws IOException { + if (fallbackCredentials.isPresent()) { + fallbackCredentials.get().refresh(); + } + + credentialCache.invalidateAll(); + } + + private static final class CredentialHelperCacheLoader + implements CacheLoader { + private final CredentialHelperProvider credentialHelperProvider; + private final CredentialHelperEnvironment credentialHelperEnvironment; + + public CredentialHelperCacheLoader( + CredentialHelperProvider credentialHelperProvider, + CredentialHelperEnvironment credentialHelperEnvironment) { + this.credentialHelperProvider = Preconditions.checkNotNull(credentialHelperProvider); + this.credentialHelperEnvironment = Preconditions.checkNotNull(credentialHelperEnvironment); + } + + @Nullable + @Override + public GetCredentialsResponse load(URI uri) throws IOException, InterruptedException { + Preconditions.checkNotNull(uri); + + Optional maybeCredentialHelper = + credentialHelperProvider.findCredentialHelper(uri); + if (maybeCredentialHelper.isEmpty()) { + return null; + } + CredentialHelper credentialHelper = maybeCredentialHelper.get(); + + return credentialHelper.getCredentials(credentialHelperEnvironment, uri); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/buildeventservice/BUILD b/src/main/java/com/google/devtools/build/lib/buildeventservice/BUILD index bfe3f367142f6f..26f91c3091e532 100644 --- a/src/main/java/com/google/devtools/build/lib/buildeventservice/BUILD +++ b/src/main/java/com/google/devtools/build/lib/buildeventservice/BUILD @@ -39,6 +39,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib:runtime", "//src/main/java/com/google/devtools/build/lib/analysis:test/test_configuration", "//src/main/java/com/google/devtools/build/lib/authandtls", + "//src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper", "//src/main/java/com/google/devtools/build/lib/bugreport", "//src/main/java/com/google/devtools/build/lib/buildeventservice/client", "//src/main/java/com/google/devtools/build/lib/buildeventstream", diff --git a/src/main/java/com/google/devtools/build/lib/buildeventservice/BazelBuildEventServiceModule.java b/src/main/java/com/google/devtools/build/lib/buildeventservice/BazelBuildEventServiceModule.java index 49878218bca925..c0f6b5e7b30125 100644 --- a/src/main/java/com/google/devtools/build/lib/buildeventservice/BazelBuildEventServiceModule.java +++ b/src/main/java/com/google/devtools/build/lib/buildeventservice/BazelBuildEventServiceModule.java @@ -24,6 +24,7 @@ import com.google.common.collect.ImmutableSet; import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions; import com.google.devtools.build.lib.authandtls.GoogleAuthUtils; +import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperEnvironment; import com.google.devtools.build.lib.buildeventservice.client.BuildEventServiceClient; import com.google.devtools.build.lib.buildeventservice.client.BuildEventServiceGrpcClient; import com.google.devtools.build.lib.runtime.CommandEnvironment; @@ -86,8 +87,13 @@ protected BuildEventServiceClient getBesClient( Credentials credentials = GoogleAuthUtils.newCredentials( - env.getReporter(), - env.getClientEnv(), + CredentialHelperEnvironment.newBuilder() + .setEventReporter(env.getReporter()) + .setWorkspacePath(env.getWorkspace()) + .setClientEnvironment(env.getClientEnv()) + .setHelperExecutionTimeout(authAndTLSOptions.credentialHelperTimeout) + .build(), + env.getCommandLinePathFactory(), env.getRuntime().getFileSystem(), newConfig.authAndTLSOptions()); diff --git a/src/main/java/com/google/devtools/build/lib/remote/BUILD b/src/main/java/com/google/devtools/build/lib/remote/BUILD index 60b32486cb14e8..9a08c88a75084e 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/BUILD +++ b/src/main/java/com/google/devtools/build/lib/remote/BUILD @@ -45,6 +45,7 @@ java_library( ":Retrier", "//src/main/java/com/google/devtools/build/lib:build-request-options", "//src/main/java/com/google/devtools/build/lib:runtime", + "//src/main/java/com/google/devtools/build/lib:runtime/command_line_path_factory", "//src/main/java/com/google/devtools/build/lib/actions", "//src/main/java/com/google/devtools/build/lib/actions:action_input_helper", "//src/main/java/com/google/devtools/build/lib/actions:artifacts", diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java index 10c80a1c46ad14..5fed432abe4d3c 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java @@ -47,6 +47,7 @@ import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions; import com.google.devtools.build.lib.authandtls.CallCredentialsProvider; import com.google.devtools.build.lib.authandtls.GoogleAuthUtils; +import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperEnvironment; import com.google.devtools.build.lib.bazel.repository.downloader.Downloader; import com.google.devtools.build.lib.buildeventstream.BuildEventArtifactUploader; import com.google.devtools.build.lib.buildeventstream.LocalFilesArtifactUploader; @@ -74,6 +75,7 @@ import com.google.devtools.build.lib.runtime.BuildEventArtifactUploaderFactory; import com.google.devtools.build.lib.runtime.Command; import com.google.devtools.build.lib.runtime.CommandEnvironment; +import com.google.devtools.build.lib.runtime.CommandLinePathFactory; import com.google.devtools.build.lib.runtime.RepositoryRemoteExecutor; import com.google.devtools.build.lib.runtime.RepositoryRemoteExecutorFactory; import com.google.devtools.build.lib.runtime.ServerBuilder; @@ -102,7 +104,6 @@ import java.net.URISyntaxException; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.ExecutorService; @@ -213,9 +214,14 @@ private void initHttpAndDiskCache( try { creds = newCredentials( - env.getClientEnv(), + CredentialHelperEnvironment.newBuilder() + .setEventReporter(env.getReporter()) + .setWorkspacePath(env.getWorkspace()) + .setClientEnvironment(env.getClientEnv()) + .setHelperExecutionTimeout(authAndTlsOptions.credentialHelperTimeout) + .build(), + env.getCommandLinePathFactory(), env.getRuntime().getFileSystem(), - env.getReporter(), authAndTlsOptions, remoteOptions); } catch (IOException e) { @@ -427,9 +433,14 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException { callCredentialsProvider = GoogleAuthUtils.newCallCredentialsProvider( newCredentials( - env.getClientEnv(), + CredentialHelperEnvironment.newBuilder() + .setEventReporter(env.getReporter()) + .setWorkspacePath(env.getWorkspace()) + .setClientEnvironment(env.getClientEnv()) + .setHelperExecutionTimeout(authAndTlsOptions.credentialHelperTimeout) + .build(), + env.getCommandLinePathFactory(), env.getRuntime().getFileSystem(), - env.getReporter(), authAndTlsOptions, remoteOptions)); } catch (IOException e) { @@ -1045,14 +1056,15 @@ RemoteActionContextProvider getActionContextProvider() { } static Credentials newCredentials( - Map clientEnv, + CredentialHelperEnvironment credentialHelperEnvironment, + CommandLinePathFactory commandLinePathFactory, FileSystem fileSystem, - Reporter reporter, AuthAndTLSOptions authAndTlsOptions, RemoteOptions remoteOptions) throws IOException { Credentials credentials = - GoogleAuthUtils.newCredentials(reporter, clientEnv, fileSystem, authAndTlsOptions); + GoogleAuthUtils.newCredentials( + credentialHelperEnvironment, commandLinePathFactory, fileSystem, authAndTlsOptions); try { if (credentials != null @@ -1060,11 +1072,13 @@ static Credentials newCredentials( && Ascii.toLowerCase(remoteOptions.remoteCache).startsWith("http://") && !credentials.getRequestMetadata(new URI(remoteOptions.remoteCache)).isEmpty()) { // TODO(yannic): Make this a error aborting the build. - reporter.handle( - Event.warn( - "Credentials are transmitted in plaintext to " - + remoteOptions.remoteCache - + ". Please consider using an HTTPS endpoint.")); + credentialHelperEnvironment + .getEventReporter() + .handle( + Event.warn( + "Credentials are transmitted in plaintext to " + + remoteOptions.remoteCache + + ". Please consider using an HTTPS endpoint.")); } } catch (URISyntaxException e) { throw new IOException(e.getMessage(), e); diff --git a/src/test/java/com/google/devtools/build/lib/remote/BUILD b/src/test/java/com/google/devtools/build/lib/remote/BUILD index 3d606046c76b37..74ac0126b4b797 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/BUILD +++ b/src/test/java/com/google/devtools/build/lib/remote/BUILD @@ -42,6 +42,7 @@ java_test( test_class = "com.google.devtools.build.lib.AllTests", deps = [ "//src/main/java/com/google/devtools/build/lib:runtime", + "//src/main/java/com/google/devtools/build/lib:runtime/command_line_path_factory", "//src/main/java/com/google/devtools/build/lib/actions", "//src/main/java/com/google/devtools/build/lib/actions:action_input_helper", "//src/main/java/com/google/devtools/build/lib/actions:artifacts", @@ -53,6 +54,7 @@ java_test( "//src/main/java/com/google/devtools/build/lib/analysis:config/core_options", "//src/main/java/com/google/devtools/build/lib/analysis:server_directories", "//src/main/java/com/google/devtools/build/lib/authandtls", + "//src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper", "//src/main/java/com/google/devtools/build/lib/buildeventstream", "//src/main/java/com/google/devtools/build/lib/clock", "//src/main/java/com/google/devtools/build/lib/collect/nestedset", diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteModuleTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteModuleTest.java index 93ea38e8c389fb..5194ac0f383c7d 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/RemoteModuleTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteModuleTest.java @@ -33,6 +33,7 @@ import com.google.devtools.build.lib.analysis.ServerDirectories; import com.google.devtools.build.lib.analysis.config.CoreOptions; import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions; +import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperEnvironment; import com.google.devtools.build.lib.events.Reporter; import com.google.devtools.build.lib.exec.BinTools; import com.google.devtools.build.lib.exec.ExecutionOptions; @@ -44,6 +45,7 @@ import com.google.devtools.build.lib.runtime.ClientOptions; import com.google.devtools.build.lib.runtime.Command; import com.google.devtools.build.lib.runtime.CommandEnvironment; +import com.google.devtools.build.lib.runtime.CommandLinePathFactory; import com.google.devtools.build.lib.runtime.CommonCommandOptions; import com.google.devtools.build.lib.runtime.commands.BuildCommand; import com.google.devtools.build.lib.testutil.Scratch; @@ -63,8 +65,8 @@ import io.grpc.util.MutableHandlerRegistry; import java.io.IOException; import java.net.URI; +import java.time.Duration; import java.util.ArrayList; -import java.util.Map; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -478,17 +480,24 @@ public void getCapabilities( @Test public void testNetrc_netrcWithoutRemoteCache() throws Exception { String netrc = "/.netrc"; - Map clientEnv = ImmutableMap.of("NETRC", netrc); FileSystem fileSystem = new InMemoryFileSystem(DigestHashFunction.SHA256); Scratch scratch = new Scratch(fileSystem); scratch.file(netrc, "machine foo.example.org login baruser password barpass"); AuthAndTLSOptions authAndTLSOptions = Options.getDefaults(AuthAndTLSOptions.class); RemoteOptions remoteOptions = Options.getDefaults(RemoteOptions.class); - Reporter reporter = new Reporter(new EventBus()); Credentials credentials = RemoteModule.newCredentials( - clientEnv, fileSystem, reporter, authAndTLSOptions, remoteOptions); + CredentialHelperEnvironment.newBuilder() + .setEventReporter(new Reporter(new EventBus())) + .setWorkspacePath(fileSystem.getPath("/workspace")) + .setClientEnvironment(ImmutableMap.of("NETRC", netrc)) + .setHelperExecutionTimeout(Duration.ZERO) + .build(), + new CommandLinePathFactory(fileSystem, ImmutableMap.of()), + fileSystem, + authAndTLSOptions, + remoteOptions); assertThat(credentials).isNotNull(); assertThat(credentials.getRequestMetadata(URI.create("https://foo.example.org"))).isNotEmpty();