Skip to content

Commit

Permalink
Add netrc support to --bes_backend (bazelbuild#15970)
Browse files Browse the repository at this point in the history 
Progress on bazelbuild#15856

Fixes bazelbuild#15928

Closes bazelbuild#15930.

PiperOrigin-RevId: 462379746
Change-Id: Ia7ae470bdcbd97c6cb42cc290c3891393ec9ce3a

Co-authored-by: Yannic Bonenberger <[email protected]>
  • Loading branch information
@tjgq @Yannic
tjgq and Yannic authored Jul 25, 2022
1 parent e4ee344 commit 96d23d3
Show file tree
Hide file tree
Showing 14 changed files with 268 additions and 200 deletions.
2 changes: 2 additions & 0 deletions src/main/java/com/google/devtools/build/lib/authandtls/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ java_library(
srcs = glob(["*.java"]),
deps = [
"//src/main/java/com/google/devtools/build/lib/concurrent",
"//src/main/java/com/google/devtools/build/lib/events",
"//src/main/java/com/google/devtools/build/lib/vfs",
"//src/main/java/com/google/devtools/common/options",
"//third_party:auth",
"//third_party:auto_value",
Expand Down
108 changes: 94 additions & 14 deletions src/main/java/com/google/devtools/build/lib/authandtls/GoogleAuthUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,10 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.events.Reporter;
import com.google.devtools.build.lib.vfs.FileSystem;
import com.google.devtools.build.lib.vfs.Path;
import io.grpc.CallCredentials;
import io.grpc.ClientInterceptor;
import io.grpc.ManagedChannel;
Expand All @@ -41,6 +45,8 @@
import java.io.IOException;
import java.io.InputStream;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.Executor;
import java.util.concurrent.TimeUnit;
import javax.annotation.Nullable;
Expand Down Expand Up @@ -186,14 +192,17 @@ private static NettyChannelBuilder newNettyChannelBuilder(String targetUrl, Stri
}

/**
* Create a new {@link CallCredentials} object.
* Create a new {@link CallCredentials} object from the authentication flags, or null if no flags
* are set.
*
* @throws IOException in case the call credentials can't be constructed.
* @throws IOException in case the credentials can't be constructed.
*/
public static CallCredentials newCallCredentials(AuthAndTLSOptions options) throws IOException {
Credentials creds = newCredentials(options);
if (creds != null) {
return MoreCallCredentials.from(creds);
@Nullable
public static CallCredentials newGoogleCallCredentials(AuthAndTLSOptions options)
throws IOException {
Optional<Credentials> creds = newGoogleCredentials(options);
if (creds.isPresent()) {
return MoreCallCredentials.from(creds.get());
}
return null;
}
Expand All @@ -210,18 +219,52 @@ public static CallCredentialsProvider newCallCredentialsProvider(@Nullable Crede
}

/**
* Create a new {@link Credentials} object, or {@code null} if no options are provided.
* Create a new {@link Credentials} with following order:
*
* <ol>
* <li>If authentication enabled by flags, use it to create credentials
* <li>Use .netrc to provide credentials if exists
* <li>Otherwise, return {@code null}
* </ol>
*
* @throws IOException in case the credentials can't be constructed.
*/
@Nullable
public static Credentials newCredentials(@Nullable AuthAndTLSOptions options) throws IOException {
public static Credentials newCredentials(
Reporter reporter,
Map<String, String> clientEnv,
FileSystem fileSystem,
AuthAndTLSOptions authAndTlsOptions)
throws IOException {
Optional<Credentials> credentials = newGoogleCredentials(authAndTlsOptions);

if (credentials.isEmpty()) {
// Fallback to .netrc if it exists.
try {
credentials = newCredentialsFromNetrc(clientEnv, fileSystem);
} catch (IOException e) {
// TODO(yannic): Make this fail the build.
reporter.handle(Event.warn(e.getMessage()));
}
}

return credentials.orElse(null);
}

/**
* Create a new {@link Credentials} object from the authentication flags, or null if no flags are
* set.
*
* @throws IOException in case the credentials can't be constructed.
*/
public static Optional<Credentials> newGoogleCredentials(@Nullable AuthAndTLSOptions options)
throws IOException {
if (options == null) {
return null;
return Optional.empty();
} else if (options.googleCredentials != null) {
// Credentials from file
try (InputStream authFile = new FileInputStream(options.googleCredentials)) {
return newCredentials(authFile, options.googleAuthScopes);
return Optional.of(newGoogleCredentialsFromFile(authFile, options.googleAuthScopes));
} catch (FileNotFoundException e) {
String message =
String.format(
Expand All @@ -230,10 +273,11 @@ public static Credentials newCredentials(@Nullable AuthAndTLSOptions options) th
throw new IOException(message, e);
}
} else if (options.useGoogleDefaultCredentials) {
return newCredentials(
null /* Google Application Default Credentials */, options.googleAuthScopes);
return Optional.of(
newGoogleCredentialsFromFile(
null /* Google Application Default Credentials */, options.googleAuthScopes));
}
return null;
return Optional.empty();
}

/**
Expand All @@ -242,7 +286,7 @@ public static Credentials newCredentials(@Nullable AuthAndTLSOptions options) th
* @throws IOException in case the credentials can't be constructed.
*/
@VisibleForTesting
public static Credentials newCredentials(
public static Credentials newGoogleCredentialsFromFile(
@Nullable InputStream credentialsFile, List<String> authScopes) throws IOException {
try {
GoogleCredentials creds =
Expand All @@ -258,4 +302,40 @@ public static Credentials newCredentials(
throw new IOException(message, e);
}
}

/**
* Create a new {@link Credentials} object by parsing the .netrc file with following order to
* search it:
*
* <ol>
* <li>If environment variable $NETRC exists, use it as the path to the .netrc file
* <li>Fallback to $HOME/.netrc
* </ol>
*
* @return the {@link Credentials} object or {@code null} if there is no .netrc file.
* @throws IOException in case the credentials can't be constructed.
*/
@VisibleForTesting
static Optional<Credentials> newCredentialsFromNetrc(
Map<String, String> clientEnv, FileSystem fileSystem) throws IOException {
Optional<String> netrcFileString =
Optional.ofNullable(clientEnv.get("NETRC"))
.or(() -> Optional.ofNullable(clientEnv.get("HOME")).map(home -> home + "/.netrc"));
if (netrcFileString.isEmpty()) {
return Optional.empty();
}

Path netrcFile = fileSystem.getPath(netrcFileString.get());
if (!netrcFile.exists()) {
return Optional.empty();
}

try {
Netrc netrc = NetrcParser.parseAndClose(netrcFile.getInputStream());
return Optional.of(new NetrcCredentials(netrc));
} catch (IOException e) {
throw new IOException(
"Failed to parse " + netrcFile.getPathString() + ": " + e.getMessage(), e);
}
}
}
1 change: 1 addition & 0 deletions src/main/java/com/google/devtools/build/lib/buildeventservice/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ java_library(
"//src/main/java/com/google/devtools/build/lib/util/io:out-err",
"//src/main/java/com/google/devtools/common/options",
"//src/main/protobuf:failure_details_java_proto",
"//third_party:auth",
"//third_party:auto_value",
"//third_party:flogger",
"//third_party:guava",
Expand Down
21 changes: 19 additions & 2 deletions ...in/java/com/google/devtools/build/lib/buildeventservice/BazelBuildEventServiceModule.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,10 @@

package com.google.devtools.build.lib.buildeventservice;

import com.google.auth.Credentials;
import com.google.auto.value.AutoValue;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
Expand All @@ -24,9 +26,11 @@
import com.google.devtools.build.lib.authandtls.GoogleAuthUtils;
import com.google.devtools.build.lib.buildeventservice.client.BuildEventServiceClient;
import com.google.devtools.build.lib.buildeventservice.client.BuildEventServiceGrpcClient;
import com.google.devtools.build.lib.runtime.CommandEnvironment;
import io.grpc.ClientInterceptor;
import io.grpc.ManagedChannel;
import io.grpc.Metadata;
import io.grpc.auth.MoreCallCredentials;
import io.grpc.stub.MetadataUtils;
import java.io.IOException;
import java.util.Map;
Expand Down Expand Up @@ -70,15 +74,28 @@ protected Class<BuildEventServiceOptions> optionsClass() {

@Override
protected BuildEventServiceClient getBesClient(
BuildEventServiceOptions besOptions, AuthAndTLSOptions authAndTLSOptions) throws IOException {
CommandEnvironment env,
BuildEventServiceOptions besOptions,
AuthAndTLSOptions authAndTLSOptions)
throws IOException {
BackendConfig newConfig = BackendConfig.create(besOptions, authAndTLSOptions);
if (client == null || !Objects.equals(config, newConfig)) {
clearBesClient();
Preconditions.checkState(config == null);
Preconditions.checkState(client == null);

Credentials credentials =
GoogleAuthUtils.newCredentials(
env.getReporter(),
env.getClientEnv(),
env.getRuntime().getFileSystem(),
newConfig.authAndTLSOptions());

config = newConfig;
client =
new BuildEventServiceGrpcClient(
newGrpcChannel(config),
GoogleAuthUtils.newCallCredentials(config.authAndTLSOptions()),
credentials != null ? MoreCallCredentials.from(credentials) : null,
makeGrpcInterceptor(config));
}
return client;
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/com/google/devtools/build/lib/buildeventservice/BuildEventServiceModule.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -677,7 +677,7 @@ private BuildEventServiceTransport createBesTransport(

final BuildEventServiceClient besClient;
try {
besClient = getBesClient(besOptions, authTlsOptions);
besClient = getBesClient(cmdEnv, besOptions, authTlsOptions);
} catch (IOException | OptionsParsingException e) {
reportError(
reporter,
Expand Down Expand Up @@ -822,7 +822,7 @@ private static AbruptExitException createAbruptExitException(
protected abstract Class<BESOptionsT> optionsClass();

protected abstract BuildEventServiceClient getBesClient(
BESOptionsT besOptions, AuthAndTLSOptions authAndTLSOptions)
CommandEnvironment env, BESOptionsT besOptions, AuthAndTLSOptions authAndTLSOptions)
throws IOException, OptionsParsingException;

protected abstract void clearBesClient();
Expand Down
120 changes: 29 additions & 91 deletions src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,6 @@
import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions.UnresolvedScopedCredentialHelper;
import com.google.devtools.build.lib.authandtls.CallCredentialsProvider;
import com.google.devtools.build.lib.authandtls.GoogleAuthUtils;
import com.google.devtools.build.lib.authandtls.Netrc;
import com.google.devtools.build.lib.authandtls.NetrcCredentials;
import com.google.devtools.build.lib.authandtls.NetrcParser;
import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperEnvironment;
import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperProvider;
import com.google.devtools.build.lib.bazel.repository.downloader.Downloader;
Expand Down Expand Up @@ -1052,94 +1049,6 @@ RemoteActionContextProvider getActionContextProvider() {
return actionContextProvider;
}

/**
* Create a new {@link Credentials} object by parsing the .netrc file with following order to
* search it:
*
* <ol>
* <li>If environment variable $NETRC exists, use it as the path to the .netrc file
* <li>Fallback to $HOME/.netrc
* </ol>
*
* @return the {@link Credentials} object or {@code null} if there is no .netrc file.
* @throws IOException in case the credentials can't be constructed.
*/
@VisibleForTesting
static Credentials newCredentialsFromNetrc(Map<String, String> clientEnv, FileSystem fileSystem)
throws IOException {
String netrcFileString =
Optional.ofNullable(clientEnv.get("NETRC"))
.orElseGet(
() ->
Optional.ofNullable(clientEnv.get("HOME"))
.map(home -> home + "/.netrc")
.orElse(null));
if (netrcFileString == null) {
return null;
}

Path netrcFile = fileSystem.getPath(netrcFileString);
if (netrcFile.exists()) {
try {
Netrc netrc = NetrcParser.parseAndClose(netrcFile.getInputStream());
return new NetrcCredentials(netrc);
} catch (IOException e) {
throw new IOException(
"Failed to parse " + netrcFile.getPathString() + ": " + e.getMessage(), e);
}
} else {
return null;
}
}

/**
* Create a new {@link Credentials} with following order:
*
* <ol>
* <li>If authentication enabled by flags, use it to create credentials
* <li>Use .netrc to provide credentials if exists
* <li>Otherwise, return {@code null}
* </ol>
*
* @throws IOException in case the credentials can't be constructed.
*/
@VisibleForTesting
static Credentials newCredentials(
Map<String, String> clientEnv,
FileSystem fileSystem,
Reporter reporter,
AuthAndTLSOptions authAndTlsOptions,
RemoteOptions remoteOptions)
throws IOException {
Credentials creds = GoogleAuthUtils.newCredentials(authAndTlsOptions);

// Fallback to .netrc if it exists
if (creds == null) {
try {
creds = newCredentialsFromNetrc(clientEnv, fileSystem);
} catch (IOException e) {
reporter.handle(Event.warn(e.getMessage()));
}

try {
if (creds != null
&& remoteOptions.remoteCache != null
&& Ascii.toLowerCase(remoteOptions.remoteCache).startsWith("http://")
&& !creds.getRequestMetadata(new URI(remoteOptions.remoteCache)).isEmpty()) {
reporter.handle(
Event.warn(
"Username and password from .netrc is transmitted in plaintext to "
+ remoteOptions.remoteCache
+ ". Please consider using an HTTPS endpoint."));
}
} catch (URISyntaxException e) {
throw new IOException(e.getMessage(), e);
}
}

return creds;
}

@VisibleForTesting
static CredentialHelperProvider newCredentialHelperProvider(
CredentialHelperEnvironment environment,
Expand All @@ -1163,6 +1072,35 @@ static CredentialHelperProvider newCredentialHelperProvider(
return builder.build();
}

static Credentials newCredentials(
Map<String, String> clientEnv,
FileSystem fileSystem,
Reporter reporter,
AuthAndTLSOptions authAndTlsOptions,
RemoteOptions remoteOptions)
throws IOException {
Credentials credentials =
GoogleAuthUtils.newCredentials(reporter, clientEnv, fileSystem, authAndTlsOptions);

try {
if (credentials != null
&& remoteOptions.remoteCache != null
&& Ascii.toLowerCase(remoteOptions.remoteCache).startsWith("http://")
&& !credentials.getRequestMetadata(new URI(remoteOptions.remoteCache)).isEmpty()) {
// TODO(yannic): Make this a error aborting the build.
reporter.handle(
Event.warn(
"Credentials are transmitted in plaintext to "
+ remoteOptions.remoteCache
+ ". Please consider using an HTTPS endpoint."));
}
} catch (URISyntaxException e) {
throw new IOException(e.getMessage(), e);
}

return credentials;
}

@VisibleForTesting
@AutoValue
abstract static class ScopedCredentialHelper {
Expand Down
Loading

0 comments on commit 96d23d3

Please sign in to comment.