Skip to content
This repository has been archived by the owner on Jan 19, 2022. It is now read-only.

Create protocol for specifying secrets' project and versions #2302

Merged
merged 20 commits into from
Apr 10, 2020
Merged

Create protocol for specifying secrets' project and versions #2302

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
222575d
Create protocol for specifying secrets
dzou Apr 3, 2020
69ea656
Fix assert notNull statements
dzou Apr 6, 2020
d9c7592
reduce visibility of SecretManagerPropertyIdentifier
dzou Apr 6, 2020
3d87209
PR Comments
dzou Apr 6, 2020
f8d5fd2
PR Comments
dzou Apr 7, 2020
231b208
update sample
dzou Apr 7, 2020
4ae2c45
Fix checkstyle
dzou Apr 7, 2020
1f20609
Fix sample
dzou Apr 7, 2020
f026809
make private visibility for sample fields
dzou Apr 7, 2020
5b01c33
fix tests
dzou Apr 7, 2020
f00e1c3
add visible for testing comment
dzou Apr 7, 2020
32a8165
documentation updates
dzou Apr 8, 2020
d54c7d6
Update secret manager enabled property name.
dzou Apr 8, 2020
8b145f1
remove enabled property from tests
dzou Apr 8, 2020
2ce82cb
PR Comments
dzou Apr 8, 2020
41c83f0
template changes + refactor
dzou Apr 8, 2020
1116fb8
secret manager prefix change to sm://
dzou Apr 8, 2020
c48a3a5
add comments
dzou Apr 8, 2020
968e44a
fix checkstyle
dzou Apr 8, 2020
a81143b
PR Comments
dzou Apr 10, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions ...amework/cloud/gcp/autoconfigure/secretmanager/GcpSecretManagerBootstrapConfiguration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,9 +98,8 @@ public SecretManagerServiceClient secretManagerClient() throws IOException {

@Bean
public PropertySourceLocator secretManagerPropertySourceLocator(SecretManagerServiceClient client) {
SecretManagerPropertySourceLocator propertySourceLocator = new SecretManagerPropertySourceLocator(
client, this.gcpProjectIdProvider, this.properties.getSecretNamePrefix());
propertySourceLocator.setVersions(this.properties.getVersions());
SecretManagerPropertySourceLocator propertySourceLocator =
new SecretManagerPropertySourceLocator(client, this.gcpProjectIdProvider);
return propertySourceLocator;
}
}
30 changes: 0 additions & 30 deletions ...org/springframework/cloud/gcp/autoconfigure/secretmanager/GcpSecretManagerProperties.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,6 @@

package org.springframework.cloud.gcp.autoconfigure.secretmanager;

import java.util.HashMap;
import java.util.Map;

import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.context.properties.NestedConfigurationProperty;
import org.springframework.cloud.gcp.core.Credentials;
Expand All @@ -39,17 +36,6 @@ public class GcpSecretManagerProperties implements CredentialsSupplier {
*/
private String projectId;

/**
* Defines a prefix String that will be prepended to the environment property names
* of secrets in Secret Manager.
*/
private String secretNamePrefix = "";
meltsufin marked this conversation as resolved.
Show resolved Hide resolved

/**
* Defines versions for specific secret-ids.
*/
private Map<String, String> versions = new HashMap<>();

public Credentials getCredentials() {
return credentials;
}
Expand All @@ -61,20 +47,4 @@ public String getProjectId() {
public void setProjectId(String projectId) {
this.projectId = projectId;
}

public String getSecretNamePrefix() {
return secretNamePrefix;
}

public void setSecretNamePrefix(String secretNamePrefix) {
this.secretNamePrefix = secretNamePrefix;
}

public Map<String, String> getVersions() {
return versions;
}

public void setVersions(Map<String, String> versions) {
this.versions = versions;
}
}
144 changes: 73 additions & 71 deletions ...rg/springframework/cloud/gcp/autoconfigure/secretmanager/SecretManagerPropertySource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,117 +16,119 @@

package org.springframework.cloud.gcp.autoconfigure.secretmanager;

import java.util.Collections;
import java.util.HashMap;
import java.util.Map;

import com.google.api.gax.rpc.NotFoundException;
import com.google.cloud.secretmanager.v1beta1.AccessSecretVersionResponse;
import com.google.cloud.secretmanager.v1beta1.ProjectName;
import com.google.cloud.secretmanager.v1beta1.Secret;
import com.google.cloud.secretmanager.v1beta1.SecretManagerServiceClient;
import com.google.cloud.secretmanager.v1beta1.SecretManagerServiceClient.ListSecretsPagedResponse;
import com.google.cloud.secretmanager.v1beta1.SecretVersionName;
import com.google.protobuf.ByteString;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import org.springframework.cloud.gcp.core.GcpProjectIdProvider;
import org.springframework.core.env.EnumerablePropertySource;

/**
* Retrieves secrets from GCP Secret Manager under the current GCP project.
* A property source for Secret Manager which accesses the Secret Manager APIs when {@link #getProperty} is called.
*
* @author Daniel Zou
* @author Eddú Meléndez
* @since 1.2.2
*/
public class SecretManagerPropertySource extends EnumerablePropertySource<SecretManagerServiceClient> {

private static final Log LOGGER = LogFactory.getLog(SecretManagerPropertySource.class);

private static final String LATEST_VERSION_STRING = "latest";

private final Map<String, Object> properties;
private static final String GCP_SECRET_PREFIX = "gcp-secret/";
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Did the Secrets Manager team comment on this prefix? Have you looked into using sm://?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Colon can't be used in a property name because it means something special in SPEL. I wouldn't want to try to escape the colon either; I think it would look worse with the back slashes.

Am open to other secret prefixes though.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wonder if we can tap into that "special meaning" and configure it for our purpose.

Copy link
Contributor Author

@dzou dzou Apr 7, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Colon is like the null-safety operator. If the left side of the colon is evaluated to null, then it defaults to the value on the right side.

So if i do gs://blah/blah - it firsts evaluates gs which it finds to be null, then returns the string //blah/blah.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is only the case if SPEL is used, such as with @Value, right?
Notice that with @Value on a Resource it's interpreted as a protocol instead.
For example, see the "gs://" implementation we have for Storage.

Copy link
Contributor Author

@dzou dzou Apr 8, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ran the code through the debugger, the colon is a special character; i don't think this is configurable. I think we should avoid it to avoid unexpected behavior in the future with null values.

I would be open to other choices of prefixes, maybe sm| or gcp-sm| these seem to all work. I recall Seth suggested using | as an alternative.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we ever return null though? Is there something wrong with throwing an exception for a property that doesn't exist?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah I can see the argument where if a user specifies a secret using our syntax, there is the expectation that a secret is there; if it's missing then that's most likely an error and they probably don't want null there.

I would be comfortable with either approach. We can still throw exception using | too by the way.

To me it seems the tradeoff here is purely cosmetic. I.e. if the cosmetic benefits of using : outweighs the risk that we need to change behavior to return null someday.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What's the prefix that the Secret Manager team suggested we use? There's value in being compatible with other libraries for this product. Let's confirm with the product team.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Based on discussion offline, looks like sm:// has precedent of being used, so I just changed it to that for now.


private final String[] propertyNames;
private final GcpProjectIdProvider projectIdProvider;

public SecretManagerPropertySource(
String propertySourceName,
SecretManagerServiceClient client,
GcpProjectIdProvider projectIdProvider,
String secretsPrefix) {
this(propertySourceName, client, projectIdProvider, secretsPrefix, Collections.EMPTY_MAP);
}

public SecretManagerPropertySource(
String propertySourceName,
SecretManagerServiceClient client,
GcpProjectIdProvider projectIdProvider,
String secretsPrefix,
Map<String, String> versions) {
GcpProjectIdProvider projectIdProvider) {
super(propertySourceName, client);

Map<String, Object> propertiesMap = createSecretsPropertiesMap(
client, projectIdProvider.getProjectId(), secretsPrefix, versions);
this.projectIdProvider = projectIdProvider;
}

this.properties = propertiesMap;
this.propertyNames = propertiesMap.keySet().toArray(new String[propertiesMap.size()]);
@Override
public Object getProperty(String name) {
SecretVersionName secretIdentifier = parseFromProperty(name, this.projectIdProvider);

if (secretIdentifier != null) {
return getSecretPayload(secretIdentifier);
}
else {
return null;
}
}

/**
* The {@link SecretManagerPropertySource} is not enumerable, so this always returns an empty array.
* @return the empty array.
*/
@Override
public String[] getPropertyNames() {
return propertyNames;
return new String[0];
meltsufin marked this conversation as resolved.
Show resolved Hide resolved
}

@Override
public Object getProperty(String name) {
return properties.get(name);
private ByteString getSecretPayload(SecretVersionName secretIdentifier) {
try {
AccessSecretVersionResponse response = getSource().accessSecretVersion(secretIdentifier);
return response.getPayload().getData();
}
catch (NotFoundException e) {
return null;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Wouldn't this be considered an error if we got here? Not sure we should swallow the exception.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure, let's throw exception, this might be good for supporting the gcp-sm:// prefix per our discussion above.

}
}

private static Map<String, Object> createSecretsPropertiesMap(
SecretManagerServiceClient client, String projectId, String secretsPrefix, Map<String, String> versions) {

ListSecretsPagedResponse response = client.listSecrets(ProjectName.of(projectId));
Map<String, Object> secretsMap = new HashMap<>();
for (Secret secret : response.iterateAll()) {
String secretId = extractSecretId(secret);
ByteString secretPayload = getSecretPayload(client, projectId, secretId, versions);
if (secretPayload != null) {
secretsMap.put(secretsPrefix + secretId, secretPayload);
}
static SecretVersionName parseFromProperty(String property, GcpProjectIdProvider projectIdProvider) {
dzou marked this conversation as resolved.
Show resolved Hide resolved
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should this move to SecretManagerTemplate? We probably want to reconsider all those additional methods for specifying project and version and use the fully qualified secret identifiers instead.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done; I added these methods for getSecret, but for createSecret and secretExists it doesn't quite fit since those methods can accept project and secretId as valid inputs but are not fixed to a version.

if (!property.startsWith(GCP_SECRET_PREFIX)) {
return null;
}

return secretsMap;
}
String resourcePath = property.substring(GCP_SECRET_PREFIX.length());
String[] tokens = resourcePath.split("/");

private static ByteString getSecretPayload(
SecretManagerServiceClient client,
String projectId,
String secretId,
Map<String, String> versions) {
String projectId = projectIdProvider.getProjectId();
String secretId = null;
String version = "latest";

String version = versions.containsKey(secretId) ? versions.get(secretId) : LATEST_VERSION_STRING;
if (tokens.length == 1) {
// property is form "gcp-secret/<secret-id>"
secretId = tokens[0];
}
else if (tokens.length == 2) {
// property is form "gcp-secret/<secret-id>/<version>"
secretId = tokens[0];
version = tokens[1];
}
else if (tokens.length == 3) {
// property is form "gcp-secret/<project-id>/<secret-id>/<version-id>"
projectId = tokens[0];
secretId = tokens[1];
version = tokens[2];
}
else if (tokens.length == 4
&& tokens[0].equals("projects")
&& tokens[2].equals("secrets")) {
// property is form "gcp-secret/projects/<project-id>/secrets/<secret-id>"
projectId = tokens[1];
secretId = tokens[3];
}
else if (tokens.length == 6
&& tokens[0].equals("projects")
&& tokens[2].equals("secrets")
&& tokens[4].equals("versions")) {
// property is form "gcp-secret/projects/<project-id>/secrets/<secret-id>/versions/<version>"
projectId = tokens[1];
secretId = tokens[3];
version = tokens[5];
}
else {
return null;
dzou marked this conversation as resolved.
Show resolved Hide resolved
}

SecretVersionName secretVersionName = SecretVersionName.newBuilder()
return SecretVersionName.newBuilder()
.setProject(projectId)
.setSecret(secretId)
.setSecretVersion(version)
.build();

AccessSecretVersionResponse response = client.accessSecretVersion(secretVersionName);
return response.getPayload().getData();
}

/**
* Extracts the Secret ID from the {@link Secret}. The secret ID refers to the unique ID
* given to the secret when it is saved under a GCP project.
*
* <p>
* The secret ID is extracted from the full secret name of the form:
* projects/${PROJECT_ID}/secrets/${SECRET_ID}
*/
private static String extractSecretId(Secret secret) {
String[] secretNameTokens = secret.getName().split("/");
return secretNameTokens[secretNameTokens.length - 1];
}
}
18 changes: 2 additions & 16 deletions ...ngframework/cloud/gcp/autoconfigure/secretmanager/SecretManagerPropertySourceLocator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,6 @@

package org.springframework.cloud.gcp.autoconfigure.secretmanager;

import java.util.Map;

import com.google.cloud.secretmanager.v1beta1.SecretManagerServiceClient;

import org.springframework.cloud.bootstrap.config.PropertySourceLocator;
Expand All @@ -41,30 +39,18 @@ public class SecretManagerPropertySourceLocator implements PropertySourceLocator

private final GcpProjectIdProvider projectIdProvider;

private final String secretsPrefix;

private Map<String, String> versions;

SecretManagerPropertySourceLocator(
SecretManagerServiceClient client,
GcpProjectIdProvider projectIdProvider,
String secretsPrefix) {
GcpProjectIdProvider projectIdProvider) {
this.client = client;
this.projectIdProvider = projectIdProvider;
this.secretsPrefix = secretsPrefix;
}

public void setVersions(Map<String, String> versions) {
this.versions = versions;
}

@Override
public PropertySource<?> locate(Environment environment) {
return new SecretManagerPropertySource(
SECRET_MANAGER_NAME,
this.client,
this.projectIdProvider,
this.secretsPrefix,
this.versions);
this.projectIdProvider);
}
}
93 changes: 93 additions & 0 deletions .../org/springframework/cloud/gcp/autoconfigure/secretmanager/SecretPropertySourceTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
/*
* Copyright 2017-2020 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.springframework.cloud.gcp.autoconfigure.secretmanager;

import com.google.cloud.secretmanager.v1beta1.SecretVersionName;
import org.junit.Test;

import org.springframework.cloud.gcp.core.GcpProjectIdProvider;

import static org.assertj.core.api.Assertions.assertThat;

public class SecretPropertySourceTests {
dzou marked this conversation as resolved.
Show resolved Hide resolved

private static final GcpProjectIdProvider DEFAULT_PROJECT_ID_PROVIDER = () -> "defaultProject";

@Test
public void testNonSecret() {
String property = "spring.cloud.datasource";
SecretVersionName secretIdentifier =
SecretManagerPropertySource.parseFromProperty(property, DEFAULT_PROJECT_ID_PROVIDER);

assertThat(secretIdentifier).isNull();
}

@Test
public void testShortProperty_secretId() {
String property = "gcp-secret/the-secret";
dzou marked this conversation as resolved.
Show resolved Hide resolved
SecretVersionName secretIdentifier =
SecretManagerPropertySource.parseFromProperty(property, DEFAULT_PROJECT_ID_PROVIDER);

assertThat(secretIdentifier.getProject()).isEqualTo("defaultProject");
assertThat(secretIdentifier.getSecret()).isEqualTo("the-secret");
assertThat(secretIdentifier.getSecretVersion()).isEqualTo("latest");
}

@Test
public void testShortProperty_projectSecretId() {
String property = "gcp-secret/the-secret/the-version";
SecretVersionName secretIdentifier =
SecretManagerPropertySource.parseFromProperty(property, DEFAULT_PROJECT_ID_PROVIDER);

assertThat(secretIdentifier.getProject()).isEqualTo("defaultProject");
assertThat(secretIdentifier.getSecret()).isEqualTo("the-secret");
assertThat(secretIdentifier.getSecretVersion()).isEqualTo("the-version");
}

@Test
public void testShortProperty_projectSecretIdVersion() {
String property = "gcp-secret/my-project/the-secret/2";
SecretVersionName secretIdentifier =
SecretManagerPropertySource.parseFromProperty(property, DEFAULT_PROJECT_ID_PROVIDER);

assertThat(secretIdentifier.getProject()).isEqualTo("my-project");
assertThat(secretIdentifier.getSecret()).isEqualTo("the-secret");
assertThat(secretIdentifier.getSecretVersion()).isEqualTo("2");
}

@Test
public void testLongProperty_projectSecret() {
String property = "gcp-secret/projects/my-project/secrets/the-secret";
SecretVersionName secretIdentifier =
SecretManagerPropertySource.parseFromProperty(property, DEFAULT_PROJECT_ID_PROVIDER);

assertThat(secretIdentifier.getProject()).isEqualTo("my-project");
assertThat(secretIdentifier.getSecret()).isEqualTo("the-secret");
assertThat(secretIdentifier.getSecretVersion()).isEqualTo("latest");
}

@Test
public void testLongProperty_projectSecretVersion() {
String property = "gcp-secret/projects/my-project/secrets/the-secret/versions/3";
SecretVersionName secretIdentifier =
SecretManagerPropertySource.parseFromProperty(property, DEFAULT_PROJECT_ID_PROVIDER);

assertThat(secretIdentifier.getProject()).isEqualTo("my-project");
assertThat(secretIdentifier.getSecret()).isEqualTo("the-secret");
assertThat(secretIdentifier.getSecretVersion()).isEqualTo("3");
}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Missing a test for getSecretPayload.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We'll cover this directly in your configuration tests.

I don't think it's worth making it non-private to write tests for it unlike the parseProperty method which had a lot more code.

}
Loading