Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Bring the Vault provider to gcp sql parity with Vault #2012

Merged
merged 13 commits into from
Oct 2, 2023
Merged

Bring the Vault provider to gcp sql parity with Vault #2012

Show file tree
Hide file tree
Changes from 12 commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
5cf4340
add auth_type and service_account_json to mysql and postgres resources
kpcraig Sep 13, 2023
a56ad55
Merge branch 'main' into VAULT-18401/gcp-sql-parity
kpcraig Sep 13, 2023
81251c9
modernize new tests
kpcraig Sep 13, 2023
8511577
docs
kpcraig Sep 13, 2023
b2e45ac
schema parameter descriptions
kpcraig Sep 13, 2023
0197faf
resource declaration for postgres test
kpcraig Sep 13, 2023
9c95e01
separate non-cloud and cloud test envvars
kpcraig Sep 13, 2023
5da6780
test fixes for mysql
kpcraig Sep 25, 2023
425a138
grab service_account_json from terraform data since we don't get it i…
kpcraig Oct 2, 2023
f5ce3ed
thread meta parameter into data functions to support version checking
kpcraig Oct 2, 2023
274f0f3
move cloud specific params into cloud-enabled getters
kpcraig Oct 2, 2023
5a7b660
add import tests
kpcraig Oct 2, 2023
4b47080
add meta field for version checking for getters
kpcraig Oct 2, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 81 additions & 8 deletions vault/resource_database_secret_backend_connection.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ type connectionStringConfig struct {
excludeUsernameTemplate bool
includeUserPass bool
includeDisableEscaping bool
isCloud bool
}

const (
Expand Down Expand Up @@ -556,6 +557,7 @@ func getDatabaseSchema(typ schema.ValueType) schemaMap {
Elem: connectionStringResource(&connectionStringConfig{
includeUserPass: true,
includeDisableEscaping: true,
isCloud: true,
}),
MaxItems: 1,
ConflictsWith: util.CalculateConflictsWith(dbEnginePostgres.Name(), dbEngineTypes),
Expand Down Expand Up @@ -752,6 +754,20 @@ func connectionStringResource(config *connectionStringConfig) *schema.Resource {
}
}

if config.isCloud {
res.Schema["auth_type"] = &schema.Schema{
Copy link
Contributor

@Zlaticanin Zlaticanin Sep 28, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Small nit, but should we maybe use constants for these fields? We can add them to the internal/consts.go where we keep most of them?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we could - it didn't match with the rest of the file so i didn't commit.

Type: schema.TypeString,
Optional: true,
Description: "Specify alternative authorization type. (Only 'gcp_iam' is valid currently)",
}
res.Schema["service_account_json"] = &schema.Schema{
kpcraig marked this conversation as resolved.
Show resolved Hide resolved
Type: schema.TypeString,
Optional: true,
Description: "A JSON encoded credential for use with IAM authorization",
Sensitive: true,
}
}

if !config.excludeUsernameTemplate {
res.Schema["username_template"] = &schema.Schema{
Type: schema.TypeString,
Expand All @@ -774,6 +790,7 @@ func connectionStringResource(config *connectionStringConfig) *schema.Resource {
func mysqlConnectionStringResource() *schema.Resource {
r := connectionStringResource(&connectionStringConfig{
includeUserPass: true,
isCloud: true,
})
r.Schema["tls_certificate_key"] = &schema.Schema{
Type: schema.TypeString,
Expand Down Expand Up @@ -850,7 +867,7 @@ func getDBEngineFromResp(engines []*dbEngine, r *api.Secret) (*dbEngine, error)
return nil, fmt.Errorf("no supported database engines found for plugin %q", pluginName)
}

func getDatabaseAPIDataForEngine(engine *dbEngine, idx int, d *schema.ResourceData) (map[string]interface{}, error) {
func getDatabaseAPIDataForEngine(engine *dbEngine, idx int, d *schema.ResourceData, meta interface{}) (map[string]interface{}, error) {
prefix := engine.ResourcePrefix(idx)
data := map[string]interface{}{}

Expand All @@ -877,7 +894,7 @@ func getDatabaseAPIDataForEngine(engine *dbEngine, idx int, d *schema.ResourceDa
case dbEngineMSSQL:
setMSSQLDatabaseConnectionData(d, prefix, data)
case dbEngineMySQL:
setMySQLDatabaseConnectionData(d, prefix, data)
setMySQLDatabaseConnectionData(d, prefix, data, meta)
case dbEngineMySQLRDS:
setDatabaseConnectionDataWithUserPass(d, prefix, data)
case dbEngineMySQLAurora:
Expand All @@ -887,7 +904,7 @@ func getDatabaseAPIDataForEngine(engine *dbEngine, idx int, d *schema.ResourceDa
case dbEngineOracle:
setDatabaseConnectionDataWithUserPass(d, prefix, data)
case dbEnginePostgres:
setDatabaseConnectionDataWithDisableEscaping(d, prefix, data)
setPostgresDatabaseConnectionData(d, prefix, data, meta)
case dbEngineElasticSearch:
setElasticsearchDatabaseConnectionData(d, prefix, data)
case dbEngineRedis:
Expand Down Expand Up @@ -1012,6 +1029,7 @@ func getConnectionDetailsFromResponse(d *schema.ResourceData, prefix string, res
result["username_template"] = v.(string)
}
}

return result
}

Expand All @@ -1033,6 +1051,29 @@ func getMSSQLConnectionDetailsFromResponse(d *schema.ResourceData, prefix string
return result, nil
}

func getPostgresConnectionDetailsFromResponse(d *schema.ResourceData, prefix string, resp *api.Secret) map[string]interface{} {
result := getConnectionDetailsFromResponseWithDisableEscaping(d, prefix, resp)
details := resp.Data["connection_details"]
data, ok := details.(map[string]interface{})
if !ok {
return nil
}

// cloud specific
kpcraig marked this conversation as resolved.
Show resolved Hide resolved
if v, ok := data["auth_type"]; ok {
result["auth_type"] = v.(string)
}
if v, ok := d.GetOk(prefix + "service_account_json"); ok {
result["service_account_json"] = v.(string)
} else {
if v, ok := data["service_account_json"]; ok {
result["service_account_json"] = v.(string)
}
}

return result
}

func getConnectionDetailsFromResponseWithDisableEscaping(d *schema.ResourceData, prefix string, resp *api.Secret) map[string]interface{} {
result := getConnectionDetailsFromResponseWithUserPass(d, prefix, resp)
if result == nil {
Expand Down Expand Up @@ -1068,6 +1109,19 @@ func getMySQLConnectionDetailsFromResponse(d *schema.ResourceData, prefix string
result["tls_ca"] = v.(string)
}
}

// cloud specific
if v, ok := data["auth_type"]; ok {
kpcraig marked this conversation as resolved.
Show resolved Hide resolved
result["auth_type"] = v.(string)
}
if v, ok := d.GetOk(prefix + "service_account_json"); ok {
result["service_account_json"] = v.(string)
} else {
if v, ok := data["service_account_json"]; ok {
result["service_account_json"] = v.(string)
}
}

return result
}

Expand Down Expand Up @@ -1351,6 +1405,18 @@ func setDatabaseConnectionData(d *schema.ResourceData, prefix string, data map[s
}
}

func setCloudDatabaseConnectionData(d *schema.ResourceData, prefix string, data map[string]interface{}, meta interface{}) {
if !provider.IsAPISupported(meta, provider.VaultVersion115) {
return
}
if v, ok := d.GetOk(prefix + "auth_type"); ok {
data["auth_type"] = v.(string)
}
if v, ok := d.GetOk(prefix + "service_account_json"); ok {
data["service_account_json"] = v.(string)
}
}

func setMSSQLDatabaseConnectionData(d *schema.ResourceData, prefix string, data map[string]interface{}) {
setDatabaseConnectionDataWithDisableEscaping(d, prefix, data)
if v, ok := d.GetOk(prefix + "contained_db"); ok {
Expand All @@ -1362,8 +1428,9 @@ func setMSSQLDatabaseConnectionData(d *schema.ResourceData, prefix string, data
}
}

func setMySQLDatabaseConnectionData(d *schema.ResourceData, prefix string, data map[string]interface{}) {
func setMySQLDatabaseConnectionData(d *schema.ResourceData, prefix string, data map[string]interface{}, meta interface{}) {
setDatabaseConnectionDataWithUserPass(d, prefix, data)
setCloudDatabaseConnectionData(d, prefix, data, meta)
if v, ok := d.GetOk(prefix + "tls_certificate_key"); ok {
data["tls_certificate_key"] = v.(string)
}
Expand All @@ -1372,6 +1439,12 @@ func setMySQLDatabaseConnectionData(d *schema.ResourceData, prefix string, data
}
}

func setPostgresDatabaseConnectionData(d *schema.ResourceData, prefix string, data map[string]interface{}, meta interface{}) {
setDatabaseConnectionDataWithDisableEscaping(d, prefix, data)
setCloudDatabaseConnectionData(d, prefix, data, meta)

}

func setRedisDatabaseConnectionData(d *schema.ResourceData, prefix string, data map[string]interface{}) {
if v, ok := d.GetOk(prefix + "host"); ok {
data["host"] = v.(string)
Expand Down Expand Up @@ -1577,7 +1650,7 @@ func databaseSecretBackendConnectionCreateOrUpdate(
path := databaseSecretBackendConnectionPath(
d.Get("backend").(string), d.Get("name").(string))
if err := writeDatabaseSecretConfig(
d, client, engine, 0, false, path); err != nil {
d, client, engine, 0, false, path, meta); err != nil {
return err
}

Expand All @@ -1588,9 +1661,9 @@ func databaseSecretBackendConnectionCreateOrUpdate(
}

func writeDatabaseSecretConfig(d *schema.ResourceData, client *api.Client,
engine *dbEngine, idx int, unifiedSchema bool, path string,
engine *dbEngine, idx int, unifiedSchema bool, path string, meta interface{},
) error {
data, err := getDatabaseAPIDataForEngine(engine, idx, d)
data, err := getDatabaseAPIDataForEngine(engine, idx, d, meta)
if err != nil {
return err
}
Expand Down Expand Up @@ -1808,7 +1881,7 @@ func getDBConnectionConfig(d *schema.ResourceData, engine *dbEngine, idx int,
case dbEngineOracle:
result = getConnectionDetailsFromResponseWithUserPass(d, prefix, resp)
case dbEnginePostgres:
result = getConnectionDetailsFromResponseWithDisableEscaping(d, prefix, resp)
result = getPostgresConnectionDetailsFromResponse(d, prefix, resp)
case dbEngineElasticSearch:
result = getElasticsearchConnectionDetailsFromResponse(d, prefix, resp)
case dbEngineSnowflake:
Expand Down
124 changes: 124 additions & 0 deletions vault/resource_database_secret_backend_connection_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -442,6 +442,47 @@ func TestAccDatabaseSecretBackendConnection_mssql(t *testing.T) {
})
}

func TestAccDatabaseSecretBackendConnection_mysql_cloud(t *testing.T) {
// wanted this to be the included with the following test, but the env-var check is different
values := testutil.SkipTestEnvUnset(t, "MYSQL_CLOUD_CONNECTION_URL", "MYSQL_CLOUD_CONNECTION_SERVICE_ACCOUNT_JSON")
connURL, saJSON := values[0], values[1]

backend := acctest.RandomWithPrefix("tf-test-db")
name := acctest.RandomWithPrefix("db")
resource.Test(t, resource.TestCase{
ProviderFactories: providerFactories,
PreCheck: func() {
testutil.TestAccPreCheck(t)
SkipIfAPIVersionLT(t, testProvider.Meta(), provider.VaultVersion115)
},
CheckDestroy: testAccDatabaseSecretBackendConnectionCheckDestroy,
Steps: []resource.TestStep{
{
Config: testAccDatabaseSecretBackendConnectionConfig_mysql_cloud(name, backend, connURL, "gcp_iam", saJSON),
Check: testComposeCheckFuncCommonDatabaseSecretBackend(name, backend, dbEngineMySQL.DefaultPluginName(),
resource.TestCheckResourceAttr(testDefaultDatabaseSecretBackendResource, "allowed_roles.#", "2"),
resource.TestCheckResourceAttr(testDefaultDatabaseSecretBackendResource, "allowed_roles.0", "dev"),
resource.TestCheckResourceAttr(testDefaultDatabaseSecretBackendResource, "allowed_roles.1", "prod"),
resource.TestCheckResourceAttr(testDefaultDatabaseSecretBackendResource, "root_rotation_statements.#", "1"),
resource.TestCheckResourceAttr(testDefaultDatabaseSecretBackendResource, "root_rotation_statements.0", "FOOBAR"),
resource.TestCheckResourceAttr(testDefaultDatabaseSecretBackendResource, "verify_connection", "true"),
resource.TestCheckResourceAttr(testDefaultDatabaseSecretBackendResource, "mysql.0.connection_url", connURL),
resource.TestCheckResourceAttr(testDefaultDatabaseSecretBackendResource, "mysql.0.auth_type", "gcp_iam"),
resource.TestCheckResourceAttr(testDefaultDatabaseSecretBackendResource, "mysql.0.max_open_connections", "2"),
resource.TestCheckResourceAttr(testDefaultDatabaseSecretBackendResource, "mysql.0.max_idle_connections", "0"),
resource.TestCheckResourceAttr(testDefaultDatabaseSecretBackendResource, "mysql.0.max_connection_lifetime", "0"),
),
},
{
ResourceName: testDefaultDatabaseSecretBackendResource,
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"verify_connection", "mysql.0.service_account_json"},
},
},
})
}

func TestAccDatabaseSecretBackendConnection_mysql(t *testing.T) {
MaybeSkipDBTests(t, dbEngineMySQL)

Expand Down Expand Up @@ -776,6 +817,45 @@ func TestAccDatabaseSecretBackendConnection_postgresql(t *testing.T) {
})
}

func TestAccDatabaseSecretBackendConnection_postgresql_cloud(t *testing.T) {
// wanted this to be the included with the following test, but the env-var check is different
values := testutil.SkipTestEnvUnset(t, "POSTGRES_CLOUD_URL", "POSTGRES_CLOUD_SERVICE_ACCOUNT_JSON")
connURL, saJSON := values[0], values[1]

backend := acctest.RandomWithPrefix("tf-test-db")
name := acctest.RandomWithPrefix("db")
resource.Test(t, resource.TestCase{
ProviderFactories: providerFactories,
PreCheck: func() {
testutil.TestAccPreCheck(t)
SkipIfAPIVersionLT(t, testProvider.Meta(), provider.VaultVersion115)
},
CheckDestroy: testAccDatabaseSecretBackendConnectionCheckDestroy,
Steps: []resource.TestStep{
{
Config: testAccDatabaseSecretBackendConnectionConfig_postgres_cloud(name, backend, connURL, "gcp_iam", saJSON),
Check: testComposeCheckFuncCommonDatabaseSecretBackend(name, backend, dbEngineMySQL.DefaultPluginName(),
resource.TestCheckResourceAttr(testDefaultDatabaseSecretBackendResource, "allowed_roles.#", "2"),
resource.TestCheckResourceAttr(testDefaultDatabaseSecretBackendResource, "allowed_roles.0", "dev"),
resource.TestCheckResourceAttr(testDefaultDatabaseSecretBackendResource, "allowed_roles.1", "prod"),
resource.TestCheckResourceAttr(testDefaultDatabaseSecretBackendResource, "root_rotation_statements.#", "1"),
resource.TestCheckResourceAttr(testDefaultDatabaseSecretBackendResource, "root_rotation_statements.0", "FOOBAR"),
resource.TestCheckResourceAttr(testDefaultDatabaseSecretBackendResource, "verify_connection", "true"),
resource.TestCheckResourceAttr(testDefaultDatabaseSecretBackendResource, "postgresql.0.connection_url", connURL),
resource.TestCheckResourceAttr(testDefaultDatabaseSecretBackendResource, "postgresql.0.disable_escaping", "true"),
resource.TestCheckResourceAttr(testDefaultDatabaseSecretBackendResource, "postgresql.0.auth_type", "gcp_iam"),
),
},
{
ResourceName: testDefaultDatabaseSecretBackendResource,
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"verify_connection", "postgres.0.service_account_json"},
},
},
})
vinay-gopalan marked this conversation as resolved.
Show resolved Hide resolved
}

func TestAccDatabaseSecretBackendConnection_elasticsearch(t *testing.T) {
MaybeSkipDBTests(t, dbEngineElasticSearch)

Expand Down Expand Up @@ -1519,6 +1599,28 @@ resource "vault_database_secret_backend_connection" "test" {
`, path, name, connURL, username, password)
}

func testAccDatabaseSecretBackendConnectionConfig_mysql_cloud(name, path, connURL, authType, serviceAccountJSON string) string {
return fmt.Sprintf(`
resource "vault_mount" "db" {
path = "%s"
type = "database"
}

resource "vault_database_secret_backend_connection" "test" {
backend = vault_mount.db.path
name = "%s"
allowed_roles = ["dev", "prod"]
root_rotation_statements = ["FOOBAR"]

mysql {
connection_url = "%s"
auth_type = "%s"
service_account_json = "%s"
}
}
`, path, name, connURL, authType, serviceAccountJSON)
}

func testAccDatabaseSecretBackendConnectionConfig_postgresql(name, path, userTempl, username, password, openConn, idleConn, maxConnLifetime string, parsedURL *url.URL) string {
return fmt.Sprintf(`
resource "vault_mount" "db" {
Expand Down Expand Up @@ -1566,6 +1668,28 @@ resource "vault_database_secret_backend_connection" "test" {
`, path, name, parsedURL.String())
}

func testAccDatabaseSecretBackendConnectionConfig_postgres_cloud(name, path, connURL, authType, serviceAccountJSON string) string {
return fmt.Sprintf(`
resource "vault_mount" "db" {
path = "%s"
type = "database"
}

resource "vault_database_secret_backend_connection" "test" {
backend = vault_mount.db.path
name = "%s"
allowed_roles = ["dev", "prod"]
root_rotation_statements = ["FOOBAR"]

postgresql {
connection_url = "%s"
auth_type = "%s"
service_account_json = "%s"
}
}
`, path, name, connURL, authType, serviceAccountJSON)
}

func testAccDatabaseSecretBackendConnectionConfig_snowflake(name, path, url, username, password, userTempl string) string {
return fmt.Sprintf(`
resource "vault_mount" "db" {
Expand Down
2 changes: 1 addition & 1 deletion vault/resource_database_secrets_mount.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -246,7 +246,7 @@ func databaseSecretsMountCreateOrUpdate(d *schema.ResourceData, meta interface{}
return fmt.Errorf("duplicate name %q for engine %#v", name, engine)
}
seen[name] = true
if err := writeDatabaseSecretConfig(d, client, engine, i, true, path); err != nil {
if err := writeDatabaseSecretConfig(d, client, engine, i, true, path, meta); err != nil {
return err
}
count++
Expand Down
8 changes: 8 additions & 0 deletions website/docs/r/database_secret_backend_connection.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -293,6 +293,10 @@ See the [Vault

* `password` - (Optional) The root credential password used in the connection URL.

* `auth_type` - (Optional) Enable IAM authentication to a Google Cloud instance when set to `gcp_iam`

* `service_account_json` - (Optional) JSON encoding of an IAM access key. Requires `auth_type` to be `gcp_iam`.

* `tls_certificate_key` - (Optional) x509 certificate for connecting to the database. This must be a PEM encoded version of the private key and the certificate combined.

* `tls_ca` - (Optional) x509 CA file for validating the certificate presented by the MySQL server. Must be PEM encoded.
Expand Down Expand Up @@ -321,6 +325,10 @@ See the [Vault

* `password` - (Optional) The root credential password used in the connection URL.

* `auth_type` - (Optional) Enable IAM authentication to a Google Cloud instance when set to `gcp_iam`

* `service_account_json` - (Optional) JSON encoding of an IAM access key. Requires `auth_type` to be `gcp_iam`.

* `disable_escaping` - (Optional) Disable special character escaping in username and password.

* `username_template` - (Optional) For Vault v1.7+. The template to use for username generation.
Expand Down