Skip to content

Commit

Permalink
feat: New --private-key-file flag when creating a resource instance (#…
Browse files Browse the repository at this point in the history 
…670)

* allow for using a private key file flag for SSH tunnels or Snowflake
Co-authored-by: Jessy Jordan <[email protected]>
  • Loading branch information
@janelletavares
janelletavares authored Apr 3, 2023
1 parent ed6e780 commit 05bda8b
Show file tree
Hide file tree
Showing 6 changed files with 389 additions and 121 deletions.
119 changes: 89 additions & 30 deletions cmd/meroxa/root/resources/create.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"context"
"encoding/json"
"fmt"
"os"

"github.com/google/uuid"
"github.com/meroxa/cli/cmd/meroxa/builder"
Expand Down Expand Up @@ -49,15 +50,16 @@ type Create struct {
Environment string `long:"env" usage:"environment (name or UUID) where resource will be created"`

// credentials
Username string `long:"username" short:"" usage:"username"`
Password string `long:"password" short:"" usage:"password"`
CaCert string `long:"ca-cert" short:"" usage:"trusted certificates for verifying resource"`
ClientCert string `long:"client-cert" short:"" usage:"client certificate for authenticating to the resource"`
ClientKey string `long:"client-key" short:"" usage:"client private key for authenticating to the resource"`
SSL bool `long:"ssl" short:"" usage:"use SSL"`
SSHURL string `long:"ssh-url" short:"" usage:"SSH tunneling address"`
SSHPrivateKey string `long:"ssh-private-key" short:"" usage:"SSH tunneling private key"`
Token string `long:"token" short:"" usage:"API Token"`
Username string `long:"username" short:"" usage:"username"`
Password string `long:"password" short:"" usage:"password"`
CaCert string `long:"ca-cert" short:"" usage:"trusted certificates for verifying resource"`
ClientCert string `long:"client-cert" short:"" usage:"client certificate for authenticating to the resource"`
ClientKey string `long:"client-key" short:"" usage:"client private key for authenticating to the resource"`
SSL bool `long:"ssl" short:"" usage:"use SSL"`
SSHURL string `long:"ssh-url" short:"" usage:"SSH tunneling address"`
SSHPrivateKey string `long:"ssh-private-key" short:"" usage:"SSH tunneling private key"`
PrivateKeyFile string `long:"private-key-file" short:"" usage:"path to private key file"`
Token string `long:"token" short:"" usage:"API Token"`
}
}

Expand All @@ -82,41 +84,72 @@ func (c *Create) Docs() builder.Docs {

// TODO: Provide example with `--env` once it's not behind a feature flag
Example: `
$ meroxa resources create store --type postgres -u "$DATABASE_URL" --metadata '{"logical_replication":"true"}'
$ meroxa resources create datalake --type s3 -u "s3://$AWS_ACCESS_KEY_ID:$AWS_ACCESS_KEY_SECRET@us-east-1/meroxa-demos"
$ meroxa resources create warehouse --type redshift -u "$REDSHIFT_URL"
$ meroxa resources create slack --type url -u "$WEBHOOK_URL"
$ meroxa resource create mysqldb \
--type mysql \
--url "mysql://$MYSQL_USER:$MYSQL_PASS@$MYSQL_URL:$MYSQL_PORT/$MYSQL_DB"
$ meroxa resource create mybigquery \
--type bigquery \
-u "bigquery://$GCP_PROJECT_ID/$GCP_DATASET_NAME" \
--client-key "$(cat $GCP_SERVICE_ACCOUNT_JSON_FILE)"
$ meroxa resource create mongo \
--type mongodb \
-u "mongodb://$MONGO_USER:$MONGO_PASS@$MONGO_URL:$MONGO_PORT"
$ meroxa resource create sourcedb \
--type confluentcloud \
--url kafka+sasl+ssl://$API_KEY:$API_SECRET@<$BOOTSTRAP_SERVER>?sasl_mechanism=plain
$ meroxa resource create meteor \
--type cosmosdb \
--url cosmosdb://user:[email protected]:443/pluto
$ meroxa resource create elasticsearch \
--type elasticsearch \
-u "https://$ES_USER:$ES_PASS@$ES_URL:$ES_PORT" \
--metadata '{"index.prefix": "$ES_INDEX","incrementing.field.name": "$ES_INCREMENTING_FIELD"}'
$ meroxa resource create mybigquery \
--type bigquery \
-u "bigquery://$GCP_PROJECT_ID/$GCP_DATASET_NAME" \
--client-key "$(cat $GCP_SERVICE_ACCOUNT_JSON_FILE)"
$ meroxa resource create sourcedb \
--type kafka \
--url kafka+sasl+ssl://$KAFKA_USER:$KAFKA_PASS@<$BOOTSTRAP_SERVER>?sasl_mechanism=plain
$ meroxa resource create mongo \
--type mongodb \
-u "mongodb://$MONGO_USER:$MONGO_PASS@$MONGO_URL:$MONGO_PORT"
$ meroxa resource create mysqldb \
--type mysql \
--url "mysql://$MYSQL_USER:$MYSQL_PASS@$MYSQL_URL:$MYSQL_PORT/$MYSQL_DB"
$ meroxa resource create workspace \
--type notion \
--token AbCdEfG123456
$ meroxa resource create workspace \
--type oracledb \
--url oracle://user:[email protected]:1521/database
$ meroxa resources create store \
--type postgres \
-u "$DATABASE_URL" \
--metadata '{"logical_replication":"true"}'
$ meroxa resources create warehouse \
--type redshift \
-u "$REDSHIFT_URL" \
--ssh-url ssh://user@[email protected]:22 \
--private-key-file ~/.ssh/my-key
$ meroxa resources create datalake \
--type s3 \
-u "s3://$AWS_ACCESS_KEY_ID:$AWS_ACCESS_KEY_SECRET@us-east-1/meroxa-demos"
$ meroxa resource create snowflake \
--type snowflakedb \
-u "snowflake://$SNOWFLAKE_URL/meroxa_db/stream_data" \
--username meroxa_user \
--password $SNOWFLAKE_PRIVATE_KEY
--private-key-file /Users/me/.ssh/snowflake_ed25519
$ meroxa resource create sourcedb \
--type kafka \
--url kafka+sasl+ssl://$KAFKA_USER:$KAFKA_PASS@<$BOOTSTRAP_SERVER>?sasl_mechanism=plain
$ meroxa resource create hr \
--type sqlserver \
--url "sqlserver://$MSSQL_USER:$MSSQL_PASS@$MSSQL_URL:$MSSQL_PORT/$MSSQL_DB"
$ meroxa resource create sourcedb \
--type confluentcloud \
--url kafka+sasl+ssl://$API_KEY:$API_SECRET@<$BOOTSTRAP_SERVER>?sasl_mechanism=plain`,
$ meroxa resources create slack \
--type url \
-u "$WEBHOOK_URL"`,
}
}

Expand Down Expand Up @@ -177,6 +210,10 @@ func (c *Create) Execute(ctx context.Context) error {
env = string(meroxa.EnvironmentTypeCommon)
}

if err := c.handlePrivateKeyFlags(ctx); err != nil {
return err
}

if c.hasCredentials() {
input.Credentials = &meroxa.Credentials{
Username: c.flags.Username,
Expand Down Expand Up @@ -233,3 +270,25 @@ func (c *Create) hasCredentials() bool {
c.flags.Token != "" ||
c.flags.SSL
}

func (c *Create) handlePrivateKeyFlags(ctx context.Context) error {
path := c.flags.PrivateKeyFile
if path != "" && c.flags.SSHPrivateKey == "" {
bytes, err := os.ReadFile(path)
if err != nil {
return fmt.Errorf("could not find SSH private key at %q."+
" Try a different path", path)
}
key := string(bytes)
c.flags.SSHPrivateKey = key

if c.flags.Type == string(meroxa.ResourceTypeSnowflake) {
if c.flags.Password != "" {
c.logger.Warnf(ctx, "ignoring value of --ssh-private-key-file (%s) in favor of value of --password", c.flags.PrivateKeyFile)
} else {
c.flags.Password = key
}
}
}
return nil
}
108 changes: 108 additions & 0 deletions cmd/meroxa/root/resources/create_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,15 @@ import (
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"reflect"
"testing"

"github.com/golang/mock/gomock"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/meroxa/cli/cmd/meroxa/builder"
"github.com/meroxa/cli/cmd/meroxa/global"
Expand Down Expand Up @@ -59,6 +64,9 @@ func TestCreateResourceFlags(t *testing.T) {
{name: "metadata", required: false, shorthand: "m"},
{name: "env", required: false},
{name: "token", required: false},
{name: "ssh-url", required: false},
{name: "ssh-private-key", required: false},
{name: "private-key-file", required: false},
}

c := builder.BuildCobraCommand(&Create{})
Expand Down Expand Up @@ -358,3 +366,103 @@ Sign up for the Beta here: https://share.hsforms.com/1Uq6UYoL8Q6eV5QzSiyIQkAc2sm
t.Fatalf("expected output:\n%s\ngot:\n%s", wantError, gotError)
}
}

func TestCreateResourceExecutionPrivateKeyFlags(t *testing.T) {
ctx := context.Background()
logger := log.NewTestLogger()

keyVal := "super-secret"
keyFile := filepath.Join("/tmp", uuid.NewString())
err := os.WriteFile(keyFile, []byte(keyVal), 0600)
require.NoError(t, err)

tests := []struct {
name string
inputType string
inputSSHPrivateKeyFlag string
inputPasswordFlag string
inputPrivateKeyFileFlag string
expectedPassword string
expectedSSHPrivateKeyVal string
}{
{
name: "create postgres resource with SSH Tunnel --ssh-private-key",
inputType: string(meroxa.ResourceTypePostgres),
inputSSHPrivateKeyFlag: keyVal,
expectedPassword: "",
expectedSSHPrivateKeyVal: keyVal,
},
{
name: "create postgres resource with SSH Tunnel --private-key-file",
inputType: string(meroxa.ResourceTypePostgres),
inputPrivateKeyFileFlag: keyFile,
expectedPassword: "",
expectedSSHPrivateKeyVal: keyVal,
},
{
name: "create postgres resource with both SSH flags",
inputType: string(meroxa.ResourceTypePostgres),
inputPrivateKeyFileFlag: keyFile,
inputSSHPrivateKeyFlag: keyVal,
expectedPassword: "",
expectedSSHPrivateKeyVal: keyVal,
},
{
name: "create snowflake resource with --password",
inputType: string(meroxa.ResourceTypeSnowflake),
inputPasswordFlag: keyVal,
expectedPassword: keyVal,
expectedSSHPrivateKeyVal: "",
},
{
name: "create snowflake resource with --private-key-file",
inputPrivateKeyFileFlag: keyFile,
inputType: string(meroxa.ResourceTypeSnowflake),
expectedPassword: keyVal,
expectedSSHPrivateKeyVal: keyVal,
},
{
name: "create snowflake resource with both secret flags",
inputPasswordFlag: keyVal,
inputPrivateKeyFileFlag: keyFile,
inputType: string(meroxa.ResourceTypeSnowflake),
expectedPassword: keyVal,
expectedSSHPrivateKeyVal: keyVal,
},
}

for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
ctrl := gomock.NewController(t)
client := mock.NewMockClient(ctrl)

c := &Create{
client: client,
logger: logger,
}

client.
EXPECT().
CreateResource(
ctx,
gomock.Any(),
).
Return(&meroxa.Resource{}, nil)

c.args.Name = "my-resource"
c.flags.Type = tc.inputType
c.flags.URL = "anything"
c.flags.Password = tc.inputPasswordFlag
c.flags.SSHPrivateKey = tc.inputSSHPrivateKeyFlag
c.flags.PrivateKeyFile = tc.inputPrivateKeyFileFlag

err := c.Execute(ctx)
if err != nil {
t.Fatalf("not expected error, got %q", err.Error())
}

assert.Equalf(t, tc.expectedSSHPrivateKeyVal, c.flags.SSHPrivateKey, "mistach in private key flag value")
assert.Equalf(t, tc.expectedPassword, c.flags.Password, "mismatch in password flag value")
})
}
}
Loading

0 comments on commit 05bda8b

Please sign in to comment.