diff --git a/cmd/security-bootstrapper/Dockerfile b/cmd/security-bootstrapper/Dockerfile index ce3b925085..816dbf1179 100644 --- a/cmd/security-bootstrapper/Dockerfile +++ b/cmd/security-bootstrapper/Dockerfile @@ -60,6 +60,9 @@ COPY --from=builder /edgex-go/cmd/security-bootstrapper/res/configuration.toml . # needed for bootstrapping Redis db COPY --from=builder /edgex-go/cmd/security-bootstrapper/res-bootstrap-redis/configuration.toml ${BOOTSTRAP_REDIS_DIR}/res/ +# copy Consul ACL related configs +COPY --from=builder /edgex-go/cmd/security-bootstrapper/consul-acl/ ${SECURITY_INIT_DIR}/consul-bootstrapper/ + # Expose the file directory as a volume since there's long-running state VOLUME ${SECURITY_INIT_DIR} diff --git a/cmd/security-bootstrapper/consul-acl/config_consul_acl.json b/cmd/security-bootstrapper/consul-acl/config_consul_acl.json new file mode 100644 index 0000000000..4eb9858a08 --- /dev/null +++ b/cmd/security-bootstrapper/consul-acl/config_consul_acl.json @@ -0,0 +1,7 @@ +{ + "acl": { + "enabled": true, + "default_policy": "allow", + "enable_token_persistence": true + } +} diff --git a/cmd/security-bootstrapper/entrypoint-scripts/consul_wait_install.sh b/cmd/security-bootstrapper/entrypoint-scripts/consul_wait_install.sh index b713786d07..5fc58b5a01 100755 --- a/cmd/security-bootstrapper/entrypoint-scripts/consul_wait_install.sh +++ b/cmd/security-bootstrapper/entrypoint-scripts/consul_wait_install.sh @@ -52,7 +52,10 @@ done DEFAULT_CONSUL_LOCAL_CONFIG=' { "enable_local_script_checks": true, - "disable_update_check": true + "disable_update_check": true, + "ports": { + "dns": -1 + } } ' @@ -63,15 +66,49 @@ export CONSUL_LOCAL_CONFIG echo "$(date) CONSUL_LOCAL_CONFIG: ${CONSUL_LOCAL_CONFIG}" -echo "$(date) Starting edgex-consul..." -exec docker-entrypoint.sh agent -ui -bootstrap -server -client 0.0.0.0 & +echo "$(date) ENABLE_REGISTRY_ACL = ${ENABLE_REGISTRY_ACL}" -# wait for the consul port -echo "$(date) Executing waitFor on Consul with waiting on its own port \ - tcp://${STAGEGATE_REGISTRY_HOST}:${STAGEGATE_REGISTRY_PORT}" -/edgex-init/security-bootstrapper --confdir=/edgex-init/res waitFor \ - -uri tcp://"${STAGEGATE_REGISTRY_HOST}":"${STAGEGATE_REGISTRY_PORT}" \ - -timeout "${STAGEGATE_WAITFOR_TIMEOUT}" +if [ "${ENABLE_REGISTRY_ACL}" == "true" ]; then + echo "$(date) Starting edgex-core-consul with ACL enabled ..." + docker-entrypoint.sh agent \ + -ui \ + -bootstrap \ + -server \ + -config-file=/edgex-init/consul-bootstrapper/config_consul_acl.json \ + -client 0.0.0.0 & + # wait for the secretstore tokens ready as we need the token for bootstrapping + echo "$(date) Executing waitFor on Consul with waiting on TokensReadyPort \ + tcp://${STAGEGATE_SECRETSTORESETUP_HOST}:${STAGEGATE_SECRETSTORESETUP_TOKENS_READYPORT}" + /edgex-init/security-bootstrapper --confdir=/edgex-init/res waitFor \ + -uri tcp://"${STAGEGATE_SECRETSTORESETUP_HOST}":"${STAGEGATE_SECRETSTORESETUP_TOKENS_READYPORT}" \ + -timeout "${STAGEGATE_WAITFOR_TIMEOUT}" + + # we don't want to exit out the whole Consul process when ACL bootstrapping failed, just that + # Consul won't have ACL to be used + set +e + # call setupRegistryACL bootstrapping command, containing both ACL bootstrapping and re-configure consul access steps + /edgex-init/security-bootstrapper --confdir=/edgex-init/res setupRegistryACL + setupACL_code=$? + if [ "${setupACL_code}" -ne 0 ]; then + echo "$(date) failed to set up Consul ACL" + fi + set -e + # no need to wait for Consul's port since it is in ready state after all ACL stuff +else + echo "$(date) Starting edgex-core-consul with ACL disabled ..." + docker-entrypoint.sh agent \ + -ui \ + -bootstrap \ + -server \ + -client 0.0.0.0 & + # wait for the consul port + # this waitFor is not necessary in the other ACL case, as it is already in the ready state + echo "$(date) Executing waitFor on Consul with waiting on its own port \ + tcp://${STAGEGATE_REGISTRY_HOST}:${STAGEGATE_REGISTRY_PORT}" + /edgex-init/security-bootstrapper --confdir=/edgex-init/res waitFor \ + -uri tcp://"${STAGEGATE_REGISTRY_HOST}":"${STAGEGATE_REGISTRY_PORT}" \ + -timeout "${STAGEGATE_WAITFOR_TIMEOUT}" +fi # Signal that Consul is ready for services blocked waiting on Consul /edgex-init/security-bootstrapper --confdir=/edgex-init/res listenTcp \ diff --git a/cmd/security-bootstrapper/res/configuration.toml b/cmd/security-bootstrapper/res/configuration.toml index 819e310be3..f1eef43f69 100644 --- a/cmd/security-bootstrapper/res/configuration.toml +++ b/cmd/security-bootstrapper/res/configuration.toml @@ -20,6 +20,14 @@ LogLevel = 'INFO' Host = "edgex-core-consul" Port = 8500 ReadyPort = 54324 + [StageGate.Registry.ACL] + Protocol = "http" + # this is the filepath for the generated Consul management token from ACL bootstrap + BootstrapTokenPath = "/tmp/edgex/secrets/edgex-consul/admin/bootstrap_token.json" + # this is the filepath for the Vault token created from secretstore-setup + SecretsAdminTokenPath = "/tmp/edgex/secrets/edgex-consul/admin/token.json" + # this is the filepath for the sentinel file to indicate the registry ACL is set up successfully + SentinelFilePath = "/edgex-init/consul-bootstrapper/consul_acl_done" [StageGate.KongDb] Host = "kong-db" Port = 5432 @@ -27,3 +35,13 @@ LogLevel = 'INFO' [StageGate.WaitFor] Timeout = "10s" RetryInterval = "1s" + +# this configuration is just part of the whole go-mod-bootstrap's secret store to have +# protocol, host, and port of secretstore using in the security-bootstrapper +# we are not really using the secret store provider from go-mod-bootstrap in the code +# also this is needed as snap does not have those environments from env-files +[SecretStore] +Type = 'vault' +Protocol = 'http' +Host = 'localhost' +Port = 8200 diff --git a/cmd/security-secretstore-setup/res-file-token-provider/configuration.toml b/cmd/security-secretstore-setup/res-file-token-provider/configuration.toml index ac532a1108..7ea1a65be4 100644 --- a/cmd/security-secretstore-setup/res-file-token-provider/configuration.toml +++ b/cmd/security-secretstore-setup/res-file-token-provider/configuration.toml @@ -3,7 +3,7 @@ LogLevel = 'DEBUG' [SecretStore] Type = "vault" Protocol = "http" -Host = "edgex-vault" +Host = "localhost" Port = 8200 ServerName = "" CaFilePath = "" diff --git a/cmd/security-secretstore-setup/res/configuration.toml b/cmd/security-secretstore-setup/res/configuration.toml index f9dc1f56f9..56cf2697dc 100644 --- a/cmd/security-secretstore-setup/res/configuration.toml +++ b/cmd/security-secretstore-setup/res/configuration.toml @@ -21,7 +21,7 @@ LogLevel = 'DEBUG' [SecretStore] Type = "vault" Protocol = "http" -Host = "edgex-vault" +Host = "localhost" Port = 8200 CertPath = "" CaFilePath = "" diff --git a/internal/security/bootstrapper/command/cmd_dispatcher.go b/internal/security/bootstrapper/command/cmd_dispatcher.go index df1b69bdc0..83fb5d889f 100644 --- a/internal/security/bootstrapper/command/cmd_dispatcher.go +++ b/internal/security/bootstrapper/command/cmd_dispatcher.go @@ -25,6 +25,7 @@ import ( "github.com/edgexfoundry/edgex-go/internal/security/bootstrapper/command/gethttpstatus" "github.com/edgexfoundry/edgex-go/internal/security/bootstrapper/command/listen" "github.com/edgexfoundry/edgex-go/internal/security/bootstrapper/command/ping" + "github.com/edgexfoundry/edgex-go/internal/security/bootstrapper/command/setupacl" "github.com/edgexfoundry/edgex-go/internal/security/bootstrapper/command/waitfor" "github.com/edgexfoundry/edgex-go/internal/security/bootstrapper/config" "github.com/edgexfoundry/edgex-go/internal/security/bootstrapper/interfaces" @@ -44,8 +45,8 @@ func NewCommand( var err error if len(args) < 1 { - return nil, fmt.Errorf("subcommand required (%s, %s, %s, %s, %s, %s)", gate.CommandName, listen.CommandName, - ping.CommandName, gethttpstatus.CommandName, genpassword.CommandName, waitfor.CommandName) + return nil, fmt.Errorf("subcommand required (%s, %s, %s, %s, %s, %s, %s)", gate.CommandName, listen.CommandName, + ping.CommandName, gethttpstatus.CommandName, genpassword.CommandName, waitfor.CommandName, setupacl.CommandName) } commandName := args[0] @@ -63,6 +64,8 @@ func NewCommand( command, err = genpassword.NewCommand(ctx, wg, lc, configuration, args[1:]) case waitfor.CommandName: command, err = waitfor.NewCommand(ctx, wg, lc, configuration, args[1:]) + case setupacl.CommandName: + command, err = setupacl.NewCommand(ctx, wg, lc, configuration, args[1:]) default: command = nil err = fmt.Errorf("unsupported command %s", commandName) diff --git a/internal/security/bootstrapper/command/cmd_dispatcher_test.go b/internal/security/bootstrapper/command/cmd_dispatcher_test.go index d497c831ed..6e18c7e2ee 100644 --- a/internal/security/bootstrapper/command/cmd_dispatcher_test.go +++ b/internal/security/bootstrapper/command/cmd_dispatcher_test.go @@ -54,6 +54,7 @@ func TestNewCommand(t *testing.T) { {"Good: genPassword command", []string{"genPassword"}, "genPassword", false}, {"Good: getHttpStatus command", []string{"getHttpStatus", "--url=http://localhost:55555"}, "getHttpStatus", false}, {"Good: waitFor command", []string{"waitFor", "--uri=http://localhost:55555"}, "waitFor", false}, + {"Good: setupRegistryACL command", []string{"setupRegistryACL"}, "setupRegistryACL", false}, {"Bad: unknown command", []string{"unknown"}, "", true}, {"Bad: empty command", []string{}, "", true}, {"Bad: listenTcp command missing required --port", []string{"listenTcp"}, "", true}, @@ -62,18 +63,20 @@ func TestNewCommand(t *testing.T) { } for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { + test := tt //capture as local copy + t.Run(test.name, func(t *testing.T) { + t.Parallel() // Act - command, err := NewCommand(ctx, wg, lc, config, tt.cmdArgs) + command, err := NewCommand(ctx, wg, lc, config, test.cmdArgs) // Assert - if tt.expectedErr { + if test.expectedErr { require.Error(t, err) require.Nil(t, command) } else { require.NoError(t, err) require.NotNil(t, command) - require.Equal(t, tt.expectedCmdName, command.GetCommandName()) + require.Equal(t, test.expectedCmdName, command.GetCommandName()) } }) } diff --git a/internal/security/bootstrapper/command/flags_common.go b/internal/security/bootstrapper/command/flags_common.go index 10326677f6..afbedb7c9a 100644 --- a/internal/security/bootstrapper/command/flags_common.go +++ b/internal/security/bootstrapper/command/flags_common.go @@ -97,13 +97,14 @@ func HelpCallback() { " --confdir Specify local configuration directory\n"+ "\n"+ "Commands:\n"+ - " help Show available commands (this text)\n"+ - " gate Do security bootstrapper gating on stages while starting services\n"+ - " listenTcp Start up a TCP listener\n"+ - " pingPgDb Test Postgres database readiness\n"+ - " getHttpStatus Do an HTTP GET call to get the status code\n"+ - " genPassword Generate a random password\n"+ - " waitfor Wait for the other services with specified URI(s) to connect:\n"+ - " the URI(s) can be communication protocols like tcp/tcp4/tcp6/http/https or files\n", + " gate Do security bootstrapper gating on stages while starting services\n"+ + " genPassword Generate a random password\n"+ + " getHttpStatus Do an HTTP GET call to get the status code\n"+ + " help Show available commands (this text)\n"+ + " listenTcp Start up a TCP listener\n"+ + " pingPgDb Test Postgres database readiness\n"+ + " setupRegistryACL Set up registry's ACL and configure the access\n"+ + " waitFor Wait for the other services with specified URI(s) to connect:\n"+ + " the URI(s) can be communication protocols like tcp/tcp4/tcp6/http/https or files\n", os.Args[0]) } diff --git a/internal/security/bootstrapper/command/setupacl/aclbootstrap.go b/internal/security/bootstrapper/command/setupacl/aclbootstrap.go new file mode 100644 index 0000000000..9c649b2d59 --- /dev/null +++ b/internal/security/bootstrapper/command/setupacl/aclbootstrap.go @@ -0,0 +1,112 @@ +/******************************************************************************* + * Copyright 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + *******************************************************************************/ + +package setupacl + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "os" + "path/filepath" + + "github.com/edgexfoundry/go-mod-secrets/v2/pkg/token/fileioperformer" +) + +// BootStrapACLTokenInfo is the key portion of the response metadata from consulACLBootstrapAPI +type BootStrapACLTokenInfo struct { + SecretID string `json:"SecretID"` + Policies []Policy `json:"Policies"` +} + +// Policy is the metadata for ACL policy +type Policy struct { + ID string `json:"ID"` + Name string `json:"Name"` +} + +// generateBootStrapACLToken should only be called once per Consul agent +func (c *cmd) generateBootStrapACLToken() (*BootStrapACLTokenInfo, error) { + aclBootstrapURL, err := c.getRegistryApiUrl(consulACLBootstrapAPI) + if err != nil { + return nil, err + } + + req, err := http.NewRequest(http.MethodPut, aclBootstrapURL, http.NoBody) + if err != nil { + return nil, fmt.Errorf("Failed to prepare request for http URL: %w", err) + } + resp, err := c.client.Do(req) + if err != nil { + return nil, fmt.Errorf("Failed to send request for http URL: %w", err) + } + + defer func() { + _ = resp.Body.Close() + }() + + responseBody, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("Failed to read response body of bootstrap ACL: %w", err) + } + + var bootstrapACLToken BootStrapACLTokenInfo + switch resp.StatusCode { + case http.StatusOK: + if err := json.NewDecoder(bytes.NewReader(responseBody)).Decode(&bootstrapACLToken); err != nil { + return nil, fmt.Errorf("failed to decode bootstrapACLToken json data: %v", err) + } + return &bootstrapACLToken, nil + default: + return nil, fmt.Errorf("failed to bootstrap Consul's ACL via URL [%s] and status code= %d: %s", aclBootstrapURL, + resp.StatusCode, string(responseBody)) + } +} + +func (c *cmd) saveBootstrapACLToken(tokenInfoToBeSaved *BootStrapACLTokenInfo) error { + // Write the token to the specified file + tokenFileAbsPath, err := filepath.Abs(c.configuration.StageGate.Registry.ACL.BootstrapTokenPath) + if err != nil { + return fmt.Errorf("failed to convert tokenFile to absolute path %s: %s", + c.configuration.StageGate.Registry.ACL.BootstrapTokenPath, err.Error()) + } + + // create the directory of tokenfile if not exists yet + dirOfToken := filepath.Dir(tokenFileAbsPath) + fileIoPerformer := fileioperformer.NewDefaultFileIoPerformer() + if err := fileIoPerformer.MkdirAll(dirOfToken, 0700); err != nil { + return fmt.Errorf("failed to create tokenpath base dir: %s", err.Error()) + } + + fileWriter, err := fileIoPerformer.OpenFileWriter(tokenFileAbsPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) + if err != nil { + return fmt.Errorf("failed to open file writer %s: %s", tokenFileAbsPath, err.Error()) + } + + if err := json.NewEncoder(fileWriter).Encode(tokenInfoToBeSaved); err != nil { + _ = fileWriter.Close() + return fmt.Errorf("failed to write bootstrap token: %s", err.Error()) + } + + if err := fileWriter.Close(); err != nil { + return fmt.Errorf("failed to close token file: %s", err.Error()) + } + + c.loggingClient.Infof("bootstrap token is written to %s", tokenFileAbsPath) + + return nil +} diff --git a/internal/security/bootstrapper/command/setupacl/command.go b/internal/security/bootstrapper/command/setupacl/command.go new file mode 100644 index 0000000000..3d13ea8f3e --- /dev/null +++ b/internal/security/bootstrapper/command/setupacl/command.go @@ -0,0 +1,363 @@ +/******************************************************************************* + * Copyright 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + *******************************************************************************/ + +package setupacl + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/edgexfoundry/edgex-go/internal" + "github.com/edgexfoundry/edgex-go/internal/security/bootstrapper/config" + "github.com/edgexfoundry/edgex-go/internal/security/bootstrapper/helper" + "github.com/edgexfoundry/edgex-go/internal/security/bootstrapper/interfaces" + + "github.com/edgexfoundry/go-mod-bootstrap/v2/bootstrap/startup" + + "github.com/edgexfoundry/go-mod-secrets/v2/pkg" + "github.com/edgexfoundry/go-mod-secrets/v2/pkg/token/authtokenloader" + "github.com/edgexfoundry/go-mod-secrets/v2/pkg/token/fileioperformer" + + "github.com/edgexfoundry/go-mod-core-contracts/v2/clients" + "github.com/edgexfoundry/go-mod-core-contracts/v2/clients/logger" +) + +const ( + // the command name for setting up registry's ACL + CommandName string = "setupRegistryACL" + + consulGetLeaderAPI = "/v1/status/leader" + consulACLBootstrapAPI = "/v1/acl/bootstrap" + consulConfigAccessVaultAPI = "/v1/consul/config/access" + consulLegacyACLModeError = "The ACL system is currently in legacy mode" + defaultRetryTimeout = 30 * time.Second + emptyLeader = `""` + emptyToken = "" +) + +type cmd struct { + loggingClient logger.LoggingClient + client internal.HttpCaller + configuration *config.ConfigurationStruct + + // internal state + retryTimeout time.Duration +} + +// NewCommand creates a new cmd and parses through options if any +func NewCommand( + _ context.Context, + _ *sync.WaitGroup, + lc logger.LoggingClient, + conf *config.ConfigurationStruct, + args []string) (interfaces.Command, error) { + cmd := cmd{ + loggingClient: lc, + client: pkg.NewRequester(lc).Insecure(), + configuration: conf, + retryTimeout: defaultRetryTimeout, + } + var dummy string + + flagSet := flag.NewFlagSet(CommandName, flag.ContinueOnError) + flagSet.StringVar(&dummy, "confdir", "", "") // handled by bootstrap; duplicated here to prevent arg parsing errors + + err := flagSet.Parse(args) + if err != nil { + return nil, fmt.Errorf("Unable to parse command: %s: %w", strings.Join(args, " "), err) + } + + return &cmd, nil +} + +// Execute implements Command and runs this command +// command setupRegistryACL sets up the ACL system of the registry, Consul in this case, preparing for generating +// Consul's agent tokens later on +func (c *cmd) Execute() (statusCode int, err error) { + c.loggingClient.Infof("Security bootstrapper running %s", CommandName) + + // need to have a sentinel file to guard against the re-run of the command once we have successfully bootstrap ACL + // if we already have a sentinelFile exists then skip this whole process since we already done this + // process successfully before, otherwise Consul's ACL bootstrap will cause a panic + sentinelFileAbsPath, err := filepath.Abs(c.configuration.StageGate.Registry.ACL.SentinelFilePath) + if err != nil { + return interfaces.StatusCodeExitWithError, fmt.Errorf("failed to get the absolute path of the sentinel file: %v", err) + } + + if helper.CheckIfFileExists(sentinelFileAbsPath) { + c.loggingClient.Info("Registry ACL had been setup successfully already, skip") + return + } + + if err := c.waitForNonEmptyConsulLeader(); err != nil { + return interfaces.StatusCodeExitWithError, fmt.Errorf("failed to wait for Consul leader: %v", err) + } + + var bootstrapACLToken *BootStrapACLTokenInfo + bootstrapACLToken, err = c.generateBootStrapACLToken() + if err != nil { + // although we have a leader, but it is a very very rare chance that we could hit an error on legacy mode + // here we will sleep a bit of time and then retry once if there is error on Legacy ACL type of message + // because Consul is still on its way to initialize the new ACL system internally + // for the details of this issue, see related issue on Consul's Github website: + // https://github.com/hashicorp/consul/issues/5218#issuecomment-457212336 + if !strings.Contains(err.Error(), consulLegacyACLModeError) { + // other type of ACL bootstrapping error, cannot continue + return interfaces.StatusCodeExitWithError, fmt.Errorf("failed to bootstrap registry's ACL: %v", err) + } + + c.loggingClient.Warnf("found Consul still in ACL legacy mode, will retry once again: %v", err) + time.Sleep(5 * time.Second) + bootstrapACLToken, err = c.generateBootStrapACLToken() + if err != nil { + return interfaces.StatusCodeExitWithError, fmt.Errorf("failed to bootstrap registry's ACL: %v", err) + } + } + + c.loggingClient.Info("successfully bootstrap registry ACL") + + // Save the bootstrap token into the file so that it can be used later on + if err := c.saveBootstrapACLToken(bootstrapACLToken); err != nil { + return interfaces.StatusCodeExitWithError, fmt.Errorf("failed to save registry's bootstrap ACL token: %v", err) + } + + // retrieve the secretstore (Vault) token from the file produced by secretstore-setup + secretstoreToken, err := c.getSecretStoreTokenFromFile() + if err != nil { + return interfaces.StatusCodeExitWithError, fmt.Errorf("failed to retrieve secretstore token: %v", err) + } + + c.loggingClient.Info("successfully get secretstore token and configuring the registry access for secretestore") + + // configure Consul access with both Secret Store token and consul's bootstrap acl token + if err := c.configureConsulAccess(secretstoreToken, bootstrapACLToken.SecretID); err != nil { + return interfaces.StatusCodeExitWithError, fmt.Errorf("failed to configure Consul access: %v", err) + } + + // write a sentinel file to indicate Consul ACL bootstrap is done so that we don't bootstrap ACL again, + // this is to avoid re-bootstrapping error and that error can cause the snap crash if restart this process + if err := c.writeSentinelFile(); err != nil { + return interfaces.StatusCodeExitWithError, fmt.Errorf("failed to write sentinel file: %v", err) + } + + c.loggingClient.Info("setupRegistryACL successfully done") + + return +} + +// GetCommandName returns the name of this command +func (c *cmd) GetCommandName() string { + return CommandName +} + +func (c *cmd) getRegistryApiUrl(api string) (string, error) { + apiURL := fmt.Sprintf("%s://%s:%d%s", c.configuration.StageGate.Registry.ACL.Protocol, + c.configuration.StageGate.Registry.Host, c.configuration.StageGate.Registry.Port, api) + _, err := url.Parse(apiURL) + if err != nil { + return "", fmt.Errorf("failed to parse API URL: %v", err) + } + return apiURL, nil +} + +// waitForNonEmptyConsulLeader is a special waitFor function on waiting for "non-empty" leader being available +// the ordinary http waitFor won't work as the returned http status code from API call is 200 even when Consul's leader +// is an empty string ("") but we need an non-empty leader; so 200 doesn't mean we have a leader +func (c *cmd) waitForNonEmptyConsulLeader() error { + timeoutInSec := int(c.retryTimeout.Seconds()) + timer := startup.NewTimer(timeoutInSec, 1) + for timer.HasNotElapsed() { + if err := c.getNonEmptyConsulLeader(); err != nil { + c.loggingClient.Warnf("error from getting Consul leader API call, will retry it again: %v", err) + timer.SleepForInterval() + continue + } + c.loggingClient.Info("found Consul leader to bootstrap ACL") + return nil + } + + return errors.New("timed out to get non-empty Consul leader") +} + +// getNonEmptyConsulLeader makes http request call to get the registry Consul leader +// the response of getting leader call could be an empty leader (represented by "") +// even if the http status code is 200 when Consul is just booting up and +// it will take a bit of time to elect the raft leader +func (c *cmd) getNonEmptyConsulLeader() error { + getLeaderURL, err := c.getRegistryApiUrl(consulGetLeaderAPI) + if err != nil { + return err + } + + req, err := http.NewRequest(http.MethodGet, getLeaderURL, http.NoBody) + if err != nil { + return fmt.Errorf("Failed to prepare request for http URL: %w", err) + } + resp, err := c.client.Do(req) + if err != nil { + return fmt.Errorf("Failed to send request for http URL: %w", err) + } + + defer func() { + _ = resp.Body.Close() + }() + + responseBody, err := ioutil.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("Failed to read response body to get leader: %w", err) + } + + switch resp.StatusCode { + case http.StatusOK: + trimmedResp := strings.TrimSpace(string(responseBody)) + // Consul's raft leader election process could take a bit of time + // before responds back with a non-empty leader + if len(trimmedResp) == 0 || emptyLeader == trimmedResp { + return errors.New("no leader yet") + } + // now we have a cluster raft leader + c.loggingClient.Infof("leader [%s] is elected", trimmedResp) + return nil + + // almost unlikely for this case unless URL is incorrect + default: + return fmt.Errorf("get Consul leader request failed with status code= %d: %s", resp.StatusCode, string(responseBody)) + } +} + +func (c *cmd) getSecretStoreTokenFromFile() (string, error) { + trimmedFilePath := strings.TrimSpace(c.configuration.StageGate.Registry.ACL.SecretsAdminTokenPath) + if len(trimmedFilePath) == 0 { + return emptyToken, errors.New("required StageGate_Registry_SecretsAdminTokenPath from configuration is empty") + } + + tokenFileAbsPath, err := filepath.Abs(trimmedFilePath) + if err != nil { + return emptyToken, fmt.Errorf("failed to convert tokenFile to absolute path %s: %v", trimmedFilePath, err) + } + + // since the secretstore token is created by another service, secretstore-setup, + // so here we want to make sure we have the file + if exists := helper.CheckIfFileExists(tokenFileAbsPath); !exists { + return emptyToken, fmt.Errorf("secretstore token file %s not found", tokenFileAbsPath) + } + + fileOpener := fileioperformer.NewDefaultFileIoPerformer() + tokenLoader := authtokenloader.NewAuthTokenLoader(fileOpener) + secretStoreToken, err := tokenLoader.Load(tokenFileAbsPath) + if err != nil { + return emptyToken, fmt.Errorf("tokenLoader failed to load secretstore token: %v", err) + } + + c.loggingClient.Infof("successfully retrieved secretstore management token from %s", trimmedFilePath) + + return secretStoreToken, nil +} + +// configureConsulAccess is to enable the Consul config access to the SecretStore via consul/config/access API +// see the reference: https://www.vaultproject.io/api-docs/secret/consul#configure-access +func (c *cmd) configureConsulAccess(secretStoreToken string, bootstrapACLToken string) error { + configAccessURL := fmt.Sprintf("%s://%s:%d%s", c.configuration.SecretStore.Protocol, + c.configuration.SecretStore.Host, c.configuration.SecretStore.Port, consulConfigAccessVaultAPI) + _, err := url.Parse(configAccessURL) + if err != nil { + return fmt.Errorf("failed to parse config Access URL: %v", err) + } + + c.loggingClient.Debugf("configAccessURL: %s", configAccessURL) + + type ConfigAccess struct { + RegistryAddress string `json:"address"` + BootstrapACLToken string `json:"token"` + } + + payload := &ConfigAccess{ + RegistryAddress: fmt.Sprintf("%s:%d", c.configuration.StageGate.Registry.Host, c.configuration.StageGate.Registry.Port), + BootstrapACLToken: bootstrapACLToken, + } + + jsonPayload, err := json.Marshal(payload) + c.loggingClient.Tracef("payload: %v", payload) + if err != nil { + return fmt.Errorf("Failed to marshal JSON string payload: %v", err) + } + + req, err := http.NewRequest(http.MethodPost, configAccessURL, bytes.NewBuffer(jsonPayload)) + if err != nil { + return fmt.Errorf("Failed to prepare POST request for http URL: %w", err) + } + + req.Header.Add("X-Vault-Token", secretStoreToken) + req.Header.Add(clients.ContentType, clients.ContentTypeJSON) + resp, err := c.client.Do(req) + if err != nil { + return fmt.Errorf("Failed to send request for http URL: %w", err) + } + + defer func() { + _ = resp.Body.Close() + }() + + switch resp.StatusCode { + case http.StatusNoContent: + // no response body returned in this case + c.loggingClient.Info("successfully configure Consul access for secretstore") + return nil + default: + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + c.loggingClient.Errorf("cannot read resp.Body: %v", err) + } + return fmt.Errorf("failed to configure Consul access for secretstore via URL [%s] and status code= %d: %s", + configAccessURL, resp.StatusCode, string(body)) + } +} + +func (c *cmd) writeSentinelFile() error { + absPath, err := filepath.Abs(c.configuration.StageGate.Registry.ACL.SentinelFilePath) + if err != nil { + return fmt.Errorf("failed to get the absolute path of the sentinel file: %v", err) + } + + dirToWrite := filepath.Dir(absPath) + filePerformer := fileioperformer.NewDefaultFileIoPerformer() + if err := filePerformer.MkdirAll(dirToWrite, 0700); err != nil { + return fmt.Errorf("failed to create sentinel base dir: %s", err.Error()) + } + + writer, err := filePerformer.OpenFileWriter(absPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) + if err != nil { + return fmt.Errorf("failed to open file writer %s: %s", absPath, err.Error()) + } + defer func() { _ = writer.Close() }() + + if _, err := writer.Write([]byte("done")); err != nil { + return fmt.Errorf("failed to write out to sentinel file %s: %v", absPath, err) + } + + return nil +} diff --git a/internal/security/bootstrapper/command/setupacl/command_test.go b/internal/security/bootstrapper/command/setupacl/command_test.go new file mode 100644 index 0000000000..886803fcf3 --- /dev/null +++ b/internal/security/bootstrapper/command/setupacl/command_test.go @@ -0,0 +1,238 @@ +/******************************************************************************* + * Copyright 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + *******************************************************************************/ + +package setupacl + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strconv" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/edgexfoundry/edgex-go/internal/security/bootstrapper/config" + "github.com/edgexfoundry/edgex-go/internal/security/bootstrapper/helper" + "github.com/edgexfoundry/edgex-go/internal/security/bootstrapper/interfaces" + + "github.com/edgexfoundry/go-mod-core-contracts/v2/clients/logger" +) + +func TestNewCommand(t *testing.T) { + // Arrange + ctx := context.Background() + wg := &sync.WaitGroup{} + lc := logger.MockLogger{} + config := &config.ConfigurationStruct{} + + tests := []struct { + name string + cmdArgs []string + expectedErr bool + }{ + {"Good:setupRegistryACL cmd empty option", []string{}, false}, + {"Bad:setupRegistryACL invalid option", []string{"--invalid=xxx"}, true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + command, err := NewCommand(ctx, wg, lc, config, tt.cmdArgs) + if tt.expectedErr { + require.Error(t, err) + } else { + require.NoError(t, err) + require.NotNil(t, command) + } + }) + } +} + +func TestExecute(t *testing.T) { + type prepareTestFunc func(aclOkResponse bool, configAccessOkResponse bool, t *testing.T) (*config.ConfigurationStruct, + *httptest.Server) + + // Arrange + ctx := context.Background() + wg := &sync.WaitGroup{} + lc := logger.MockLogger{} + + tests := []struct { + name string + adminDir string + prepare prepareTestFunc + aclOkResponse bool + configAccessOkResponse bool + expectedErr bool + }{ + {"Good:setupRegistryACL with ok response from server", "test1", prepareTestRegistryServer, true, true, false}, + {"Bad:setupRegistryACL with bootstrap ACL API failed response from server", "test2", + prepareTestRegistryServer, false, false, true}, + {"Bad:setupRegistryACL with non-existing server", "test3", + func(_ bool, _ bool, _ *testing.T) (*config.ConfigurationStruct, *httptest.Server) { + return &config.ConfigurationStruct{ + StageGate: config.StageGateInfo{ + Registry: config.RegistryInfo{ + Host: "non-existing", + Port: 10001, + ACL: config.ACLInfo{Protocol: "http"}, + }, + }}, httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) + }, false, false, true}, + {"Bad:setupRegistryACL with empty api protocol", "test4", + func(_ bool, _ bool, _ *testing.T) (*config.ConfigurationStruct, *httptest.Server) { + return &config.ConfigurationStruct{ + StageGate: config.StageGateInfo{ + Registry: config.RegistryInfo{ + Host: "localhost", + Port: 10001, + ACL: config.ACLInfo{Protocol: ""}, + }, + }}, httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) + }, false, false, true}, + {"Bad:setupRegistryACL with timed out on waiting for secret token file", "", + prepareTestRegistryServer, true, false, true}, + {"Bad:setupRegistryACL with config access API failed response from server", "test5", + prepareTestRegistryServer, true, false, true}, + } + + for _, tt := range tests { + test := tt // capture as local copy + t.Run(test.name, func(t *testing.T) { + t.Parallel() + // prepare test + conf, testServer := test.prepare(test.aclOkResponse, test.configAccessOkResponse, t) + defer testServer.Close() + // setup token related configs + conf.StageGate.Registry.ACL.SecretsAdminTokenPath = filepath.Join(test.adminDir, "secret_token.json") + conf.StageGate.Registry.ACL.BootstrapTokenPath = filepath.Join(test.adminDir, "bootstrap_token.json") + conf.StageGate.Registry.ACL.SentinelFilePath = filepath.Join(test.adminDir, "sentinel_test_file") + + setupRegistryACL, err := NewCommand(ctx, wg, lc, conf, []string{}) + require.NoError(t, err) + require.NotNil(t, setupRegistryACL) + require.Equal(t, "setupRegistryACL", setupRegistryACL.GetCommandName()) + + // create test secret token file + if test.adminDir != "" { + err = helper.CreateDirectoryIfNotExists(test.adminDir) + require.NoError(t, err) + err = ioutil.WriteFile(conf.StageGate.Registry.ACL.SecretsAdminTokenPath, + []byte(secretstoreTokenJsonStub), 0600) + require.NoError(t, err) + } + + // to speed up the test timeout + localcmd := setupRegistryACL.(*cmd) + localcmd.retryTimeout = 3 * time.Second + statusCode, err := setupRegistryACL.Execute() + defer func() { + if test.adminDir == "" { + // empty test dir case don't have the directory to clean up + _ = os.Remove(conf.StageGate.Registry.ACL.BootstrapTokenPath) + } else { + _ = os.RemoveAll(test.adminDir) + } + }() + + if test.expectedErr { + require.Error(t, err) + require.Equal(t, interfaces.StatusCodeExitWithError, statusCode) + } else { + require.NoError(t, err) + require.Equal(t, interfaces.StatusCodeExitNormal, statusCode) + require.FileExists(t, conf.StageGate.Registry.ACL.BootstrapTokenPath) + require.FileExists(t, conf.StageGate.Registry.ACL.SecretsAdminTokenPath) + require.FileExists(t, conf.StageGate.Registry.ACL.SentinelFilePath) + } + }) + } +} + +func prepareTestRegistryServer(aclOkResponse bool, configAccessOkResponse bool, t *testing.T) (*config.ConfigurationStruct, + *httptest.Server) { + registryTestConf := &config.ConfigurationStruct{} + + respCnt := 0 + testSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.EscapedPath() { + case consulGetLeaderAPI: + require.Equal(t, http.MethodGet, r.Method) + respCnt++ + w.WriteHeader(http.StatusOK) + var err error + if respCnt == 2 { + _, err = w.Write([]byte("127.0.0.1:12345")) + } else { + _, err = w.Write([]byte("")) + } + require.NoError(t, err) + case consulACLBootstrapAPI: + require.Equal(t, http.MethodPut, r.Method) + if aclOkResponse { + w.WriteHeader(http.StatusOK) + jsonResponse := map[string]interface{}{ + "AccessorID": "bad060a9-0e2b-47ba-98d5-9d622e2322b5", + "SecretID": "7240fdd9-1665-419b-a8c5-5691ca03af7c", + "Description": "Bootstrap Token (Global Management)", + "Policies": []map[string]interface{}{ + { + "ID": "00000000-0000-0000-0000-000000000001", + "Name": "global-management", + }, + }, + "Local": false, + "CreateTime": "2021-03-01T10:34:20.843397-07:00", + } + err := json.NewEncoder(w).Encode(jsonResponse) + require.NoError(t, err) + } else { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte("The ACL system is currently in legacy mode.")) + } + case consulConfigAccessVaultAPI: + require.Equal(t, http.MethodPost, r.Method) + if configAccessOkResponse { + w.WriteHeader(http.StatusNoContent) + } else { + w.WriteHeader(http.StatusForbidden) + } + default: + t.Fatal(fmt.Sprintf("Unexpected call to URL %s", r.URL.EscapedPath())) + } + })) + tsURL, err := url.Parse(testSrv.URL) + require.NoError(t, err) + portNum, _ := strconv.Atoi(tsURL.Port()) + registryTestConf.StageGate.Registry.ACL.Protocol = tsURL.Scheme + registryTestConf.StageGate.Registry.Host = tsURL.Hostname() + registryTestConf.StageGate.Registry.Port = portNum + registryTestConf.StageGate.WaitFor.Timeout = "1m" + registryTestConf.StageGate.WaitFor.RetryInterval = "1s" + // for the sake of simplicity, we use the same test server as the secret store server + registryTestConf.SecretStore.Protocol = tsURL.Scheme + registryTestConf.SecretStore.Host = tsURL.Hostname() + registryTestConf.SecretStore.Port = portNum + + return registryTestConf, testSrv +} diff --git a/internal/security/bootstrapper/command/setupacl/stubdata_test.go b/internal/security/bootstrapper/command/setupacl/stubdata_test.go new file mode 100644 index 0000000000..94d745cb06 --- /dev/null +++ b/internal/security/bootstrapper/command/setupacl/stubdata_test.go @@ -0,0 +1,51 @@ +/******************************************************************************* + * Copyright 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + *******************************************************************************/ + +package setupacl + +// this is just the stub for test data related +const ( + secretstoreTokenJsonStub = ` + { + "auth": { + "accessor": "xxxxxxxxxxxxxxxxxxxxxxx", + "client_token": "yyyyyyyyyyyyyyyyyyyyyyyyyy", + "entity_id": "", + "lease_duration": 3600, + "metadata": { + "description": "Consul secrets engine management token" + }, + "orphan": true, + "policies": [ + "consul_secrets_engine_management_policy", + "default" + ], + "renewable": true, + "token_policies": [ + "consul_secrets_engine_management_policy", + "default" + ], + "token_type": "service" + }, + "data": null, + "lease_duration": 0, + "lease_id": "", + "renewable": false, + "request_id": "aaaaaaaa-1111-2222-bbbb-cccccccccccc", + "warnings": null, + "wrap_info": null + } + ` +) diff --git a/internal/security/bootstrapper/config/config.go b/internal/security/bootstrapper/config/config.go index cc4f62648c..3f6b657e35 100644 --- a/internal/security/bootstrapper/config/config.go +++ b/internal/security/bootstrapper/config/config.go @@ -20,8 +20,9 @@ import ( ) type ConfigurationStruct struct { - LogLevel string - StageGate StageGateInfo + LogLevel string + StageGate StageGateInfo + SecretStore bootstrapConfig.SecretStoreInfo } // UpdateFromRaw converts configuration received from the registry to a service-specific configuration struct which is diff --git a/internal/security/bootstrapper/config/types.go b/internal/security/bootstrapper/config/types.go index 18bda476f8..694b7170af 100644 --- a/internal/security/bootstrapper/config/types.go +++ b/internal/security/bootstrapper/config/types.go @@ -56,6 +56,19 @@ type RegistryInfo struct { Host string Port int ReadyPort int + ACL ACLInfo +} + +// ACLInfo defines the fields related to Registry's ACL process +type ACLInfo struct { + // the protocol used for registry's API calls, usually it is different from the protocol of waitFor, i.e. TCP + Protocol string + // filepath to save the registry's token generated from ACL bootstrapping + BootstrapTokenPath string + // filepath for the secretstore's token created from secretstore-setup + SecretsAdminTokenPath string + // filepath for the sentinel file to indicate the registry ACL is set up successfully + SentinelFilePath string } // KongDBInfo defines the fields related to diff --git a/internal/security/bootstrapper/helper/helper.go b/internal/security/bootstrapper/helper/helper.go index 4260cbd207..0de1cdfe4f 100644 --- a/internal/security/bootstrapper/helper/helper.go +++ b/internal/security/bootstrapper/helper/helper.go @@ -26,7 +26,7 @@ import ( // MarkComplete creates a doneFile file func MarkComplete(dirPath, doneFile string) error { doneFilePath := filepath.Join(dirPath, doneFile) - if !checkIfFileExists(doneFilePath) { + if !CheckIfFileExists(doneFilePath) { if err := writeFile(doneFilePath); err != nil { return err } @@ -48,7 +48,8 @@ func CreateDirectoryIfNotExists(dirName string) (err error) { return } -func checkIfFileExists(fileName string) bool { +// CheckIfFileExists returns true if the specified fileName exists +func CheckIfFileExists(fileName string) bool { fileInfo, statErr := os.Stat(fileName) if os.IsNotExist(statErr) { return false