From 7f9dcefbb9b19b3196c2abccd3a717795d0359f6 Mon Sep 17 00:00:00 2001 From: Jim Wang Date: Thu, 25 Feb 2021 17:04:14 -0700 Subject: [PATCH 1/5] feat(security): Implementation to Set up Consul ACL Mainly feature added is to enable Consul's ACL and bootstrap ACL. The following are detailed changes: - Add Consul's ACL configuration file with "allow" policy - Disable Consul's DNS port - Add ACL related configuration toml on security-bootstrapper - Add setupRegistryACL subcommand to security-bootstrapper - Add checking and retry logic for "non-empty" Consul leader - Add implementation for setting up Consul's ACL, including bootstrap ACL and configure Consul secrets access for Vault Closes: #3156 Signed-off-by: Jim Wang --- cmd/security-bootstrapper/Dockerfile | 3 + .../consul-acl/config_consul_acl.json | 7 + .../entrypoint-scripts/consul_wait_install.sh | 53 ++- .../res/configuration.toml | 16 + .../bootstrapper/command/cmd_dispatcher.go | 7 +- .../command/cmd_dispatcher_test.go | 11 +- .../bootstrapper/command/flags_common.go | 17 +- .../command/setupacl/aclbootstrap.go | 114 ++++++ .../bootstrapper/command/setupacl/command.go | 364 ++++++++++++++++++ .../command/setupacl/command_test.go | 236 ++++++++++++ .../command/setupacl/stubdata_test.go | 51 +++ .../security/bootstrapper/config/config.go | 5 +- .../security/bootstrapper/config/types.go | 11 + .../security/bootstrapper/helper/helper.go | 5 +- .../bootstrapper/interfaces/command.go | 2 + 15 files changed, 875 insertions(+), 27 deletions(-) create mode 100644 cmd/security-bootstrapper/consul-acl/config_consul_acl.json create mode 100644 internal/security/bootstrapper/command/setupacl/aclbootstrap.go create mode 100644 internal/security/bootstrapper/command/setupacl/command.go create mode 100644 internal/security/bootstrapper/command/setupacl/command_test.go create mode 100644 internal/security/bootstrapper/command/setupacl/stubdata_test.go diff --git a/cmd/security-bootstrapper/Dockerfile b/cmd/security-bootstrapper/Dockerfile index ce3b925085..816dbf1179 100644 --- a/cmd/security-bootstrapper/Dockerfile +++ b/cmd/security-bootstrapper/Dockerfile @@ -60,6 +60,9 @@ COPY --from=builder /edgex-go/cmd/security-bootstrapper/res/configuration.toml . # needed for bootstrapping Redis db COPY --from=builder /edgex-go/cmd/security-bootstrapper/res-bootstrap-redis/configuration.toml ${BOOTSTRAP_REDIS_DIR}/res/ +# copy Consul ACL related configs +COPY --from=builder /edgex-go/cmd/security-bootstrapper/consul-acl/ ${SECURITY_INIT_DIR}/consul-bootstrapper/ + # Expose the file directory as a volume since there's long-running state VOLUME ${SECURITY_INIT_DIR} diff --git a/cmd/security-bootstrapper/consul-acl/config_consul_acl.json b/cmd/security-bootstrapper/consul-acl/config_consul_acl.json new file mode 100644 index 0000000000..4eb9858a08 --- /dev/null +++ b/cmd/security-bootstrapper/consul-acl/config_consul_acl.json @@ -0,0 +1,7 @@ +{ + "acl": { + "enabled": true, + "default_policy": "allow", + "enable_token_persistence": true + } +} diff --git a/cmd/security-bootstrapper/entrypoint-scripts/consul_wait_install.sh b/cmd/security-bootstrapper/entrypoint-scripts/consul_wait_install.sh index b713786d07..3225ec0737 100755 --- a/cmd/security-bootstrapper/entrypoint-scripts/consul_wait_install.sh +++ b/cmd/security-bootstrapper/entrypoint-scripts/consul_wait_install.sh @@ -52,7 +52,10 @@ done DEFAULT_CONSUL_LOCAL_CONFIG=' { "enable_local_script_checks": true, - "disable_update_check": true + "disable_update_check": true, + "ports": { + "dns": -1 + } } ' @@ -63,15 +66,47 @@ export CONSUL_LOCAL_CONFIG echo "$(date) CONSUL_LOCAL_CONFIG: ${CONSUL_LOCAL_CONFIG}" -echo "$(date) Starting edgex-consul..." -exec docker-entrypoint.sh agent -ui -bootstrap -server -client 0.0.0.0 & +echo "$(date) ENABLE_REGISTRY_ACL = ${ENABLE_REGISTRY_ACL}" -# wait for the consul port -echo "$(date) Executing waitFor on Consul with waiting on its own port \ - tcp://${STAGEGATE_REGISTRY_HOST}:${STAGEGATE_REGISTRY_PORT}" -/edgex-init/security-bootstrapper --confdir=/edgex-init/res waitFor \ - -uri tcp://"${STAGEGATE_REGISTRY_HOST}":"${STAGEGATE_REGISTRY_PORT}" \ - -timeout "${STAGEGATE_WAITFOR_TIMEOUT}" +if [ "${ENABLE_REGISTRY_ACL}" == "true" ]; then + echo "$(date) Starting edgex-core-consul with ACL enabled ..." + docker-entrypoint.sh agent \ + -ui \ + -bootstrap \ + -server \ + -config-file=/edgex-init/consul-bootstrapper/config_consul_acl.json \ + -client 0.0.0.0 & + # wait for the secretstore tokens ready as we need the token for bootstrapping + echo "$(date) Executing waitFor on Consul with waiting on TokensReadyPort \ + tcp://${STAGEGATE_SECRETSTORESETUP_HOST}:${STAGEGATE_SECRETSTORESETUP_TOKENS_READYPORT}" + /edgex-init/security-bootstrapper --confdir=/edgex-init/res waitFor \ + -uri tcp://"${STAGEGATE_SECRETSTORESETUP_HOST}":"${STAGEGATE_SECRETSTORESETUP_TOKENS_READYPORT}" \ + -timeout "${STAGEGATE_WAITFOR_TIMEOUT}" + + # we don't want to exit out the whole Consul process when ACL bootstrapping failed, just that + # Consul won't have ACL to be used + set +e + # call setupRegistryACL bootstrapping command, containing both ACL bootstrapping and re-configure consul access steps + /edgex-init/security-bootstrapper --confdir=/edgex-init/res setupRegistryACL + setupACL_code=$? + if [ "${setupACL_code}" -ne 0 ]; then + echo "$(date) failed to set up Consul ACL" + fi + set -e +else + echo "$(date) Starting edgex-core-consul with ACL disabled ..." + docker-entrypoint.sh agent \ + -ui \ + -bootstrap \ + -server \ + -client 0.0.0.0 & + # wait for the consul port + echo "$(date) Executing waitFor on Consul with waiting on its own port \ + tcp://${STAGEGATE_REGISTRY_HOST}:${STAGEGATE_REGISTRY_PORT}" + /edgex-init/security-bootstrapper --confdir=/edgex-init/res waitFor \ + -uri tcp://"${STAGEGATE_REGISTRY_HOST}":"${STAGEGATE_REGISTRY_PORT}" \ + -timeout "${STAGEGATE_WAITFOR_TIMEOUT}" +fi # Signal that Consul is ready for services blocked waiting on Consul /edgex-init/security-bootstrapper --confdir=/edgex-init/res listenTcp \ diff --git a/cmd/security-bootstrapper/res/configuration.toml b/cmd/security-bootstrapper/res/configuration.toml index 819e310be3..194a3ff7bd 100644 --- a/cmd/security-bootstrapper/res/configuration.toml +++ b/cmd/security-bootstrapper/res/configuration.toml @@ -20,6 +20,12 @@ LogLevel = 'INFO' Host = "edgex-core-consul" Port = 8500 ReadyPort = 54324 + [StageGate.Registry.ACL] + Protocol = "http" + # this is the filepath for the generated Consul management token from ACL bootstrap + BootstrapTokenPath = "/tmp/edgex/secrets/edgex-consul/admin/bootstrap_token.json" + # this is the filepath for the Vault token created from secretstore-setup + SecretsAdminTokenPath = "/tmp/edgex/secrets/edgex-consul/admin/token.json" [StageGate.KongDb] Host = "kong-db" Port = 5432 @@ -27,3 +33,13 @@ LogLevel = 'INFO' [StageGate.WaitFor] Timeout = "10s" RetryInterval = "1s" + +# this configuration is just part of the whole go-mod-bootstrap's secret store to have +# protocol, host, and port of secretstore using in the security-bootstrapper +# we are not really using the secret store provider from go-mod-bootstrap in the code +# also this is needed as snap does not have those environments from env-files +[SecretStore] +Type = 'vault' +Protocol = 'http' +Host = 'edgex-vault' +Port = 8200 diff --git a/internal/security/bootstrapper/command/cmd_dispatcher.go b/internal/security/bootstrapper/command/cmd_dispatcher.go index df1b69bdc0..83fb5d889f 100644 --- a/internal/security/bootstrapper/command/cmd_dispatcher.go +++ b/internal/security/bootstrapper/command/cmd_dispatcher.go @@ -25,6 +25,7 @@ import ( "github.com/edgexfoundry/edgex-go/internal/security/bootstrapper/command/gethttpstatus" "github.com/edgexfoundry/edgex-go/internal/security/bootstrapper/command/listen" "github.com/edgexfoundry/edgex-go/internal/security/bootstrapper/command/ping" + "github.com/edgexfoundry/edgex-go/internal/security/bootstrapper/command/setupacl" "github.com/edgexfoundry/edgex-go/internal/security/bootstrapper/command/waitfor" "github.com/edgexfoundry/edgex-go/internal/security/bootstrapper/config" "github.com/edgexfoundry/edgex-go/internal/security/bootstrapper/interfaces" @@ -44,8 +45,8 @@ func NewCommand( var err error if len(args) < 1 { - return nil, fmt.Errorf("subcommand required (%s, %s, %s, %s, %s, %s)", gate.CommandName, listen.CommandName, - ping.CommandName, gethttpstatus.CommandName, genpassword.CommandName, waitfor.CommandName) + return nil, fmt.Errorf("subcommand required (%s, %s, %s, %s, %s, %s, %s)", gate.CommandName, listen.CommandName, + ping.CommandName, gethttpstatus.CommandName, genpassword.CommandName, waitfor.CommandName, setupacl.CommandName) } commandName := args[0] @@ -63,6 +64,8 @@ func NewCommand( command, err = genpassword.NewCommand(ctx, wg, lc, configuration, args[1:]) case waitfor.CommandName: command, err = waitfor.NewCommand(ctx, wg, lc, configuration, args[1:]) + case setupacl.CommandName: + command, err = setupacl.NewCommand(ctx, wg, lc, configuration, args[1:]) default: command = nil err = fmt.Errorf("unsupported command %s", commandName) diff --git a/internal/security/bootstrapper/command/cmd_dispatcher_test.go b/internal/security/bootstrapper/command/cmd_dispatcher_test.go index d497c831ed..6e18c7e2ee 100644 --- a/internal/security/bootstrapper/command/cmd_dispatcher_test.go +++ b/internal/security/bootstrapper/command/cmd_dispatcher_test.go @@ -54,6 +54,7 @@ func TestNewCommand(t *testing.T) { {"Good: genPassword command", []string{"genPassword"}, "genPassword", false}, {"Good: getHttpStatus command", []string{"getHttpStatus", "--url=http://localhost:55555"}, "getHttpStatus", false}, {"Good: waitFor command", []string{"waitFor", "--uri=http://localhost:55555"}, "waitFor", false}, + {"Good: setupRegistryACL command", []string{"setupRegistryACL"}, "setupRegistryACL", false}, {"Bad: unknown command", []string{"unknown"}, "", true}, {"Bad: empty command", []string{}, "", true}, {"Bad: listenTcp command missing required --port", []string{"listenTcp"}, "", true}, @@ -62,18 +63,20 @@ func TestNewCommand(t *testing.T) { } for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { + test := tt //capture as local copy + t.Run(test.name, func(t *testing.T) { + t.Parallel() // Act - command, err := NewCommand(ctx, wg, lc, config, tt.cmdArgs) + command, err := NewCommand(ctx, wg, lc, config, test.cmdArgs) // Assert - if tt.expectedErr { + if test.expectedErr { require.Error(t, err) require.Nil(t, command) } else { require.NoError(t, err) require.NotNil(t, command) - require.Equal(t, tt.expectedCmdName, command.GetCommandName()) + require.Equal(t, test.expectedCmdName, command.GetCommandName()) } }) } diff --git a/internal/security/bootstrapper/command/flags_common.go b/internal/security/bootstrapper/command/flags_common.go index 10326677f6..afbedb7c9a 100644 --- a/internal/security/bootstrapper/command/flags_common.go +++ b/internal/security/bootstrapper/command/flags_common.go @@ -97,13 +97,14 @@ func HelpCallback() { " --confdir Specify local configuration directory\n"+ "\n"+ "Commands:\n"+ - " help Show available commands (this text)\n"+ - " gate Do security bootstrapper gating on stages while starting services\n"+ - " listenTcp Start up a TCP listener\n"+ - " pingPgDb Test Postgres database readiness\n"+ - " getHttpStatus Do an HTTP GET call to get the status code\n"+ - " genPassword Generate a random password\n"+ - " waitfor Wait for the other services with specified URI(s) to connect:\n"+ - " the URI(s) can be communication protocols like tcp/tcp4/tcp6/http/https or files\n", + " gate Do security bootstrapper gating on stages while starting services\n"+ + " genPassword Generate a random password\n"+ + " getHttpStatus Do an HTTP GET call to get the status code\n"+ + " help Show available commands (this text)\n"+ + " listenTcp Start up a TCP listener\n"+ + " pingPgDb Test Postgres database readiness\n"+ + " setupRegistryACL Set up registry's ACL and configure the access\n"+ + " waitFor Wait for the other services with specified URI(s) to connect:\n"+ + " the URI(s) can be communication protocols like tcp/tcp4/tcp6/http/https or files\n", os.Args[0]) } diff --git a/internal/security/bootstrapper/command/setupacl/aclbootstrap.go b/internal/security/bootstrapper/command/setupacl/aclbootstrap.go new file mode 100644 index 0000000000..12a70212e9 --- /dev/null +++ b/internal/security/bootstrapper/command/setupacl/aclbootstrap.go @@ -0,0 +1,114 @@ +/******************************************************************************* + * Copyright 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + *******************************************************************************/ + +package setupacl + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "os" + "path/filepath" + + "github.com/edgexfoundry/go-mod-secrets/v2/pkg/token/fileioperformer" +) + +// BootStrapACLTokenInfo is the key portion of the response metadata from consulACLBootstrapAPI +type BootStrapACLTokenInfo struct { + AccessorID string `json:"AccessorID"` + SecretID string `json:"SecretID"` + Policies []Policy `json:"Policies"` + CreateTime string `json:"CreateTime"` +} + +// Policy is the metadata for ACL policy +type Policy struct { + ID string `json:"ID"` + Name string `json:"Name"` +} + +// this should only be called once per Consul agent +func (c *cmd) callConsulACLBootstrapAPI() (*BootStrapACLTokenInfo, error) { + aclBootstrapURL, err := c.getAPI_URL(consulACLBootstrapAPI) + if err != nil { + return nil, err + } + + req, err := http.NewRequest(http.MethodPut, aclBootstrapURL, http.NoBody) + if err != nil { + return nil, fmt.Errorf("Failed to prepare request for http URL: %w", err) + } + resp, err := c.client.Do(req) + if err != nil { + return nil, fmt.Errorf("Failed to send request for http URL: %w", err) + } + + defer func() { + _ = resp.Body.Close() + }() + + responseBody, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("Failed to read response body of bootstrap ACL: %w", err) + } + + var bootstrapACLToken BootStrapACLTokenInfo + switch resp.StatusCode { + case http.StatusOK: + if err := json.NewDecoder(bytes.NewReader(responseBody)).Decode(&bootstrapACLToken); err != nil { + return nil, fmt.Errorf("failed to decode bootstrapACLToken json data: %v", err) + } + return &bootstrapACLToken, nil + default: + return nil, fmt.Errorf("failed to bootstrap Consul's ACL via URL [%s] and status code= %d: %s", aclBootstrapURL, + resp.StatusCode, string(responseBody)) + } +} + +func (c *cmd) saveBootstrapToken(tokenInfoToBeSaved *BootStrapACLTokenInfo) error { + // Write the token to the specified file + tokenFileAbsPath, err := filepath.Abs(c.configuration.StageGate.Registry.ACL.BootstrapTokenPath) + if err != nil { + return fmt.Errorf("failed to convert tokenFile to absolute path %s: %s", + c.configuration.StageGate.Registry.ACL.BootstrapTokenPath, err.Error()) + } + + // create the directory of tokenfile if not exists yet + dirOfToken := filepath.Dir(tokenFileAbsPath) + fileIoPerformer := fileioperformer.NewDefaultFileIoPerformer() + if err := fileIoPerformer.MkdirAll(dirOfToken, 0700); err != nil { + return fmt.Errorf("failed to create tokenpath base dir: %s", err.Error()) + } + + fileWriter, err := fileIoPerformer.OpenFileWriter(tokenFileAbsPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) + if err != nil { + return fmt.Errorf("failed to open file writer %s: %s", tokenFileAbsPath, err.Error()) + } + + if err := json.NewEncoder(fileWriter).Encode(tokenInfoToBeSaved); err != nil { + _ = fileWriter.Close() + return fmt.Errorf("failed to write bootstrap token: %s", err.Error()) + } + + if err := fileWriter.Close(); err != nil { + return fmt.Errorf("failed to close token file: %s", err.Error()) + } + + c.loggingClient.Infof("bootstrap token is written to %s", tokenFileAbsPath) + + return nil +} diff --git a/internal/security/bootstrapper/command/setupacl/command.go b/internal/security/bootstrapper/command/setupacl/command.go new file mode 100644 index 0000000000..1b2e9f6a96 --- /dev/null +++ b/internal/security/bootstrapper/command/setupacl/command.go @@ -0,0 +1,364 @@ +/******************************************************************************* + * Copyright 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + *******************************************************************************/ + +package setupacl + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/edgexfoundry/edgex-go/internal" + "github.com/edgexfoundry/edgex-go/internal/security/bootstrapper/config" + "github.com/edgexfoundry/edgex-go/internal/security/bootstrapper/helper" + "github.com/edgexfoundry/edgex-go/internal/security/bootstrapper/interfaces" + + "github.com/edgexfoundry/go-mod-secrets/v2/pkg" + + "github.com/edgexfoundry/go-mod-core-contracts/v2/clients" + "github.com/edgexfoundry/go-mod-core-contracts/v2/clients/logger" +) + +const ( + // the command name for setting up registry's ACL + CommandName string = "setupRegistryACL" + + consulGetLeaderAPI = "/v1/status/leader" + consulACLBootstrapAPI = "/v1/acl/bootstrap" + consulConfigAccessVaultAPI = "/v1/consul/config/access" + consulLegacyACLModeError = "The ACL system is currently in legacy mode" + defaultRetryTimeout = 30 * time.Second + emptyLeader = `""` + emptyToken = "" +) + +type cmd struct { + waitGroup *sync.WaitGroup + loggingClient logger.LoggingClient + client internal.HttpCaller + configuration *config.ConfigurationStruct + + // internal state + errs chan error + retryTimeout time.Duration +} + +// NewCommand creates a new cmd and parses through options if any +func NewCommand( + _ context.Context, + _ *sync.WaitGroup, + lc logger.LoggingClient, + conf *config.ConfigurationStruct, + args []string) (interfaces.Command, error) { + cmd := cmd{ + waitGroup: &sync.WaitGroup{}, + loggingClient: lc, + client: pkg.NewRequester(lc).Insecure(), + configuration: conf, + retryTimeout: defaultRetryTimeout, + } + var dummy string + + flagSet := flag.NewFlagSet(CommandName, flag.ContinueOnError) + flagSet.StringVar(&dummy, "confdir", "", "") // handled by bootstrap; duplicated here to prevent arg parsing errors + + err := flagSet.Parse(args) + if err != nil { + return nil, fmt.Errorf("Unable to parse command: %s: %w", strings.Join(args, " "), err) + } + + return &cmd, nil +} + +// Execute implements Command and runs this command +// command setupRegistryACL sets up the ACL system of the registry, Consul in this case, preparing for generating +// Consul's agent tokens later on +func (c *cmd) Execute() (statusCode int, err error) { + c.loggingClient.Infof("Security bootstrapper running %s", CommandName) + + if err := c.waitForNonEmptyConsulLeader(); err != nil { + return interfaces.StatusCodeExitWithError, fmt.Errorf("failed to wait for Consul leader: %v", err) + } + + var bootstrapTokenInfo *BootStrapACLTokenInfo + bootstrapTokenInfo, err = c.callConsulACLBootstrapAPI() + if err != nil { + // although we have a leader, but it is a very very rare chance that we could hit an error on legacy mode + // here we will sleep a bit of time and then retry once if there is error on Legacy ACL type of message + // because Consul is still on its way to initialize the new ACL system internally + // for the details of this issue, see related issue on Consul's Github website: + // https://github.com/hashicorp/consul/issues/5218#issuecomment-457212336 + if !strings.Contains(err.Error(), consulLegacyACLModeError) { + // other type of ACL bootstrapping error, cannot continue + return interfaces.StatusCodeExitWithError, fmt.Errorf("failed to bootstrap registry's ACL: %v", err) + } + + c.loggingClient.Warnf("found Consul still in ACL legacy mode, will retry once again: %v", err) + time.Sleep(5 * time.Second) + bootstrapTokenInfo, err = c.callConsulACLBootstrapAPI() + if err != nil { + return interfaces.StatusCodeExitWithError, fmt.Errorf("failed to bootstrap registry's ACL: %v", err) + } + } + + c.loggingClient.Info("successfully bootstrap registry ACL") + + // Save the bootstrap token into the file so that it can be used later on + if err := c.saveBootstrapToken(bootstrapTokenInfo); err != nil { + return interfaces.StatusCodeExitWithError, fmt.Errorf("failed to save registry's bootstrap token: %v", err) + } + + // retrieve the secretstore (Vault) token from the file produced by secretstore-setup + secretstoreToken, err := c.getSecretStoreTokenFromFile() + if err != nil { + return interfaces.StatusCodeExitWithError, fmt.Errorf("failed to retrieve secretstore token: %v", err) + } + + c.loggingClient.Info("successfully get secretstore token and configuring the registry access for secretestore") + + // configure Consul access with both Vault mgmt token and consul's bootstrap token + if err := c.configureConsulAccess(secretstoreToken, bootstrapTokenInfo.SecretID); err != nil { + return interfaces.StatusCodeExitWithError, fmt.Errorf("failed to configure Consul access: %v", err) + } + + return +} + +// GetCommandName returns the name of this command +func (c *cmd) GetCommandName() string { + return CommandName +} + +func (c *cmd) getAPI_URL(api string) (string, error) { + apiURL := fmt.Sprintf("%s://%s:%d%s", c.configuration.StageGate.Registry.ACL.Protocol, + c.configuration.StageGate.Registry.Host, c.configuration.StageGate.Registry.Port, api) + _, err := url.Parse(apiURL) + if err != nil { + return "", fmt.Errorf("failed to parse API URL: %v", err) + } + return apiURL, nil +} + +// waitForNonEmptyConsulLeader is a special waitFor function on waiting for "non-empty" leader being available +// the ordinary http waitFor won't work as the returned http status code from API call is 200 even when Consul's leader +// is an empty string ("") but we need an non-empty leader; so 200 doesn't mean we have a leader +func (c *cmd) waitForNonEmptyConsulLeader() error { + c.errs = make(chan error, 1) + // check Consul whether we have non-empty leader elected before proceed to Consul's ACL bootstrapping process + // as it requires non-empty leader to start with + go func() { + c.checkConsulLeader() + c.waitGroup.Wait() + }() + + gotLeader := make(chan bool) + // a separate goroutine to keep waiting for a leader got elected from the checkConsulLeader goroutine + // until retryTimeout is reached otherwise + go func() { + for { + if err := <-c.errs; err != nil { + continue + } + gotLeader <- true + close(gotLeader) + return + } + }() + + // block here until either we get a leader, or timeout is reached in which case ACL bootstrapping cannot continue + select { + case <-gotLeader: + c.loggingClient.Info("found Consul leader to bootstrap ACL") + return nil + case <-time.After(c.retryTimeout): + return errors.New("timed out to get non-empty Consul leader") + } +} + +// checkConsulLeader is a goroutine constantly running if there is an error from API call +func (c *cmd) checkConsulLeader() { + c.waitGroup.Add(1) + go func() { + defer c.waitGroup.Done() + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + // ticks every second to retry if we receive an error from API call + for range ticker.C { + if err := c.callConsulLeaderAPI(); err != nil { + c.loggingClient.Warnf("error from getting Consul leader API call, will retry it again: %v", err) + c.errs <- err + continue + } + // once reached here, we have a leader and terminate this goroutine + c.errs <- nil + return + } + }() +} + +// callConsulLeaderAPI makes http request call to get the registry Consul leader +// the response of getting leader call could be an empty leader (represented by "") +// even if the http status code is 200 when Consul is just booting up and +// it will take a bit of time to elect the raft leader +func (c *cmd) callConsulLeaderAPI() error { + getLeaderURL, err := c.getAPI_URL(consulGetLeaderAPI) + if err != nil { + return err + } + + req, err := http.NewRequest(http.MethodGet, getLeaderURL, http.NoBody) + if err != nil { + return fmt.Errorf("Failed to prepare request for http URL: %w", err) + } + resp, err := c.client.Do(req) + if err != nil { + return fmt.Errorf("Failed to send request for http URL: %w", err) + } + + defer func() { + _ = resp.Body.Close() + }() + + responseBody, err := ioutil.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("Failed to read response body to get leader: %w", err) + } + + switch resp.StatusCode { + case http.StatusOK: + trimmedResp := strings.TrimSpace(string(responseBody)) + // Consul's raft leader election process could take a bit of time + // before responds back with a non-empty leader + if len(trimmedResp) == 0 || emptyLeader == trimmedResp { + return errors.New("no leader yet") + } + // now we have a cluster raft leader + c.loggingClient.Infof("leader [%s] is elected", trimmedResp) + return nil + + // almost unlikely for this case unless URL is incorrect + default: + return fmt.Errorf("get Consul leader request failed with status code= %d: %s", resp.StatusCode, string(responseBody)) + } +} + +func (c *cmd) getSecretStoreTokenFromFile() (string, error) { + trimmedFilePath := strings.TrimSpace(c.configuration.StageGate.Registry.ACL.SecretsAdminTokenPath) + if len(trimmedFilePath) == 0 { + return emptyToken, errors.New("required StageGate_Registry_SecretsAdminTokenPath from configuration is empty") + } + + tokenFileAbsPath, err := filepath.Abs(trimmedFilePath) + if err != nil { + return emptyToken, fmt.Errorf("failed to convert tokenFile to absolute path %s: %v", trimmedFilePath, err) + } + + // since the secretstore token is created by another service, secretstore-setup, + // so here we want to make sure we have the file + if exists := helper.CheckIfFileExists(tokenFileAbsPath); !exists { + return emptyToken, fmt.Errorf("secretstore token file %s not found", tokenFileAbsPath) + } + + type SecretStoreToken struct { + Authentication struct { + Token string `json:"client_token"` + } `json:"auth"` + } + + tokenJSONFile, err := os.Open(trimmedFilePath) + + if err != nil { + return emptyToken, fmt.Errorf("failed to open secretstore token file: %v", err) + } + + var tokenData SecretStoreToken + if err := json.NewDecoder(tokenJSONFile).Decode(&tokenData); err != nil { + return emptyToken, fmt.Errorf("failed to decode secretstore json token file: %v", err) + } + + c.loggingClient.Infof("successfully retrieved secretstore management token from %s", trimmedFilePath) + + return tokenData.Authentication.Token, nil +} + +// configureConsulAccess is to enable the Consul config access to Vault via consul/config/access API +// see the reference: https://www.vaultproject.io/api-docs/secret/consul#configure-access +func (c *cmd) configureConsulAccess(secretStoreToken string, consulToken string) error { + configAccessURL := fmt.Sprintf("%s://%s:%d%s", c.configuration.SecretStore.Protocol, + c.configuration.SecretStore.Host, c.configuration.SecretStore.Port, consulConfigAccessVaultAPI) + _, err := url.Parse(configAccessURL) + if err != nil { + return fmt.Errorf("failed to parse config Access URL: %v", err) + } + + c.loggingClient.Debugf("configAccessURL: %s", configAccessURL) + + type ConfigAccess struct { + RegistryAddress string `json:"address"` + BootstrapToken string `json:"token"` + } + + payload := &ConfigAccess{ + RegistryAddress: fmt.Sprintf("%s:%d", c.configuration.StageGate.Registry.Host, c.configuration.StageGate.Registry.Port), + BootstrapToken: consulToken, + } + + jsonPayload, err := json.Marshal(payload) + c.loggingClient.Tracef("payload: %v", payload) + if err != nil { + return fmt.Errorf("Failed to marshal JSON string payload: %v", err) + } + + req, err := http.NewRequest(http.MethodPost, configAccessURL, bytes.NewBuffer(jsonPayload)) + if err != nil { + return fmt.Errorf("Failed to prepare POST request for http URL: %w", err) + } + + req.Header.Add("X-Vault-Token", secretStoreToken) + req.Header.Add(clients.ContentType, interfaces.JSONContentType) + resp, err := c.client.Do(req) + if err != nil { + return fmt.Errorf("Failed to send request for http URL: %w", err) + } + + defer func() { + _ = resp.Body.Close() + }() + + switch resp.StatusCode { + case http.StatusNoContent: + // no response body returned in this case + c.loggingClient.Info("successfully configure Consul access for secretstore") + return nil + default: + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + c.loggingClient.Errorf("cannot read resp.Body: %v", err) + } + return fmt.Errorf("failed to configure Consul access for secretstore via URL [%s] and status code= %d: %s", + configAccessURL, resp.StatusCode, string(body)) + } +} diff --git a/internal/security/bootstrapper/command/setupacl/command_test.go b/internal/security/bootstrapper/command/setupacl/command_test.go new file mode 100644 index 0000000000..5071a9e6ea --- /dev/null +++ b/internal/security/bootstrapper/command/setupacl/command_test.go @@ -0,0 +1,236 @@ +/******************************************************************************* + * Copyright 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + *******************************************************************************/ + +package setupacl + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strconv" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/edgexfoundry/edgex-go/internal/security/bootstrapper/config" + "github.com/edgexfoundry/edgex-go/internal/security/bootstrapper/helper" + "github.com/edgexfoundry/edgex-go/internal/security/bootstrapper/interfaces" + + "github.com/edgexfoundry/go-mod-core-contracts/v2/clients/logger" +) + +func TestNewCommand(t *testing.T) { + // Arrange + ctx := context.Background() + wg := &sync.WaitGroup{} + lc := logger.MockLogger{} + config := &config.ConfigurationStruct{} + + tests := []struct { + name string + cmdArgs []string + expectedErr bool + }{ + {"Good:setupRegistryACL cmd empty option", []string{}, false}, + {"Bad:setupRegistryACL invalid option", []string{"--invalid=xxx"}, true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + command, err := NewCommand(ctx, wg, lc, config, tt.cmdArgs) + if tt.expectedErr { + require.Error(t, err) + } else { + require.NoError(t, err) + require.NotNil(t, command) + } + }) + } +} + +func TestExecute(t *testing.T) { + type prepareTestFunc func(aclOkResponse bool, configAccessOkResponse bool, t *testing.T) (*config.ConfigurationStruct, + *httptest.Server) + + // Arrange + ctx := context.Background() + wg := &sync.WaitGroup{} + lc := logger.MockLogger{} + + tests := []struct { + name string + adminDir string + prepare prepareTestFunc + aclOkResponse bool + configAccessOkResponse bool + expectedErr bool + }{ + {"Good:setupRegistryACL with ok response from server", "test1", prepareTestRegistryServer, true, true, false}, + {"Bad:setupRegistryACL with bootstrap ACL API failed response from server", "test2", + prepareTestRegistryServer, false, false, true}, + {"Bad:setupRegistryACL with non-existing server", "test3", + func(_ bool, _ bool, _ *testing.T) (*config.ConfigurationStruct, *httptest.Server) { + return &config.ConfigurationStruct{ + StageGate: config.StageGateInfo{ + Registry: config.RegistryInfo{ + Host: "non-existing", + Port: 10001, + ACL: config.ACLInfo{Protocol: "http"}, + }, + }}, httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) + }, false, false, true}, + {"Bad:setupRegistryACL with empty api protocol", "test4", + func(_ bool, _ bool, _ *testing.T) (*config.ConfigurationStruct, *httptest.Server) { + return &config.ConfigurationStruct{ + StageGate: config.StageGateInfo{ + Registry: config.RegistryInfo{ + Host: "localhost", + Port: 10001, + ACL: config.ACLInfo{Protocol: ""}, + }, + }}, httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) + }, false, false, true}, + {"Bad:setupRegistryACL with timed out on waiting for secret token file", "", + prepareTestRegistryServer, true, false, true}, + {"Bad:setupRegistryACL with config access API failed response from server", "test5", + prepareTestRegistryServer, true, false, true}, + } + + for _, tt := range tests { + test := tt // capture as local copy + t.Run(test.name, func(t *testing.T) { + t.Parallel() + // prepare test + conf, testServer := test.prepare(test.aclOkResponse, test.configAccessOkResponse, t) + defer testServer.Close() + // setup token related configs + conf.StageGate.Registry.ACL.SecretsAdminTokenPath = filepath.Join(test.adminDir, "secret_token.json") + conf.StageGate.Registry.ACL.BootstrapTokenPath = filepath.Join(test.adminDir, "bootstrap_token.json") + + setupRegistryACL, err := NewCommand(ctx, wg, lc, conf, []string{}) + require.NoError(t, err) + require.NotNil(t, setupRegistryACL) + require.Equal(t, "setupRegistryACL", setupRegistryACL.GetCommandName()) + + // create test secret token file + if test.adminDir != "" { + err = helper.CreateDirectoryIfNotExists(test.adminDir) + require.NoError(t, err) + err = ioutil.WriteFile(conf.StageGate.Registry.ACL.SecretsAdminTokenPath, + []byte(secretstoreTokenJsonStub), 0600) + require.NoError(t, err) + } + + // to speed up the test timeout + localcmd := setupRegistryACL.(*cmd) + localcmd.retryTimeout = 3 * time.Second + statusCode, err := setupRegistryACL.Execute() + defer func() { + if test.adminDir == "" { + // empty test dir case don't have the directory to clean up + _ = os.Remove(conf.StageGate.Registry.ACL.BootstrapTokenPath) + } else { + _ = os.RemoveAll(test.adminDir) + } + }() + + if test.expectedErr { + require.Error(t, err) + require.Equal(t, interfaces.StatusCodeExitWithError, statusCode) + } else { + require.NoError(t, err) + require.Equal(t, interfaces.StatusCodeExitNormal, statusCode) + require.FileExists(t, conf.StageGate.Registry.ACL.BootstrapTokenPath) + require.FileExists(t, conf.StageGate.Registry.ACL.SecretsAdminTokenPath) + } + }) + } +} + +func prepareTestRegistryServer(aclOkResponse bool, configAccessOkResponse bool, t *testing.T) (*config.ConfigurationStruct, + *httptest.Server) { + registryTestConf := &config.ConfigurationStruct{} + + respCnt := 0 + testSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.EscapedPath() { + case consulGetLeaderAPI: + require.Equal(t, http.MethodGet, r.Method) + respCnt++ + w.WriteHeader(http.StatusOK) + var err error + if respCnt == 2 { + _, err = w.Write([]byte("127.0.0.1:12345")) + } else { + _, err = w.Write([]byte("")) + } + require.NoError(t, err) + case consulACLBootstrapAPI: + require.Equal(t, http.MethodPut, r.Method) + if aclOkResponse { + w.WriteHeader(http.StatusOK) + jsonResponse := map[string]interface{}{ + "AccessorID": "bad060a9-0e2b-47ba-98d5-9d622e2322b5", + "SecretID": "7240fdd9-1665-419b-a8c5-5691ca03af7c", + "Description": "Bootstrap Token (Global Management)", + "Policies": []map[string]interface{}{ + { + "ID": "00000000-0000-0000-0000-000000000001", + "Name": "global-management", + }, + }, + "Local": false, + "CreateTime": "2021-03-01T10:34:20.843397-07:00", + } + err := json.NewEncoder(w).Encode(jsonResponse) + require.NoError(t, err) + } else { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte("The ACL system is currently in legacy mode.")) + } + case consulConfigAccessVaultAPI: + require.Equal(t, http.MethodPost, r.Method) + if configAccessOkResponse { + w.WriteHeader(http.StatusNoContent) + } else { + w.WriteHeader(http.StatusForbidden) + } + default: + t.Fatal(fmt.Sprintf("Unexpected call to URL %s", r.URL.EscapedPath())) + } + })) + tsURL, err := url.Parse(testSrv.URL) + require.NoError(t, err) + portNum, _ := strconv.Atoi(tsURL.Port()) + registryTestConf.StageGate.Registry.ACL.Protocol = tsURL.Scheme + registryTestConf.StageGate.Registry.Host = tsURL.Hostname() + registryTestConf.StageGate.Registry.Port = portNum + registryTestConf.StageGate.WaitFor.Timeout = "1m" + registryTestConf.StageGate.WaitFor.RetryInterval = "1s" + // for the sake of simplicity, we use the same test server as the secret store server + registryTestConf.SecretStore.Protocol = tsURL.Scheme + registryTestConf.SecretStore.Host = tsURL.Hostname() + registryTestConf.SecretStore.Port = portNum + + return registryTestConf, testSrv +} diff --git a/internal/security/bootstrapper/command/setupacl/stubdata_test.go b/internal/security/bootstrapper/command/setupacl/stubdata_test.go new file mode 100644 index 0000000000..94d745cb06 --- /dev/null +++ b/internal/security/bootstrapper/command/setupacl/stubdata_test.go @@ -0,0 +1,51 @@ +/******************************************************************************* + * Copyright 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + *******************************************************************************/ + +package setupacl + +// this is just the stub for test data related +const ( + secretstoreTokenJsonStub = ` + { + "auth": { + "accessor": "xxxxxxxxxxxxxxxxxxxxxxx", + "client_token": "yyyyyyyyyyyyyyyyyyyyyyyyyy", + "entity_id": "", + "lease_duration": 3600, + "metadata": { + "description": "Consul secrets engine management token" + }, + "orphan": true, + "policies": [ + "consul_secrets_engine_management_policy", + "default" + ], + "renewable": true, + "token_policies": [ + "consul_secrets_engine_management_policy", + "default" + ], + "token_type": "service" + }, + "data": null, + "lease_duration": 0, + "lease_id": "", + "renewable": false, + "request_id": "aaaaaaaa-1111-2222-bbbb-cccccccccccc", + "warnings": null, + "wrap_info": null + } + ` +) diff --git a/internal/security/bootstrapper/config/config.go b/internal/security/bootstrapper/config/config.go index cc4f62648c..3f6b657e35 100644 --- a/internal/security/bootstrapper/config/config.go +++ b/internal/security/bootstrapper/config/config.go @@ -20,8 +20,9 @@ import ( ) type ConfigurationStruct struct { - LogLevel string - StageGate StageGateInfo + LogLevel string + StageGate StageGateInfo + SecretStore bootstrapConfig.SecretStoreInfo } // UpdateFromRaw converts configuration received from the registry to a service-specific configuration struct which is diff --git a/internal/security/bootstrapper/config/types.go b/internal/security/bootstrapper/config/types.go index 18bda476f8..ca24eb690c 100644 --- a/internal/security/bootstrapper/config/types.go +++ b/internal/security/bootstrapper/config/types.go @@ -56,6 +56,17 @@ type RegistryInfo struct { Host string Port int ReadyPort int + ACL ACLInfo +} + +// ACLInfo defines the fields related to Registry's ACL process +type ACLInfo struct { + // the protocol used for registry's API calls, usually it is different from the protocol of waitFor, i.e. TCP + Protocol string + // filepath to save the registry's token generated from ACL bootstrapping + BootstrapTokenPath string + // filepath for the secretstore's token created from secretstore-setup + SecretsAdminTokenPath string } // KongDBInfo defines the fields related to diff --git a/internal/security/bootstrapper/helper/helper.go b/internal/security/bootstrapper/helper/helper.go index 4260cbd207..0de1cdfe4f 100644 --- a/internal/security/bootstrapper/helper/helper.go +++ b/internal/security/bootstrapper/helper/helper.go @@ -26,7 +26,7 @@ import ( // MarkComplete creates a doneFile file func MarkComplete(dirPath, doneFile string) error { doneFilePath := filepath.Join(dirPath, doneFile) - if !checkIfFileExists(doneFilePath) { + if !CheckIfFileExists(doneFilePath) { if err := writeFile(doneFilePath); err != nil { return err } @@ -48,7 +48,8 @@ func CreateDirectoryIfNotExists(dirName string) (err error) { return } -func checkIfFileExists(fileName string) bool { +// CheckIfFileExists returns true if the specified fileName exists +func CheckIfFileExists(fileName string) bool { fileInfo, statErr := os.Stat(fileName) if os.IsNotExist(statErr) { return false diff --git a/internal/security/bootstrapper/interfaces/command.go b/internal/security/bootstrapper/interfaces/command.go index ee81dddf49..e2534d0094 100644 --- a/internal/security/bootstrapper/interfaces/command.go +++ b/internal/security/bootstrapper/interfaces/command.go @@ -22,6 +22,8 @@ const ( StatusCodeNoOptionSelected = 1 // StatusCodeExitWithError is exit code for error case StatusCodeExitWithError = 2 + // JSONContentType is the content type for JSON based body/payload + JSONContentType = "application/json" ) // Command implement the Command pattern From 08b54ac766c3fe3d3ca63a08240694b4d417effa Mon Sep 17 00:00:00 2001 From: Jim Wang Date: Wed, 3 Mar 2021 16:15:29 -0700 Subject: [PATCH 2/5] feat(security): Add sentinel file mechanism to avoid error Added the acl done sentinel file once Consul's registry had been setup successfully once This is to prevent the error if it is re-run second time or later. Like 2nd time compose-up or re-run the consul service in the snap. Signed-off-by: Jim Wang --- .../bootstrapper/command/setupacl/command.go | 47 +++++++++++++++++++ .../command/setupacl/command_test.go | 1 + 2 files changed, 48 insertions(+) diff --git a/internal/security/bootstrapper/command/setupacl/command.go b/internal/security/bootstrapper/command/setupacl/command.go index 1b2e9f6a96..87e282e20a 100644 --- a/internal/security/bootstrapper/command/setupacl/command.go +++ b/internal/security/bootstrapper/command/setupacl/command.go @@ -53,6 +53,7 @@ const ( defaultRetryTimeout = 30 * time.Second emptyLeader = `""` emptyToken = "" + sentinelFile = "consul_acl_done" ) type cmd struct { @@ -99,6 +100,14 @@ func NewCommand( func (c *cmd) Execute() (statusCode int, err error) { c.loggingClient.Infof("Security bootstrapper running %s", CommandName) + // need to have a sentinel file to guard against the re-run of the command once we have successfully bootstrap ACL + // if we already have a sentinelFile exists then skip this whole process since we already done this + // process successfully before, otherwise Consul's ACL bootstrap will cause a panic + if c.hasSentinelFile() { + c.loggingClient.Info("Registry ACL had been setup successfully already, skip") + return + } + if err := c.waitForNonEmptyConsulLeader(); err != nil { return interfaces.StatusCodeExitWithError, fmt.Errorf("failed to wait for Consul leader: %v", err) } @@ -144,6 +153,14 @@ func (c *cmd) Execute() (statusCode int, err error) { return interfaces.StatusCodeExitWithError, fmt.Errorf("failed to configure Consul access: %v", err) } + // write a sentinel file to indicate Consul ACL bootstrap is done so that we don't bootstrap ACL again, + // this is to avoid re-bootstrapping error and that error can cause the snap crash if restart this process + if err := c.writeSentinelFile(); err != nil { + return interfaces.StatusCodeExitWithError, fmt.Errorf("failed to write sentinel file: %v", err) + } + + c.loggingClient.Info("setupRegistryACL successfully done") + return } @@ -289,6 +306,7 @@ func (c *cmd) getSecretStoreTokenFromFile() (string, error) { } tokenJSONFile, err := os.Open(trimmedFilePath) + defer func() { tokenJSONFile.Close() }() if err != nil { return emptyToken, fmt.Errorf("failed to open secretstore token file: %v", err) @@ -362,3 +380,32 @@ func (c *cmd) configureConsulAccess(secretStoreToken string, consulToken string) configAccessURL, resp.StatusCode, string(body)) } } + +func (c *cmd) writeSentinelFile() error { + filePathToSave := c.getSentinelFilePath() + fileHandle, err := os.OpenFile(filePathToSave, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600) + defer func() { fileHandle.Close() }() + + if err != nil { + return fmt.Errorf("failed to open sentinel file %s: %v", filePathToSave, err) + } + + if _, err := fileHandle.Write([]byte("done")); err != nil { + return fmt.Errorf("failed to write out to sentinel file %s: %v", filePathToSave, err) + } + + return nil +} + +func (c *cmd) hasSentinelFile() bool { + sentinelFilePath := c.getSentinelFilePath() + return helper.CheckIfFileExists(sentinelFilePath) +} + +func (c *cmd) getSentinelFilePath() string { + // use the same directory as the bootstrap acl token for this sentinel file directory + // as it is the only writable directory in terms of volume mount in docker + absPath, _ := filepath.Abs(c.configuration.StageGate.Registry.ACL.BootstrapTokenPath) + sentinelDir := filepath.Dir(absPath) + return filepath.Join(sentinelDir, sentinelFile) +} diff --git a/internal/security/bootstrapper/command/setupacl/command_test.go b/internal/security/bootstrapper/command/setupacl/command_test.go index 5071a9e6ea..34814bfb4e 100644 --- a/internal/security/bootstrapper/command/setupacl/command_test.go +++ b/internal/security/bootstrapper/command/setupacl/command_test.go @@ -162,6 +162,7 @@ func TestExecute(t *testing.T) { require.Equal(t, interfaces.StatusCodeExitNormal, statusCode) require.FileExists(t, conf.StageGate.Registry.ACL.BootstrapTokenPath) require.FileExists(t, conf.StageGate.Registry.ACL.SecretsAdminTokenPath) + require.FileExists(t, localcmd.getSentinelFilePath()) } }) } From b8970f07553857e13bfec2dc87e20020855b1788 Mon Sep 17 00:00:00 2001 From: Jim Wang Date: Wed, 3 Mar 2021 17:07:52 -0700 Subject: [PATCH 3/5] fix: Address PR feedback and comments Address Lenny's PR comments and feedback Signed-off-by: Jim Wang --- .../command/setupacl/aclbootstrap.go | 8 +- .../bootstrapper/command/setupacl/command.go | 93 ++++++------------- .../bootstrapper/interfaces/command.go | 2 - 3 files changed, 32 insertions(+), 71 deletions(-) diff --git a/internal/security/bootstrapper/command/setupacl/aclbootstrap.go b/internal/security/bootstrapper/command/setupacl/aclbootstrap.go index 12a70212e9..ff87f7dafd 100644 --- a/internal/security/bootstrapper/command/setupacl/aclbootstrap.go +++ b/internal/security/bootstrapper/command/setupacl/aclbootstrap.go @@ -41,9 +41,9 @@ type Policy struct { Name string `json:"Name"` } -// this should only be called once per Consul agent -func (c *cmd) callConsulACLBootstrapAPI() (*BootStrapACLTokenInfo, error) { - aclBootstrapURL, err := c.getAPI_URL(consulACLBootstrapAPI) +// generateBootStrapACLToken should only be called once per Consul agent +func (c *cmd) generateBootStrapACLToken() (*BootStrapACLTokenInfo, error) { + aclBootstrapURL, err := c.getRegistryApiUrl(consulACLBootstrapAPI) if err != nil { return nil, err } @@ -79,7 +79,7 @@ func (c *cmd) callConsulACLBootstrapAPI() (*BootStrapACLTokenInfo, error) { } } -func (c *cmd) saveBootstrapToken(tokenInfoToBeSaved *BootStrapACLTokenInfo) error { +func (c *cmd) saveBootstrapACLToken(tokenInfoToBeSaved *BootStrapACLTokenInfo) error { // Write the token to the specified file tokenFileAbsPath, err := filepath.Abs(c.configuration.StageGate.Registry.ACL.BootstrapTokenPath) if err != nil { diff --git a/internal/security/bootstrapper/command/setupacl/command.go b/internal/security/bootstrapper/command/setupacl/command.go index 87e282e20a..3f9b648ea2 100644 --- a/internal/security/bootstrapper/command/setupacl/command.go +++ b/internal/security/bootstrapper/command/setupacl/command.go @@ -36,6 +36,8 @@ import ( "github.com/edgexfoundry/edgex-go/internal/security/bootstrapper/helper" "github.com/edgexfoundry/edgex-go/internal/security/bootstrapper/interfaces" + "github.com/edgexfoundry/go-mod-bootstrap/v2/bootstrap/startup" + "github.com/edgexfoundry/go-mod-secrets/v2/pkg" "github.com/edgexfoundry/go-mod-core-contracts/v2/clients" @@ -63,7 +65,6 @@ type cmd struct { configuration *config.ConfigurationStruct // internal state - errs chan error retryTimeout time.Duration } @@ -112,8 +113,8 @@ func (c *cmd) Execute() (statusCode int, err error) { return interfaces.StatusCodeExitWithError, fmt.Errorf("failed to wait for Consul leader: %v", err) } - var bootstrapTokenInfo *BootStrapACLTokenInfo - bootstrapTokenInfo, err = c.callConsulACLBootstrapAPI() + var bootstrapACLToken *BootStrapACLTokenInfo + bootstrapACLToken, err = c.generateBootStrapACLToken() if err != nil { // although we have a leader, but it is a very very rare chance that we could hit an error on legacy mode // here we will sleep a bit of time and then retry once if there is error on Legacy ACL type of message @@ -127,7 +128,7 @@ func (c *cmd) Execute() (statusCode int, err error) { c.loggingClient.Warnf("found Consul still in ACL legacy mode, will retry once again: %v", err) time.Sleep(5 * time.Second) - bootstrapTokenInfo, err = c.callConsulACLBootstrapAPI() + bootstrapACLToken, err = c.generateBootStrapACLToken() if err != nil { return interfaces.StatusCodeExitWithError, fmt.Errorf("failed to bootstrap registry's ACL: %v", err) } @@ -136,8 +137,8 @@ func (c *cmd) Execute() (statusCode int, err error) { c.loggingClient.Info("successfully bootstrap registry ACL") // Save the bootstrap token into the file so that it can be used later on - if err := c.saveBootstrapToken(bootstrapTokenInfo); err != nil { - return interfaces.StatusCodeExitWithError, fmt.Errorf("failed to save registry's bootstrap token: %v", err) + if err := c.saveBootstrapACLToken(bootstrapACLToken); err != nil { + return interfaces.StatusCodeExitWithError, fmt.Errorf("failed to save registry's bootstrap ACL token: %v", err) } // retrieve the secretstore (Vault) token from the file produced by secretstore-setup @@ -148,8 +149,8 @@ func (c *cmd) Execute() (statusCode int, err error) { c.loggingClient.Info("successfully get secretstore token and configuring the registry access for secretestore") - // configure Consul access with both Vault mgmt token and consul's bootstrap token - if err := c.configureConsulAccess(secretstoreToken, bootstrapTokenInfo.SecretID); err != nil { + // configure Consul access with both Secret Store token and consul's bootstrap acl token + if err := c.configureConsulAccess(secretstoreToken, bootstrapACLToken.SecretID); err != nil { return interfaces.StatusCodeExitWithError, fmt.Errorf("failed to configure Consul access: %v", err) } @@ -169,7 +170,7 @@ func (c *cmd) GetCommandName() string { return CommandName } -func (c *cmd) getAPI_URL(api string) (string, error) { +func (c *cmd) getRegistryApiUrl(api string) (string, error) { apiURL := fmt.Sprintf("%s://%s:%d%s", c.configuration.StageGate.Registry.ACL.Protocol, c.configuration.StageGate.Registry.Host, c.configuration.StageGate.Registry.Port, api) _, err := url.Parse(apiURL) @@ -183,65 +184,27 @@ func (c *cmd) getAPI_URL(api string) (string, error) { // the ordinary http waitFor won't work as the returned http status code from API call is 200 even when Consul's leader // is an empty string ("") but we need an non-empty leader; so 200 doesn't mean we have a leader func (c *cmd) waitForNonEmptyConsulLeader() error { - c.errs = make(chan error, 1) - // check Consul whether we have non-empty leader elected before proceed to Consul's ACL bootstrapping process - // as it requires non-empty leader to start with - go func() { - c.checkConsulLeader() - c.waitGroup.Wait() - }() - - gotLeader := make(chan bool) - // a separate goroutine to keep waiting for a leader got elected from the checkConsulLeader goroutine - // until retryTimeout is reached otherwise - go func() { - for { - if err := <-c.errs; err != nil { - continue - } - gotLeader <- true - close(gotLeader) - return + timeoutInSec := int(c.retryTimeout.Seconds()) + timer := startup.NewTimer(timeoutInSec, 1) + for timer.HasNotElapsed() { + if err := c.getNonEmptyConsulLeader(); err != nil { + c.loggingClient.Warnf("error from getting Consul leader API call, will retry it again: %v", err) + timer.SleepForInterval() + continue } - }() - - // block here until either we get a leader, or timeout is reached in which case ACL bootstrapping cannot continue - select { - case <-gotLeader: c.loggingClient.Info("found Consul leader to bootstrap ACL") return nil - case <-time.After(c.retryTimeout): - return errors.New("timed out to get non-empty Consul leader") } -} -// checkConsulLeader is a goroutine constantly running if there is an error from API call -func (c *cmd) checkConsulLeader() { - c.waitGroup.Add(1) - go func() { - defer c.waitGroup.Done() - ticker := time.NewTicker(time.Second) - defer ticker.Stop() - // ticks every second to retry if we receive an error from API call - for range ticker.C { - if err := c.callConsulLeaderAPI(); err != nil { - c.loggingClient.Warnf("error from getting Consul leader API call, will retry it again: %v", err) - c.errs <- err - continue - } - // once reached here, we have a leader and terminate this goroutine - c.errs <- nil - return - } - }() + return errors.New("timed out to get non-empty Consul leader") } -// callConsulLeaderAPI makes http request call to get the registry Consul leader +// getNonEmptyConsulLeader makes http request call to get the registry Consul leader // the response of getting leader call could be an empty leader (represented by "") // even if the http status code is 200 when Consul is just booting up and // it will take a bit of time to elect the raft leader -func (c *cmd) callConsulLeaderAPI() error { - getLeaderURL, err := c.getAPI_URL(consulGetLeaderAPI) +func (c *cmd) getNonEmptyConsulLeader() error { + getLeaderURL, err := c.getRegistryApiUrl(consulGetLeaderAPI) if err != nil { return err } @@ -322,9 +285,9 @@ func (c *cmd) getSecretStoreTokenFromFile() (string, error) { return tokenData.Authentication.Token, nil } -// configureConsulAccess is to enable the Consul config access to Vault via consul/config/access API +// configureConsulAccess is to enable the Consul config access to the SecretStore via consul/config/access API // see the reference: https://www.vaultproject.io/api-docs/secret/consul#configure-access -func (c *cmd) configureConsulAccess(secretStoreToken string, consulToken string) error { +func (c *cmd) configureConsulAccess(secretStoreToken string, bootstrapACLToken string) error { configAccessURL := fmt.Sprintf("%s://%s:%d%s", c.configuration.SecretStore.Protocol, c.configuration.SecretStore.Host, c.configuration.SecretStore.Port, consulConfigAccessVaultAPI) _, err := url.Parse(configAccessURL) @@ -335,13 +298,13 @@ func (c *cmd) configureConsulAccess(secretStoreToken string, consulToken string) c.loggingClient.Debugf("configAccessURL: %s", configAccessURL) type ConfigAccess struct { - RegistryAddress string `json:"address"` - BootstrapToken string `json:"token"` + RegistryAddress string `json:"address"` + BootstrapACLToken string `json:"token"` } payload := &ConfigAccess{ - RegistryAddress: fmt.Sprintf("%s:%d", c.configuration.StageGate.Registry.Host, c.configuration.StageGate.Registry.Port), - BootstrapToken: consulToken, + RegistryAddress: fmt.Sprintf("%s:%d", c.configuration.StageGate.Registry.Host, c.configuration.StageGate.Registry.Port), + BootstrapACLToken: bootstrapACLToken, } jsonPayload, err := json.Marshal(payload) @@ -356,7 +319,7 @@ func (c *cmd) configureConsulAccess(secretStoreToken string, consulToken string) } req.Header.Add("X-Vault-Token", secretStoreToken) - req.Header.Add(clients.ContentType, interfaces.JSONContentType) + req.Header.Add(clients.ContentType, clients.ContentTypeJSON) resp, err := c.client.Do(req) if err != nil { return fmt.Errorf("Failed to send request for http URL: %w", err) diff --git a/internal/security/bootstrapper/interfaces/command.go b/internal/security/bootstrapper/interfaces/command.go index e2534d0094..ee81dddf49 100644 --- a/internal/security/bootstrapper/interfaces/command.go +++ b/internal/security/bootstrapper/interfaces/command.go @@ -22,8 +22,6 @@ const ( StatusCodeNoOptionSelected = 1 // StatusCodeExitWithError is exit code for error case StatusCodeExitWithError = 2 - // JSONContentType is the content type for JSON based body/payload - JSONContentType = "application/json" ) // Command implement the Command pattern From 7c9b397cfff3d75d4c741b6aac7875ac9d2b4cdb Mon Sep 17 00:00:00 2001 From: Jim Wang Date: Thu, 4 Mar 2021 07:58:08 -0700 Subject: [PATCH 4/5] fix: code clean up Remove unused waitGroup Signed-off-by: Jim Wang --- internal/security/bootstrapper/command/setupacl/command.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/internal/security/bootstrapper/command/setupacl/command.go b/internal/security/bootstrapper/command/setupacl/command.go index 3f9b648ea2..1420b18031 100644 --- a/internal/security/bootstrapper/command/setupacl/command.go +++ b/internal/security/bootstrapper/command/setupacl/command.go @@ -59,7 +59,6 @@ const ( ) type cmd struct { - waitGroup *sync.WaitGroup loggingClient logger.LoggingClient client internal.HttpCaller configuration *config.ConfigurationStruct @@ -76,7 +75,6 @@ func NewCommand( conf *config.ConfigurationStruct, args []string) (interfaces.Command, error) { cmd := cmd{ - waitGroup: &sync.WaitGroup{}, loggingClient: lc, client: pkg.NewRequester(lc).Insecure(), configuration: conf, From 191f7d1831594804951f64ff5fb62e04ec87ee0f Mon Sep 17 00:00:00 2001 From: Jim Wang Date: Thu, 4 Mar 2021 14:57:41 -0700 Subject: [PATCH 5/5] feat(security): Add sentinel file config and also address PR feedback - add Sentinel filepath configuration in toml Address Bryon's PR feedback/comments Signed-off-by: Jim Wang --- .../entrypoint-scripts/consul_wait_install.sh | 2 + .../res/configuration.toml | 4 +- .../configuration.toml | 2 +- .../res/configuration.toml | 2 +- .../command/setupacl/aclbootstrap.go | 6 +- .../bootstrapper/command/setupacl/command.go | 65 ++++++++----------- .../command/setupacl/command_test.go | 3 +- .../security/bootstrapper/config/types.go | 2 + 8 files changed, 41 insertions(+), 45 deletions(-) diff --git a/cmd/security-bootstrapper/entrypoint-scripts/consul_wait_install.sh b/cmd/security-bootstrapper/entrypoint-scripts/consul_wait_install.sh index 3225ec0737..5fc58b5a01 100755 --- a/cmd/security-bootstrapper/entrypoint-scripts/consul_wait_install.sh +++ b/cmd/security-bootstrapper/entrypoint-scripts/consul_wait_install.sh @@ -93,6 +93,7 @@ if [ "${ENABLE_REGISTRY_ACL}" == "true" ]; then echo "$(date) failed to set up Consul ACL" fi set -e + # no need to wait for Consul's port since it is in ready state after all ACL stuff else echo "$(date) Starting edgex-core-consul with ACL disabled ..." docker-entrypoint.sh agent \ @@ -101,6 +102,7 @@ else -server \ -client 0.0.0.0 & # wait for the consul port + # this waitFor is not necessary in the other ACL case, as it is already in the ready state echo "$(date) Executing waitFor on Consul with waiting on its own port \ tcp://${STAGEGATE_REGISTRY_HOST}:${STAGEGATE_REGISTRY_PORT}" /edgex-init/security-bootstrapper --confdir=/edgex-init/res waitFor \ diff --git a/cmd/security-bootstrapper/res/configuration.toml b/cmd/security-bootstrapper/res/configuration.toml index 194a3ff7bd..f1eef43f69 100644 --- a/cmd/security-bootstrapper/res/configuration.toml +++ b/cmd/security-bootstrapper/res/configuration.toml @@ -26,6 +26,8 @@ LogLevel = 'INFO' BootstrapTokenPath = "/tmp/edgex/secrets/edgex-consul/admin/bootstrap_token.json" # this is the filepath for the Vault token created from secretstore-setup SecretsAdminTokenPath = "/tmp/edgex/secrets/edgex-consul/admin/token.json" + # this is the filepath for the sentinel file to indicate the registry ACL is set up successfully + SentinelFilePath = "/edgex-init/consul-bootstrapper/consul_acl_done" [StageGate.KongDb] Host = "kong-db" Port = 5432 @@ -41,5 +43,5 @@ LogLevel = 'INFO' [SecretStore] Type = 'vault' Protocol = 'http' -Host = 'edgex-vault' +Host = 'localhost' Port = 8200 diff --git a/cmd/security-secretstore-setup/res-file-token-provider/configuration.toml b/cmd/security-secretstore-setup/res-file-token-provider/configuration.toml index ac532a1108..7ea1a65be4 100644 --- a/cmd/security-secretstore-setup/res-file-token-provider/configuration.toml +++ b/cmd/security-secretstore-setup/res-file-token-provider/configuration.toml @@ -3,7 +3,7 @@ LogLevel = 'DEBUG' [SecretStore] Type = "vault" Protocol = "http" -Host = "edgex-vault" +Host = "localhost" Port = 8200 ServerName = "" CaFilePath = "" diff --git a/cmd/security-secretstore-setup/res/configuration.toml b/cmd/security-secretstore-setup/res/configuration.toml index f9dc1f56f9..56cf2697dc 100644 --- a/cmd/security-secretstore-setup/res/configuration.toml +++ b/cmd/security-secretstore-setup/res/configuration.toml @@ -21,7 +21,7 @@ LogLevel = 'DEBUG' [SecretStore] Type = "vault" Protocol = "http" -Host = "edgex-vault" +Host = "localhost" Port = 8200 CertPath = "" CaFilePath = "" diff --git a/internal/security/bootstrapper/command/setupacl/aclbootstrap.go b/internal/security/bootstrapper/command/setupacl/aclbootstrap.go index ff87f7dafd..9c649b2d59 100644 --- a/internal/security/bootstrapper/command/setupacl/aclbootstrap.go +++ b/internal/security/bootstrapper/command/setupacl/aclbootstrap.go @@ -29,10 +29,8 @@ import ( // BootStrapACLTokenInfo is the key portion of the response metadata from consulACLBootstrapAPI type BootStrapACLTokenInfo struct { - AccessorID string `json:"AccessorID"` - SecretID string `json:"SecretID"` - Policies []Policy `json:"Policies"` - CreateTime string `json:"CreateTime"` + SecretID string `json:"SecretID"` + Policies []Policy `json:"Policies"` } // Policy is the metadata for ACL policy diff --git a/internal/security/bootstrapper/command/setupacl/command.go b/internal/security/bootstrapper/command/setupacl/command.go index 1420b18031..3d13ea8f3e 100644 --- a/internal/security/bootstrapper/command/setupacl/command.go +++ b/internal/security/bootstrapper/command/setupacl/command.go @@ -39,6 +39,8 @@ import ( "github.com/edgexfoundry/go-mod-bootstrap/v2/bootstrap/startup" "github.com/edgexfoundry/go-mod-secrets/v2/pkg" + "github.com/edgexfoundry/go-mod-secrets/v2/pkg/token/authtokenloader" + "github.com/edgexfoundry/go-mod-secrets/v2/pkg/token/fileioperformer" "github.com/edgexfoundry/go-mod-core-contracts/v2/clients" "github.com/edgexfoundry/go-mod-core-contracts/v2/clients/logger" @@ -55,7 +57,6 @@ const ( defaultRetryTimeout = 30 * time.Second emptyLeader = `""` emptyToken = "" - sentinelFile = "consul_acl_done" ) type cmd struct { @@ -102,7 +103,12 @@ func (c *cmd) Execute() (statusCode int, err error) { // need to have a sentinel file to guard against the re-run of the command once we have successfully bootstrap ACL // if we already have a sentinelFile exists then skip this whole process since we already done this // process successfully before, otherwise Consul's ACL bootstrap will cause a panic - if c.hasSentinelFile() { + sentinelFileAbsPath, err := filepath.Abs(c.configuration.StageGate.Registry.ACL.SentinelFilePath) + if err != nil { + return interfaces.StatusCodeExitWithError, fmt.Errorf("failed to get the absolute path of the sentinel file: %v", err) + } + + if helper.CheckIfFileExists(sentinelFileAbsPath) { c.loggingClient.Info("Registry ACL had been setup successfully already, skip") return } @@ -260,27 +266,16 @@ func (c *cmd) getSecretStoreTokenFromFile() (string, error) { return emptyToken, fmt.Errorf("secretstore token file %s not found", tokenFileAbsPath) } - type SecretStoreToken struct { - Authentication struct { - Token string `json:"client_token"` - } `json:"auth"` - } - - tokenJSONFile, err := os.Open(trimmedFilePath) - defer func() { tokenJSONFile.Close() }() - + fileOpener := fileioperformer.NewDefaultFileIoPerformer() + tokenLoader := authtokenloader.NewAuthTokenLoader(fileOpener) + secretStoreToken, err := tokenLoader.Load(tokenFileAbsPath) if err != nil { - return emptyToken, fmt.Errorf("failed to open secretstore token file: %v", err) - } - - var tokenData SecretStoreToken - if err := json.NewDecoder(tokenJSONFile).Decode(&tokenData); err != nil { - return emptyToken, fmt.Errorf("failed to decode secretstore json token file: %v", err) + return emptyToken, fmt.Errorf("tokenLoader failed to load secretstore token: %v", err) } c.loggingClient.Infof("successfully retrieved secretstore management token from %s", trimmedFilePath) - return tokenData.Authentication.Token, nil + return secretStoreToken, nil } // configureConsulAccess is to enable the Consul config access to the SecretStore via consul/config/access API @@ -343,30 +338,26 @@ func (c *cmd) configureConsulAccess(secretStoreToken string, bootstrapACLToken s } func (c *cmd) writeSentinelFile() error { - filePathToSave := c.getSentinelFilePath() - fileHandle, err := os.OpenFile(filePathToSave, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600) - defer func() { fileHandle.Close() }() - + absPath, err := filepath.Abs(c.configuration.StageGate.Registry.ACL.SentinelFilePath) if err != nil { - return fmt.Errorf("failed to open sentinel file %s: %v", filePathToSave, err) + return fmt.Errorf("failed to get the absolute path of the sentinel file: %v", err) } - if _, err := fileHandle.Write([]byte("done")); err != nil { - return fmt.Errorf("failed to write out to sentinel file %s: %v", filePathToSave, err) + dirToWrite := filepath.Dir(absPath) + filePerformer := fileioperformer.NewDefaultFileIoPerformer() + if err := filePerformer.MkdirAll(dirToWrite, 0700); err != nil { + return fmt.Errorf("failed to create sentinel base dir: %s", err.Error()) } - return nil -} + writer, err := filePerformer.OpenFileWriter(absPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) + if err != nil { + return fmt.Errorf("failed to open file writer %s: %s", absPath, err.Error()) + } + defer func() { _ = writer.Close() }() -func (c *cmd) hasSentinelFile() bool { - sentinelFilePath := c.getSentinelFilePath() - return helper.CheckIfFileExists(sentinelFilePath) -} + if _, err := writer.Write([]byte("done")); err != nil { + return fmt.Errorf("failed to write out to sentinel file %s: %v", absPath, err) + } -func (c *cmd) getSentinelFilePath() string { - // use the same directory as the bootstrap acl token for this sentinel file directory - // as it is the only writable directory in terms of volume mount in docker - absPath, _ := filepath.Abs(c.configuration.StageGate.Registry.ACL.BootstrapTokenPath) - sentinelDir := filepath.Dir(absPath) - return filepath.Join(sentinelDir, sentinelFile) + return nil } diff --git a/internal/security/bootstrapper/command/setupacl/command_test.go b/internal/security/bootstrapper/command/setupacl/command_test.go index 34814bfb4e..886803fcf3 100644 --- a/internal/security/bootstrapper/command/setupacl/command_test.go +++ b/internal/security/bootstrapper/command/setupacl/command_test.go @@ -126,6 +126,7 @@ func TestExecute(t *testing.T) { // setup token related configs conf.StageGate.Registry.ACL.SecretsAdminTokenPath = filepath.Join(test.adminDir, "secret_token.json") conf.StageGate.Registry.ACL.BootstrapTokenPath = filepath.Join(test.adminDir, "bootstrap_token.json") + conf.StageGate.Registry.ACL.SentinelFilePath = filepath.Join(test.adminDir, "sentinel_test_file") setupRegistryACL, err := NewCommand(ctx, wg, lc, conf, []string{}) require.NoError(t, err) @@ -162,7 +163,7 @@ func TestExecute(t *testing.T) { require.Equal(t, interfaces.StatusCodeExitNormal, statusCode) require.FileExists(t, conf.StageGate.Registry.ACL.BootstrapTokenPath) require.FileExists(t, conf.StageGate.Registry.ACL.SecretsAdminTokenPath) - require.FileExists(t, localcmd.getSentinelFilePath()) + require.FileExists(t, conf.StageGate.Registry.ACL.SentinelFilePath) } }) } diff --git a/internal/security/bootstrapper/config/types.go b/internal/security/bootstrapper/config/types.go index ca24eb690c..694b7170af 100644 --- a/internal/security/bootstrapper/config/types.go +++ b/internal/security/bootstrapper/config/types.go @@ -67,6 +67,8 @@ type ACLInfo struct { BootstrapTokenPath string // filepath for the secretstore's token created from secretstore-setup SecretsAdminTokenPath string + // filepath for the sentinel file to indicate the registry ACL is set up successfully + SentinelFilePath string } // KongDBInfo defines the fields related to