From 9ce36fbc24aa71139e2638edf63ea1299a37b8ff Mon Sep 17 00:00:00 2001 From: "R.B. Boyer" Date: Wed, 8 Mar 2023 15:11:14 -0600 Subject: [PATCH] adding support for consul-dataplane etc --- .github/workflows/check.yml | 54 +- .golangci.yml | 2 +- Dockerfile-cdp | 8 + Dockerfile-tool | 5 + Makefile | 8 +- app/app.go | 6 + app/boot.go | 525 ++++++++++++++---- app/boot_templates.go | 4 - app/debug.go | 204 +++++-- app/docker.go | 74 ++- app/dumpconfig.go | 31 +- app/gen.go | 24 +- app/gen_catalog.go | 165 ++++++ app/runner/exec.go | 25 +- app/tfgen/agent.go | 26 +- app/tfgen/gateway.go | 14 +- app/tfgen/infra.go | 62 +++ app/tfgen/mesh.go | 204 +++++-- app/tfgen/nodes.go | 44 +- app/tfgen/o11y.go | 7 +- .../templates/container-app-dataplane.tf.tmpl | 43 ++ .../templates/container-catalog-sync.tf.tmpl | 37 ++ .../templates/prometheus-config.yml.tmpl | 5 + app/tfgen/tfgen.go | 16 +- app/util.go | 40 +- cachestore/store.go | 4 + clustertool/catalog-sync.go | 405 ++++++++++++++ clustertool/cliapi/api.go | 19 + clustertool/main.go | 69 +++ config/config.go | 18 +- config/config_test.go | 48 +- config/default_cdp.go | 3 + config/default_envoy.go | 2 +- config/parse.go | 35 +- config/raw.go | 40 +- consulfunc/catalog.go | 38 +- consulfunc/config.go | 4 +- consulfunc/ns.go | 56 ++ consulfunc/part.go | 4 +- dataplane-boot.sh | 40 ++ example-config.peer.hcl | 10 +- go.mod | 33 +- go.sum | 164 +----- infra/infra.go | 59 +- infra/infra_test.go | 234 ++++++-- infra/topology.go | 116 +++- main.go | 15 +- mesh-gateway-sidecar-boot.sh | 15 + sidecar-boot.sh | 8 +- structs/catalog_defs.go | 201 +++++++ test-configs/config.dataplane.hcl | 44 ++ test-configs/config.prepared-query.hcl | 3 +- test-configs/config.prometheus.hcl | 3 +- test-configs/config.simple-no-acls.hcl | 3 +- test-configs/config.simple-peering.hcl | 11 +- test-configs/config.simple.hcl | 3 +- test-configs/config.tls-api.hcl | 3 +- test-configs/config.vault-ca.hcl | 3 +- test-configs/config.vault.hcl | 3 +- test-configs/config.wanfed-mgw.hcl | 3 +- test-configs/test.sh | 23 +- util/files.go | 57 ++ util/util.go | 36 +- 63 files changed, 2830 insertions(+), 638 deletions(-) create mode 100644 Dockerfile-cdp create mode 100644 Dockerfile-tool create mode 100644 app/gen_catalog.go create mode 100644 app/tfgen/infra.go create mode 100644 app/tfgen/templates/container-app-dataplane.tf.tmpl create mode 100644 app/tfgen/templates/container-catalog-sync.tf.tmpl create mode 100644 clustertool/catalog-sync.go create mode 100644 clustertool/cliapi/api.go create mode 100644 clustertool/main.go create mode 100644 config/default_cdp.go create mode 100644 consulfunc/ns.go create mode 100755 dataplane-boot.sh create mode 100644 structs/catalog_defs.go create mode 100644 test-configs/config.dataplane.hcl create mode 100644 util/files.go diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 4f5fd2c..511805a 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -5,8 +5,9 @@ on: push: env: - GO_VERSION: 1.19 - GOLANGCI_LINT_VERSION: v1.50.1 + DOCKER_BUILDKIT: '1' + GO_VERSION: '1.20' + GOLANGCI_LINT_VERSION: v1.51.1 # This workflow runs for not-yet-reviewed external contributions and so it # intentionally has no write access and only limited read access to the @@ -132,15 +133,24 @@ jobs: run: | make mv $(which devconsul) /usr/local/bin + mv ./bin/clustertool /usr/local/bin devconsul help + clustertool catalog-sync -h - - name: upload binary + - name: upload devconsul binary uses: actions/upload-artifact@v3 with: name: devconsul path: /usr/local/bin/devconsul if-no-files-found: error + - name: upload clustertool binary + uses: actions/upload-artifact@v3 + with: + name: clustertool + path: /usr/local/bin/clustertool + if-no-files-found: error + get-consul-binary: runs-on: ubuntu-22.04 @@ -148,10 +158,10 @@ jobs: - name: fetch consul shell: bash run: | - docker pull consul:latest - docker tag consul:latest consul-dev:latest + docker pull hashicorp/consul:latest + docker tag hashicorp/consul:latest consul-dev:latest docker rm -f consul-extract || true - docker create --name consul-extract consul:latest + docker create --name consul-extract consul-dev:latest docker cp consul-extract:/bin/consul "/usr/local/bin/consul" docker rm -f consul-extract || true @@ -180,6 +190,12 @@ jobs: name: devconsul path: /usr/local/bin/ + - name: download clustertool binary + uses: actions/download-artifact@v3 + with: + name: clustertool + path: ./bin/ + - name: download consul binary uses: actions/download-artifact@v3 with: @@ -189,9 +205,10 @@ jobs: - name: fix permissions shell: bash run: | - chmod 755 /usr/local/bin/devconsul /usr/local/bin/consul + chmod 755 /usr/local/bin/devconsul /usr/local/bin/consul ./bin/clustertool consul version devconsul help + ./bin/clustertool catalog-sync -h - name: test container timeout-minutes: 10 @@ -200,3 +217,26 @@ jobs: docker pull consul:latest docker tag consul:latest consul-dev:latest ./test-configs/test.sh "${{matrix.config_file}}" + + - name: capture failed container logs + if: failure() + shell: bash + run: | + cd test-configs + devconsul dump-logs || true + cp -a cache cache-output + devconsul down &>/dev/null || true + + - uses: actions/upload-artifact@v3 + if: failure() + with: + name: "docker-logs-${{matrix.config_file}}" + path: ./test-configs/logs + if-no-files-found: ignore + + - uses: actions/upload-artifact@v3 + if: failure() + with: + name: "run-cache-${{matrix.config_file}}" + path: ./test-configs/cache-output + if-no-files-found: ignore diff --git a/.golangci.yml b/.golangci.yml index ae5e598..b834343 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -19,4 +19,4 @@ linters-settings: run: timeout: 10m - concurrency: 4 + concurrency: 1 diff --git a/Dockerfile-cdp b/Dockerfile-cdp new file mode 100644 index 0000000..5cdb478 --- /dev/null +++ b/Dockerfile-cdp @@ -0,0 +1,8 @@ +ARG DATAPLANE_IMAGE +FROM busybox:latest +FROM ${DATAPLANE_IMAGE} +COPY --from=0 /bin/busybox /bin/busybox +USER 0:0 +RUN ["busybox", "--install", "/bin", "-s"] +USER 100:0 +ENTRYPOINT [] diff --git a/Dockerfile-tool b/Dockerfile-tool new file mode 100644 index 0000000..5b1abb0 --- /dev/null +++ b/Dockerfile-tool @@ -0,0 +1,5 @@ +FROM alpine:latest +RUN addgroup clustertool && adduser -S -G clustertool clustertool +COPY clustertool /bin/clustertool +USER clustertool +ENTRYPOINT ["/bin/clustertool"] diff --git a/Makefile b/Makefile index db509e2..feac7c6 100644 --- a/Makefile +++ b/Makefile @@ -4,9 +4,15 @@ SHELL := /bin/bash all: install .PHONY: install -install: +install: clustertool @go install +.PHONY: clustertool +clustertool: + @mkdir -p bin + @CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -tags netgo -ldflags '-w' -o bin ./clustertool + @#docker build -t local/clustertool -f Dockerfile-tool ./bin + .PHONY: tidy tidy: go mod tidy diff --git a/app/app.go b/app/app.go index 0f1e2af..5d4ff96 100644 --- a/app/app.go +++ b/app/app.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path/filepath" + "time" "github.com/hashicorp/go-hclog" @@ -25,6 +26,7 @@ type Core = App type App struct { logger hclog.Logger rootDir string + timeout time.Duration // check-mesh config *config.Config topology *infra.Topology @@ -34,6 +36,10 @@ type App struct { BootInfo // for boot } +func (c *App) SetTimeout(v time.Duration) { + c.timeout = v +} + func New(logger hclog.Logger) (*App, error) { c := &App{ logger: logger, diff --git a/app/boot.go b/app/boot.go index ba66111..812a6cf 100644 --- a/app/boot.go +++ b/app/boot.go @@ -22,6 +22,7 @@ type BootInfo struct { masterToken string clients map[string]*api.Client replicationSecretID string + serverClients map[string]*api.Client tokens map[string]string @@ -43,12 +44,16 @@ func (c *Core) runBoot(primaryOnly bool) error { c.primaryOnly = primaryOnly if c.primaryOnly { + if c.topology.LinkMode != infra.ClusterLinkModeFederate { + return fmt.Errorf("primary boot mode only applies to traditional federation") + } c.logger.Info("only bootstrapping the primary cluster", "cluster", config.PrimaryCluster) } var err error c.clients = make(map[string]*api.Client) + c.serverClients = make(map[string]*api.Client) for _, cluster := range c.topology.Clusters() { if c.primaryOnly && !cluster.Primary { continue @@ -69,7 +74,7 @@ func (c *Core) runBoot(primaryOnly bool) error { } else { switch c.topology.LinkMode { case infra.ClusterLinkModeFederate: - if err := c.bootstrap(c.primaryClient()); err != nil { + if err := c.bootstrap(config.PrimaryCluster, c.clientForCluster(config.PrimaryCluster)); err != nil { return fmt.Errorf("bootstrap: %v", err) } // now we have master token set we can do anything @@ -79,7 +84,7 @@ func (c *Core) runBoot(primaryOnly bool) error { } case infra.ClusterLinkModePeer: for _, cluster := range c.topology.Clusters() { - if err := c.bootstrap(c.clientForCluster(cluster.Name)); err != nil { + if err := c.bootstrap(cluster.Name, c.clientForCluster(cluster.Name)); err != nil { return fmt.Errorf("bootstrap[%q]: %w", cluster.Name, err) } // now we have master token set we can do anything @@ -128,11 +133,18 @@ func (c *Core) runBoot(primaryOnly bool) error { } } + for _, cluster := range c.topology.Clusters() { + if c.primaryOnly && !cluster.Primary { + continue + } + c.waitForCompletion(cluster.Name) + } + return nil } func (c *Core) peerClusters() error { - primaryClient := c.primaryClient() + primaryClient := c.clientForCluster(config.PrimaryCluster) pc := primaryClient.Peerings() for _, cluster := range c.topology.Clusters() { if cluster.Name == config.PrimaryCluster { @@ -187,6 +199,47 @@ func (c *Core) peerClusters() error { return nil } +func (c *Core) waitForTokenOnServers(cluster string, tokenName, tokenSecret string) { + if c.config.SecurityDisableACLs || tokenName == "" || tokenSecret == "" { + return + } + + if c.masterToken == "" { + panic("no master token defined") + } + + // Try each server in turn + start := time.Now() + c.topology.WalkSilent(func(n *infra.Node) { + if n.Cluster != cluster || !n.IsServer() { + return + } + + logger := c.logger.With("cluster", cluster, "server", n.Name) + + for { + client, ok := c.serverClients[n.Name] + if !ok { + panic("server client for " + n.Name + " was not created") + } + + tok, _, err := client.ACL().TokenReadSelf(&api.QueryOptions{ + Token: tokenSecret, + AllowStale: true, + }) + if err != nil || tok == nil { + logger.Trace("token not ready on server", "token-name", tokenName, "error", err) + time.Sleep(250 * time.Millisecond) + continue + } + + logger.Info("token ready on server", "token-name", tokenName, "duration", time.Since(start)) + + return + } + }) +} + func (c *Core) waitForCrossDatacenterKV(fromCluster, toCluster string) { client := c.clients[fromCluster] @@ -202,7 +255,7 @@ func (c *Core) waitForCrossDatacenterKV(fromCluster, toCluster string) { c.logger.Info("kv write success", "from_cluster", fromCluster, "to_cluster", toCluster, ) - return + break } c.logger.Warn("kv write failed; wan not converged yet", @@ -212,9 +265,133 @@ func (c *Core) waitForCrossDatacenterKV(fromCluster, toCluster string) { } } +func (c *Core) waitForCompletion(cluster string) { + var ( + client = c.clientForCluster(cluster) + logger = c.logger.With("cluster", cluster) + ) + + tryKV := func() error { + _, err := client.KV().Put(&api.KVPair{ + Key: "local-test", + Value: []byte("payload-for-local-test-in-" + cluster), + }, nil) + return err + } + + healthValue := map[string]int{ + api.HealthPassing: 0, + api.HealthWarning: 1, + api.HealthCritical: 2, + api.HealthMaint: 3, + } + mergeHealth := func(a, b string) string { + av := healthValue[a] + bv := healthValue[b] + if av > bv { + return a + } + return b + } + + checkCatalog := func() error { + services, err := consulfunc.ListAllServices(client, c.config.EnterpriseEnabled) + if err != nil { + return fmt.Errorf("error listing services in the catalog: %w", err) + } + + var failingInstances []string + for _, sid := range services { + opts := &api.QueryOptions{} + if c.config.EnterpriseEnabled { + opts.Namespace = sid.Namespace + opts.Partition = sid.Partition + } + nodes, _, err := client.Health().Service(sid.Name, "", false, opts) + if err != nil { + return fmt.Errorf("error listing health information for service %q: %w", sid, err) + } + + for _, se := range nodes { + nid := util.NewIdentifier2(se.Node.Node, se.Node.Partition) + + overallHealth := api.HealthPassing + for _, chk := range se.Checks { + overallHealth = mergeHealth(overallHealth, chk.Status) + } + + if overallHealth != api.HealthPassing { + failingInstances = append(failingInstances, fmt.Sprintf( + "[%s @ %s is %s]", + sid, nid, overallHealth, + )) + } + } + } + + if len(failingInstances) > 0 { + return fmt.Errorf("unhealthy instances: " + strings.Join(failingInstances, ", ")) + } + return nil + } + + start := time.Now() + for { + // 1. do local KV write + if err := tryKV(); err != nil { + logger.Warn("local kv write failed; something is not ready yet", "error", err) + time.Sleep(500 * time.Millisecond) + continue + } else { + dur := time.Since(start) + logger.Info("local kv write success", "elapsed", dur) + } + + break + } + + start = time.Now() + for { + // 2. ensure all services and proxies are healthy + if err := checkCatalog(); err != nil { + logger.Warn("local catalog is not healthy yet", "error", err) + time.Sleep(500 * time.Millisecond) + continue + } else { + dur := time.Since(start) + logger.Info("local catalog is healthy", "elapsed", dur) + } + + break + } +} + +func (c *Core) createClientsForServersInCluster(cluster string) error { + if c.serverClients == nil { + c.serverClients = make(map[string]*api.Client) + } + return c.topology.Walk(func(n *infra.Node) error { + if n.Cluster != cluster || !n.IsServer() { + return nil + } + var err error + c.serverClients[n.Name], err = consulfunc.GetClient(n.LocalAddress(), c.masterToken) + if err != nil { + return fmt.Errorf("error creating final client for server=%s: %w", n.Name, err) + } + + return nil + }) +} + func (c *Core) initPrimaryCluster(cluster string, peered bool) error { var err error + err = c.createClientsForServersInCluster(cluster) + if err != nil { + return fmt.Errorf("createClientsForServersInCluster[%s]: %w", cluster, err) + } + err = c.createPartitions(cluster) if err != nil { return fmt.Errorf("createPartitions[%s]: %w", cluster, err) @@ -226,17 +403,19 @@ func (c *Core) initPrimaryCluster(cluster string, peered bool) error { } if !c.config.SecurityDisableACLs { - err = c.createReplicationToken(cluster) - if err != nil { - return fmt.Errorf("createReplicationToken[%s]: %w", cluster, err) + if c.topology.LinkMode == infra.ClusterLinkModeFederate { + err = c.createReplicationToken(cluster) + if err != nil { + return fmt.Errorf("createReplicationToken[%s]: %w", cluster, err) + } } - err = c.createMeshGatewayToken(cluster, peered) + err = c.createMeshGatewayToken(cluster, cluster, peered) if err != nil { return fmt.Errorf("createMeshGatewayToken[%s]: %w", cluster, err) } - err = c.createAgentTokens(cluster) + err = c.createAgentTokens(cluster, cluster) if err != nil { return fmt.Errorf("createAgentTokens[%s]: %w", cluster, err) } @@ -273,7 +452,7 @@ func (c *Core) initPrimaryCluster(cluster string, peered bool) error { return fmt.Errorf("initializeKubernetes[%s]: %w", cluster, err) } } else { - err = c.createServiceTokens(cluster) + err = c.createServiceTokens(cluster, cluster) if err != nil { return fmt.Errorf("createServiceTokens[%s]: %w", cluster, err) } @@ -300,11 +479,69 @@ func (c *Core) initSecondaryDCs() error { if cluster.Primary { continue } + c.clients[cluster.Name], err = consulfunc.GetClient(c.topology.LeaderIP(cluster.Name, false), c.masterToken) if err != nil { return fmt.Errorf("error creating final client for cluster=%s: %v", cluster.Name, err) } + err = c.createClientsForServersInCluster(cluster.Name) + if err != nil { + return fmt.Errorf("createClientsForServersInCluster[%s]: %w", cluster.Name, err) + } + + // When we create tokens in one dc intended for another, we have to + // wait to write them to the cache until after they work in the target + // DC. + var ( + mgwDelay func() error + agentDelay func() error + svcDelay func() error + ) + if !c.config.SecurityDisableACLs { + // initialize some acl tokens unique to this dc + mgwDelay, err = c.createMeshGatewayTokenDelayWrite(config.PrimaryCluster, cluster.Name, false) + if err != nil { + return fmt.Errorf("createMeshGatewayToken[%s]: %w", cluster.Name, err) + } + + agentDelay, err = c.createAgentTokensDelayWrite(config.PrimaryCluster, cluster.Name) + if err != nil { + return fmt.Errorf("createAgentTokens[%s]: %w", cluster.Name, err) + } + + if c.config.KubernetesEnabled { + return fmt.Errorf("currently the k8s ACL mode is incompatible with secondary datacenters with this tool") + } else { + svcDelay, err = c.createServiceTokensDelayWrite(config.PrimaryCluster, cluster.Name) + if err != nil { + return fmt.Errorf("createServiceTokens[%s]: %w", cluster.Name, err) + } + } + } + + if !c.config.SecurityDisableACLs { + // ensure management token works here + c.waitForTokenOnServers(cluster.Name, "master", c.masterToken) + + if mgwDelay != nil { + if err = mgwDelay(); err != nil { + return fmt.Errorf("createMeshGatewayToken.delay[%s]: %w", cluster.Name, err) + } + } + + if agentDelay != nil { + if err = agentDelay(); err != nil { + return fmt.Errorf("createAgentTokens.delay[%s]: %w", cluster.Name, err) + } + } + if svcDelay != nil { + if err = svcDelay(); err != nil { + return fmt.Errorf("createServiceTokens.delay[%s]: %w", cluster.Name, err) + } + } + } + err = c.injectAgentTokensAndWaitForNodeUpdates(cluster.Name, false) if err != nil { return fmt.Errorf("injectAgentTokensAndWaitForNodeUpdates[%s]: %v", cluster.Name, err) @@ -327,15 +564,35 @@ func (c *Core) initSecondaryDCs() error { return nil } -func (c *Core) primaryClient() *api.Client { - return c.clients[config.PrimaryCluster] -} - func (c *Core) clientForCluster(cluster string) *api.Client { return c.clients[cluster] } -func (c *Core) bootstrap(client *api.Client) error { +func isACLNotBootstrapped(err error) bool { + switch { + case strings.Index(err.Error(), "ACL system must be bootstrapped before making any requests that require authorization") != -1: + return true + case strings.Index(err.Error(), "The ACL system is currently in legacy mode") != -1: + return true + } + return false +} + +// TODO: call this in secondaries +func (c *Core) isACLBootstrapped(client *api.Client) (bool, error) { + policy, _, err := client.ACL().PolicyReadByName("global-management", &api.QueryOptions{ + Token: c.masterToken, + }) + if err != nil { + if strings.Index(err.Error(), "Unexpected response code: 403 (ACL not found)") != -1 { + return false, nil + } + return false, err + } + return policy != nil, nil +} + +func (c *Core) bootstrap(cluster string, client *api.Client) error { // TODO: peering var err error c.masterToken, err = c.cache.LoadValue("master-token") @@ -353,11 +610,21 @@ func (c *Core) bootstrap(client *api.Client) error { ac := client.ACL() if c.masterToken != "" { + NOT_BOOTED: + ready, err := c.isACLBootstrapped(client) + if err != nil { + return fmt.Errorf("error checking if the acl system is bootstrapped: %w", err) + } else if !ready { + c.logger.Warn("ACL system is not ready yet") + time.Sleep(250 * time.Millisecond) + goto NOT_BOOTED + } + TRYAGAIN: // check to see if it works - _, _, err := ac.TokenReadSelf(&api.QueryOptions{Token: c.masterToken}) + _, _, err = ac.TokenReadSelf(&api.QueryOptions{Token: c.masterToken}) if err != nil { - if strings.Index(err.Error(), "The ACL system is currently in legacy mode") != -1 { + if isACLNotBootstrapped(err) { c.logger.Warn("system is rebooting", "error", err) time.Sleep(250 * time.Millisecond) goto TRYAGAIN @@ -374,7 +641,7 @@ TRYAGAIN2: c.logger.Info("bootstrapping ACLs") tok, _, err := ac.Bootstrap() if err != nil { - if strings.Index(err.Error(), "The ACL system is currently in legacy mode") != -1 { + if isACLNotBootstrapped(err) { c.logger.Warn("system is rebooting", "error", err) time.Sleep(250 * time.Millisecond) goto TRYAGAIN2 @@ -389,6 +656,8 @@ TRYAGAIN2: c.logger.Info("current master token", "token", c.masterToken) + c.waitForTokenOnServers(cluster, "initial-management", c.masterToken) + return nil } @@ -396,9 +665,6 @@ func (c *Core) createPartitions(cluster string) error { if !c.config.EnterpriseEnabled { return nil } - if c.config.EnterpriseDisablePartitions { - return nil - } var ( client = c.clientForCluster(cluster) @@ -582,16 +848,26 @@ func (c *Core) createReplicationToken(cluster string) error { logger.Info("replication token", "secretID", token.SecretID) + c.waitForTokenOnServers(config.PrimaryCluster, "replication", token.SecretID) + return nil } -func (c *Core) createMeshGatewayToken(cluster string, peered bool) error { +func (c *Core) createMeshGatewayToken(fromCluster, forCluster string, peered bool) error { + delay, err := c.createMeshGatewayTokenDelayWrite(fromCluster, forCluster, peered) + if err != nil { + return err + } + return delay() +} + +func (c *Core) createMeshGatewayTokenDelayWrite(fromCluster, forCluster string, peered bool) (func() error, error) { var ( - client = c.clientForCluster(cluster) - logger = c.logger.With("cluster", cluster) + client = c.clientForCluster(fromCluster) + logger = c.logger.With("cluster", forCluster) ) - const meshGatewayName = "mesh-gateway" + meshGatewayName := fmt.Sprintf("mesh-gateway--" + forCluster) p := &api.ACLPolicy{ Name: meshGatewayName, @@ -615,9 +891,11 @@ func (c *Core) createMeshGatewayToken(cluster string, peered bool) error { policy = "read" } ` - if !c.config.EnterpriseDisablePartitions { - p.Rules = ` partition "default" { ` + p.Rules + ` } ` + if peered { + p.Rules += ` mesh = "write" ` } + // Wrap with default partition. + p.Rules = ` partition "default" { ` + p.Rules + ` } ` } else { p.Rules = ` @@ -634,10 +912,13 @@ func (c *Core) createMeshGatewayToken(cluster string, peered bool) error { policy = "read" } ` + if peered { + p.Rules += ` mesh = "write" ` + } } p, err := consulfunc.CreateOrUpdatePolicy(client, p, nil) if err != nil { - return err + return nil, fmt.Errorf("could not create policy: %w", err) } token := &api.ACLToken{ @@ -648,28 +929,23 @@ func (c *Core) createMeshGatewayToken(cluster string, peered bool) error { token, err = consulfunc.CreateOrUpdateToken(client, token, nil) if err != nil { - return err - } - - if err := c.cache.SaveValue("mesh-gateway--"+cluster, token.SecretID); err != nil { - return err - } - if !peered { - for _, altCluster := range c.topology.Clusters() { - if altCluster.Name == cluster { - continue // skip - } - if err := c.cache.SaveValue("mesh-gateway--"+altCluster.Name, token.SecretID); err != nil { - return err - } - } + return nil, fmt.Errorf("could not create token: %w", err) } // c.setToken("mesh-gateway", "", token.SecretID) logger.Info("mesh-gateway token", "secretID", token.SecretID) - return nil + return func() error { + // Make sure we wait for it before letting it manifest in the cache store. + c.waitForTokenOnServers(forCluster, "mesh-gateway--"+forCluster, token.SecretID) + + if err := c.cache.SaveValue("mesh-gateway--"+forCluster, token.SecretID); err != nil { + return err + } + logger.Info("mesh-gateway token written to cache", "secretID", token.SecretID) + return nil + }, nil } func (c *Core) injectReplicationToken() error { @@ -681,7 +957,7 @@ func (c *Core) injectReplicationToken() error { agentMasterToken := c.config.AgentMasterToken return c.topology.Walk(func(node *infra.Node) error { - if node.Cluster == config.PrimaryCluster || !node.Server { + if node.Cluster == config.PrimaryCluster || !node.IsServer() { return nil } @@ -708,17 +984,30 @@ func (c *Core) injectReplicationToken() error { } // each agent will get a minimal policy configured -func (c *Core) createAgentTokens(cluster string) error { +func (c *Core) createAgentTokens(fromCluster, forCluster string) error { + delay, err := c.createAgentTokensDelayWrite(fromCluster, forCluster) + if err != nil { + return err + } + return delay() +} + +func (c *Core) createAgentTokensDelayWrite(fromCluster, forCluster string) (func() error, error) { var ( - client = c.clientForCluster(cluster) - logger = c.logger.With("cluster", cluster) + client = c.clientForCluster(fromCluster) + logger = c.logger.With("cluster", forCluster) ) - return c.topology.Walk(func(node *infra.Node) error { - if c.topology.LinkWithPeering() && node.Cluster != cluster { + var funcs []func() error + err := c.topology.Walk(func(node *infra.Node) error { + if node.Cluster != forCluster { return nil // skip } + if !node.IsAgent() { + return nil + } + policyName := "agent--" + node.Name p := &api.ACLPolicy{ @@ -728,15 +1017,14 @@ func (c *Core) createAgentTokens(cluster string) error { if c.config.EnterpriseEnabled { p.Rules = ` - node "` + node.Name + `-pod" { policy = "write" } + node "` + node.PodName() + `" { policy = "write" } service_prefix "" { policy = "read" } ` - if !c.config.EnterpriseDisablePartitions { - p.Rules = ` partition "` + node.Partition + `" { ` + p.Rules + ` } ` - } + // Wrap with default partition. + p.Rules = ` partition "` + node.Partition + `" { ` + p.Rules + ` } ` } else { p.Rules = ` - node "` + node.Name + `-pod" { policy = "write" } + node "` + node.PodName() + `" { policy = "write" } service_prefix "" { policy = "read" } ` } @@ -765,8 +1053,29 @@ func (c *Core) createAgentTokens(cluster string) error { c.setToken("agent", node.Name, token.SecretID) + funcs = append(funcs, func() error { + c.waitForTokenOnServers(forCluster, "agent--"+node.Name, token.SecretID) + return nil + }) + return nil }) + if err != nil { + return nil, err + } + + return delayFuncs(funcs), nil +} + +func delayFuncs(funcs []func() error) func() error { + return func() error { + for _, fn := range funcs { + if err := fn(); err != nil { + return err + } + } + return nil + } } // TALK TO EACH AGENT @@ -776,6 +1085,9 @@ func (c *Core) injectAgentTokens(datacenter string) error { if node.Cluster != datacenter { return nil } + if !node.IsAgent() { + return nil + } agentClient, err := consulfunc.GetClient(node.LocalAddress(), agentMasterToken) if err != nil { return err @@ -861,9 +1173,8 @@ func (c *Core) createAnonymousPolicy(cluster string) error { service_prefix "" { policy = "read" } } ` - if !c.config.EnterpriseDisablePartitions { - p.Rules = ` partition_prefix "" { ` + p.Rules + ` } ` - } + // Wrap with default partition. + p.Rules = ` partition_prefix "" { ` + p.Rules + ` } ` } else { p.Rules = ` node_prefix "" { policy = "read" } @@ -922,27 +1233,37 @@ func apiOptionsFromConfigPartition(pc *config.Partition, ns string) *consulfunc. } } -func (c *Core) createServiceTokens(cluster string) error { +func (c *Core) createServiceTokens(fromCluster, forCluster string) error { + delay, err := c.createServiceTokensDelayWrite(fromCluster, forCluster) + if err != nil { + return err + } + return delay() +} + +func (c *Core) createServiceTokensDelayWrite(fromCluster, forCluster string) (func() error, error) { var ( - client = c.clientForCluster(cluster) - logger = c.logger.With("cluster", cluster) + client = c.clientForCluster(fromCluster) + logger = c.logger.With("cluster", forCluster) ) - done := make(map[util.Identifier]struct{}) - - return c.topology.Walk(func(n *infra.Node) error { + var ( + funcs []func() error + done = make(map[util.Identifier]struct{}) + ) + err := c.topology.Walk(func(n *infra.Node) error { + if n.Cluster != forCluster { + return nil // skip + } if n.Service == nil { return nil } if _, ok := done[n.Service.ID]; ok { return nil } - if c.topology.LinkWithPeering() && n.Cluster != cluster { - return nil // skip - } token := &api.ACLToken{ - Description: "service--" + n.Service.ID.ID(), + Description: "service--" + forCluster + "--" + n.Service.ID.ID(), Local: false, ServiceIdentities: []*api.ACLServiceIdentity{ { @@ -954,9 +1275,6 @@ func (c *Core) createServiceTokens(cluster string) error { token.Namespace = n.Service.ID.Namespace token.Partition = n.Service.ID.Partition } - if c.config.EnterpriseDisablePartitions { - token.Partition = "" - } token, err := consulfunc.CreateOrUpdateToken(client, token, nil) if err != nil { @@ -970,15 +1288,24 @@ func (c *Core) createServiceTokens(cluster string) error { "token", token.SecretID, ) - if err := c.cache.SaveValue("service-token--"+n.Service.ID.ID(), token.SecretID); err != nil { - return err - } + funcs = append(funcs, func() error { + c.waitForTokenOnServers(forCluster, "service--"+forCluster+"--"+n.Service.ID.ID(), token.SecretID) - // c.setToken("service", sn.ID(), token.SecretID) + if err := c.cache.SaveValue("service--"+forCluster+"--"+n.Service.ID.ID(), token.SecretID); err != nil { + return err + } + return nil + }) done[n.Service.ID] = struct{}{} return nil }) + + if err != nil { + return nil, err + } + + return delayFuncs(funcs), nil } func (c *Core) writeCentralConfigs(cluster string) error { @@ -987,9 +1314,7 @@ func (c *Core) writeCentralConfigs(cluster string) error { logger = c.logger.With("cluster", cluster) ) - currentEntries, err := consulfunc.ListAllConfigEntries(client, - c.config.EnterpriseEnabled, - c.config.EnterpriseDisablePartitions) + currentEntries, err := consulfunc.ListAllConfigEntries(client, c.config.EnterpriseEnabled) if err != nil { return err } @@ -1109,19 +1434,6 @@ func (c *Core) writeCentralConfigs(cluster string) error { } } } - if c.config.EnterpriseDisablePartitions { - switch entry.GetKind() { - case api.ProxyDefaults: - thisEntry := entry.(*api.ProxyConfigEntry) - thisEntry.Partition = "" - case api.ServiceIntentions: - thisEntry := entry.(*api.ServiceIntentionsConfigEntry) - thisEntry.Partition = "" - for _, src := range thisEntry.Sources { - src.Partition = "" - } - } - } if _, _, err := ce.Set(entry, nil); err != nil { return err } @@ -1191,19 +1503,21 @@ func (c *Core) writeServiceRegistrationFiles() error { return nil } + if !n.IsAgent() { + return nil + } + type templateOpts struct { - Service *infra.Service - EnterpriseEnabled bool - EnterpriseDisablePartitions bool - LinkWithFederation bool - LinkWithPeering bool + Service *infra.Service + EnterpriseEnabled bool + LinkWithFederation bool + LinkWithPeering bool } opts := templateOpts{ - Service: n.Service, - EnterpriseEnabled: c.config.EnterpriseEnabled, - EnterpriseDisablePartitions: c.config.EnterpriseDisablePartitions, - LinkWithFederation: c.topology.LinkWithFederation(), - LinkWithPeering: c.topology.LinkWithPeering(), + Service: n.Service, + EnterpriseEnabled: c.config.EnterpriseEnabled, + LinkWithFederation: c.topology.LinkWithFederation(), + LinkWithPeering: c.topology.LinkWithPeering(), } var buf bytes.Buffer @@ -1336,9 +1650,7 @@ func (c *Core) injectAgentTokensAndWaitForNodeUpdates(cluster string, isPrimaryC err error ) if c.config.EnterpriseEnabled && isPrimaryCluster { - allNodes, err = consulfunc.ListAllNodes(client, cluster, - c.config.EnterpriseEnabled, - c.config.EnterpriseDisablePartitions) + allNodes, err = consulfunc.ListAllNodes(client, cluster, c.config.EnterpriseEnabled) if err != nil { return fmt.Errorf("consulfunc.ListAllNodes: %w", err) } @@ -1373,8 +1685,11 @@ func (c *Core) determineNodeUpdateStragglers(nodes []*api.Node, cluster string) if n.Cluster != cluster { return } + if !n.IsAgent() { + return + } - catNode, ok := nm[n.Name+"-pod"] + catNode, ok := nm[n.PodName()] if ok && len(catNode.TaggedAddresses) > 0 { return } diff --git a/app/boot_templates.go b/app/boot_templates.go index 6bd113b..2307602 100644 --- a/app/boot_templates.go +++ b/app/boot_templates.go @@ -8,9 +8,7 @@ services = [ name = "{{.Service.ID.Name}}" {{- if .EnterpriseEnabled }} namespace = "{{.Service.ID.Namespace}}" -{{- if not .EnterpriseDisablePartitions }} partition = "{{.Service.ID.Partition}}" -{{- end }} {{- end }} port = {{.Service.Port}} @@ -38,9 +36,7 @@ services = [ destination_name = "{{.Service.UpstreamID.Name}}" {{- if .EnterpriseEnabled }} destination_namespace = "{{.Service.UpstreamID.Namespace}}" -{{- if not .EnterpriseDisablePartitions }} destination_partition = "{{.Service.UpstreamID.Partition}}" -{{- end }} {{- end }} local_bind_port = {{.Service.UpstreamLocalPort}} {{- if .Service.UpstreamDatacenter }} diff --git a/app/debug.go b/app/debug.go index 17c31e0..082af01 100644 --- a/app/debug.go +++ b/app/debug.go @@ -34,6 +34,110 @@ var knownConfigEntryKinds = []string{ api.ExportedServices, } +func (a *App) RunDumpLogs() error { + // This only makes sense to run after you've configured it once. + if err := checkHasInitRunOnce(); err != nil { + return err + } + + if err := os.RemoveAll("logs"); err != nil { + return fmt.Errorf("problem clearing logs directory: %w", err) + } + + if err := os.MkdirAll("logs", 0755); err != nil { + return fmt.Errorf("could not create logs output directory: %w", err) + } + + logDockerCmd := func(fn string, args []string) error { + var err error + fn, err = filepath.Abs(fn) + if err != nil { + return err + } + w, err := safeio.OpenFile(fn, 0644) + if err != nil { + return err + } + defer w.Close() + + if err := a.runner.DockerExec(args, w); err != nil { + return err + } + + return w.Commit() + } + + writeLogs := func(c string) error { + fn := filepath.Join("logs", c+".log") + if err := logDockerCmd(fn, []string{"logs", c}); err != nil { + return err + } + + a.logger.Info("captured docker logs", "container", c, "path", fn) + + return nil + } + + dumpRoute := func(c string) error { + fn := filepath.Join("logs", c+".route.txt") + args := []string{ + "exec", + c, + "route", "-n", + // d exec dc1-server1 route -n + } + if err := logDockerCmd(fn, args); err != nil { + return err + } + + a.logger.Info("captured docker route table", "container", c, "path", fn) + + return nil + } + + doStuff := func(c string) error { + if err := writeLogs(c); err != nil { + return err + } + if err := dumpRoute(c); err != nil { + return err + } + return nil + } + + a.topology.WalkSilent(func(n *infra.Node) { + var containers []string + + containers = append(containers, n.Name) + + if n.MeshGateway { + containers = append( + containers, + n.Name+"-mesh-gateway", + ) + } + if n.Service != nil { + containers = append( + containers, + n.Name+"-"+n.Service.ID.Name+"-sidecar-proxy", + ) + } + + for _, c := range containers { + if err := doStuff(c); err != nil { + switch { + case strings.Contains(err.Error(), `Error response from daemon: No such container:`): + case strings.Contains(err.Error(), `Error: No such container:`): + default: + a.logger.Error("could not capture docker logs", "container", c, "error", err) + } + } + } + }) + + return nil +} + func (c *Core) RunDebugListConfigs() error { client, err := c.debugPrimaryClient() if err != nil { @@ -187,51 +291,65 @@ func (c *Core) runDebugSaveGrafana(client *grafana.Client, uid, fileName string) func (c *Core) RunCheckMesh() error { client := cleanhttp.DefaultClient() - now := time.Now() + var stopCh <-chan time.Time + if c.timeout > 0 { + stopCh = time.After(c.timeout) + } - c.topology.WalkSilent(func(n *infra.Node) { - if n.Server || n.MeshGateway { - return + successMap := make(map[string]map[string]struct{}) + for { + select { + case <-stopCh: + return errors.New("did not complete") + default: } - addr := n.LocalAddress() - logger := c.logger.Named(n.Name) - logger = logger.With("addr", addr) + anyFailed := false + c.topology.WalkSilent(func(n *infra.Node) { + if !n.RunsWorkloads() || n.MeshGateway || n.Service == nil { + return + } + addr := n.LocalAddress() + sid := n.Service.ID.String() - logger.Info("Checking pingpong mesh instance") + nodeSuccessMap, ok := successMap[n.Name] + if !ok { + nodeSuccessMap = make(map[string]struct{}) + successMap[n.Name] = nodeSuccessMap + } - ppr, err := fetchPingPongPage(client, addr) - if err != nil { - logger.Error("fetching endpoint failed", "error", err) - return - } - if ppr.Name != "" { - logger = logger.Named("app__" + ppr.Name) - } + if _, ok := nodeSuccessMap[sid]; ok { + return + } - logger.Info("found application", "app", ppr.Name) - if len(ppr.Pings) > 0 { - evt := ppr.Pings[0] + logger := c.logger.With( + "node", n.Name, + "service", sid, + "addr", addr, + ) - if evt.Err != "" { - logger.Error("last ping", "error", evt.Err) - } else { - logger.Info("last ping", - "started", prettyTime(now, evt.Start), - "ended", prettyTime(now, evt.End)) - } - } - if len(ppr.Pongs) > 0 { - evt := ppr.Pongs[0] + // logger.Info("Checking pingpong mesh instance") - if evt.Err != "" { - logger.Error("last pong", "error", evt.Err) + status, err := fetchPingHealthz(client, addr) + if err != nil { + logger.Error("fetching endpoint failed", "error", err) + anyFailed = true + return + } + if status == "OK" { + logger.Info("last ping", "status", status) + nodeSuccessMap[sid] = struct{}{} } else { - logger.Info("last pong", "received", - prettyTime(now, evt.Recv)) + logger.Error("last ping", "status", status) + anyFailed = true } + }) + if !anyFailed { + break } - }) + time.Sleep(100 * time.Millisecond) + } + c.logger.Info("mesh check complete", "status", "OK") return nil } @@ -259,6 +377,24 @@ type Ping struct { // DurSec int `json:",omitempty"` } +func fetchPingHealthz(client *http.Client, addr string) (string, error) { + resp, err := client.Get("http://" + addr + ":8080/pinghealthz") + if err != nil { + return "", err + } + if resp.Body == nil { + return "", errors.New("no response body") + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + + return strings.TrimSpace(string(body)), nil +} + func fetchPingPongPage(client *http.Client, addr string) (*PingPongResponse, error) { resp, err := client.Get("http://" + addr + ":8080") if err != nil { diff --git a/app/docker.go b/app/docker.go index caa8b83..9522a77 100644 --- a/app/docker.go +++ b/app/docker.go @@ -36,11 +36,19 @@ func (a *App) buildDockerImages(force bool) error { if err := addFileToHash("Dockerfile-envoy", hash); err != nil { return err } + if err := addFileToHash("Dockerfile-cdp", hash); err != nil { + return err + } + if err := addFileToHash("Dockerfile-tool", hash); err != nil { + return err + } - hash.Write([]byte(a.config.EnvoyVersion)) - hash.Write([]byte(a.config.ConsulImage)) - hash.Write([]byte(a.config.CanaryEnvoyVersion)) - hash.Write([]byte(a.config.CanaryConsulImage)) + hash.Write([]byte(a.config.Versions.Envoy)) + hash.Write([]byte(a.config.Versions.ConsulImage)) + hash.Write([]byte(a.config.Versions.DataplaneImage)) + hash.Write([]byte(a.config.CanaryVersions.Envoy)) + hash.Write([]byte(a.config.CanaryVersions.ConsulImage)) + hash.Write([]byte(a.config.CanaryVersions.DataplaneImage)) currentHash = fmt.Sprintf("%x", hash.Sum(nil)) } @@ -65,16 +73,16 @@ func (a *App) buildDockerImages(force bool) error { // tag base if err := a.runner.DockerExec([]string{ "tag", - a.config.ConsulImage, + a.config.Versions.ConsulImage, "local/consul-base:latest", }, nil); err != nil { return err } - if a.config.CanaryConsulImage != "" { + if a.config.CanaryVersions.ConsulImage != "" { if err := a.runner.DockerExec([]string{ "tag", - a.config.CanaryConsulImage, + a.config.CanaryVersions.ConsulImage, "local/consul-base-canary:latest", }, nil); err != nil { return err @@ -87,7 +95,7 @@ func (a *App) buildDockerImages(force bool) error { "--build-arg", "CONSUL_IMAGE=local/consul-base:latest", "--build-arg", - "ENVOY_VERSION=" + a.config.EnvoyVersion, + "ENVOY_VERSION=" + a.config.Versions.Envoy, "-t", "local/consul-envoy", "-f", "Dockerfile-envoy", ".", @@ -95,13 +103,13 @@ func (a *App) buildDockerImages(force bool) error { return err } - if a.config.CanaryEnvoyVersion != "" { + if a.config.CanaryVersions.Envoy != "" { if err := a.runner.DockerExec([]string{ "build", "--build-arg", "CONSUL_IMAGE=local/consul-base-canary:latest", "--build-arg", - "ENVOY_VERSION=" + a.config.CanaryEnvoyVersion, + "ENVOY_VERSION=" + a.config.CanaryVersions.Envoy, "-t", "local/consul-envoy-canary", "-f", "Dockerfile-envoy", ".", @@ -110,6 +118,50 @@ func (a *App) buildDockerImages(force bool) error { } } + // build cdp + if err := a.runner.DockerExec([]string{ + "build", + "--build-arg", + "DATAPLANE_IMAGE=" + a.config.Versions.DataplaneImage, + "-t", "local/consul-dataplane", + "-f", "Dockerfile-cdp", + ".", + }, nil); err != nil { + return err + } + + if a.config.CanaryVersions.DataplaneImage != "" { + if err := a.runner.DockerExec([]string{ + "build", + "--build-arg", + "DATAPLANE_IMAGE=" + a.config.CanaryVersions.DataplaneImage, + "-t", "local/consul-dataplane-canary", + "-f", "Dockerfile-cdp", + ".", + }, nil); err != nil { + return err + } + } + + // build tool + { + _, err := os.Stat("./bin/clustertool") + if os.IsNotExist(err) { + return fmt.Errorf("clustertool binary not present in bin/ ; please run 'make'") + } else if err != nil { + return err + } + + if err := a.runner.DockerExec([]string{ + "build", + "-t", "local/clustertool", + "-f", "Dockerfile-tool", + "./bin", + }, nil); err != nil { + return err + } + } + // Checkpoint if _, err := safeio.WriteToFile(bytes.NewReader([]byte(currentHash)), "cache/docker.hash", 0644); err != nil { return err @@ -129,7 +181,7 @@ func (a *App) runStopDC2() error { ) a.topology.WalkSilent(func(n *infra.Node) { - pods[n.Cluster] = append(pods[n.Cluster], n.Name+"-pod") + pods[n.Cluster] = append(pods[n.Cluster], n.PodName()) containers[n.Cluster] = append(containers[n.Cluster], n.Name) if n.MeshGateway { containers[n.Cluster] = append(containers[n.Cluster], n.Name+"-mesh-gateway") diff --git a/app/dumpconfig.go b/app/dumpconfig.go index 9b7ef2a..0fd8020 100644 --- a/app/dumpconfig.go +++ b/app/dumpconfig.go @@ -13,24 +13,40 @@ func (c *Core) RunConfigDump() error { var ( servers = make(map[string]int) clients = make(map[string]int) + dataplanes = make(map[string]int) localAddrs = make(map[string]string) clusters []string pods = make(map[string][]string) containers = make(map[string][]string) ) c.topology.WalkSilent(func(n *infra.Node) { - if n.Server { + switch n.Kind { + case infra.NodeKindServer: servers[n.Cluster]++ - } else { + case infra.NodeKindClient: clients[n.Cluster]++ + case infra.NodeKindDataplane: + dataplanes[n.Cluster]++ } localAddrs[n.Name] = n.LocalAddress() - pods[n.Cluster] = append(pods[n.Cluster], n.Name+"-pod") - containers[n.Cluster] = append(containers[n.Cluster], n.Name) + pods[n.Cluster] = append(pods[n.Cluster], n.PodName()) + if n.IsAgent() { + containers[n.Cluster] = append(containers[n.Cluster], n.Name) + } if n.MeshGateway { containers[n.Cluster] = append(containers[n.Cluster], n.Name+"-mesh-gateway") } + if n.Kind == infra.NodeKindInfra { + containers[n.Cluster] = append(containers[n.Cluster], n.Name+"-catalog-sync") + } + + if n.RunsWorkloads() && n.Service != nil { + s := n.Service + + containers[n.Cluster] = append(containers[n.Cluster], n.Name+"-"+s.ID.Name) + containers[n.Cluster] = append(containers[n.Cluster], n.Name+"-"+s.ID.Name+"-sidecar") + } }) for _, dc := range c.topology.Clusters() { @@ -39,8 +55,8 @@ func (c *Core) RunConfigDump() error { m := map[string]interface{}{ "confName": c.config.ConfName, - "image": c.config.ConsulImage, - "envoyVersion": c.config.EnvoyVersion, + "image": c.config.Versions.ConsulImage, + "envoyVersion": c.config.Versions.Envoy, "tls": bool2str(c.config.EncryptionTLS), "gossip": bool2str(c.config.EncryptionGossip), "k8s": bool2str(c.config.KubernetesEnabled), @@ -60,6 +76,9 @@ func (c *Core) RunConfigDump() error { for dc, n := range clients { m["topology.clients."+dc] = n } + for dc, n := range dataplanes { + m["topology.dataplanes."+dc] = n + } if len(args) == 0 { fmt.Printf(jsonPretty(m) + "\n") diff --git a/app/gen.go b/app/gen.go index 14c2222..d899093 100644 --- a/app/gen.go +++ b/app/gen.go @@ -15,13 +15,17 @@ func (c *Core) runGenerate(primaryOnly bool) error { c.topology.WalkSilent(func(node *infra.Node) { c.logger.Info("Generating node", + "kind", node.Kind, "name", node.Name, - "server", node.Server, "dc", node.Cluster, "ip", node.LocalAddress(), ) }) + if err := c.generateCatalogInfo(primaryOnly); err != nil { + return err + } + if err := c.generateConfigs(primaryOnly); err != nil { return err } @@ -82,24 +86,32 @@ func (c *Core) generateConfigs(primaryOnly bool) error { addVolume("grafana-data") } - addImage("pause", "k8s.gcr.io/pause:3.3") - addImage("consul", c.config.ConsulImage) + addImage("pause", "registry.k8s.io/pause:3.3") + addImage("consul", c.config.Versions.ConsulImage) addImage("consul-envoy", "local/consul-envoy:latest") + addImage("consul-dataplane", "local/consul-dataplane:latest") //c.config.Versions.DataplaneImage) addImage("pingpong", "rboyer/pingpong:latest") + addImage("clustertool", "local/clustertool:latest") - if c.config.CanaryEnvoyVersion != "" { + if c.config.CanaryVersions.Envoy != "" { addImage("consul-envoy-canary", "local/consul-envoy-canary:latest") } + if c.config.CanaryVersions.DataplaneImage != "" { + addImage("consul-dataplane-canary", "local/consul-dataplane-canary:latest") //c.config.CanaryVersions.DataplaneImage) + } + if err := c.topology.Walk(func(node *infra.Node) error { - addVolume(node.Name) + if node.IsAgent() { + addVolume(node.Name) + } // NOTE: primaryOnly implies we still generate empty pods in the remote datacenters populatePodContents := true if primaryOnly { populatePodContents = node.Cluster == config.PrimaryCluster } - myContainers, err := tfgen.GenerateAgentContainers(c.config, c.topology, node, populatePodContents) + myContainers, err := tfgen.GenerateNodeContainers(c.config, c.topology, c.cache, node, populatePodContents) if err != nil { return err } diff --git a/app/gen_catalog.go b/app/gen_catalog.go new file mode 100644 index 0000000..f69db4f --- /dev/null +++ b/app/gen_catalog.go @@ -0,0 +1,165 @@ +package app + +import ( + "encoding/json" + "fmt" + "strconv" + + "github.com/rboyer/devconsul/config" + "github.com/rboyer/devconsul/infra" + "github.com/rboyer/devconsul/structs" + "github.com/rboyer/devconsul/util" +) + +func (c *Core) generateCatalogInfo(primaryOnly bool) error { + genLogger := c.logger.Named("generate-catalog-info") + + for _, cluster := range c.topology.Clusters() { + if primaryOnly && cluster.Name != config.PrimaryCluster { + continue + } + + logger := genLogger.With("cluster", cluster.Name) + + var ( + nodes = make(map[util.Identifier2]*structs.CatalogNode) + services = make(map[util.Identifier2]map[util.Identifier]*structs.CatalogService) + proxies = make(map[util.Identifier2]map[util.Identifier]*structs.CatalogProxy) + ) + err := c.topology.Walk(func(n *infra.Node) error { + consulNodeName := n.PodName() + + if n.Cluster != cluster.Name { + return nil + } + if n.Service == nil { + return nil + } + + if n.Kind != infra.NodeKindDataplane { + return nil + } + + if n.Service.UpstreamExtraHCL != "" { + panic("cannot do this") + } + + var ( + nid = util.NewIdentifier2(consulNodeName, n.Partition) + sidApp = n.Service.ID + sidProxy = util.NewIdentifier( + n.Service.ID.Name+"-sidecar-proxy", + n.Service.ID.Namespace, + n.Service.ID.Partition, + ) + ) + + if _, ok := nodes[nid]; !ok { + nodes[nid] = &structs.CatalogNode{ + Node: consulNodeName, + Address: n.LocalAddress(), + // TaggedAddresses map[string]string + NodeMeta: map[string]string{"devconsul-virtual": "1"}, + Partition: n.Partition, + } + logger.Info("agentless node defined", + "node", consulNodeName, + "partition", n.Service.ID.Partition, + ) + + services[nid] = make(map[util.Identifier]*structs.CatalogService) + proxies[nid] = make(map[util.Identifier]*structs.CatalogProxy) + } + + // register app on node + services[nid][sidApp] = &structs.CatalogService{ + Node: consulNodeName, + Partition: n.Partition, + // + Service: n.Service.ID.Name, + Meta: n.Service.Meta, + Port: n.Service.Port, + Address: n.LocalAddress(), + Namespace: n.Service.ID.Namespace, + // + CheckID: n.Service.ID.String(), + TCPCheck: n.LocalAddress() + ":" + strconv.Itoa(n.Service.Port), + } + + logger.Info("agentless service defined", + "service", n.Service.ID.Name, + "node", consulNodeName, + "namespace", n.Service.ID.Namespace, + "partition", n.Service.ID.Partition, + ) + + // register proxy for service + proxies[nid][sidProxy] = &structs.CatalogProxy{ + CatalogService: structs.CatalogService{ + Node: consulNodeName, + Partition: n.Partition, + // + Service: n.Service.ID.Name + "-sidecar-proxy", + Meta: n.Service.Meta, + Port: 20000, // TODO: fake + Address: n.LocalAddress(), + Namespace: n.Service.ID.Namespace, + // + CheckID: n.Service.ID.String(), + TCPCheck: n.LocalAddress() + ":20000", + }, + ProxyDestinationServiceName: n.Service.ID.Name, + ProxyLocalServicePort: n.Service.Port, + ProxyUpstreams: []*structs.CatalogProxyUpstream{{ + DestinationName: n.Service.UpstreamID.Name, + DestinationNamespace: n.Service.UpstreamID.Namespace, + DestinationPartition: n.Service.UpstreamID.Partition, + DestinationPeer: n.Service.UpstreamPeer, + LocalBindPort: n.Service.UpstreamLocalPort, + Datacenter: n.Service.UpstreamDatacenter, + }}, + } + + logger.Info("agentless proxy defined", + "service", n.Service.ID.Name+"-sidecar-proxy", + "node", consulNodeName, + "namespace", n.Service.ID.Namespace, + "namespace", n.Service.ID.Namespace, + "partition", n.Service.ID.Partition, + ) + + return nil + }) + if err != nil { + return fmt.Errorf("error generating catalog info details for cluster %q: %w", cluster.Name, err) + } + + out := &structs.CatalogDefinition{} + for _, n := range nodes { + out.Nodes = append(out.Nodes, n) + } + for _, m := range services { + for _, svc := range m { + out.Services = append(out.Services, svc) + } + } + for _, m := range proxies { + for _, p := range m { + out.Proxies = append(out.Proxies, p) + } + } + out.Sort() + + d, err := json.MarshalIndent(out, "", " ") + if err != nil { + return fmt.Errorf("error rendering catalog definition for cluster %q: %w", cluster.Name, err) + } + + filename := "catalog_def." + cluster.Name + ".json" + if err := c.cache.WriteStringFile(filename, string(d)); err != nil { + return fmt.Errorf("error writing catalog definition for cluster %q: %w", cluster.Name, err) + } + } + + return nil +} diff --git a/app/runner/exec.go b/app/runner/exec.go index 004930e..b995e10 100644 --- a/app/runner/exec.go +++ b/app/runner/exec.go @@ -150,12 +150,35 @@ func cmdExec(name, binary string, args []string, stdout io.Writer, dir string) e cmd.Stderr = &errWriter cmd.Stdin = nil if err := cmd.Run(); err != nil { - return fmt.Errorf("could not invoke %q: %v : %s", name, err, errWriter.String()) + return &ExecError{ + BinaryName: name, + Err: err, + ErrorOutput: errWriter.String(), + } } return nil } +type ExecError struct { + BinaryName string + ErrorOutput string + Err error +} + +func (e *ExecError) Unwrap() error { + return e.Err +} + +func (e *ExecError) Error() string { + return fmt.Sprintf( + "could not invoke %q: %v : %s", + e.BinaryName, + e.Err, + e.ErrorOutput, + ) +} + func (r *Runner) GetPathToSelf() string { if r.consulBinFlavor == "" { panic("our own binary was not analyzed") diff --git a/app/tfgen/agent.go b/app/tfgen/agent.go index 0bd2cb0..3c7bbe1 100644 --- a/app/tfgen/agent.go +++ b/app/tfgen/agent.go @@ -17,14 +17,18 @@ func GenerateAgentHCL( topology *infra.Topology, node *infra.Node, ) (string, error) { + if !node.IsAgent() { + return "", fmt.Errorf("not an agent") + } + var inSecondaryDatacenter bool - if node.Server && topology.LinkWithFederation() { + if node.IsServer() && topology.LinkWithFederation() { inSecondaryDatacenter = node.Cluster != config.PrimaryCluster } var b HCLBuilder - b.add("server", node.Server) + b.add("server", node.IsServer()) b.add("client_addr", "0.0.0.0") b.add("advertise_addr", node.LocalAddress()) b.add("datacenter", node.Cluster) @@ -41,7 +45,7 @@ func GenerateAgentHCL( b.addSlice("retry_join", topology.ServerIPs(node.Cluster)) - if node.Server && topology.LinkWithPeering() { + if node.IsServer() && topology.LinkWithPeering() { b.addBlock("peering", func() { b.add("enabled", true) }) @@ -67,7 +71,7 @@ func GenerateAgentHCL( if cfg.EncryptionTLS { prefix := node.Cluster + "-client-consul-" + strconv.Itoa(node.Index) - if node.Server { + if node.IsServer() { prefix = node.Cluster + "-server-consul-" + strconv.Itoa(node.Index) } b.addBlock("tls", func() { @@ -87,7 +91,7 @@ func GenerateAgentHCL( // b.add("verify_incoming", true) }) } - if cfg.EncryptionTLSGRPC || (node.Server && cfg.EncryptionServerTLSGRPC) { + if cfg.EncryptionTLSGRPC || (node.IsServer() && cfg.EncryptionServerTLSGRPC) { b.addBlock("grpc", func() { b.add("ca_file", "/tls/consul-agent-ca.pem") b.add("cert_file", "/tls/"+prefix+".pem") @@ -102,7 +106,7 @@ func GenerateAgentHCL( if cfg.EncryptionTLSAPI { b.add("https", 8501) } - if cfg.EncryptionTLSGRPC || (node.Server && cfg.EncryptionServerTLSGRPC) { + if cfg.EncryptionTLSGRPC || (node.IsServer() && cfg.EncryptionServerTLSGRPC) { b.add("grpc_tls", 8503) b.add("grpc", -1) } else { @@ -110,7 +114,7 @@ func GenerateAgentHCL( b.add("grpc_tls", -1) } - if !node.Server && node.Segment != "" { + if !node.IsServer() && node.Segment != "" { port := cfg.EnterpriseSegments[node.Segment] b.add("serf_lan", port) } @@ -122,11 +126,11 @@ func GenerateAgentHCL( b.add("default_policy", "deny") b.add("down_policy", "extend-cache") b.add("enable_token_persistence", true) - if node.Server && inSecondaryDatacenter { + if node.IsServer() && inSecondaryDatacenter { b.add("enable_token_replication", true) } b.addBlock("tokens", func() { - if node.Server && !inSecondaryDatacenter { + if node.IsServer() && !inSecondaryDatacenter { b.add("initial_management", cfg.InitialMasterToken) } b.add("agent_recovery", cfg.AgentMasterToken) @@ -134,7 +138,7 @@ func GenerateAgentHCL( }) } - if node.Server { + if node.IsServer() { useWANIP := false switch topology.NetworkShape { case infra.NetworkShapeIslands: @@ -222,7 +226,7 @@ func GenerateAgentHCL( } else { b.add("segment", node.Segment) - if !cfg.EnterpriseDisablePartitions && cfg.EnterpriseEnabled { + if cfg.EnterpriseEnabled { b.add("partition", node.Partition) } } diff --git a/app/tfgen/gateway.go b/app/tfgen/gateway.go index 3baa9b1..2102e5a 100644 --- a/app/tfgen/gateway.go +++ b/app/tfgen/gateway.go @@ -17,6 +17,12 @@ func GenerateMeshGatewayContainer( return nil } + switch node.Kind { + case infra.NodeKindClient: + default: + panic("figure this out: " + node.Kind) + } + type tfMeshGatewayInfo struct { PodName string NodeName string @@ -56,10 +62,8 @@ func GenerateMeshGatewayContainer( } if config.EnterpriseEnabled && node.Partition != "" { - if !config.EnterpriseDisablePartitions { - mgi.SidecarBootEnvVars = append(mgi.SidecarBootEnvVars, - "SBOOT_PARTITION="+node.Partition) - } + mgi.SidecarBootEnvVars = append(mgi.SidecarBootEnvVars, + "SBOOT_PARTITION="+node.Partition) } if config.EncryptionTLSAPI { @@ -81,7 +85,7 @@ func GenerateMeshGatewayContainer( mgi.EnableWAN = true mgi.LANAddress = `{{ GetInterfaceIP \"eth0\" }}:8443` if node.MeshGatewayUseDNSWANAddress { - mgi.WANAddress = node.Name + "-pod:8443" + mgi.WANAddress = node.PodName() + ":8443" } else { mgi.WANAddress = `{{ GetInterfaceIP \"eth0\" }}:8443` } diff --git a/app/tfgen/infra.go b/app/tfgen/infra.go new file mode 100644 index 0000000..ea561d5 --- /dev/null +++ b/app/tfgen/infra.go @@ -0,0 +1,62 @@ +package tfgen + +import ( + "strings" + "text/template" + + "github.com/rboyer/devconsul/cachestore" + "github.com/rboyer/devconsul/config" + "github.com/rboyer/devconsul/infra" + "github.com/rboyer/devconsul/util" +) + +type catalogSyncInfo struct { + PodName string + NodeName string + Args []string + HashValue string +} + +func GenerateInfraContainers( + config *config.Config, + topology *infra.Topology, + cache *cachestore.Store, + podName string, + node *infra.Node, +) ([]Resource, error) { + if node.Kind != infra.NodeKindInfra { + return nil, nil + } + + filename := "catalog_def." + node.Cluster + ".json" + hv, err := util.HashFile(cache.GetPathToStringFile(filename)) + if err != nil { + return nil, err + } + + info := catalogSyncInfo{ + PodName: node.PodName(), + NodeName: node.Name, + HashValue: hv, + Args: []string{ + "-cluster", node.Cluster, + "-config-file", "/secrets/" + filename, + "-consul-ip", strings.Join(topology.ServerIPs(node.Cluster), ","), + }, + } + + if config.EnterpriseEnabled { + info.Args = append(info.Args, "-enterprise") + } + + if !config.SecurityDisableACLs { + // TODO: switch this to its own token + info.Args = append(info.Args, "-token-file", "/secrets/master-token.val") + } + + res := Eval(tfCatalogSyncT, &info) + + return []Resource{res}, nil +} + +var tfCatalogSyncT = template.Must(template.ParseFS(content, "templates/container-catalog-sync.tf.tmpl")) diff --git a/app/tfgen/mesh.go b/app/tfgen/mesh.go index 144a64e..f77061d 100644 --- a/app/tfgen/mesh.go +++ b/app/tfgen/mesh.go @@ -9,20 +9,30 @@ import ( "github.com/rboyer/devconsul/infra" ) -type pingpongInfo struct { - PodName string - NodeName string - PingPong string // ping or pong - MetaString string +type pingpongAppInfo struct { + PodName string + NodeName string + PingPong string // ping or pong + MetaString string +} + +type pingpongSidecarInfo struct { + pingpongAppInfo + EnvoyImageResource string SidecarBootEnvVars []string UseBuiltinProxy bool EnvoyLogLevel string - EnvoyImageResource string - EnableACLs bool +} + +type pingpongDataplaneInfo struct { + pingpongAppInfo + DataplaneImageResource string + EnvVars []string } func GeneratePingPongContainers( config *config.Config, + topology *infra.Topology, podName string, node *infra.Node, ) []Resource { @@ -31,18 +41,18 @@ func GeneratePingPongContainers( } svc := node.Service - ppi := pingpongInfo{ - PodName: podName, - NodeName: node.Name, - PingPong: svc.ID.Name, - UseBuiltinProxy: node.UseBuiltinProxy, - EnvoyLogLevel: config.EnvoyLogLevel, - EnvoyImageResource: "docker_image.consul-envoy.latest", - EnableACLs: !config.SecurityDisableACLs, + switch node.Kind { + case infra.NodeKindClient, infra.NodeKindDataplane: + default: + return nil } - if node.Canary { - ppi.EnvoyImageResource = "docker_image.consul-envoy-canary.latest" + + appinfo := pingpongAppInfo{ + PodName: podName, + NodeName: node.Name, + PingPong: svc.ID.Name, } + if len(svc.Meta) > 0 { var kvs []struct{ K, V string } for k, v := range svc.Meta { @@ -55,63 +65,143 @@ func GeneratePingPongContainers( for _, kv := range kvs { parts = append(parts, kv.K+"-"+kv.V) } - ppi.MetaString = strings.Join(parts, "--") + appinfo.MetaString = strings.Join(parts, "--") + // ppi.MetaString = strings.Join(parts, "--") } - proxyType := "envoy" - if node.UseBuiltinProxy { - proxyType = "builtin" - } + res := make([]Resource, 0, 2) + res = append(res, Eval(tfPingPongAppT, &appinfo)) - if config.SecurityDisableACLs { - ppi.SidecarBootEnvVars = []string{ - "SBOOT_PROXY_TYPE=" + proxyType, - "SBOOT_REGISTER_FILE=/secrets/servicereg__" + node.Name + "__" + svc.ID.Name + ".hcl", - // - "SBOOT_MODE=insecure", + if node.Kind == infra.NodeKindDataplane { + if node.UseBuiltinProxy { + panic("not possible") } - } else if config.KubernetesEnabled { - ppi.SidecarBootEnvVars = []string{ - "SBOOT_PROXY_TYPE=" + proxyType, - "SBOOT_REGISTER_FILE=/secrets/servicereg__" + node.Name + "__" + svc.ID.Name + ".hcl", - // - "SBOOT_MODE=login", - "SBOOT_BEARER_TOKEN_FILE=/secrets/k8s/service_jwt_token." + svc.ID.Name, - "SBOOT_TOKEN_SINK_FILE=/tmp/consul.token", + + dataplaneInfo := pingpongDataplaneInfo{ + pingpongAppInfo: appinfo, + DataplaneImageResource: "docker_image.consul-dataplane.latest", + } + + if node.Canary { + dataplaneInfo.DataplaneImageResource = "docker_image.consul-dataplane-canary.latest" + } + + env := make(map[string]string) + // --- REQUIRED --- + env["DP_CONSUL_ADDRESSES"] = topology.ServerIPs(node.Cluster)[0] + // env["DP_CONSUL_ADDRESSES"] = "exec=/bin/echo " + + // strings.Join(topology.ServerIPs(node.Cluster), " ") + env["DP_SERVICE_NODE_NAME"] = node.PodName() // TODO(cdp): is this enough? + env["DP_PROXY_SERVICE_ID"] = svc.ID.Name + "-sidecar-proxy" + // --- enterprise required --- + if config.EnterpriseEnabled { + env["DP_SERVICE_NAMESPACE"] = svc.ID.Namespace + env["DP_SERVICE_PARTITION"] = svc.ID.Partition + } + + env["DP_LOG_LEVEL"] = config.EnvoyLogLevel + + // envoy + env["DP_ENVOY_ADMIN_BIND_ADDRESS"] = "0.0.0.0" // for demo purposes + env["DP_ENVOY_ADMIN_BIND_PORT"] = "19000" + + // acls + if config.SecurityDisableACLs { + // nothing + } else if config.KubernetesEnabled { + env["DP_CREDENTIAL_TYPE"] = "login" + env["DP_CREDENTIAL_LOGIN_AUTH_METHOD"] = "minikube" + env["DP_CREDENTIAL_LOGIN_BEARER_TOKEN_PATH"] = "/secrets/k8s/service_jwt_token." + svc.ID.Name + } else { + env["DP_CREDENTIAL_TYPE"] = "static" + env["SBOOT_TOKEN_FILE"] = "/secrets/service--" + node.Cluster + "--" + svc.ID.ID() + ".val" + // env["DP_CREDENTIAL_STATIC_TOKEN"] = "" + } + + if config.EncryptionTLSGRPC { + env["SBOOT_AGENT_GRPC_TLS"] = "1" + + // The path to a file or directory containing CA certificates used to + // verify the server's certificate. Environment variable: DP_CA_CERTS. + // env["DP_CA_CERTS"] = "/tls" + env["DP_CA_CERTS"] = "/tls/consul-agent-ca.pem" + env["DP_CONSUL_GRPC_PORT"] = "8503" + + env["DP_TLS_SERVER_NAME"] = "server." + node.Cluster + ".consul" + } else { + env["DP_TLS_INSECURE_SKIP_VERIFY"] = "1" + env["DP_CONSUL_GRPC_PORT"] = "8502" } + + dataplaneInfo.EnvVars = renderEnv(env) + + res = append(res, Eval(tfPingPongDataplaneT, &dataplaneInfo)) } else { - ppi.SidecarBootEnvVars = []string{ - "SBOOT_PROXY_TYPE=" + proxyType, - "SBOOT_REGISTER_FILE=/secrets/servicereg__" + node.Name + "__" + svc.ID.Name + ".hcl", - // - "SBOOT_MODE=direct", - "SBOOT_TOKEN_FILE=/secrets/service-token--" + svc.ID.ID() + ".val", + sidecarInfo := pingpongSidecarInfo{ + pingpongAppInfo: appinfo, + EnvoyImageResource: "docker_image.consul-envoy.latest", + UseBuiltinProxy: node.UseBuiltinProxy, + EnvoyLogLevel: config.EnvoyLogLevel, } - } - if config.EnterpriseEnabled && node.Partition != "" { - if !config.EnterpriseDisablePartitions { - ppi.SidecarBootEnvVars = append(ppi.SidecarBootEnvVars, - "SBOOT_PARTITION="+node.Partition) + if node.Canary { + sidecarInfo.EnvoyImageResource = "docker_image.consul-envoy-canary.latest" + // ppi.DataplaneImageResource = "docker_image.consul-dataplane-canary.latest" + } + + proxyType := "envoy" + if node.UseBuiltinProxy { + proxyType = "builtin" + } + + env := make(map[string]string) + env["SBOOT_PROXY_TYPE"] = proxyType + env["SBOOT_REGISTER_FILE"] = "/secrets/servicereg__" + node.Name + "__" + svc.ID.Name + ".hcl" + + if config.SecurityDisableACLs { + env["SBOOT_MODE"] = "insecure" + } else if config.KubernetesEnabled { + env["SBOOT_MODE"] = "login" + env["SBOOT_BEARER_TOKEN_FILE"] = "/secrets/k8s/service_jwt_token." + svc.ID.Name + env["SBOOT_TOKEN_SINK_FILE"] = "/tmp/consul.token" + } else { + env["SBOOT_MODE"] = "direct" + env["SBOOT_TOKEN_FILE"] = "/secrets/service--" + node.Cluster + "--" + svc.ID.ID() + ".val" } - } - if config.EncryptionTLSAPI { - ppi.SidecarBootEnvVars = append(ppi.SidecarBootEnvVars, - "SBOOT_AGENT_TLS=1") + if config.EnterpriseEnabled && node.Partition != "" { + env["SBOOT_PARTITION"] = node.Partition + } + + if config.EncryptionTLSAPI { + env["SBOOT_AGENT_TLS"] = "1" + } + if config.EncryptionTLSGRPC { + env["SBOOT_AGENT_GRPC_TLS"] = "1" + } + + sidecarInfo.SidecarBootEnvVars = renderEnv(env) + + res = append(res, Eval(tfPingPongSidecarT, &sidecarInfo)) } - if config.EncryptionTLSGRPC { - ppi.SidecarBootEnvVars = append(ppi.SidecarBootEnvVars, - "SBOOT_AGENT_GRPC_TLS=1") + + return res +} + +func renderEnv(m map[string]string) []string { + if len(m) == 0 { + return nil } - return []Resource{ - Eval(tfPingPongAppT, &ppi), - Eval(tfPingPongSidecarT, &ppi), + out := make([]string, 0, len(m)) + for k, v := range m { + out = append(out, k+"="+v) } + return out } // TODO: make chaos opt-in // "-pong-chaos", var tfPingPongAppT = template.Must(template.ParseFS(content, "templates/container-app.tf.tmpl")) var tfPingPongSidecarT = template.Must(template.ParseFS(content, "templates/container-app-sidecar.tf.tmpl")) +var tfPingPongDataplaneT = template.Must(template.ParseFS(content, "templates/container-app-dataplane.tf.tmpl")) diff --git a/app/tfgen/nodes.go b/app/tfgen/nodes.go index 2a14111..d7fbb1f 100644 --- a/app/tfgen/nodes.go +++ b/app/tfgen/nodes.go @@ -3,6 +3,7 @@ package tfgen import ( "text/template" + "github.com/rboyer/devconsul/cachestore" "github.com/rboyer/devconsul/config" "github.com/rboyer/devconsul/infra" ) @@ -15,23 +16,16 @@ type terraformPod struct { EnterpriseLicensePath string } -func GenerateAgentContainers( +func GenerateNodeContainers( cfg *config.Config, topology *infra.Topology, + cache *cachestore.Store, node *infra.Node, podContents bool, ) ([]Resource, error) { - podName := node.Name + "-pod" - - podHCL, err := GenerateAgentHCL(cfg, topology, node) - if err != nil { - return nil, err - } - pod := terraformPod{ - PodName: podName, + PodName: node.PodName(), Node: node, - HCL: podHCL, Labels: map[string]string{ // }, @@ -45,14 +39,34 @@ func GenerateAgentContainers( containers = append(containers, Eval(tfPauseT, &pod)) if podContents { - containers = append(containers, Eval(tfConsulT, &pod)) + if node.IsAgent() { + podHCL, err := GenerateAgentHCL(cfg, topology, node) + if err != nil { + return nil, err + } + pod.HCL = podHCL + + containers = append(containers, Eval(tfConsulT, &pod)) + } + + if node.RunsWorkloads() { + if gwRes := GenerateMeshGatewayContainer(cfg, topology, pod.PodName, pod.Node); gwRes != nil { + containers = append(containers, gwRes) + } - if gwRes := GenerateMeshGatewayContainer(cfg, topology, podName, pod.Node); gwRes != nil { - containers = append(containers, gwRes) + if resources := GeneratePingPongContainers(cfg, topology, pod.PodName, pod.Node); len(resources) > 0 { + containers = append(containers, resources...) + } } - if resources := GeneratePingPongContainers(cfg, podName, pod.Node); len(resources) > 0 { - containers = append(containers, resources...) + if node.Kind == infra.NodeKindInfra { + resources, err := GenerateInfraContainers(cfg, topology, cache, pod.PodName, pod.Node) + if err != nil { + return nil, err + } + if len(resources) > 0 { + containers = append(containers, resources...) + } } } diff --git a/app/tfgen/o11y.go b/app/tfgen/o11y.go index a99ede6..23fdcb8 100644 --- a/app/tfgen/o11y.go +++ b/app/tfgen/o11y.go @@ -38,6 +38,7 @@ func GeneratePrometheusConfigFile(cfg *config.Config, topology *infra.Topology) Params map[string][]string Targets []string Labels []kv + Token string } jobs := make(map[string]*job) @@ -54,13 +55,13 @@ func GeneratePrometheusConfigFile(cfg *config.Config, topology *infra.Topology) } topology.WalkSilent(func(node *infra.Node) { - if node.Server { + if node.IsServer() { add(&job{ Name: "consul-server--" + node.Name, MetricsPath: "/v1/agent/metrics", + Token: cfg.AgentMasterToken, Params: map[string][]string{ "format": {"prometheus"}, - "token": {cfg.AgentMasterToken}, }, Targets: []string{ net.JoinHostPort(node.LocalAddress(), "8500"), @@ -76,9 +77,9 @@ func GeneratePrometheusConfigFile(cfg *config.Config, topology *infra.Topology) add(&job{ Name: "consul-client--" + node.Name, MetricsPath: "/v1/agent/metrics", + Token: cfg.AgentMasterToken, Params: map[string][]string{ "format": {"prometheus"}, - "token": {cfg.AgentMasterToken}, }, Targets: []string{ net.JoinHostPort(node.LocalAddress(), "8500"), diff --git a/app/tfgen/templates/container-app-dataplane.tf.tmpl b/app/tfgen/templates/container-app-dataplane.tf.tmpl new file mode 100644 index 0000000..d4b8b83 --- /dev/null +++ b/app/tfgen/templates/container-app-dataplane.tf.tmpl @@ -0,0 +1,43 @@ +resource "docker_container" "{{.NodeName}}-{{.PingPong}}-sidecar" { + name = "{{.NodeName}}-{{.PingPong}}-sidecar" + network_mode = "container:${docker_container.{{.PodName}}.id}" + image = {{ .DataplaneImageResource }} + restart = "on-failure" + + labels { + label = "devconsul" + value = "1" + } + labels { + label = "devconsul.type" + value = "dataplane" + } + + volumes { + host_path = abspath("cache") + container_path = "/secrets" + read_only = true + } + volumes { + host_path = abspath("dataplane-boot.sh") + container_path = "/bin/dataplane-boot.sh" + read_only = true + } + volumes { + host_path = abspath("cache/tls") + container_path = "/tls" + read_only = true + } + + env = [ +{{- range .EnvVars }} + "{{.}}", +{{- end}} + ] + + #"/usr/local/bin/consul-dataplane" + command = [ + "/usr/local/bin/dumb-init", + "/bin/dataplane-boot.sh", + ] +} diff --git a/app/tfgen/templates/container-catalog-sync.tf.tmpl b/app/tfgen/templates/container-catalog-sync.tf.tmpl new file mode 100644 index 0000000..f244ee3 --- /dev/null +++ b/app/tfgen/templates/container-catalog-sync.tf.tmpl @@ -0,0 +1,37 @@ +resource "docker_container" "{{.NodeName}}-catalog-sync" { + name = "{{.NodeName}}-catalog-sync" + network_mode = "container:${docker_container.{{.PodName}}.id}" + image = docker_image.clustertool.latest + restart = "on-failure" + + labels { + label = "devconsul" + value = "1" + } + labels { + label = "devconsul.type" + value = "infra" + } + + volumes { + host_path = abspath("cache") + container_path = "/secrets" + read_only = true + } + volumes { + host_path = abspath("cache/tls") + container_path = "/tls" + read_only = true + } + + env = [ + "DEVCONSUL_HASH_VALUE={{.HashValue}}", + ] + + command = [ + "catalog-sync", +{{- range .Args }} + "{{.}}", +{{- end}} + ] +} diff --git a/app/tfgen/templates/prometheus-config.yml.tmpl b/app/tfgen/templates/prometheus-config.yml.tmpl index 82b10fb..c9ed2a2 100644 --- a/app/tfgen/templates/prometheus-config.yml.tmpl +++ b/app/tfgen/templates/prometheus-config.yml.tmpl @@ -31,6 +31,11 @@ scrape_configs: - job_name: {{.Name}} metrics_path: "{{.MetricsPath}}" honor_labels: false +{{ if .Token }} + authorization: + type: Bearer + credentials: "{{ .Token }}" +{{ end }} params: {{- range $k, $vl := .Params }} {{ $k }}: diff --git a/app/tfgen/tfgen.go b/app/tfgen/tfgen.go index 517e72c..16dc722 100644 --- a/app/tfgen/tfgen.go +++ b/app/tfgen/tfgen.go @@ -4,16 +4,18 @@ import ( "embed" ) -//go:embed templates/container-pause.tf.tmpl +//go:embed templates/container-app-dataplane.tf.tmpl +//go:embed templates/container-app-sidecar.tf.tmpl +//go:embed templates/container-app.tf.tmpl +//go:embed templates/container-catalog-sync.tf.tmpl //go:embed templates/container-consul.tf.tmpl +//go:embed templates/container-grafana.tf //go:embed templates/container-mgw.tf.tmpl -//go:embed templates/container-app.tf.tmpl -//go:embed templates/container-app-sidecar.tf.tmpl -//go:embed templates/prometheus-config.yml.tmpl +//go:embed templates/container-pause.tf.tmpl //go:embed templates/container-prometheus.tf -//go:embed templates/container-grafana.tf -//go:embed templates/grafana-prometheus.yml -//go:embed templates/grafana.ini //go:embed templates/container-vault.tf +//go:embed templates/grafana.ini +//go:embed templates/grafana-prometheus.yml +//go:embed templates/prometheus-config.yml.tmpl //go:embed templates/vault-config.hcl var content embed.FS diff --git a/app/util.go b/app/util.go index 1603381..e4e52f3 100644 --- a/app/util.go +++ b/app/util.go @@ -3,42 +3,28 @@ package app import ( "encoding/json" "io" - "os" - "path/filepath" + + "github.com/rboyer/devconsul/util" ) +// Deprecated: x func filesExist(parent string, paths ...string) (bool, error) { - for _, p := range paths { - ok, err := fileExists(filepath.Join(parent, p)) - if err != nil { - return false, err - } - if !ok { - return false, nil - } - } - return true, nil + return util.FilesExist(parent, paths...) } +// Deprecated: x func fileExists(path string) (bool, error) { - _, err := os.Stat(path) - if os.IsNotExist(err) { - return false, nil - } else if err != nil { - return false, err - } else { - return true, nil - } + return util.FileExists(path) } +// Deprecated: x +func hashFile(path string) (string, error) { + return util.HashFile(path) +} + +// Deprecated: x func addFileToHash(path string, w io.Writer) error { - f, err := os.Open(path) - if err != nil { - return err - } - defer f.Close() - _, err = io.Copy(w, f) - return err + return util.AddFileToHash(path, w) } func jsonPretty(val interface{}) string { diff --git a/cachestore/store.go b/cachestore/store.go index e47494a..f1a9ecc 100644 --- a/cachestore/store.go +++ b/cachestore/store.go @@ -83,6 +83,10 @@ func (s *Store) DelValuePrefix(prefix string) ([]string, error) { return out, nil } +func (s *Store) GetPathToStringFile(filename string) string { + return filepath.Join(s.Dir, filename) +} + func (s *Store) LoadStringFile(filename string) (string, error) { fn := filepath.Join(s.Dir, filename) b, err := os.ReadFile(fn) diff --git a/clustertool/catalog-sync.go b/clustertool/catalog-sync.go new file mode 100644 index 0000000..9b5e5d7 --- /dev/null +++ b/clustertool/catalog-sync.go @@ -0,0 +1,405 @@ +package main + +import ( + "encoding/json" + "errors" + "flag" + "fmt" + "math/rand" + "net" + "os" + "strings" + "time" + + "github.com/hashicorp/consul/api" + "github.com/hashicorp/go-hclog" + + "github.com/rboyer/devconsul/consulfunc" + "github.com/rboyer/devconsul/structs" + "github.com/rboyer/devconsul/util" +) + +const ( + checkTimeout = 2 * time.Second +) + +type NodeID = util.Identifier2 +type ServiceID = util.Identifier + +func init() { + allCommands["catalog-sync"] = &catalogSyncCommand{} +} + +type catalogSyncCommand struct { + logger hclog.Logger + conf *structs.CatalogDefinition + client *api.Client + + // lazy + dialer *net.Dialer + last healthResults + + cluster string + flagConfig string + flagConfigFile string + flagFreq time.Duration + enterprise bool + token string + tokenFile string + rawConsulIPs string + + consulIPs []string +} + +func (c *catalogSyncCommand) RegisterFlags() { + flag.StringVar(&c.flagConfig, "config", "", "json configuration") + flag.StringVar(&c.flagConfigFile, "config-file", "", "json configuration file") + flag.DurationVar(&c.flagFreq, "freq", 1*time.Second, "frequency of checks") + flag.StringVar(&c.token, "token", "", "consul token") + flag.StringVar(&c.tokenFile, "token-file", "", "consul token file") + flag.StringVar(&c.rawConsulIPs, "consul-ip", "", "consul address") + flag.BoolVar(&c.enterprise, "enterprise", false, "should we care about enterprise") + flag.StringVar(&c.cluster, "cluster", "", "cluster name") +} + +func pickRandomIP(ips []string) string { + switch len(ips) { + case 0: + return "" + case 1: + return ips[0] + default: + idx := rand.Intn(len(ips)) + return ips[idx] + } +} + +func (c *catalogSyncCommand) Run(logger hclog.Logger) error { + c.logger = logger + + if c.flagFreq <= 0 { + return fmt.Errorf("freq flag is invalid: %v", c.flagFreq) + } + if c.rawConsulIPs == "" { + return errors.New("missing required 'consul-ip' argument") + } else { + c.consulIPs = strings.Split(c.rawConsulIPs, ",") + c.rawConsulIPs = "" + } + if c.cluster == "" { + return errors.New("missing required 'cluster' argument") + } + + if c.flagConfig == "" && c.flagConfigFile == "" { + return errors.New("one of 'config' or 'config-file' is required") + } + if c.flagConfig != "" && c.flagConfigFile != "" { + return errors.New("ONLY one of 'config' or 'config-file' is required") + } + + if c.token != "" && c.tokenFile != "" { + return errors.New("ONLY one of 'token' or 'token-file' can be set at once") + } + + if c.tokenFile != "" { + for { + _, err := os.Stat(c.tokenFile) + if err == nil { + break + } else if os.IsNotExist(err) { + c.logger.Info("token-file not ready yet") + time.Sleep(250 * time.Millisecond) + } else if err != nil { + return fmt.Errorf("'token-file' is not readable %q: %w", c.tokenFile, err) + } + } + + b, err := os.ReadFile(c.tokenFile) + if err != nil { + return fmt.Errorf("error reading token file %q: %w", c.tokenFile, err) + } + c.token = strings.TrimSpace(string(b)) + c.tokenFile = "" + } + + if c.flagConfigFile != "" { + b, err := os.ReadFile(c.flagConfigFile) + if err != nil { + return fmt.Errorf("error reading from config file %q: %w", c.flagConfigFile, err) + } + c.flagConfig = string(b) + c.flagConfigFile = "" + } + + c.conf = &structs.CatalogDefinition{} + if err := json.Unmarshal([]byte(c.flagConfig), c.conf); err != nil { + return fmt.Errorf("error parsing provided config: %w", err) + } + + serverIP := pickRandomIP(c.consulIPs) + c.logger.Info("pinning to server", "ip", serverIP) + + var err error + c.client, err = consulfunc.GetClient(serverIP, c.token) + if err != nil { + return fmt.Errorf("could not create consul api client: %w", err) + } + + if err := c.initialSync(); err != nil { + return fmt.Errorf("initial catalog sync failed: %w", err) + } + + return c.detectAndSyncHealth() + +} + +func (c *catalogSyncCommand) initialSync() error { + c.logger.Info("syncing initial catalog data", + "num_nodes", len(c.conf.Nodes), + "num_services", len(c.conf.Services), + "num_proxies", len(c.conf.Proxies), + ) + + curr, err := c.dumpCurrentCatalog(c.cluster) + if err != nil { + return fmt.Errorf("error dumping current catalog state: %w", err) + } + + // register missing nodes + expect := make(map[NodeID]map[ServiceID]struct{}) + for _, n := range c.conf.Nodes { + nid := n.ID() + + // register synthetic node + if _, err := c.client.Catalog().Register(n.ToAPI(c.enterprise), nil); err != nil { + return fmt.Errorf("error registering virtual node %s: %w", nid, err) + } + expect[nid] = make(map[ServiceID]struct{}) + + c.logger.Info("agentless node created", + "node", n.Node, + "partition", n.Partition, + ) + } + + // register missing services + for _, svc := range c.conf.Services { + sid := svc.ID() + nid := svc.NodeID() + + if _, err := c.client.Catalog().Register(svc.ToAPI(c.enterprise), nil); err != nil { + return fmt.Errorf("error registering service %s to node %s: %w", sid, nid, err) + } + + expect[nid][sid] = struct{}{} + + c.logger.Info("agentless service created", + "service", sid.Name, + "node", nid.Name, + "namespace", sid.Namespace, + "partition", sid.Partition, + ) + } + + // register missing proxies + for _, svc := range c.conf.Proxies { + sid := svc.ID() + nid := svc.NodeID() + + if _, err := c.client.Catalog().Register(svc.ToAPI(c.enterprise), nil); err != nil { + return fmt.Errorf("error registering proxy for service %s to node %s: %w", sid, nid, err) + } + + expect[nid][sid] = struct{}{} + + c.logger.Info("agentless proxy created", + // "service", n.Service.ID.Name+"-sidecar-proxy", + "service", sid.Name, + "node", nid.Name, + "namespace", sid.Namespace, + "partition", sid.Partition, + ) + } + + // remove all strays + for nid, m := range curr { + expectM, ok := expect[nid] + if !ok { + _, err := c.client.Catalog().Deregister(&api.CatalogDeregistration{ + Node: nid.Name, + Partition: nid.Partition, + }, nil) + if err != nil { + return fmt.Errorf("error deregistering virtual node %s: %w", nid, err) + } + c.logger.Info("agentless node removed", + "node", nid.Name, + "partition", nid.Partition, + ) + continue + } + + for sid := range m { + if _, ok := expectM[sid]; !ok { + _, err := c.client.Catalog().Deregister(&api.CatalogDeregistration{ + Node: nid.Name, + Namespace: sid.Namespace, + Partition: sid.Partition, + ServiceID: sid.Name, + }, nil) + if err != nil { + return fmt.Errorf("error deregistering virtual service %s from node %s: %w", sid, nid, err) + } + + c.logger.Info("agentless service removed", + "service", sid.Name, + "node", nid.Name, + "namespace", sid.Namespace, + "partition", sid.Partition, + ) + } + } + } + + return nil +} + +func (c *catalogSyncCommand) dumpCurrentCatalog(cluster string) (map[NodeID]map[ServiceID]struct{}, error) { + // dump virtual nodes + rawNodes, err := consulfunc.ListAllNodesWithFilter(c.client, cluster, c.enterprise, `"devconsul-virtual" in Meta`) + if err != nil { + return nil, err + } + + curr := make(map[NodeID]map[ServiceID]struct{}) + for _, n := range rawNodes { + nid := util.NewIdentifier2(n.Node, n.Partition) + + m, ok := curr[nid] + if !ok { + m = make(map[ServiceID]struct{}) + curr[nid] = m + } + + rawSvc, _, err := c.client.Catalog().NodeServiceList(n.Node, &api.QueryOptions{ + Partition: n.Partition, + Namespace: "*", + }) + if err != nil { + return nil, err + } + + for _, svc := range rawSvc.Services { + sid := util.NewIdentifier(svc.Service, svc.Namespace, svc.Partition) + m[sid] = struct{}{} + } + } + // $ curl -sLi --get 10.0.1.11:8500/v1/catalog/nodes?token=root --data-urlencode 'filter="devconsul-virtual" in Meta' + + return curr, nil +} + +func (c *catalogSyncCommand) detectAndSyncHealth() error { + for { + c.detectAndSyncHealthOnce() + time.Sleep(c.flagFreq) + } +} + +type healthResults struct { + data map[NodeID]map[ServiceID]string +} + +func (h *healthResults) getResult(nid NodeID, sid ServiceID) string { + if h.data == nil { + return "" + } + + if m, ok := h.data[nid]; ok { + if v, ok := m[sid]; ok { + return v + } + } + return "" +} + +func (h *healthResults) setResult(nid NodeID, sid ServiceID, value string) { + if h.data == nil { + h.data = make(map[NodeID]map[ServiceID]string) + } + + m, ok := h.data[nid] + if !ok { + m = make(map[ServiceID]string) + h.data[nid] = m + } + m[sid] = value +} + +func (c *catalogSyncCommand) detectAndSyncHealthOnce() { + for _, svc := range c.conf.Services { + if svc.TCPCheck == "" || svc.CheckID == "" { + continue + } + + logger := c.logger.With( + "node", svc.NodeID().String(), + "service", svc.ID().String(), + ) + + var ( + nid = svc.NodeID() + sid = svc.ID() + ) + + lastResult := c.last.getResult(nid, sid) + + if err := c.checkTCP(svc.TCPCheck); err != nil { + if lastResult != api.HealthCritical { + logger.Warn("health check status is now failing", "p", lastResult, "n", api.HealthCritical) + + if err := c.syncHealth(svc, api.HealthCritical); err != nil { + logger.Error("could not update catalog for health status change", "error", err) + continue + } + + c.last.setResult(nid, sid, api.HealthCritical) + } + } else { + if lastResult != api.HealthPassing { + logger.Info("health check status is now passing", "p", lastResult, "n", api.HealthPassing) + + if err := c.syncHealth(svc, api.HealthPassing); err != nil { + logger.Error("could not update catalog for health status change", "error", err) + continue + } + + c.last.setResult(nid, sid, api.HealthPassing) + } + } + } +} + +func (c *catalogSyncCommand) syncHealth(svc *structs.CatalogService, status string) error { + reg := svc.ToAPI(c.enterprise) + reg.Check.Status = status + + _, err := c.client.Catalog().Register(reg, nil) + return err +} + +func (c *catalogSyncCommand) checkTCP(addr string) error { + if c.dialer == nil { + // Create the socket dialer + c.dialer = &net.Dialer{ + Timeout: checkTimeout, + } + } + + conn, err := c.dialer.Dial("tcp", addr) + if err != nil { + return fmt.Errorf("tcp check failed: %w", err) + } + conn.Close() + return nil +} diff --git a/clustertool/cliapi/api.go b/clustertool/cliapi/api.go new file mode 100644 index 0000000..488d82b --- /dev/null +++ b/clustertool/cliapi/api.go @@ -0,0 +1,19 @@ +package cliapi + +// Deprecated: delete +type HealthCheckConfig struct { + Checks []*HealthCheck `json:",omitempty"` +} + +// Deprecated: delete +type HealthCheck struct { + Node string `json:",omitempty"` + Service string `json:",omitempty"` + Namespace string `json:",omitempty"` + Partition string `json:",omitempty"` + + HTTP string `json:",omitempty"` + TCP string `json:",omitempty"` + + LastResultOK *bool `json:"-"` +} diff --git a/clustertool/main.go b/clustertool/main.go new file mode 100644 index 0000000..7f7c796 --- /dev/null +++ b/clustertool/main.go @@ -0,0 +1,69 @@ +package main + +import ( + "flag" + "io" + "log" + "os" + "sort" + "strings" + + "github.com/hashicorp/go-hclog" +) + +const programName = "clustertool" + +type Command interface { + RegisterFlags() + Run(hclog.Logger) error +} + +var allCommands = make(map[string]Command) + +func main() { + log.SetOutput(io.Discard) + + // Create logger + logger := hclog.New(&hclog.LoggerOptions{ + Name: programName, + Level: hclog.Debug, + Output: os.Stderr, + JSONFormat: false, + }) + + if len(os.Args) == 1 { + logger.Error("Missing required subcommand") + os.Exit(1) + } + subcommand := os.Args[1] + os.Args = os.Args[1:] + os.Args[0] = programName + + if subcommand == "help" { + var keys []string + for name := range allCommands { + keys = append(keys, name) + } + sort.Strings(keys) + + logger.Info("available commands: [" + strings.Join(keys, ", ") + "]") + os.Exit(0) + } + + for name, cmd := range allCommands { + if name == subcommand { + cmd.RegisterFlags() + flag.Parse() + + if err := cmd.Run(logger); err != nil { + logger.Error(err.Error()) + os.Exit(1) + } + os.Exit(0) + return + } + } + + logger.Error("Unknown subcommand", "subcommand", subcommand) + os.Exit(1) +} diff --git a/config/config.go b/config/config.go index 80f7930..eb13537 100644 --- a/config/config.go +++ b/config/config.go @@ -4,13 +4,17 @@ import ( "github.com/hashicorp/consul/api" ) +type Versions struct { + ConsulImage string + Envoy string + DataplaneImage string +} + // Config is the runtime configuration struct derived from rawConfig. type Config struct { ConfName string // name from config.hcl - ConsulImage string - EnvoyVersion string - CanaryConsulImage string - CanaryEnvoyVersion string + Versions Versions + CanaryVersions Versions CanaryNodes []string EncryptionTLS bool EncryptionTLSAPI bool @@ -32,16 +36,17 @@ type Config struct { EnterpriseEnabled bool EnterpriseSegments map[string]int EnterprisePartitions []*Partition - EnterpriseDisablePartitions bool EnterpriseLicensePath string TopologyNetworkShape string TopologyLinkMode string + TopologyNodeMode string TopologyClusters []*Cluster TopologyNodes []*Node } func (c *Config) CanaryInfo() (configured bool, nodes map[string]struct{}) { - configured = c.CanaryConsulImage != "" && c.CanaryEnvoyVersion != "" + // TODO(cdp): how is this supposed to work? + configured = c.CanaryVersions.ConsulImage != "" && c.CanaryVersions.Envoy != "" nodes = make(map[string]struct{}) for _, n := range c.CanaryNodes { @@ -76,6 +81,7 @@ type Cluster struct { type Node struct { NodeName string `hcl:"name,label"` + Mode string `hcl:"mode,optional"` Segment string `hcl:"segment,optional"` Partition string `hcl:"partition,optional"` UpstreamName string `hcl:"upstream_name,optional"` diff --git a/config/config_test.go b/config/config_test.go index 30bb894..eaf55c4 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -13,16 +13,20 @@ func TestParseConfig_EmptyInferDefaults(t *testing.T) { require.Equal(t, &Config{ ConfName: "legacy", - ConsulImage: "consul-dev:latest", EnvoyLogLevel: "info", - EnvoyVersion: "v1.23.1", TopologyNetworkShape: "flat", TopologyLinkMode: "federate", + TopologyNodeMode: "agent", TopologyClusters: []*Cluster{ {Name: "dc1", Servers: 1, Clients: 2}, }, ConfigEntries: map[string][]api.ConfigEntry{}, VaultAsMeshCA: make(map[string]struct{}), + Versions: Versions{ + ConsulImage: "consul-dev:latest", + DataplaneImage: "hashicorp/consul-dataplane:1.0.0", + Envoy: "v1.24.0", + }, }, fc) } @@ -35,16 +39,20 @@ func TestParseConfig_BothFormats(t *testing.T) { require.NoError(t, err) require.Equal(t, &Config{ ConfName: "legacy", - ConsulImage: "consul-dev:latest", EnvoyLogLevel: "info", - EnvoyVersion: "v1.18.3", TopologyNetworkShape: "flat", TopologyLinkMode: "federate", + TopologyNodeMode: "agent", TopologyClusters: []*Cluster{ {Name: "dc1", Servers: 1, Clients: 2}, }, ConfigEntries: map[string][]api.ConfigEntry{}, VaultAsMeshCA: make(map[string]struct{}), + Versions: Versions{ + ConsulImage: "consul-dev:latest", + DataplaneImage: "hashicorp/consul-dataplane:1.0.0", + Envoy: "v1.18.3", + }, }, fc) }) t.Run("new 1", func(t *testing.T) { @@ -61,16 +69,20 @@ func TestParseConfig_BothFormats(t *testing.T) { require.NoError(t, err) require.Equal(t, &Config{ ConfName: "beta", - ConsulImage: "consul-dev:latest", EnvoyLogLevel: "info", - EnvoyVersion: "v1.18.3", TopologyNetworkShape: "flat", TopologyLinkMode: "federate", + TopologyNodeMode: "agent", TopologyClusters: []*Cluster{ {Name: "dc1", Servers: 1, Clients: 2}, }, ConfigEntries: map[string][]api.ConfigEntry{}, VaultAsMeshCA: make(map[string]struct{}), + Versions: Versions{ + ConsulImage: "consul-dev:latest", + Envoy: "v1.18.3", + DataplaneImage: "hashicorp/consul-dataplane:1.0.0", + }, }, fc) }) t.Run("new 2", func(t *testing.T) { @@ -87,16 +99,20 @@ func TestParseConfig_BothFormats(t *testing.T) { require.NoError(t, err) require.Equal(t, &Config{ ConfName: "alpha", - ConsulImage: "consul-dev:latest", EnvoyLogLevel: "info", - EnvoyVersion: "v1.17.3", TopologyNetworkShape: "flat", TopologyLinkMode: "federate", + TopologyNodeMode: "agent", TopologyClusters: []*Cluster{ {Name: "dc1", Servers: 1, Clients: 2}, }, ConfigEntries: map[string][]api.ConfigEntry{}, VaultAsMeshCA: make(map[string]struct{}), + Versions: Versions{ + ConsulImage: "consul-dev:latest", + Envoy: "v1.17.3", + DataplaneImage: "hashicorp/consul-dataplane:1.0.0", + }, }, fc) }) } @@ -213,11 +229,16 @@ EOF require.NoError(t, err) expected := &Config{ - ConfName: "legacy", - ConsulImage: "my-dev-image:blah", - EnvoyVersion: "v1.18.3", - CanaryConsulImage: "consul:1.9.5", - CanaryEnvoyVersion: "v1.17.2", + ConfName: "legacy", + Versions: Versions{ + ConsulImage: "my-dev-image:blah", + Envoy: "v1.18.3", + DataplaneImage: "hashicorp/consul-dataplane:1.0.0", + }, + CanaryVersions: Versions{ + ConsulImage: "consul:1.9.5", + Envoy: "v1.17.2", + }, CanaryNodes: []string{"abc", "def"}, EncryptionTLS: true, EncryptionTLSAPI: true, @@ -246,6 +267,7 @@ EOF EnterpriseSegments: map[string]int{"seg1": 8303, "seg2": 8304}, TopologyNetworkShape: "islands", TopologyLinkMode: "peer", + TopologyNodeMode: "agent", TopologyClusters: []*Cluster{ {Name: "dc1", Servers: 3, Clients: 2, MeshGateways: 1}, {Name: "dc2", Servers: 3, Clients: 2, MeshGateways: 1}, diff --git a/config/default_cdp.go b/config/default_cdp.go new file mode 100644 index 0000000..8114b60 --- /dev/null +++ b/config/default_cdp.go @@ -0,0 +1,3 @@ +package config + +const DefaultDataplaneImage = "hashicorp/consul-dataplane:1.0.0" diff --git a/config/default_envoy.go b/config/default_envoy.go index 056fe2b..7461cb7 100644 --- a/config/default_envoy.go +++ b/config/default_envoy.go @@ -1,3 +1,3 @@ package config -const DefaultEnvoyVersion = "v1.23.1" +const DefaultEnvoyVersion = "v1.24.0" diff --git a/config/parse.go b/config/parse.go index 94b0357..8494ca0 100644 --- a/config/parse.go +++ b/config/parse.go @@ -43,6 +43,9 @@ func parseConfig(pathname string, contents []byte) (*Config, error) { if uc.EnvoyVersion == "" { uc.EnvoyVersion = DefaultEnvoyVersion } + if uc.DataplaneImage == "" { + uc.DataplaneImage = DefaultDataplaneImage + } if uc.Envoy.LogLevel == "" { uc.Envoy.LogLevel = "info" } @@ -52,6 +55,9 @@ func parseConfig(pathname string, contents []byte) (*Config, error) { if uc.Topology.LinkMode == "" { uc.Topology.LinkMode = "federate" } + if uc.Topology.NodeMode == "" { + uc.Topology.NodeMode = "agent" + } if !uc.Security.DisableACLs { if uc.Topology.LinkMode == "peer" && uc.Security.InitialMasterToken == "" { @@ -101,12 +107,18 @@ func parseConfig(pathname string, contents []byte) (*Config, error) { } cfg := &Config{ - ConfName: uc.Name, - ConsulImage: uc.ConsulImage, - EnvoyVersion: uc.EnvoyVersion, - CanaryConsulImage: uc.CanaryProxies.ConsulImage, - CanaryEnvoyVersion: uc.CanaryProxies.EnvoyVersion, - CanaryNodes: uc.CanaryProxies.Nodes, + ConfName: uc.Name, + Versions: Versions{ + ConsulImage: uc.ConsulImage, + Envoy: uc.EnvoyVersion, + DataplaneImage: uc.DataplaneImage, + }, + CanaryNodes: uc.CanaryProxies.Nodes, + CanaryVersions: Versions{ + ConsulImage: uc.CanaryProxies.ConsulImage, + Envoy: uc.CanaryProxies.EnvoyVersion, + DataplaneImage: uc.CanaryProxies.DataplaneImage, + }, EncryptionTLS: uc.Security.Encryption.TLS, EncryptionTLSAPI: uc.Security.Encryption.TLSAPI, EncryptionTLSGRPC: uc.Security.Encryption.TLSGRPC, @@ -123,10 +135,10 @@ func parseConfig(pathname string, contents []byte) (*Config, error) { InitialMasterToken: uc.Security.InitialMasterToken, EnterpriseEnabled: uc.Enterprise.Enabled, EnterprisePartitions: uc.Enterprise.Partitions, - EnterpriseDisablePartitions: uc.Enterprise.DisablePartitions, EnterpriseLicensePath: uc.Enterprise.LicensePath, TopologyNetworkShape: uc.Topology.NetworkShape, TopologyLinkMode: uc.Topology.LinkMode, + TopologyNodeMode: uc.Topology.NodeMode, TopologyClusters: uc.Topology.Cluster, TopologyNodes: uc.Topology.Nodes, ConfigEntries: make(map[string][]api.ConfigEntry), @@ -306,11 +318,14 @@ func validateConfig(cfg *Config) error { return fmt.Errorf("encryption.tls_grpc=true requires encryption.tls=true") } - if cfg.CanaryConsulImage == "" && cfg.CanaryEnvoyVersion != "" { + if cfg.CanaryVersions.ConsulImage == "" && cfg.CanaryVersions.Envoy != "" { return fmt.Errorf("canary_proxies.consul_image must be set if canary_proxies.envoy_verison is set") } - if cfg.CanaryConsulImage != "" && cfg.CanaryEnvoyVersion == "" { - return fmt.Errorf("canary_proxies.envoy_image must be set if canary_proxies.consul_image is set") + if cfg.CanaryVersions.ConsulImage == "" && cfg.CanaryVersions.DataplaneImage != "" { + return fmt.Errorf("canary_proxies.consul_image must be set if canary_proxies.dataplane_version is set") + } + if cfg.CanaryVersions.ConsulImage != "" && cfg.CanaryVersions.Envoy == "" && cfg.CanaryVersions.DataplaneImage == "" { + return fmt.Errorf("canary_proxies.envoy_image and/or canary_proxies.dataplane_version must be set if canary_proxies.consul_image is set") } if cfg.PrometheusEnabled && cfg.SecurityDisableACLs { diff --git a/config/raw.go b/config/raw.go index e92947a..e20372b 100644 --- a/config/raw.go +++ b/config/raw.go @@ -17,17 +17,18 @@ func (e *rawConfigEnvelope) GetActive() (*rawConfig, bool) { // rawConfig is the top level structure representing the contents of a user // provided config file. It is what the file is initially decoded into. type rawConfig struct { - Name string `hcl:"name,label"` - ConsulImage string `hcl:"consul_image,optional"` - EnvoyVersion string `hcl:"envoy_version,optional"` - CanaryProxies *rawConfigCanaryProxies `hcl:"canary_proxies,block"` - Security *rawConfigSecurity `hcl:"security,block"` - Kubernetes *rawConfigK8S `hcl:"kubernetes,block"` - Envoy *rawConfigEnvoy `hcl:"envoy,block"` - Monitor *rawConfigMonitor `hcl:"monitor,block"` - Enterprise *rawConfigEnterprise `hcl:"enterprise,block"` - Topology *rawTopology `hcl:"topology,block"` - Clusters []*rawClusterConfig `hcl:"cluster_config,block"` + Name string `hcl:"name,label"` + ConsulImage string `hcl:"consul_image,optional"` + EnvoyVersion string `hcl:"envoy_version,optional"` + DataplaneImage string `hcl:"dataplane_image,optional"` + CanaryProxies *rawConfigCanaryProxies `hcl:"canary_proxies,block"` + Security *rawConfigSecurity `hcl:"security,block"` + Kubernetes *rawConfigK8S `hcl:"kubernetes,block"` + Envoy *rawConfigEnvoy `hcl:"envoy,block"` + Monitor *rawConfigMonitor `hcl:"monitor,block"` + Enterprise *rawConfigEnterprise `hcl:"enterprise,block"` + Topology *rawTopology `hcl:"topology,block"` + Clusters []*rawClusterConfig `hcl:"cluster_config,block"` DeprecatedRawConfigEntries []string `hcl:"config_entries,optional"` } @@ -102,22 +103,23 @@ type rawConfigEncryption struct { } type rawConfigCanaryProxies struct { - ConsulImage string `hcl:"consul_image,optional"` - EnvoyVersion string `hcl:"envoy_version,optional"` - Nodes []string `hcl:"nodes,optional"` + ConsulImage string `hcl:"consul_image,optional"` + EnvoyVersion string `hcl:"envoy_version,optional"` + DataplaneImage string `hcl:"dataplane_image,optional"` + Nodes []string `hcl:"nodes,optional"` } type rawConfigEnterprise struct { - Enabled bool `hcl:"enabled,optional"` - Partitions []*Partition `hcl:"partition,block"` - Segments []string `hcl:"segments,optional"` - LicensePath string `hcl:"license_path,optional"` - DisablePartitions bool `hcl:"disable_partitions,optional"` + Enabled bool `hcl:"enabled,optional"` + Partitions []*Partition `hcl:"partition,block"` + Segments []string `hcl:"segments,optional"` + LicensePath string `hcl:"license_path,optional"` } type rawTopology struct { NetworkShape string `hcl:"network_shape,optional"` LinkMode string `hcl:"link_mode,optional"` + NodeMode string `hcl:"node_mode,optional"` Cluster []*Cluster `hcl:"cluster,block"` Nodes []*Node `hcl:"node,block"` diff --git a/consulfunc/catalog.go b/consulfunc/catalog.go index 0dd0193..b5e99ee 100644 --- a/consulfunc/catalog.go +++ b/consulfunc/catalog.go @@ -1,9 +1,17 @@ package consulfunc -import "github.com/hashicorp/consul/api" +import ( + "github.com/hashicorp/consul/api" -func ListAllNodes(client *api.Client, dc string, enterprise, partitionsDisabled bool) ([]*api.Node, error) { - queryOptionList, err := PartitionQueryOptionsList(client, enterprise, partitionsDisabled) + "github.com/rboyer/devconsul/util" +) + +func ListAllNodes(client *api.Client, dc string, enterprise bool) ([]*api.Node, error) { + return ListAllNodesWithFilter(client, dc, enterprise, "") +} + +func ListAllNodesWithFilter(client *api.Client, dc string, enterprise bool, filter string) ([]*api.Node, error) { + queryOptionList, err := PartitionQueryOptionsList(client, enterprise) if err != nil { return nil, err } @@ -13,6 +21,7 @@ func ListAllNodes(client *api.Client, dc string, enterprise, partitionsDisabled var out []*api.Node for _, queryOpts := range queryOptionList { queryOpts.Datacenter = dc + queryOpts.Filter = filter nodes, _, err := cc.Nodes(queryOpts) if err != nil { return nil, err @@ -21,3 +30,26 @@ func ListAllNodes(client *api.Client, dc string, enterprise, partitionsDisabled } return out, nil } + +func ListAllServices(client *api.Client, enterprise bool) ([]util.Identifier, error) { + queryOptionList, err := TenantQueryOptionsList(client, enterprise) + if err != nil { + return nil, err + } + + cc := client.Catalog() + + var out []util.Identifier + for _, queryOpts := range queryOptionList { + names, _, err := cc.Services(queryOpts) + if err != nil { + return nil, err + } + + for name := range names { + sid := util.NewIdentifier(name, queryOpts.Namespace, queryOpts.Partition) + out = append(out, sid) + } + } + return out, nil +} diff --git a/consulfunc/config.go b/consulfunc/config.go index 1cb3ed3..14974a7 100644 --- a/consulfunc/config.go +++ b/consulfunc/config.go @@ -27,10 +27,10 @@ var kinds = []string{ api.ExportedServices, } -func ListAllConfigEntries(client *api.Client, enterprise, partitionsDisabled bool) (map[ConfigKindName]api.ConfigEntry, error) { +func ListAllConfigEntries(client *api.Client, enterprise bool) (map[ConfigKindName]api.ConfigEntry, error) { ce := client.ConfigEntries() - queryOptionList, err := PartitionQueryOptionsList(client, enterprise, partitionsDisabled) + queryOptionList, err := PartitionQueryOptionsList(client, enterprise) if err != nil { return nil, fmt.Errorf("error listing partitions: %w", err) } diff --git a/consulfunc/ns.go b/consulfunc/ns.go new file mode 100644 index 0000000..b5f1cc7 --- /dev/null +++ b/consulfunc/ns.go @@ -0,0 +1,56 @@ +package consulfunc + +import ( + "github.com/hashicorp/consul/api" +) + +func ListNamespaces(client *api.Client, partition string) ([]*api.Namespace, error) { + nsClient := client.Namespaces() + nsList, _, err := nsClient.List(&api.QueryOptions{ + Partition: partition, + }) + if err != nil { + return nil, err + } + return nsList, nil +} + +func ListNamespaceNames(client *api.Client, partition string) ([]string, error) { + nsList, err := ListNamespaces(client, partition) + if err != nil { + return nil, err + } + + var out []string + for _, ns := range nsList { + out = append(out, ns.Name) + } + return out, nil +} + +func TenantQueryOptionsList(client *api.Client, enterprise bool) ([]*api.QueryOptions, error) { + if !enterprise { + return []*api.QueryOptions{{}}, nil + } + + partitions, err := ListPartitionNames(client) + if err != nil { + return nil, err + } + + var out []*api.QueryOptions + for _, partition := range partitions { + namespaces, err := ListNamespaceNames(client, partition) + if err != nil { + return nil, err + } + + for _, ns := range namespaces { + out = append(out, &api.QueryOptions{ + Partition: partition, + Namespace: ns, + }) + } + } + return out, nil +} diff --git a/consulfunc/part.go b/consulfunc/part.go index df8add1..3979fd5 100644 --- a/consulfunc/part.go +++ b/consulfunc/part.go @@ -28,8 +28,8 @@ func ListPartitionNames(client *api.Client) ([]string, error) { return out, nil } -func PartitionQueryOptionsList(client *api.Client, enterprise, partitionsDisabled bool) ([]*api.QueryOptions, error) { - if !enterprise || partitionsDisabled { +func PartitionQueryOptionsList(client *api.Client, enterprise bool) ([]*api.QueryOptions, error) { + if !enterprise { return []*api.QueryOptions{{}}, nil } diff --git a/dataplane-boot.sh b/dataplane-boot.sh new file mode 100755 index 0000000..09f0c21 --- /dev/null +++ b/dataplane-boot.sh @@ -0,0 +1,40 @@ +#!/bin/sh + +set -euo pipefail + +case "${DP_CREDENTIAL_TYPE:-}" in + static) + # read from a token file + readonly token_file="${SBOOT_TOKEN_FILE:-}" + if [[ -z "${token_file}" ]]; then + echo "missing required env var SBOOT_TOKEN_FILE" >&2 + exit 1 + fi + if [[ ! -f "${token_file}" ]]; then + echo "token file does not exist yet: ${token_file}" >&2 + exit 1 + fi + + token="" + set +e + read -r token < "${token_file}" + set -e + # trim any whitespace; this overdoes it in the middle, but tokens don't have + # whitespace in the middle so :shrug: + token="${token//[[:space:]]}" + + export DP_CREDENTIAL_STATIC_TOKEN="${token}" + ;; + *) + ;; +esac + +# if [[ -n "${DP_CA_CERTS:-}" ]]; then +# mkdir -p /tmp/ca +# cp "${DP_CA_CERTS}/consul-agent-ca.pem" /tmp/ca +# export DP_CA_CERTS="/tmp/ca" +# fi + +env | sort + +exec consul-dataplane "$@" diff --git a/example-config.peer.hcl b/example-config.peer.hcl index 66419c9..6463c76 100644 --- a/example-config.peer.hcl +++ b/example-config.peer.hcl @@ -124,9 +124,8 @@ config "simple-no-acls" { } enterprise { - enabled = true - license_path = "/home/rboyer/.consul.dev.licence" - disable_partitions = true + enabled = true + license_path = "/home/rboyer/.consul.dev.licence" } envoy { @@ -165,9 +164,8 @@ config "simple" { } enterprise { - enabled = true - license_path = "/home/rboyer/.consul.dev.licence" - disable_partitions = true + enabled = true + license_path = "/home/rboyer/.consul.dev.licence" } envoy { diff --git a/go.mod b/go.mod index 59a46b5..596854a 100644 --- a/go.mod +++ b/go.mod @@ -1,67 +1,52 @@ module github.com/rboyer/devconsul -go 1.19 +go 1.20 require ( github.com/google/go-cmp v0.5.9 - github.com/hashicorp/consul/api v1.18.0 + github.com/hashicorp/consul/api v1.20.0 github.com/hashicorp/go-cleanhttp v0.5.2 github.com/hashicorp/go-hclog v1.4.0 github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/go-uuid v1.0.3 - github.com/hashicorp/hcl/v2 v2.16.0 - github.com/hashicorp/vault/api v1.8.3 + github.com/hashicorp/hcl/v2 v2.16.1 + github.com/hashicorp/vault/api v1.9.0 github.com/mitchellh/copystructure v1.2.0 github.com/rboyer/safeio v0.2.2 - github.com/stretchr/testify v1.8.1 - golang.org/x/crypto v0.5.0 + github.com/stretchr/testify v1.8.2 + golang.org/x/crypto v0.7.0 ) require ( github.com/agext/levenshtein v1.2.1 // indirect github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect github.com/armon/go-metrics v0.3.10 // indirect - github.com/armon/go-radix v1.0.0 // indirect github.com/cenkalti/backoff/v3 v3.0.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/fatih/color v1.13.0 // indirect - github.com/golang/protobuf v1.5.2 // indirect - github.com/golang/snappy v0.0.4 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-immutable-radix v1.3.1 // indirect - github.com/hashicorp/go-plugin v1.4.5 // indirect github.com/hashicorp/go-retryablehttp v0.6.6 // indirect github.com/hashicorp/go-rootcerts v1.0.2 // indirect - github.com/hashicorp/go-secure-stdlib/mlock v0.1.1 // indirect github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6 // indirect github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect github.com/hashicorp/go-sockaddr v1.0.2 // indirect - github.com/hashicorp/go-version v1.2.1 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/hashicorp/serf v0.10.1 // indirect - github.com/hashicorp/vault/sdk v0.7.0 // indirect - github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect - github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/mitchellh/go-wordwrap v1.0.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect - github.com/oklog/run v1.0.0 // indirect - github.com/pierrec/lz4 v2.5.2+incompatible // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/ryanuber/go-glob v1.0.0 // indirect github.com/zclconf/go-cty v1.12.1 // indirect - go.uber.org/atomic v1.9.0 // indirect - golang.org/x/net v0.5.0 // indirect - golang.org/x/sys v0.4.0 // indirect - golang.org/x/text v0.6.0 // indirect + golang.org/x/net v0.8.0 // indirect + golang.org/x/sys v0.6.0 // indirect + golang.org/x/text v0.8.0 // indirect golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 // indirect - google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 // indirect - google.golang.org/grpc v1.41.0 // indirect - google.golang.org/protobuf v1.26.0 // indirect gopkg.in/square/go-jose.v2 v2.5.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 69cc69b..bcc5c65 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,3 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/agext/levenshtein v1.2.1 h1:QmvMAjj2aEICytGiWzmxoE0x2KZvE0fvmqMOfy2tjT8= github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= @@ -8,7 +5,6 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw= github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= @@ -16,7 +12,6 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-metrics v0.3.10 h1:FR+drcQStOe+32sYyJYyZ7FIdgoGGBnwLl+flodp8Uo= github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -24,31 +19,16 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/cenkalti/backoff/v3 v3.0.0 h1:ske+9nBpD9qZsTBoF41nW5L+AIuFBKMeze18XQ3eG1c= github.com/cenkalti/backoff/v3 v3.0.0/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= -github.com/frankban/quicktest v1.13.0 h1:yNZif1OkDfNoDfb9zZa9aXIpejNR4F23Wely0c+Qdqk= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= @@ -56,43 +36,19 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= -github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/consul/api v1.18.0 h1:R7PPNzTCeN6VuQNDwwhZWJvzCtGSrNpJqfb22h3yH9g= -github.com/hashicorp/consul/api v1.18.0/go.mod h1:owRRGJ9M5xReDC5nfT8FTJrNAPbT4NM6p/k+d03q2v4= -github.com/hashicorp/consul/sdk v0.13.0 h1:lce3nFlpv8humJL8rNrrGHYSKc3q+Kxfeg3Ii1m6ZWU= -github.com/hashicorp/consul/sdk v0.13.0/go.mod h1:0hs/l5fOVhJy/VdcoaNqUSi2AUs95eF5WKtv+EYIQqE= +github.com/hashicorp/consul/api v1.20.0 h1:9IHTjNVSZ7MIwjlW3N3a7iGiykCMDpxZu8jsxFJh0yc= +github.com/hashicorp/consul/api v1.20.0/go.mod h1:nR64eD44KQ59Of/ECwt2vUmIK2DKsDzAwTmwmLl8Wpo= +github.com/hashicorp/consul/sdk v0.13.1 h1:EygWVWWMczTzXGpO93awkHFzfUka6hLYJ0qhETd+6lY= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -101,7 +57,6 @@ github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtng github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= -github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-hclog v1.4.0 h1:ctuWFGrhFha8BnnzxqeRGidlEcQkDyL5u8J8t5eA11I= github.com/hashicorp/go-hclog v1.4.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= @@ -113,15 +68,11 @@ github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHh github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/hashicorp/go-plugin v1.4.5 h1:oTE/oQR4eghggRg8VY7PAz3dr++VwDNBGCcOfIvHpBo= -github.com/hashicorp/go-plugin v1.4.5/go.mod h1:viDMjcLJuDui6pXb8U4HVfb8AamCWhHGUjr2IrTF67s= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-retryablehttp v0.6.6 h1:HJunrbHTDDbBb/ay4kxa1n+dLmttUlnP3V9oNE4hmsM= github.com/hashicorp/go-retryablehttp v0.6.6/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= -github.com/hashicorp/go-secure-stdlib/mlock v0.1.1 h1:cCRo8gK7oq6A2L6LICkUZ+/a5rLiRXFMf1Qd4xSwxTc= -github.com/hashicorp/go-secure-stdlib/mlock v0.1.1/go.mod h1:zq93CJChV6L9QTfGKtfBxKqD7BqqXx5O04A/ns2p5+I= github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6 h1:om4Al8Oy7kCm/B86rLCLah4Dt5Aa0Fr5rYBG60OzwHQ= github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6/go.mod h1:QmrqtbKuxxSWTN3ETMPuB+VtEiBJ/A9XhoYGv8E1uD8= github.com/hashicorp/go-secure-stdlib/strutil v0.1.1/go.mod h1:gKOamz3EwoIoJq7mlMIRBpVTAUn8qPCrEclOKKWhD3U= @@ -133,44 +84,34 @@ github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjG github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.2.1 h1:zEfKbn2+PDgroKdiOzqiE8rsmLqU2uwi5PB5pBJ3TkI= -github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/hcl/v2 v2.16.0 h1:MPq1q615H+9wBAdE3EbwEd6imSohElrIguuasbQruB0= -github.com/hashicorp/hcl/v2 v2.16.0/go.mod h1:JRmR89jycNkrrqnMmvPDMd56n1rQJ2Q6KocSLCMCXng= +github.com/hashicorp/hcl/v2 v2.16.1 h1:BwuxEMD/tsYgbhIW7UuI3crjovf3MzuFWiVgiv57iHg= +github.com/hashicorp/hcl/v2 v2.16.1/go.mod h1:JRmR89jycNkrrqnMmvPDMd56n1rQJ2Q6KocSLCMCXng= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= github.com/hashicorp/memberlist v0.5.0 h1:EtYPN8DpAURiapus508I4n9CzHs2W+8NZGbmmR/prTM= github.com/hashicorp/memberlist v0.5.0/go.mod h1:yvyXLpo0QaGE59Y7hDTsTzDD25JYBZ4mHgHUZ8lrOI0= github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY= github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4= -github.com/hashicorp/vault/api v1.8.3 h1:cHQOLcMhBR+aVI0HzhPxO62w2+gJhIrKguQNONPzu6o= -github.com/hashicorp/vault/api v1.8.3/go.mod h1:4g/9lj9lmuJQMtT6CmVMHC5FW1yENaVv+Nv4ZfG8fAg= -github.com/hashicorp/vault/sdk v0.7.0 h1:2pQRO40R1etpKkia5fb4kjrdYMx3BHklPxl1pxpxDHg= -github.com/hashicorp/vault/sdk v0.7.0/go.mod h1:KyfArJkhooyba7gYCKSq8v66QdqJmnbAxtV/OX1+JTs= -github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb h1:b5rjCoWHc7eqmAS4/qyk21ZsHyb6Mxv/jykxvNTkU4M= -github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= -github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE= +github.com/hashicorp/vault/api v1.9.0 h1:ab7dI6W8DuCY7yCU8blo0UCYl2oHre/dloCmzMWg9w8= +github.com/hashicorp/vault/api v1.9.0/go.mod h1:lloELQP4EyhjnCQhF8agKvWIVTmxbpEJj70b98959sM= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= @@ -180,7 +121,6 @@ github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZb github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= @@ -195,8 +135,6 @@ github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa1 github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-wordwrap v1.0.0 h1:6GlHJ/LTGMrIJbwgdqdl2eEH8o+Exx/0m8ir9Gns0u4= github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -210,17 +148,12 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/oklog/run v1.0.0 h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw= -github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUMhxq9m9ZXI= -github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= @@ -230,7 +163,6 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= @@ -239,7 +171,6 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/rboyer/safeio v0.2.2 h1:XhtqyUTRleMYGyBt3ni4j2BtEh669U2ry2INnnd+B4k= github.com/rboyer/safeio v0.2.2/go.mod h1:pSnr2LFXyn/c/fotxotyOdYy7pP/XSh6MpBmzXPjiNc= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= @@ -257,125 +188,68 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/zclconf/go-cty v1.12.1 h1:PcupnljUm9EIvbgSHQnHhUr3fO6oFmkOrvs2BAFNXXY= github.com/zclconf/go-cty v1.12.1/go.mod h1:s9IfD1LK5ccNMSWCVFCE2rJfHiZgi7JijgeWIMfhLvA= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= -go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE= -golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= +golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= -golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw= -golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= -golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= -golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 h1:NusfzzA6yGQ+ua51ck7E3omNUX/JuqbFSaRGqU8CcLI= golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.41.0 h1:f+PlOh7QV4iIJkPrx5NQ7qaNGFQ3OTse67yaDHfju4E= -google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= @@ -384,12 +258,8 @@ gopkg.in/square/go-jose.v2 v2.5.1 h1:7odma5RETjNHWJnR32wx8t+Io4djHE1PqxCFx3iiZ2w gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/infra/infra.go b/infra/infra.go index 1b26628..09661ea 100644 --- a/infra/infra.go +++ b/infra/infra.go @@ -30,6 +30,23 @@ func CompileTopology(cfg *config.Config) (*Topology, error) { return nil, fmt.Errorf("unknown network_shape: %s", cfg.TopologyNetworkShape) } + switch cfg.TopologyNodeMode { + case string(NodeModeAgent): + topology.NodeMode = NodeModeAgent + case string(NodeModeDataplane): + topology.NodeMode = NodeModeDataplane + default: + return nil, fmt.Errorf("unknown node_mode: %s", cfg.TopologyNodeMode) + } + + for _, n := range cfg.TopologyNodes { + switch n.Mode { + case string(NodeModeAgent), string(NodeModeDataplane), "": + default: + return nil, fmt.Errorf("unknown node[%q].mode: %s", n.NodeName, n.Mode) + } + } + // TODO(peering): require TLS for peering switch cfg.TopologyLinkMode { case string(ClusterLinkModePeer): @@ -92,8 +109,8 @@ func CompileTopology(cfg *config.Config) (*Topology, error) { node := &Node{ Cluster: clusterName, + Kind: NodeKindServer, Name: clusterName + "-server" + id, - Server: true, Partition: "default", Addresses: []Address{ { @@ -104,6 +121,12 @@ func CompileTopology(cfg *config.Config) (*Topology, error) { Index: idx - 1, } + if c := getNode(node.Name); c != nil { + if c.Mode != string(NodeModeAgent) { + return fmt.Errorf("a consul server cannot be agentless") + } + } + switch topology.NetworkShape { case NetworkShapeIslands: if clusterName != config.PrimaryCluster { @@ -124,6 +147,25 @@ func CompileTopology(cfg *config.Config) (*Topology, error) { topology.AddNode(node) } + { // add special pod + const idx = 100 + ip := baseIP + ".100" + + nodeName := clusterName + "-infra1" + node := &Node{ + Cluster: clusterName, + Name: nodeName, + Kind: NodeKindInfra, + Partition: "default", + Addresses: []Address{{ + Network: topology.NetworkShape.GetNetworkName(clusterName), + IPAddress: ip, + }}, + Index: idx - 1, + } + topology.AddNode(node) + } + numServiceClients := clients - meshGateways for idx := 1; idx <= clients; idx++ { isGatewayClient := (idx > numServiceClients) @@ -136,7 +178,7 @@ func CompileTopology(cfg *config.Config) (*Topology, error) { node := &Node{ Cluster: clusterName, Name: nodeName, - Server: false, + Kind: NodeKindClient, Addresses: []Address{ { Network: topology.NetworkShape.GetNetworkName(clusterName), @@ -151,6 +193,16 @@ func CompileTopology(cfg *config.Config) (*Topology, error) { nodeConfig = *c } + if topology.NodeMode == NodeModeDataplane { + node.Kind = NodeKindDataplane + } + switch nodeConfig.Mode { + case string(NodeModeAgent): + node.Kind = NodeKindClient + case string(NodeModeDataplane): + node.Kind = NodeKindDataplane + } + { // handle partition defaulting regardless of OSS/ENT nodeConfig.Partition = util.PartitionOrDefault(nodeConfig.Partition) node.Partition = nodeConfig.Partition @@ -262,6 +314,9 @@ func CompileTopology(cfg *config.Config) (*Topology, error) { if c.Clients <= 0 { return nil, fmt.Errorf("%s: must always have at least one client", c.Name) } + if c.Clients > 50 { + return nil, fmt.Errorf("%s: must always have no more than 50 clients", c.Name) + } m := clusterNamePatt.FindStringSubmatch(c.Name) if m == nil { diff --git a/infra/infra_test.go b/infra/infra_test.go index 51d7d1a..55f631d 100644 --- a/infra/infra_test.go +++ b/infra/infra_test.go @@ -18,11 +18,29 @@ func TestCompileTopology(t *testing.T) { expectFn func(t *testing.T, topo *Topology) } + run := func(t *testing.T, tc testcase) { + topo, err := CompileTopology(tc.cfg) + if tc.expectExactErr != "" { + require.EqualError(t, err, tc.expectExactErr) + require.Nil(t, topo) + } else if tc.expectErr { + require.Error(t, err) + require.Nil(t, topo) + // primary cluster "dc1" is missing from config + + } else { + require.NoError(t, err) + require.NotNil(t, topo) + tc.expectFn(t, topo) + } + } + cases := map[string]testcase{ "missing-primary": { cfg: &config.Config{ TopologyNetworkShape: "flat", TopologyLinkMode: "federate", + TopologyNodeMode: "agent", TopologyClusters: []*config.Cluster{ { Name: "dc2", @@ -33,14 +51,146 @@ func TestCompileTopology(t *testing.T) { }, expectExactErr: `primary cluster "dc1" is missing from config`, }, + "mixed-agentless": { + cfg: &config.Config{ + TopologyNetworkShape: "flat", + TopologyLinkMode: "federate", + TopologyNodeMode: "dataplane", + TopologyClusters: []*config.Cluster{ + { + Name: "dc1", + Servers: 1, + Clients: 2, + }, + }, + TopologyNodes: []*config.Node{ + { + NodeName: "dc1-client1", + Mode: "agent", + }, + }, + }, + expectFn: func(t *testing.T, topo *Topology) { + expect := &Topology{ + NetworkShape: NetworkShapeFlat, + LinkMode: ClusterLinkModeFederate, + NodeMode: NodeModeDataplane, + networks: map[string]*Network{ + "lan": { + Name: "lan", + CIDR: "10.0.0.0/16", + }, + }, + clusters: []*Cluster{ + { + Name: "dc1", + Primary: true, + Index: 1, + Servers: 1, + Clients: 2, + BaseIP: "10.0.1", + WANBaseIP: "10.1.1", + }, + }, + nm: map[string]*Node{ + // ============= dc1 ============== + "dc1-infra1": { + Index: 99, + Kind: NodeKindInfra, + Cluster: "dc1", + Name: "dc1-infra1", + Partition: "default", + Addresses: []Address{ + { + Network: "lan", + IPAddress: "10.0.1.100", + }, + }, + }, + "dc1-server1": { + Kind: NodeKindServer, + Index: 0, + Cluster: "dc1", + Name: "dc1-server1", + Partition: "default", + Addresses: []Address{ + { + Network: "lan", + IPAddress: "10.0.1.11", + }, + }, + }, + "dc1-client1": { + Kind: NodeKindClient, + Index: 0, + Cluster: "dc1", + Name: "dc1-client1", + Partition: "default", + Addresses: []Address{ + { + Network: "lan", + IPAddress: "10.0.1.21", + }, + }, + Service: &Service{ + ID: util.NewIdentifier("ping", "", ""), + Port: 8080, + UpstreamID: util.NewIdentifier("pong", "", ""), + UpstreamLocalPort: 9090, + Meta: map[string]string{}, + }, + }, + "dc1-client2": { + Kind: NodeKindDataplane, + Index: 1, + Cluster: "dc1", + Name: "dc1-client2", + Partition: "default", + Addresses: []Address{ + { + Network: "lan", + IPAddress: "10.0.1.22", + }, + }, + Service: &Service{ + ID: util.NewIdentifier("pong", "", ""), + Port: 8080, + UpstreamID: util.NewIdentifier("ping", "", ""), + UpstreamLocalPort: 9090, + Meta: map[string]string{}, + }, + }, + }, + additionalPrimaryGateways: []string(nil), + } + require.Equal(t, expect, topo) + + // Check helper methods + require.Len(t, topo.Networks(), 1) + require.Len(t, topo.Clusters(), 1) + + require.Len(t, topo.ClusterNodes("dc1"), 4) + require.Len(t, topo.ServerIPs("dc1"), 1) + require.Len(t, topo.GatewayAddrs("dc1"), 0) + + node1 := topo.Node("dc1-client1") + require.NotNil(t, node1) + require.Equal(t, "dc1-client1", node1.Name) + + // TODO(cdp): check node mode + }, + }, "full-islands": { cfg: &config.Config{ - CanaryConsulImage: "consul-canary:latest", - CanaryEnvoyVersion: "v1.16.0", + CanaryVersions: config.Versions{ + ConsulImage: "consul-canary:latest", + Envoy: "v1.16.0", + }, CanaryNodes: []string{"dc2-client2"}, EncryptionTLS: true, TopologyNetworkShape: "islands", TopologyLinkMode: "federate", + TopologyNodeMode: "agent", TopologyClusters: []*config.Cluster{ { Name: "dc1", @@ -80,6 +230,7 @@ func TestCompileTopology(t *testing.T) { expect := &Topology{ NetworkShape: NetworkShapeIslands, LinkMode: ClusterLinkModeFederate, + NodeMode: NodeModeAgent, networks: map[string]*Network{ "dc1": { Name: "dc1", @@ -118,12 +269,25 @@ func TestCompileTopology(t *testing.T) { }, nm: map[string]*Node{ // ============= dc1 ============== + "dc1-infra1": { + Index: 99, + Kind: NodeKindInfra, + Cluster: "dc1", + Name: "dc1-infra1", + Partition: "default", + Addresses: []Address{ + { + Network: "dc1", + IPAddress: "10.0.1.100", + }, + }, + }, "dc1-server1": { Index: 0, + Kind: NodeKindServer, Cluster: "dc1", Name: "dc1-server1", Partition: "default", - Server: true, Addresses: []Address{ { Network: "dc1", @@ -133,10 +297,10 @@ func TestCompileTopology(t *testing.T) { }, "dc1-server2": { Index: 1, + Kind: NodeKindServer, Cluster: "dc1", Name: "dc1-server2", Partition: "default", - Server: true, Addresses: []Address{ { Network: "dc1", @@ -146,10 +310,10 @@ func TestCompileTopology(t *testing.T) { }, "dc1-server3": { Index: 2, + Kind: NodeKindServer, Cluster: "dc1", Name: "dc1-server3", Partition: "default", - Server: true, Addresses: []Address{ { Network: "dc1", @@ -159,6 +323,7 @@ func TestCompileTopology(t *testing.T) { }, "dc1-client1": { Index: 0, + Kind: NodeKindClient, Cluster: "dc1", Name: "dc1-client1", Partition: "default", @@ -181,6 +346,7 @@ func TestCompileTopology(t *testing.T) { }, "dc1-client2": { Index: 1, + Kind: NodeKindClient, Cluster: "dc1", Name: "dc1-client2", Partition: "default", @@ -200,6 +366,7 @@ func TestCompileTopology(t *testing.T) { }, "dc1-client3": { Index: 2, + Kind: NodeKindClient, Cluster: "dc1", Name: "dc1-client3", Partition: "default", @@ -216,12 +383,25 @@ func TestCompileTopology(t *testing.T) { MeshGateway: true, }, // ============= dc2 ============== + "dc2-infra1": { + Index: 99, + Kind: NodeKindInfra, + Cluster: "dc2", + Name: "dc2-infra1", + Partition: "default", + Addresses: []Address{ + { + Network: "dc2", + IPAddress: "10.0.2.100", + }, + }, + }, "dc2-server1": { Index: 0, + Kind: NodeKindServer, Cluster: "dc2", Name: "dc2-server1", Partition: "default", - Server: true, Addresses: []Address{ { Network: "dc2", @@ -235,10 +415,10 @@ func TestCompileTopology(t *testing.T) { }, "dc2-server2": { Index: 1, + Kind: NodeKindServer, Cluster: "dc2", Name: "dc2-server2", Partition: "default", - Server: true, Addresses: []Address{ { Network: "dc2", @@ -252,10 +432,10 @@ func TestCompileTopology(t *testing.T) { }, "dc2-server3": { Index: 2, + Kind: NodeKindServer, Cluster: "dc2", Name: "dc2-server3", Partition: "default", - Server: true, Addresses: []Address{ { Network: "dc2", @@ -269,6 +449,7 @@ func TestCompileTopology(t *testing.T) { }, "dc2-client1": { Index: 0, + Kind: NodeKindClient, Cluster: "dc2", Name: "dc2-client1", Partition: "default", @@ -288,6 +469,7 @@ func TestCompileTopology(t *testing.T) { }, "dc2-client2": { Index: 1, + Kind: NodeKindClient, Cluster: "dc2", Name: "dc2-client2", Partition: "default", @@ -313,6 +495,7 @@ func TestCompileTopology(t *testing.T) { }, "dc2-client3": { Index: 2, + Kind: NodeKindClient, Cluster: "dc2", Name: "dc2-client3", Partition: "default", @@ -329,22 +512,6 @@ func TestCompileTopology(t *testing.T) { MeshGateway: true, }, }, - servers: []string{ - "dc1-server1", - "dc1-server2", - "dc1-server3", - "dc2-server1", - "dc2-server2", - "dc2-server3", - }, - clients: []string{ - "dc1-client1", - "dc1-client2", - "dc1-client3", - "dc2-client1", - "dc2-client2", - "dc2-client3", - }, additionalPrimaryGateways: []string(nil), } require.Equal(t, expect, topo) @@ -353,7 +520,7 @@ func TestCompileTopology(t *testing.T) { require.Len(t, topo.Networks(), 3) require.Len(t, topo.Clusters(), 2) - require.Len(t, topo.ClusterNodes("dc1"), 6) + require.Len(t, topo.ClusterNodes("dc1"), 7) require.Len(t, topo.ServerIPs("dc1"), 3) require.Len(t, topo.GatewayAddrs("dc1"), 1) @@ -361,7 +528,7 @@ func TestCompileTopology(t *testing.T) { require.NotNil(t, node1) require.Equal(t, "dc1-client1", node1.Name) - require.Len(t, topo.ClusterNodes("dc2"), 6) + require.Len(t, topo.ClusterNodes("dc2"), 7) require.Len(t, topo.ServerIPs("dc2"), 3) require.Len(t, topo.GatewayAddrs("dc2"), 1) @@ -375,20 +542,7 @@ func TestCompileTopology(t *testing.T) { for name, tc := range cases { tc := tc t.Run(name, func(t *testing.T) { - topo, err := CompileTopology(tc.cfg) - if tc.expectExactErr != "" { - require.EqualError(t, err, tc.expectExactErr) - require.Nil(t, topo) - } else if tc.expectErr { - require.Error(t, err) - require.Nil(t, topo) - // primary cluster "dc1" is missing from config - - } else { - require.NoError(t, err) - require.NotNil(t, topo) - tc.expectFn(t, topo) - } + run(t, tc) }) } } diff --git a/infra/topology.go b/infra/topology.go index 332b148..5eecb47 100644 --- a/infra/topology.go +++ b/infra/topology.go @@ -41,16 +41,22 @@ const ( ClusterLinkModeFederate = ClusterLinkMode("federate") ) +type NodeMode string + +const ( + NodeModeAgent = NodeMode("agent") + NodeModeDataplane = NodeMode("dataplane") +) + type Topology struct { NetworkShape NetworkShape LinkMode ClusterLinkMode + NodeMode NodeMode networks map[string]*Network clusters []*Cluster - nm map[string]*Node - servers []string // node names - clients []string // node names + nm map[string]*Node additionalPrimaryGateways []string } @@ -61,7 +67,7 @@ func (t *Topology) LinkWithFederation() bool { return t.LinkMode == ClusterLinkM func (t *Topology) LinkWithPeering() bool { return t.LinkMode == ClusterLinkModePeer } func (t *Topology) LeaderIP(cluster string, wan bool) string { - for _, name := range t.servers { + for _, name := range t.sortedNodeKind(NodeKindServer) { n := t.Node(name) if n.Cluster == cluster { if wan { @@ -104,7 +110,7 @@ func (t *Topology) Cluster(name string) *Cluster { func (t *Topology) ServerIPs(cluster string) []string { var out []string - for _, name := range t.servers { + for _, name := range t.sortedNodeKind(NodeKindServer) { n := t.Node(name) if n.Cluster == cluster { out = append(out, n.LocalAddress()) @@ -115,23 +121,50 @@ func (t *Topology) ServerIPs(cluster string) []string { func (t *Topology) GatewayAddrs(cluster string) []string { var out []string - for _, name := range t.clients { - n := t.Node(name) - if n.Cluster == cluster && n.MeshGateway { - out = append(out, n.PublicAddress()+":8443") + t.WalkSilent(func(n *Node) { + switch n.Kind { + case NodeKindClient, NodeKindDataplane: + if n.Cluster == cluster && n.MeshGateway { + out = append(out, n.PublicAddress()+":8443") + } } - } + }) out = append(out, t.additionalPrimaryGateways...) return out } -func (t *Topology) all() []string { - o := make([]string, 0, len(t.servers)+len(t.clients)) - o = append(o, t.servers...) - o = append(o, t.clients...) +func (t *Topology) sortedNodeKind(kind NodeKind) []string { + var o []string + for name, n := range t.nm { + if n.Kind == kind { + o = append(o, name) + } + } + sort.Strings(o) return o } +func (t *Topology) sortedNodes() []string { + var o1, o2, o3 []string + for name, n := range t.nm { + switch n.Kind { + case NodeKindInfra: + o1 = append(o1, name) + case NodeKindServer: + o2 = append(o2, name) + case NodeKindClient, NodeKindDataplane: + o3 = append(o3, name) + } + } + sort.Strings(o1) + sort.Strings(o2) + sort.Strings(o3) + + o1 = append(o1, o2...) + o1 = append(o1, o3...) + return o1 +} + func (t *Topology) Node(name string) *Node { if t.nm == nil { panic("node not found: " + name) @@ -162,7 +195,7 @@ func (t *Topology) ClusterNodes(cluster string) []*Node { } func (t *Topology) Walk(f func(n *Node) error) error { - for _, nodeName := range t.all() { + for _, nodeName := range t.sortedNodes() { node := t.Node(nodeName) if err := f(node); err != nil { return err @@ -171,7 +204,7 @@ func (t *Topology) Walk(f func(n *Node) error) error { return nil } func (t *Topology) WalkSilent(f func(n *Node)) { - for _, nodeName := range t.all() { + for _, nodeName := range t.sortedNodes() { node := t.Node(nodeName) f(node) } @@ -185,15 +218,13 @@ func (t *Topology) AddNetwork(n *Network) { } func (t *Topology) AddNode(node *Node) { + if node.Kind == "" { + panic("missing node kind") + } if t.nm == nil { t.nm = make(map[string]*Node) } t.nm[node.Name] = node - if node.Server { - t.servers = append(t.servers, node.Name) - } else { - t.clients = append(t.clients, node.Name) - } } func (t *Topology) AddAdditionalPrimaryGateway(addr string) { @@ -222,12 +253,22 @@ func (n *Network) DockerName() string { return "devconsul-" + n.Name } +type NodeKind string + +const ( + NodeKindUnknown NodeKind = "" + NodeKindServer NodeKind = "server" + NodeKindClient NodeKind = "client" + NodeKindDataplane NodeKind = "dataplane" + NodeKindInfra NodeKind = "infra" +) + type Node struct { + Kind NodeKind Cluster string Name string Segment string // may be empty Partition string // will be not empty - Server bool Addresses []Address Service *Service MeshGateway bool @@ -238,18 +279,33 @@ type Node struct { MeshGatewayUseDNSWANAddress bool } -func (n *Node) AddLabels(m map[string]string) { - m["devconsul.cluster"] = n.Cluster +func (n *Node) PodName() string { return n.Name + "-pod" } +func (n *Node) Hostname() string { return n.PodName() } - var agentType string - if n.Server { - agentType = "server" - } else { - agentType = "client" +func (n *Node) IsServer() bool { + return n.Kind == NodeKindServer +} + +func (n *Node) IsAgent() bool { + switch n.Kind { + case NodeKindServer, NodeKindClient: + return true + } + return false +} + +func (n *Node) RunsWorkloads() bool { + switch n.Kind { + case NodeKindServer, NodeKindClient, NodeKindDataplane: + return true } - m["devconsul.agentType"] = agentType + return false +} +func (n *Node) AddLabels(m map[string]string) { + m["devconsul.cluster"] = n.Cluster m["devconsul.node"] = n.Name + m["devconsul.kind"] = string(n.Kind) } func (n *Node) TokenName() string { return "agent--" + n.Name } diff --git a/main.go b/main.go index 4b4b6ef..e8ed6f8 100644 --- a/main.go +++ b/main.go @@ -6,6 +6,7 @@ import ( "log" "os" "strings" + "time" "github.com/hashicorp/go-hclog" @@ -32,6 +33,7 @@ var allCommands = []command{ {"config-entries", (*app.App).RunDebugListConfigs, nil}, {"grpc-check", (*app.App).RunGRPCCheck, nil}, {"check-mesh", (*app.App).RunCheckMesh, nil}, + {"dump-logs", (*app.App).RunDumpLogs, nil}, } func main() { @@ -40,7 +42,7 @@ func main() { // Create logger logger := hclog.New(&hclog.LoggerOptions{ Name: app.ProgramName, - Level: hclog.Debug, + Level: hclog.Trace, Output: os.Stderr, JSONFormat: false, }) @@ -53,10 +55,18 @@ func main() { os.Args = os.Args[1:] os.Args[0] = app.ProgramName - var resetOnce bool + var ( + resetOnce bool + timeout time.Duration + ) flag.BoolVar(&resetOnce, "force", false, "force one time operations to run again") + flag.DurationVar(&timeout, "timeout", 1*time.Minute, "[check-mesh] total runtime") flag.Parse() + if timeout < 0 { + timeout = 0 + } + if resetOnce { if err := app.ResetRunOnceMemory(); err != nil { logger.Error(err.Error()) @@ -79,6 +89,7 @@ func main() { logger.Error(err.Error()) os.Exit(1) } + core.SetTimeout(timeout) commandMap := make(map[string]func(core *app.App) error) for _, cmd := range allCommands { diff --git a/mesh-gateway-sidecar-boot.sh b/mesh-gateway-sidecar-boot.sh index c9bf041..2dcf128 100755 --- a/mesh-gateway-sidecar-boot.sh +++ b/mesh-gateway-sidecar-boot.sh @@ -8,6 +8,7 @@ readonly agent_grpc_tls="${SBOOT_AGENT_GRPC_TLS:-}" readonly partition="${SBOOT_PARTITION:-}" api_args=() +acl_api_args=() case "${mode}" in insecure) ;; @@ -29,6 +30,7 @@ case "${mode}" in done api_args+=( -token-file "${token_file}" ) + acl_api_args+=( -token-file "${token_file}" ) ;; *) @@ -39,6 +41,7 @@ esac if [[ -n "${partition}" ]]; then api_args+=( -partition "${partition}" ) + acl_api_args+=( -partition "${partition}" ) fi if [[ -n "$agent_tls" ]]; then @@ -49,6 +52,7 @@ if [[ -n "$agent_tls" ]]; then else api_args+=( -http-addr http://127.0.0.1:8500 ) fi +acl_api_args+=( -http-addr http://127.0.0.1:8500 ) grpc_args=() if [[ -n "$agent_grpc_tls" ]]; then @@ -57,6 +61,17 @@ else grpc_args+=( -grpc-addr http://127.0.0.1:8502 ) fi +if [[ "${mode}" != "insecure" ]]; then + while : ; do + if consul acl token read "${acl_api_args[@]}" -self ; then + break + fi + + echo "waiting for ACLs to work..." + sleep 0.1 + done +fi + echo "Launching mesh-gateway proxy..." exec consul connect envoy \ -register \ diff --git a/sidecar-boot.sh b/sidecar-boot.sh index 57ac563..9cf5e3a 100755 --- a/sidecar-boot.sh +++ b/sidecar-boot.sh @@ -17,6 +17,7 @@ if [[ -z "${service_register_file}" ]]; then fi api_args=() +acl_api_args=() case "${mode}" in insecure) ;; @@ -28,6 +29,7 @@ case "${mode}" in fi api_args+=( -token-file "${token_file}" ) + acl_api_args+=( -token-file "${token_file}" ) ;; login) @@ -53,6 +55,7 @@ case "${mode}" in echo "Wrote new token to ${token_sink_file}" api_args+=( -token-file "${token_sink_file}" ) + acl_api_args+=( -token-file "${token_sink_file}" ) ;; *) @@ -63,6 +66,7 @@ esac if [[ -n "${partition}" ]]; then api_args+=( -partition "${partition}" ) + acl_api_args+=( -partition "${partition}" ) fi if [[ -n "$agent_tls" ]]; then @@ -73,6 +77,7 @@ if [[ -n "$agent_tls" ]]; then else api_args+=( -http-addr http://127.0.0.1:8500 ) fi +acl_api_args+=( -http-addr http://127.0.0.1:8500 ) grpc_args=() if [[ -n "$agent_grpc_tls" ]]; then @@ -83,8 +88,7 @@ fi if [[ "${mode}" != "insecure" ]]; then while : ; do - # if consul acl token read "${api_args[@]}" -self &> /dev/null ; then - if consul acl token read "${api_args[@]}" -self ; then + if consul acl token read "${acl_api_args[@]}" -self ; then break fi diff --git a/structs/catalog_defs.go b/structs/catalog_defs.go new file mode 100644 index 0000000..8cd4e92 --- /dev/null +++ b/structs/catalog_defs.go @@ -0,0 +1,201 @@ +package structs + +import ( + "sort" + + "github.com/hashicorp/consul/api" + + "github.com/rboyer/devconsul/util" +) + +type CatalogDefinition struct { + Nodes []*CatalogNode + Services []*CatalogService + Proxies []*CatalogProxy +} + +type CatalogNode struct { + Node string `json:",omitempty"` + Address string `json:",omitempty"` + TaggedAddresses map[string]string `json:",omitempty"` + NodeMeta map[string]string `json:",omitempty"` + Partition string `json:",omitempty"` +} + +func (n *CatalogNode) ID() util.Identifier2 { + return util.NewIdentifier2(n.Node, n.Partition) +} + +func (n *CatalogNode) ToAPI(enterprise bool) *api.CatalogRegistration { + r := &api.CatalogRegistration{ + Node: n.Node, + Address: n.Address, + TaggedAddresses: n.TaggedAddresses, + NodeMeta: n.NodeMeta, + } + if enterprise { + r.Partition = n.Partition + } + return r +} + +type CatalogService struct { + Node string `json:",omitempty"` + Partition string `json:",omitempty"` + // + Service string `json:",omitempty"` + Meta map[string]string `json:",omitempty"` + Port int `json:",omitempty"` + Address string `json:",omitempty"` + Namespace string `json:",omitempty"` + // + CheckID string `json:",omitempty"` + TCPCheck string `json:",omitempty"` +} + +func (s *CatalogService) NodeID() util.Identifier2 { + return util.NewIdentifier2(s.Node, s.Partition) +} + +func (s *CatalogService) ID() util.Identifier { + return util.NewIdentifier(s.Service, s.Namespace, s.Partition) +} + +func (s *CatalogService) ToAPI(enterprise bool) *api.CatalogRegistration { + r := &api.CatalogRegistration{ + Node: s.Node, + SkipNodeUpdate: true, + Service: &api.AgentService{ + Kind: api.ServiceKindTypical, + ID: s.Service, + Service: s.Service, + Meta: s.Meta, + Port: s.Port, + Address: s.Address, + }, + Check: &api.AgentCheck{ + CheckID: s.CheckID, + Name: "external sync", + Type: "external-sync", + Status: "passing", // TODO + ServiceID: s.Service, + Definition: api.HealthCheckDefinition{ + TCP: s.TCPCheck, + }, + Output: "", + }, + } + if enterprise { + r.Partition = s.Partition + r.Service.Namespace = s.Namespace + r.Service.Partition = s.Partition + r.Check.Namespace = s.Namespace + r.Check.Partition = s.Partition + } + return r +} + +type CatalogProxy struct { + CatalogService + ProxyDestinationServiceName string `json:",omitempty"` + ProxyLocalServicePort int `json:",omitempty"` + ProxyUpstreams []*CatalogProxyUpstream `json:",omitempty"` +} + +type CatalogProxyUpstream struct { + DestinationPartition string `json:",omitempty"` + DestinationNamespace string `json:",omitempty"` + DestinationPeer string `json:",omitempty"` + DestinationName string `json:",omitempty"` + Datacenter string `json:",omitempty"` + LocalBindPort int `json:",omitempty"` +} + +func (p *CatalogProxy) ToAPI(enterprise bool) *api.CatalogRegistration { + r := p.CatalogService.ToAPI(enterprise) + r.Service.Kind = api.ServiceKindConnectProxy + r.Service.Proxy = &api.AgentServiceConnectProxyConfig{ + DestinationServiceName: p.ProxyDestinationServiceName, + DestinationServiceID: p.ProxyDestinationServiceName, + LocalServicePort: p.ProxyLocalServicePort, + } + for _, u := range p.ProxyUpstreams { + newU := api.Upstream{ + DestinationName: u.DestinationName, + DestinationPeer: u.DestinationPeer, + LocalBindPort: u.LocalBindPort, + Datacenter: u.Datacenter, + } + if enterprise { + newU.DestinationNamespace = u.DestinationNamespace + newU.DestinationPartition = u.DestinationPartition + } + r.Service.Proxy.Upstreams = append(r.Service.Proxy.Upstreams, newU) + } + return r +} + +func (d *CatalogDefinition) Sort() { + sort.Slice(d.Nodes, func(i, j int) bool { + a := d.Nodes[i] + b := d.Nodes[j] + + if a.Partition < b.Partition { + return true + } else if a.Partition > b.Partition { + return false + } + + return a.Node < b.Node + }) + + sort.Slice(d.Services, func(i, j int) bool { + a := d.Services[i] + b := d.Services[j] + + if a.Partition < b.Partition { + return true + } else if a.Partition > b.Partition { + return false + } + + if a.Node < b.Node { + return true + } else if a.Node > b.Node { + return false + } + + if a.Namespace < b.Namespace { + return true + } else if a.Namespace > b.Namespace { + return false + } + + return a.Service < b.Service + }) + + sort.Slice(d.Proxies, func(i, j int) bool { + a := d.Proxies[i] + b := d.Proxies[j] + + if a.Partition < b.Partition { + return true + } else if a.Partition > b.Partition { + return false + } + + if a.Node < b.Node { + return true + } else if a.Node > b.Node { + return false + } + + if a.Namespace < b.Namespace { + return true + } else if a.Namespace > b.Namespace { + return false + } + + return a.Service < b.Service + }) +} diff --git a/test-configs/config.dataplane.hcl b/test-configs/config.dataplane.hcl new file mode 100644 index 0000000..0e47dc3 --- /dev/null +++ b/test-configs/config.dataplane.hcl @@ -0,0 +1,44 @@ +active = "simple" + +config "simple" { + consul_image = "consul-dev:latest" + dataplane_image = "hashicorp/consul-dataplane:1.0.0" + + security { + initial_master_token = "root" + encryption { + tls = true + tls_grpc = true + gossip = true + server_tls_grpc = true + } + } + + kubernetes { + enabled = false + } + + envoy { + # log_level = "trace" + # log_level = "info" + log_level = "debug" + } + + topology { + cluster "dc1" { + servers = 1 + clients = 2 + } + cluster "dc2" { + servers = 1 + clients = 2 + } + + ### default toggle + # node_mode = "dataplane" + + node "dc1-client1" { + mode = "dataplane" + } + } +} diff --git a/test-configs/config.prepared-query.hcl b/test-configs/config.prepared-query.hcl index 3379220..83abc77 100644 --- a/test-configs/config.prepared-query.hcl +++ b/test-configs/config.prepared-query.hcl @@ -1,8 +1,7 @@ active = "prepared-query" config "prepared-query" { - consul_image = "consul-dev:latest" - envoy_version = "v1.22.5" + consul_image = "consul-dev:latest" security { initial_master_token = "root" diff --git a/test-configs/config.prometheus.hcl b/test-configs/config.prometheus.hcl index 24b616b..8522a33 100644 --- a/test-configs/config.prometheus.hcl +++ b/test-configs/config.prometheus.hcl @@ -1,8 +1,7 @@ active = "prometheus" config "prometheus" { - consul_image = "consul-dev:latest" - envoy_version = "v1.22.5" + consul_image = "consul-dev:latest" security { initial_master_token = "root" diff --git a/test-configs/config.simple-no-acls.hcl b/test-configs/config.simple-no-acls.hcl index e361e47..d36a6de 100644 --- a/test-configs/config.simple-no-acls.hcl +++ b/test-configs/config.simple-no-acls.hcl @@ -1,8 +1,7 @@ active = "simple-no-acls" config "simple-no-acls" { - consul_image = "consul-dev:latest" - envoy_version = "v1.22.5" + consul_image = "consul-dev:latest" security { disable_acls = true diff --git a/test-configs/config.simple-peering.hcl b/test-configs/config.simple-peering.hcl index d17c9a0..deeff8e 100644 --- a/test-configs/config.simple-peering.hcl +++ b/test-configs/config.simple-peering.hcl @@ -1,8 +1,7 @@ active = "simple-peering" config "simple-peering" { # TODO finish - consul_image = "consul-dev:latest" - envoy_version = "v1.22.5" + consul_image = "consul-dev:latest" security { encryption { @@ -87,11 +86,11 @@ config "simple-peering" { # TODO finish "Services": [ { "Name": "ping", - "Consumers": [ { "PeerName": "peer-dc2" } ] + "Consumers": [ { "Peer": "peer-dc2" } ] }, { "Name": "pong", - "Consumers": [ { "PeerName": "peer-dc2" } ] + "Consumers": [ { "Peer": "peer-dc2" } ] } ] } @@ -160,11 +159,11 @@ config "simple-peering" { # TODO finish "Services": [ { "Name": "ping", - "Consumers": [ { "PeerName": "peer-dc1" } ] + "Consumers": [ { "Peer": "peer-dc1" } ] }, { "Name": "pong", - "Consumers": [ { "PeerName": "peer-dc1" } ] + "Consumers": [ { "Peer": "peer-dc1" } ] } ] } diff --git a/test-configs/config.simple.hcl b/test-configs/config.simple.hcl index 88f2abf..04989d8 100644 --- a/test-configs/config.simple.hcl +++ b/test-configs/config.simple.hcl @@ -1,8 +1,7 @@ active = "simple" config "simple" { - consul_image = "consul-dev:latest" - envoy_version = "v1.22.5" + consul_image = "consul-dev:latest" security { initial_master_token = "root" diff --git a/test-configs/config.tls-api.hcl b/test-configs/config.tls-api.hcl index 8d5fde8..b38f8ba 100644 --- a/test-configs/config.tls-api.hcl +++ b/test-configs/config.tls-api.hcl @@ -1,8 +1,7 @@ active = "tls-api" config "tls-api" { - consul_image = "consul-dev:latest" - envoy_version = "v1.22.5" + consul_image = "consul-dev:latest" security { initial_master_token = "root" diff --git a/test-configs/config.vault-ca.hcl b/test-configs/config.vault-ca.hcl index 580f8bd..3487d96 100644 --- a/test-configs/config.vault-ca.hcl +++ b/test-configs/config.vault-ca.hcl @@ -1,8 +1,7 @@ active = "simple" config "simple" { - consul_image = "consul-dev:latest" - envoy_version = "v1.22.5" + consul_image = "consul-dev:latest" security { initial_master_token = "root" diff --git a/test-configs/config.vault.hcl b/test-configs/config.vault.hcl index f7a071b..a831821 100644 --- a/test-configs/config.vault.hcl +++ b/test-configs/config.vault.hcl @@ -1,8 +1,7 @@ active = "simple" config "simple" { - consul_image = "consul-dev:latest" - envoy_version = "v1.22.5" + consul_image = "consul-dev:latest" security { initial_master_token = "root" diff --git a/test-configs/config.wanfed-mgw.hcl b/test-configs/config.wanfed-mgw.hcl index 4bd2244..007e8c0 100644 --- a/test-configs/config.wanfed-mgw.hcl +++ b/test-configs/config.wanfed-mgw.hcl @@ -1,8 +1,7 @@ active = "wanfed-mgw" config "wanfed-mgw" { - consul_image = "consul-dev:latest" - envoy_version = "v1.22.5" + consul_image = "consul-dev:latest" security { initial_master_token = "root" diff --git a/test-configs/test.sh b/test-configs/test.sh index d33dd04..aee222e 100755 --- a/test-configs/test.sh +++ b/test-configs/test.sh @@ -18,10 +18,15 @@ fi rm -f config.hcl +mkdir -p bin +cp -f ../bin/clustertool ./bin cp -f ../Dockerfile-envoy . +cp -f ../Dockerfile-cdp . +cp -f ../Dockerfile-tool . cp -f ../versions.tf . cp -f ../mesh-gateway-sidecar-boot.sh . cp -f ../sidecar-boot.sh . +cp -f ../dataplane-boot.sh . terraform init @@ -38,19 +43,35 @@ for fn in config.*.hcl; do devconsul primary || { echo "FAIL: error bringing up primary environment: $fn" >&2 failed="${failed},$fn(primary)" + + devconsul dump-logs || true } fi devconsul up || { echo "FAIL: error bringing up rest of environment: $fn" >&2 failed="${failed},$fn(up)" + + devconsul dump-logs || true } + # devconsul check-mesh || { + # echo "WARN: various mesh resources are not actually healthy: $fn" >&2 + # } devconsul down &>/dev/null || { echo "FAIL: error tearing down environment: $fn" >&2 # failed="${failed},$fn(down)" } done -rm -f config.hcl Dockerfile-envoy versions.tf mesh-gateway-sidecar-boot.sh sidecar-boot.sh +rm -f \ + config.hcl \ + Dockerfile-envoy \ + Dockerfile-cdp \ + Dockerfile-tool \ + versions.tf \ + mesh-gateway-sidecar-boot.sh \ + sidecar-boot.sh \ + dataplane-boot.sh \ + bin/clustertool echo "=========================" if [[ -n "${failed}" ]]; then diff --git a/util/files.go b/util/files.go new file mode 100644 index 0000000..fad1109 --- /dev/null +++ b/util/files.go @@ -0,0 +1,57 @@ +package util + +import ( + "fmt" + "io" + "os" + "path/filepath" + + "golang.org/x/crypto/blake2b" +) + +func FilesExist(parent string, paths ...string) (bool, error) { + for _, p := range paths { + ok, err := FileExists(filepath.Join(parent, p)) + if err != nil { + return false, err + } + if !ok { + return false, nil + } + } + return true, nil +} + +func FileExists(path string) (bool, error) { + _, err := os.Stat(path) + if os.IsNotExist(err) { + return false, nil + } else if err != nil { + return false, err + } else { + return true, nil + } +} + +func HashFile(path string) (string, error) { + hash, err := blake2b.New256(nil) + if err != nil { + return "", err + } + + if err := AddFileToHash(path, hash); err != nil { + return "", err + } + + return fmt.Sprintf("%x", hash.Sum(nil)), nil +} + +func AddFileToHash(path string, w io.Writer) error { + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + _, err = io.Copy(w, f) + return err +} diff --git a/util/util.go b/util/util.go index 02cad58..660b872 100644 --- a/util/util.go +++ b/util/util.go @@ -3,9 +3,9 @@ package util import "fmt" type Identifier struct { - Name string - Namespace string - Partition string + Name string `json:",omitempty"` + Namespace string `json:",omitempty"` + Partition string `json:",omitempty"` } func NewIdentifier(name, namespace, partition string) Identifier { @@ -23,11 +23,11 @@ func (id *Identifier) Normalize() { id.Partition = PartitionOrDefault(id.Partition) } -func (id *Identifier) String() string { +func (id Identifier) String() string { return fmt.Sprintf("%s/%s/%s", id.Partition, id.Namespace, id.Name) } -func (id *Identifier) ID() string { +func (id Identifier) ID() string { return fmt.Sprintf("%s.%s.%s", id.Partition, id.Namespace, id.Name) } @@ -43,3 +43,29 @@ func NamespaceOrDefault(name string) string { } return name } + +type Identifier2 struct { + Name string `json:",omitempty"` + Partition string `json:",omitempty"` +} + +func NewIdentifier2(name, partition string) Identifier2 { + id := Identifier2{ + Name: name, + Partition: partition, + } + id.Normalize() + return id +} + +func (id *Identifier2) Normalize() { + id.Partition = PartitionOrDefault(id.Partition) +} + +func (id Identifier2) String() string { + return fmt.Sprintf("%s/%s", id.Partition, id.Name) +} + +func (id Identifier2) ID() string { + return fmt.Sprintf("%s.%s", id.Partition, id.Name) +}