Merge branch 'main' into docs/multi-port-v1-17-ga

.github/workflows/test-integrations.yml

@@ -496,6 +496,8 @@ jobs:
       contents: read
     strategy:
       fail-fast: false
+    env:
+      DEPLOYER_CONSUL_DATAPLANE_IMAGE: "docker.mirror.hashicorp.services/hashicorppreview/consul-dataplane:1.3-dev"
     steps:
       - name: Checkout code
         uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
@@ -513,20 +515,22 @@ jobs:
         run: |
           mkdir -p "${{ env.TEST_RESULTS_DIR }}"
           export NOLOGBUFFER=1
-          cd ./test-integ/connect
+          cd ./test-integ
           go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \
             --raw-command \
             --format=standard-verbose \
             --debug \
-            --packages="./..." \
             -- \
             go test \
             -tags "${{ env.GOTAGS }}" \
             -timeout=20m \
             -parallel=2 \
-            -json . \
+            -json \
+            `go list -tags "${{ env.GOTAGS }}" ./... | grep -v peering_commontopo` \
             --target-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \
-            --target-version local
+            --target-version local \
+            --latest-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \
+            --latest-version latest
         env:
           # this is needed because of incompatibility between RYUK container and GHA
           GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml

agent/agent.go

@@ -812,6 +812,7 @@ func (a *Agent) Start(ctx context.Context) error {
 			Logger:          a.proxyConfig.Logger.Named("agent-state"),
 			Tokens:          a.baseDeps.Tokens,
 			NodeName:        a.config.NodeName,
+			NodeLocality:    a.config.StructLocality(),
 			ResyncFrequency: a.config.LocalProxyConfigResyncInterval,
 		},
 	)
@@ -3686,6 +3687,13 @@ func (a *Agent) loadServices(conf *config.RuntimeConfig, snap map[structs.CheckI
 		}
 
 		ns := service.NodeService()
+
+		// We currently do not persist locality inherited from the node service
+		// (it is inherited at runtime). See agent/proxycfg-sources/local/sync.go.
+		// To support locality-aware service discovery in the future, persisting
+		// this data may be necessary. This does not impact agent-less deployments
+		// because locality is explicitly set on service registration there.
+
 		chkTypes, err := service.CheckTypes()
 		if err != nil {
 			return fmt.Errorf("Failed to validate checks for service %q: %v", service.Name, err)

agent/agent_endpoint.go

@@ -1166,6 +1166,13 @@ func (s *HTTPHandlers) AgentRegisterService(resp http.ResponseWriter, req *http.
 
 	// Get the node service.
 	ns := args.NodeService()
+
+	// We currently do not persist locality inherited from the node service
+	// (it is inherited at runtime). See agent/proxycfg-sources/local/sync.go.
+	// To support locality-aware service discovery in the future, persisting
+	// this data may be necessary. This does not impact agent-less deployments
+	// because locality is explicitly set on service registration there.
+
 	if ns.Weights != nil {
 		if err := structs.ValidateWeights(ns.Weights); err != nil {
 			return nil, HTTPError{StatusCode: http.StatusBadRequest, Reason: fmt.Sprintf("Invalid Weights: %v", err)}

agent/config/builder.go

@@ -1732,10 +1732,21 @@ func (b *builder) serviceVal(v *ServiceDefinition) *structs.ServiceDefinition {
 		Checks:            checks,
 		Proxy:             b.serviceProxyVal(v.Proxy),
 		Connect:           b.serviceConnectVal(v.Connect),
+		Locality:          b.serviceLocalityVal(v.Locality),
 		EnterpriseMeta:    v.EnterpriseMeta.ToStructs(),
 	}
 }
 
+func (b *builder) serviceLocalityVal(l *Locality) *structs.Locality {
+	if l == nil {
+		return nil
+	}
+	return &structs.Locality{
+		Region: stringVal(l.Region),
+		Zone:   stringVal(l.Zone),
+	}
+}
+
 func (b *builder) serviceKindVal(v *string) structs.ServiceKind {
 	if v == nil {
 		return structs.ServiceKindTypical

agent/config/config.go

@@ -404,6 +404,7 @@ type ServiceDefinition struct {
 	EnableTagOverride *bool                     `mapstructure:"enable_tag_override"`
 	Proxy             *ServiceProxy             `mapstructure:"proxy"`
 	Connect           *ServiceConnect           `mapstructure:"connect"`
+	Locality          *Locality                 `mapstructure:"locality"`
 
 	EnterpriseMeta `mapstructure:",squash"`
 }

agent/config/runtime_test.go

@@ -6576,6 +6576,10 @@ func TestLoad_FullConfig(t *testing.T) {
 		KVMaxValueSize:        1234567800,
 		LeaveDrainTime:        8265 * time.Second,
 		LeaveOnTerm:           true,
+		Locality: &Locality{
+			Region: strPtr("us-east-2"),
+			Zone:   strPtr("us-east-2b"),
+		},
 		Logging: logging.Config{
 			LogLevel:       "k1zo9Spt",
 			LogJSON:        true,
@@ -6678,6 +6682,10 @@ func TestLoad_FullConfig(t *testing.T) {
 						},
 					},
 				},
+				Locality: &structs.Locality{
+					Region: "us-east-1",
+					Zone:   "us-east-1a",
+				},
 			},
 			{
 				ID:      "MRHVMZuD",
@@ -6836,6 +6844,10 @@ func TestLoad_FullConfig(t *testing.T) {
 				Connect: &structs.ServiceConnect{
 					Native: true,
 				},
+				Locality: &structs.Locality{
+					Region: "us-west-1",
+					Zone:   "us-west-1a",
+				},
 				Checks: structs.CheckTypes{
 					&structs.CheckType{
 						CheckID:    "Zv99e9Ka",

agent/config/testdata/full-config.hcl

@@ -317,6 +317,10 @@ limits {
         write_rate = 101.0
     }
 }
+locality = {
+    region = "us-east-2"
+    zone = "us-east-2b"
+}
 log_level = "k1zo9Spt"
 log_json = true
 max_query_time = "18237s"
@@ -510,6 +514,10 @@ service = {
     connect {
         native = true
     }
+    locality = {
+        region = "us-west-1"
+        zone = "us-west-1a"
+    }
 }
 services = [
     {
@@ -550,6 +558,10 @@ services = [
         connect {
             sidecar_service {}
         }
+        locality = {
+            region = "us-east-1"
+            zone = "us-east-1a"
+        }
     },
     {
         id = "MRHVMZuD"

agent/config/testdata/full-config.json

@@ -366,6 +366,10 @@
       "write_rate": 101.0
     }
   },
+  "locality": {
+    "region": "us-east-2",
+    "zone": "us-east-2b"
+  },
   "log_level": "k1zo9Spt",
   "log_json": true,
   "max_query_time": "18237s",
@@ -598,6 +602,10 @@
     ],
     "connect": {
       "native": true
+    },
+    "locality": {
+      "region": "us-west-1",
+      "zone": "us-west-1a"
     }
   },
   "services": [
@@ -649,6 +657,10 @@
       },
       "connect": {
         "sidecar_service": {}
+      },
+      "locality": {
+        "region": "us-east-1",
+        "zone": "us-east-1a"
       }
     },
     {

agent/proxycfg-sources/local/sync.go

@@ -5,9 +5,10 @@ package local
 
 import (
 	"context"
-	proxysnapshot "github.com/hashicorp/consul/internal/mesh/proxy-snapshot"
 	"time"
 
+	proxysnapshot "github.com/hashicorp/consul/internal/mesh/proxy-snapshot"
+
 	"github.com/hashicorp/go-hclog"
 
 	"github.com/hashicorp/consul/agent/local"
@@ -35,6 +36,9 @@ type SyncConfig struct {
 	// NodeName is the name of the local agent node.
 	NodeName string
 
+	// NodeLocality
+	NodeLocality *structs.Locality
+
 	// Logger will be used to write log messages.
 	Logger hclog.Logger
 
@@ -110,6 +114,14 @@ func sync(cfg SyncConfig) {
 			Token: "",
 		}
 
+		// We inherit the node's locality at runtime (not persisted).
+		// The service locality takes precedence if it was set directly during
+		// registration.
+		svc = svc.DeepCopy()
+		if svc.Locality == nil {
+			svc.Locality = cfg.NodeLocality
+		}
+
 		// TODO(banks): need to work out when to default some stuff. For example
 		// Proxy.LocalServicePort is practically necessary for any sidecar and can
 		// default to the port of the sidecar service, but only if it's already

agent/proxycfg-sources/local/sync_test.go

@@ -72,8 +72,12 @@ func TestSync(t *testing.T) {
 	go Sync(ctx, SyncConfig{
 		Manager: cfgMgr,
 		State:   state,
-		Tokens:  tokens,
-		Logger:  hclog.NewNullLogger(),
+		NodeLocality: &structs.Locality{
+			Region: "some-region",
+			Zone:   "some-zone",
+		},
+		Tokens: tokens,
+		Logger: hclog.NewNullLogger(),
 	})
 
 	// Expect the service in the local state to be registered.
@@ -107,6 +111,13 @@ func TestSync(t *testing.T) {
 	select {
 	case reg := <-registerCh:
 		require.Equal(t, serviceID, reg.service.ID)
+		require.Equal(t,
+			&structs.Locality{
+				Region: "some-region",
+				Zone:   "some-zone",
+			},
+			reg.service.Locality,
+		)
 		require.Equal(t, userToken, reg.token)
 	case <-time.After(100 * time.Millisecond):
 		t.Fatal("timeout waiting for service to be registered")

command/services/register/register_test.go

@@ -9,6 +9,7 @@ import (
 	"testing"
 
 	"github.com/hashicorp/consul/agent"
+	"github.com/hashicorp/consul/api"
 	"github.com/hashicorp/consul/sdk/testutil"
 	"github.com/mitchellh/cli"
 	"github.com/stretchr/testify/require"
@@ -75,7 +76,7 @@ func TestCommand_File(t *testing.T) {
 	ui := cli.NewMockUi()
 	c := New(ui)
 
-	contents := `{ "Service": { "Name": "web" } }`
+	contents := `{ "Service": { "Name": "web", "Locality": { "Region": "us-east-1", "Zone": "us-east-1a" } } }`
 	f := testFile(t, "json")
 	defer os.Remove(f.Name())
 	if _, err := f.WriteString(contents); err != nil {
@@ -93,8 +94,11 @@ func TestCommand_File(t *testing.T) {
 	require.NoError(t, err)
 	require.Len(t, svcs, 1)
 
-	svc := svcs["web"]
-	require.NotNil(t, svc)
+	require.NotNil(t, svcs["web"])
+
+	svc, _, err := client.Agent().Service("web", nil)
+	require.NoError(t, err)
+	require.Equal(t, &api.Locality{Region: "us-east-1", Zone: "us-east-1a"}, svc.Locality)
 }
 
 func TestCommand_Flags(t *testing.T) {

test-integ/Makefile

@@ -0,0 +1,42 @@
+SHELL := /bin/bash
+
+.PHONY: noop
+noop:
+
+##@ Build
+
+.PHONY: tidy
+tidy: ## Run go mod tidy.
+	go mod tidy
+
+##@ Checks
+
+.PHONY: format
+format: ## Format the go files.
+	@for f in $$(find . -name '*.go' -print); do \
+		gofmt -s -w $$f ; \
+	done
+
+.PHONY: lint
+lint: ## Run the full linting rules.
+	golangci-lint run -v
+
+.PHONY: vet
+vet: ## Run go vet.
+	go vet ./...
+
+##@ Help
+
+# The help target prints out all targets with their descriptions organized
+# beneath their categories. The categories are represented by '##@' and the
+# target descriptions by '##'. The awk commands is responsible for reading the
+# entire set of makefiles included in this invocation, looking for lines of the
+# file as xyz: ## something, and then pretty-format the target and help. Then,
+# if there's a line with ##@ something, that gets pretty-printed as a category.
+# More info on the usage of ANSI control characters for terminal formatting:
+# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters
+# More info on the awk command:
+# http://linuxcommand.org/lc3_adv_awk.php
+.PHONY: help
+help: ## Display this help.
+	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)