Merge branch 'main' into fixes/connect-ca-secondary-update

.changelog/17739.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+http: fixed API endpoint `PUT /acl/token/:AccessorID` (update token), no longer requires `AccessorID` in the request body. Web UI can now update tokens.
+ ```

.changelog/17755.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+mesh: Stop jwt providers referenced by intentions from being deleted.
+```

.changelog/17757.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+connect: Improve transparent proxy support for virtual services and failovers.
+```

.changelog/17759.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+extensions: Improve validation and error feedback for `property-override` builtin Envoy extension
+```

.changelog/17775.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+connect: Fix issue where changes to service exports were not reflected in proxies.
+```

.changelog/17831.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+ca: Vault CA provider config no longer requires root_pki_path for secondary datacenters
+```

.github/workflows/build-artifacts.yml

@@ -13,7 +13,7 @@ permissions:
   contents: read
 
 env:
-  GOPRIVATE: github.com/hashicorp
+  GOPRIVATE: github.com/hashicorp # Required for enterprise deps
 
 jobs:
   setup:

.github/workflows/build-distros.yml

@@ -15,6 +15,7 @@ permissions:
 
 env:
   GOTAGS: ${{ endsWith(github.repository, '-enterprise') && 'consulent' || '' }}
+  GOPRIVATE: github.com/hashicorp # Required for enterprise deps
 
 jobs:
   setup:

.github/workflows/build.yml

@@ -14,6 +14,7 @@ on:
 env:
   PKG_NAME: consul
   METADATA: oss
+  GOPRIVATE: github.com/hashicorp # Required for enterprise deps
 
 jobs:
   set-product-version:

.github/workflows/go-tests.yml

@@ -21,6 +21,7 @@ permissions:
 
 env:
   TEST_RESULTS: /tmp/test-results
+  GOPRIVATE: github.com/hashicorp # Required for enterprise deps
 
 jobs:
   setup:
@@ -50,7 +51,7 @@ jobs:
   check-generated-protobuf:
     needs: 
     - setup   
-    runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }}
+    runs-on: ${{ fromJSON(needs.setup.outputs.compute-medium) }}
     steps:
     - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
     # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos.

.github/workflows/nightly-test-1.13.x.yaml

@@ -8,9 +8,10 @@ on:
   workflow_dispatch: {}
 
 env:
-  EMBER_PARTITION_TOTAL: 4      # Has to be changed in tandem with the matrix.partition
+  EMBER_PARTITION_TOTAL: 4        # Has to be changed in tandem with the matrix.partition
   BRANCH: "release/1.13.x"
-  BRANCH_NAME: "release-1.13.x" # Used for naming artifacts
+  BRANCH_NAME: "release-1.13.x"   # Used for naming artifacts
+  GOPRIVATE: github.com/hashicorp # Required for enterprise deps
 
 jobs:
   frontend-test-workspace-node:

.github/workflows/nightly-test-1.14.x.yaml

@@ -8,9 +8,10 @@ on:
   workflow_dispatch: {}
 
 env:
-  EMBER_PARTITION_TOTAL: 4      # Has to be changed in tandem with the matrix.partition
+  EMBER_PARTITION_TOTAL: 4        # Has to be changed in tandem with the matrix.partition
   BRANCH: "release/1.14.x"
-  BRANCH_NAME: "release-1.14.x" # Used for naming artifacts
+  BRANCH_NAME: "release-1.14.x"   # Used for naming artifacts
+  GOPRIVATE: github.com/hashicorp # Required for enterprise deps
 
 jobs:
   frontend-test-workspace-node:

.github/workflows/nightly-test-1.15.x.yaml

@@ -8,9 +8,10 @@ on:
   workflow_dispatch: {}
 
 env:
-  EMBER_PARTITION_TOTAL: 4      # Has to be changed in tandem with the matrix.partition
+  EMBER_PARTITION_TOTAL: 4        # Has to be changed in tandem with the matrix.partition
   BRANCH: "release/1.15.x"
-  BRANCH_NAME: "release-1.15.x" # Used for naming artifacts
+  BRANCH_NAME: "release-1.15.x"   # Used for naming artifacts
+  GOPRIVATE: github.com/hashicorp # Required for enterprise deps
 
 jobs:
   frontend-test-workspace-node:

.github/workflows/nightly-test-1.16.x.yaml

@@ -8,9 +8,10 @@ on:
   workflow_dispatch: {}
 
 env:
-  EMBER_PARTITION_TOTAL: 4      # Has to be changed in tandem with the matrix.partition
+  EMBER_PARTITION_TOTAL: 4        # Has to be changed in tandem with the matrix.partition
   BRANCH: "release/1.16.x"
-  BRANCH_NAME: "release-1.16.x" # Used for naming artifacts
+  BRANCH_NAME: "release-1.16.x"   # Used for naming artifacts
+  GOPRIVATE: github.com/hashicorp # Required for enterprise deps
 
 jobs:
   frontend-test-workspace-node:

.github/workflows/nightly-test-main.yaml

@@ -8,9 +8,10 @@ on:
   workflow_dispatch: {}
 
 env:
-  EMBER_PARTITION_TOTAL: 4      # Has to be changed in tandem with the matrix.partition
+  EMBER_PARTITION_TOTAL: 4        # Has to be changed in tandem with the matrix.partition
   BRANCH: "main"
-  BRANCH_NAME: "main"           # Used for naming artifacts
+  BRANCH_NAME: "main"             # Used for naming artifacts
+  GOPRIVATE: github.com/hashicorp # Required for enterprise deps
 
 jobs:
   frontend-test-workspace-node:

.github/workflows/reusable-lint.yml

@@ -20,6 +20,7 @@ on:
 env:
   GOTAGS: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}"
   GOARCH: ${{inputs.go-arch}}
+  GOPRIVATE: github.com/hashicorp # Required for enterprise deps
 
 jobs:
   lint:

.github/workflows/reusable-unit-split.yml

@@ -51,6 +51,7 @@ env:
   TOTAL_RUNNERS: ${{inputs.runner-count}}
   CONSUL_LICENSE: ${{secrets.consul-license}}
   GOTAGS: ${{ inputs.go-tags}}
+  GOPRIVATE: github.com/hashicorp # Required for enterprise deps
   DATADOG_API_KEY: ${{secrets.datadog-api-key}}
 
 jobs:

.github/workflows/reusable-unit.yml

@@ -46,6 +46,7 @@ env:
   GOARCH: ${{inputs.go-arch}}
   CONSUL_LICENSE: ${{secrets.consul-license}}
   GOTAGS: ${{ inputs.go-tags}}
+  GOPRIVATE: github.com/hashicorp # Required for enterprise deps
   DATADOG_API_KEY: ${{secrets.datadog-api-key}}
 
 jobs:

.github/workflows/test-integrations.yml

@@ -23,6 +23,7 @@ env:
   CONSUL_BINARY_UPLOAD_NAME: consul-bin
   # strip the hashicorp/ off the front of github.repository for consul
   CONSUL_LATEST_IMAGE_NAME: ${{ endsWith(github.repository, '-enterprise') && github.repository || 'consul' }}
+  GOPRIVATE: github.com/hashicorp # Required for enterprise deps
 
 jobs:
   setup:
@@ -365,6 +366,10 @@ jobs:
       ENVOY_VERSION: "1.25.4"
     steps:
       - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
+      # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos.
+      - name: Setup Git
+        if: ${{ endsWith(github.repository, '-enterprise') }}
+        run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com"
       - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
         with:
           go-version-file: 'go.mod'
@@ -476,6 +481,10 @@ jobs:
       ENVOY_VERSION: "1.24.6"
     steps:
       - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
+      # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos.
+      - name: Setup Git
+        if: ${{ endsWith(github.repository, '-enterprise') }}
+        run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com"
       - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
         with:
           go-version-file: 'go.mod'

GNUmakefile

@@ -3,6 +3,7 @@
 
 SHELL = bash
 
+
 GO_MODULES := $(shell find . -name go.mod -exec dirname {} \; | grep -v "proto-gen-rpc-glue/e2e" | sort)
 
 ###
@@ -72,6 +73,7 @@ CI_DEV_DOCKER_NAMESPACE?=hashicorpdev
 CI_DEV_DOCKER_IMAGE_NAME?=consul
 CI_DEV_DOCKER_WORKDIR?=bin/
 ################
+CONSUL_VERSION?=$(shell cat version/VERSION)
 
 TEST_MODCACHE?=1
 TEST_BUILDCACHE?=1
@@ -188,8 +190,11 @@ dev-docker: linux dev-build
 	@docker buildx use default && docker buildx build -t 'consul:local' -t '$(CONSUL_DEV_IMAGE)' \
        --platform linux/$(GOARCH) \
 	   --build-arg CONSUL_IMAGE_VERSION=$(CONSUL_IMAGE_VERSION) \
+		--label org.opencontainers.image.version=$(CONSUL_VERSION) \
+		--label version=$(CONSUL_VERSION) \
        --load \
        -f $(CURDIR)/build-support/docker/Consul-Dev-Multiarch.dockerfile $(CURDIR)/pkg/bin/
+	docker tag 'consul:local'  '$(CONSUL_COMPAT_TEST_IMAGE):local'
 
 check-remote-dev-image-env:
 ifndef REMOTE_DEV_IMAGE
@@ -208,6 +213,8 @@ remote-docker: check-remote-dev-image-env
 	@docker buildx use consul-builder && docker buildx build -t '$(REMOTE_DEV_IMAGE)' \
        --platform linux/amd64,linux/arm64 \
 	   --build-arg CONSUL_IMAGE_VERSION=$(CONSUL_IMAGE_VERSION) \
+		--label org.opencontainers.image.version=$(CONSUL_VERSION) \
+		--label version=$(CONSUL_VERSION) \
        --push \
        -f $(CURDIR)/build-support/docker/Consul-Dev-Multiarch.dockerfile $(CURDIR)/pkg/bin/
 
@@ -351,16 +358,17 @@ lint/%:
 	@echo "--> Running enumcover ($*)"
 	@cd $* && GOWORK=off enumcover ./...
 
+# check that the test-container module only imports allowlisted packages
+# from the root consul module. Generally we don't want to allow these imports.
+# In a few specific instances though it is okay to import test definitions and
+# helpers from some of the packages in the root module.
 .PHONY: lint-container-test-deps
 lint-container-test-deps:
 	@echo "--> Checking container tests for bad dependencies"
-	@cd test/integration/consul-container && ( \
-		found="$$(go list -m all | grep -c '^github.com/hashicorp/consul ')" ; \
-		if [[ "$$found" != "0" ]]; then \
-			echo "test/integration/consul-container: This project should not depend on the root consul module" >&2 ; \
-			exit 1 ; \
-		fi \
-	)
+	@cd test/integration/consul-container && \
+		$(CURDIR)/build-support/scripts/check-allowed-imports.sh \
+			github.com/hashicorp/consul \
+			internal/catalog/catalogtest
 
 # Build the static web ui inside a Docker container. For local testing only; do not commit these assets.
 ui: ui-docker

agent/acl_endpoint.go

@@ -441,8 +441,16 @@ func (s *HTTPHandlers) aclTokenSetInternal(req *http.Request, tokenAccessorID st
 		return nil, HTTPError{StatusCode: http.StatusBadRequest, Reason: fmt.Sprintf("Token decoding failed: %v", err)}
 	}
 
-	if !create && args.ACLToken.AccessorID != tokenAccessorID {
-		return nil, HTTPError{StatusCode: http.StatusBadRequest, Reason: "Token Accessor ID in URL and payload do not match"}
+	if !create {
+		// NOTE: AccessorID in the request body is optional when not creating a new token.
+		// If not present in the body and only in the URL then it will be filled in by Consul.
+		if args.ACLToken.AccessorID == "" {
+			args.ACLToken.AccessorID = tokenAccessorID
+		}
+
+		if args.ACLToken.AccessorID != tokenAccessorID {
+			return nil, HTTPError{StatusCode: http.StatusBadRequest, Reason: "Token Accessor ID in URL and payload do not match"}
+		}
 	}
 
 	var out structs.ACLToken

agent/acl_endpoint_test.go

@@ -907,6 +907,48 @@ func TestACL_HTTP(t *testing.T) {
 			tokenMap[token.AccessorID] = token
 		})
 
+		t.Run("Update without AccessorID in request body", func(t *testing.T) {
+			originalToken := tokenMap[idMap["token-cloned"]]
+
+			// Secret will be filled in
+			tokenInput := &structs.ACLToken{
+				Description: "Even Better description for this cloned token",
+				Policies: []structs.ACLTokenPolicyLink{
+					{
+						ID:   idMap["policy-read-all-nodes"],
+						Name: policyMap[idMap["policy-read-all-nodes"]].Name,
+					},
+				},
+				NodeIdentities: []*structs.ACLNodeIdentity{
+					{
+						NodeName:   "foo",
+						Datacenter: "bar",
+					},
+				},
+			}
+
+			req, _ := http.NewRequest("PUT", "/v1/acl/token/"+originalToken.AccessorID, jsonBody(tokenInput))
+			req.Header.Add("X-Consul-Token", "root")
+			resp := httptest.NewRecorder()
+			obj, err := a.srv.ACLTokenCRUD(resp, req)
+			require.NoError(t, err)
+			token, ok := obj.(*structs.ACLToken)
+			require.True(t, ok)
+
+			require.Equal(t, originalToken.AccessorID, token.AccessorID)
+			require.Equal(t, originalToken.SecretID, token.SecretID)
+			require.Equal(t, tokenInput.Description, token.Description)
+			require.Equal(t, tokenInput.Policies, token.Policies)
+			require.Equal(t, tokenInput.NodeIdentities, token.NodeIdentities)
+			require.True(t, token.CreateIndex > 0)
+			require.True(t, token.CreateIndex < token.ModifyIndex)
+			require.NotNil(t, token.Hash)
+			require.NotEqual(t, token.Hash, []byte{})
+			require.NotEqual(t, token.Hash, originalToken.Hash)
+
+			tokenMap[token.AccessorID] = token
+		})
+
 		t.Run("CRUD Missing Token Accessor ID", func(t *testing.T) {
 			req, _ := http.NewRequest("GET", "/v1/acl/token/", nil)
 			req.Header.Add("X-Consul-Token", "root")

agent/agent.go

@@ -4555,7 +4555,11 @@ func (a *Agent) proxyDataSources() proxycfg.DataSources {
 		sources.ExportedPeeredServices = proxycfgglue.ServerExportedPeeredServices(deps)
 		sources.FederationStateListMeshGateways = proxycfgglue.ServerFederationStateListMeshGateways(deps)
 		sources.GatewayServices = proxycfgglue.ServerGatewayServices(deps)
-		sources.Health = proxycfgglue.ServerHealth(deps, proxycfgglue.ClientHealth(a.rpcClientHealth))
+		// We do not use this health check currently due to a bug with the way that service exports
+		// interact with ACLs and the streaming backend. See comments in `proxycfgglue.ServerHealthBlocking`
+		// for more details.
+		// sources.Health = proxycfgglue.ServerHealth(deps, proxycfgglue.ClientHealth(a.rpcClientHealth))
+		sources.Health = proxycfgglue.ServerHealthBlocking(deps, proxycfgglue.ClientHealth(a.rpcClientHealth), server.FSM().State())
 		sources.HTTPChecks = proxycfgglue.ServerHTTPChecks(deps, a.config.NodeName, proxycfgglue.CacheHTTPChecks(a.cache), a.State)
 		sources.Intentions = proxycfgglue.ServerIntentions(deps)
 		sources.IntentionUpstreams = proxycfgglue.ServerIntentionUpstreams(deps)

agent/config/builder.go

@@ -984,7 +984,7 @@ func (b *builder) build() (rt RuntimeConfig, err error) {
 		AutoEncryptIPSAN:                       autoEncryptIPSAN,
 		AutoEncryptAllowTLS:                    autoEncryptAllowTLS,
 		AutoConfig:                             autoConfig,
-		Cloud:                                  b.cloudConfigVal(c.Cloud),
+		Cloud:                                  b.cloudConfigVal(c),
 		ConnectEnabled:                         connectEnabled,
 		ConnectCAProvider:                      connectCAProvider,
 		ConnectCAConfig:                        connectCAConfig,
@@ -1290,6 +1290,10 @@ func (b *builder) validate(rt RuntimeConfig) error {
 			"1 and 63 bytes.", rt.NodeName)
 	}
 
+	if err := rt.StructLocality().Validate(); err != nil {
+		return fmt.Errorf("locality is invalid: %s", err)
+	}
+
 	if ipaddr.IsAny(rt.AdvertiseAddrLAN.IP) {
 		return fmt.Errorf("Advertise address cannot be 0.0.0.0, :: or [::]")
 	}
@@ -1469,7 +1473,7 @@ func (b *builder) validate(rt RuntimeConfig) error {
 				return err
 			}
 		case structs.VaultCAProvider:
-			if _, err := ca.ParseVaultCAConfig(rt.ConnectCAConfig); err != nil {
+			if _, err := ca.ParseVaultCAConfig(rt.ConnectCAConfig, rt.PrimaryDatacenter == rt.Datacenter); err != nil {
 				return err
 			}
 		case structs.AWSCAProvider:
@@ -2541,21 +2545,26 @@ func validateAutoConfigAuthorizer(rt RuntimeConfig) error {
 	return nil
 }
 
-func (b *builder) cloudConfigVal(v *CloudConfigRaw) hcpconfig.CloudConfig {
+func (b *builder) cloudConfigVal(v Config) hcpconfig.CloudConfig {
 	val := hcpconfig.CloudConfig{
 		ResourceID: os.Getenv("HCP_RESOURCE_ID"),
 	}
-	if v == nil {
+	// Node id might get overriden in setup.go:142
+	nodeID := stringVal(v.NodeID)
+	val.NodeID = types.NodeID(nodeID)
+	val.NodeName = b.nodeName(v.NodeName)
+
+	if v.Cloud == nil {
 		return val
 	}
 
-	val.ClientID = stringVal(v.ClientID)
-	val.ClientSecret = stringVal(v.ClientSecret)
-	val.AuthURL = stringVal(v.AuthURL)
-	val.Hostname = stringVal(v.Hostname)
-	val.ScadaAddress = stringVal(v.ScadaAddress)
+	val.ClientID = stringVal(v.Cloud.ClientID)
+	val.ClientSecret = stringVal(v.Cloud.ClientSecret)
+	val.AuthURL = stringVal(v.Cloud.AuthURL)
+	val.Hostname = stringVal(v.Cloud.Hostname)
+	val.ScadaAddress = stringVal(v.Cloud.ScadaAddress)
 
-	if resourceID := stringVal(v.ResourceID); resourceID != "" {
+	if resourceID := stringVal(v.Cloud.ResourceID); resourceID != "" {
 		val.ResourceID = resourceID
 	}
 	return val