Backport of NET-5397 - wire up destination golden tests from sidecar-proxy controller for xds controller and xdsv2 into release/1.17.x

.changelog/19225.txt

@@ -0,0 +1,9 @@
+```release-note:security
+Upgrade Go to 1.20.10.
+This resolves vulnerability [CVE-2023-39325](https://nvd.nist.gov/vuln/detail/CVE-2023-39325)
+/ [CVE-2023-44487](https://nvd.nist.gov/vuln/detail/CVE-2023-44487)(`net/http`).
+```
+```release-note:security
+Update `golang.org/x/net` to v0.17.0 to address [CVE-2023-39325](https://nvd.nist.gov/vuln/detail/CVE-2023-39325)
+/ [CVE-2023-44487](https://nvd.nist.gov/vuln/detail/CVE-2023-44487)(`x/net/http2`).
+```

.changelog/19274.txt

@@ -0,0 +1,3 @@
+```release-note:security
+connect: update supported envoy versions to 1.24.12, 1.25.11, 1.26.6, 1.27.2 to address [CVE-2023-44487](https://github.com/envoyproxy/envoy/security/advisories/GHSA-jhv4-f7mr-xx76)
+```

.changelog/19285.txt

@@ -0,0 +1,7 @@
+```release-note:bug
+ca: Fix bug with Vault CA provider where token renewal goroutines could leak if CA failed to initialize.
+```
+
+```release-note:bug
+ca: Fix bug with Vault CA provider where renewing a retracted token would cause retries in a tight loop, degrading performance.
+```

.changelog/19339.txt

@@ -0,0 +1,4 @@
+```release-note:bug
+connect: Fix bug where uncleanly closed xDS connections would influence connection balancing for too long and prevent envoy instances from starting. Two new configuration fields
+`performance.grpc_keepalive_timeout` and `performance.grpc_keepalive_interval` now exist to allow for configuration on how often these dead connections will be cleaned up.
+```

.changelog/_7406.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+server: **(Enterprise Only)** Fixed an issue where snake case keys were rejected when configuring the control-plane-request-limit config entry
+```

.github/scripts/verify_envoy_version.sh

@@ -4,7 +4,7 @@
 
 set -euo pipefail
 
-current_branch=$GITHUB_REF
+current_branch=$GITHUB_REF_NAME
 GITHUB_DEFAULT_BRANCH='main'
 
 if [ -z "$GITHUB_TOKEN" ]; then
@@ -13,10 +13,15 @@ if [ -z "$GITHUB_TOKEN" ]; then
 fi
 
 if [ -z "$current_branch" ]; then
-  echo "GITHUB_REF must be set"
+  echo "GITHUB_REF_NAME must be set"
   exit 1
 fi
 
+if [[ "$SKIP_VERIFY_ENVOY_VERSION" = "true" ]]; then
+  echo -e "*************** VERIFY ENVOY VERSION IS DISABLED. To enable, update environment variable in Github settings *****************"
+  exit 0
+fi
+
 # Get Consul and Envoy version 
 SCRIPT_DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
 pushd $SCRIPT_DIR/../.. # repository root
@@ -76,7 +81,6 @@ released_envoy_version=$(get_latest_envoy_version)
 major_released_envoy_version="${released_envoy_version[@]:1:4}"
 
 validate_envoy_version_main(){
-  echo "verify "main" GitHub branch has latest envoy version"
   # Get envoy version for current branch
   ENVOY_VERSIONS=$(sanitize_consul_envoy_version | awk '{print $2}' | tr ',' ' ')
   envoy_version_main_branch=$(get_major_version ${ENVOY_VERSIONS})
@@ -118,8 +122,8 @@ echo checking out branch: "${current_branch}"
 git checkout "${current_branch}"
 
 echo
-echo "Branch ${current_branch} =>Consul version: ${CONSUL_VERSION}; Envoy Version: ${ENVOY_VERSIONS}" 
-echo "Branch ${GITHUB_DEFAULT_BRANCH} =>Consul version: ${CONSUL_VERSION_DEFAULT_BRANCH}; Envoy Version: ${ENVOY_VERSIONS_DEFAULT_BRANCH}" 
+echo "Branch ${current_branch} => Consul version: ${CONSUL_VERSION}; Envoy Version: ${ENVOY_VERSIONS}" 
+echo "Branch ${GITHUB_DEFAULT_BRANCH} => Consul version: ${CONSUL_VERSION_DEFAULT_BRANCH}; Envoy Version: ${ENVOY_VERSIONS_DEFAULT_BRANCH}" 
 
 ## Get major Consul and Envoy versions on release and default branch
 MAJOR_CONSUL_VERSION=$(get_major_version ${CONSUL_VERSION})

.github/workflows/build.yml

@@ -28,14 +28,15 @@ jobs:
       shared-ldflags: ${{ steps.shared-ldflags.outputs.shared-ldflags }}
     steps:
       - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+      # action-set-product-version implicitly sets fields like 'product-version' using version/VERSION
+      # https://github.com/hashicorp/actions-set-product-version
       - name: set product version
         id: set-product-version
         uses: hashicorp/actions-set-product-version@v1
       - name: get product version
         id: get-product-version
         run: |
           CONSUL_DATE=$(build-support/scripts/build-date.sh)
-          ## TODO: This assumes `make version` outputs 1.1.1+ent-prerel
           echo "product-date=${CONSUL_DATE}" >> "$GITHUB_OUTPUT"
 
       - name: Set shared -ldflags
@@ -85,15 +86,15 @@ jobs:
     strategy:
       matrix:
         include:
-          - {go: "1.20.8", goos: "linux", goarch: "386"}
-          - {go: "1.20.8", goos: "linux", goarch: "amd64"}
-          - {go: "1.20.8", goos: "linux", goarch: "arm"}
-          - {go: "1.20.8", goos: "linux", goarch: "arm64"}
-          - {go: "1.20.8", goos: "freebsd", goarch: "386"}
-          - {go: "1.20.8", goos: "freebsd", goarch: "amd64"}
-          - {go: "1.20.8", goos: "windows", goarch: "386"}
-          - {go: "1.20.8", goos: "windows", goarch: "amd64"}
-          - {go: "1.20.8", goos: "solaris", goarch: "amd64"}
+          - {go: "1.20.10", goos: "linux", goarch: "386"}
+          - {go: "1.20.10", goos: "linux", goarch: "amd64"}
+          - {go: "1.20.10", goos: "linux", goarch: "arm"}
+          - {go: "1.20.10", goos: "linux", goarch: "arm64"}
+          - {go: "1.20.10", goos: "freebsd", goarch: "386"}
+          - {go: "1.20.10", goos: "freebsd", goarch: "amd64"}
+          - {go: "1.20.10", goos: "windows", goarch: "386"}
+          - {go: "1.20.10", goos: "windows", goarch: "amd64"}
+          - {go: "1.20.10", goos: "solaris", goarch: "amd64"}
       fail-fast: true
 
     name: Go ${{ matrix.go }} ${{ matrix.goos }} ${{ matrix.goarch }} build
@@ -182,7 +183,7 @@ jobs:
     strategy:
       matrix:
         include:
-          - {go: "1.20.8", goos: "linux", goarch: "s390x"}
+          - {go: "1.20.10", goos: "linux", goarch: "s390x"}
       fail-fast: true
 
     name: Go ${{ matrix.go }} ${{ matrix.goos }} ${{ matrix.goarch }} build
@@ -233,7 +234,7 @@ jobs:
       matrix:
         goos: [ darwin ]
         goarch: [ "amd64", "arm64" ]
-        go: [ "1.20.8" ]
+        go: [ "1.20.10" ]
       fail-fast: true
 
     name: Go ${{ matrix.go }} ${{ matrix.goos }} ${{ matrix.goarch }} build
@@ -299,8 +300,10 @@ jobs:
       # This naming convention will be used ONLY for per-commit dev images
       - name: Set docker dev tag
         run: |
-          version="${{ env.version }}"
-          echo "dev_tag=${version%.*}-dev" >> $GITHUB_ENV
+          echo "full_dev_tag=${{ env.version }}"
+          echo "full_dev_tag=${{ env.version }}" >> $GITHUB_ENV
+          echo "minor_dev_tag=$(echo ${{ env.version }}| sed -E 's/([0-9]+\.[0-9]+)\.[0-9]+(-[0-9a-zA-Z\+\.]+)?$/\1\2/')" 
+          echo "minor_dev_tag=$(echo ${{ env.version }}| sed -E 's/([0-9]+\.[0-9]+)\.[0-9]+(-[0-9a-zA-Z\+\.]+)?$/\1\2/')" >> $GITHUB_ENV
 
       - name: Docker Build (Action)
         uses: hashicorp/actions-docker-build@v1
@@ -312,8 +315,10 @@ jobs:
             docker.io/hashicorp/${{env.repo}}:${{env.version}}
             public.ecr.aws/hashicorp/${{env.repo}}:${{env.version}}
           dev_tags: |
-            docker.io/hashicorppreview/${{ env.repo }}:${{ env.dev_tag }}
-            docker.io/hashicorppreview/${{ env.repo }}:${{ env.dev_tag }}-${{ github.sha }}
+            docker.io/hashicorppreview/${{ env.repo }}:${{ env.full_dev_tag }}
+            docker.io/hashicorppreview/${{ env.repo }}:${{ env.full_dev_tag }}-${{ github.sha }}
+            docker.io/hashicorppreview/${{ env.repo }}:${{ env.minor_dev_tag }}
+            docker.io/hashicorppreview/${{ env.repo }}:${{ env.minor_dev_tag }}-${{ github.sha }}
           smoke_test: .github/scripts/verify_docker.sh v${{ env.version }}
 
   build-docker-ubi-redhat:
@@ -353,8 +358,10 @@ jobs:
       # This naming convention will be used ONLY for per-commit dev images
       - name: Set docker dev tag
         run: |
-          version="${{ env.version }}"
-          echo "dev_tag=${version%.*}-dev" >> $GITHUB_ENV
+          echo "full_dev_tag=${{ env.version }}"
+          echo "full_dev_tag=${{ env.version }}" >> $GITHUB_ENV
+          echo "minor_dev_tag=$(echo ${{ env.version }}| sed -E 's/([0-9]+\.[0-9]+)\.[0-9]+(-[0-9a-zA-Z\+\.]+)?$/\1\2/')" 
+          echo "minor_dev_tag=$(echo ${{ env.version }}| sed -E 's/([0-9]+\.[0-9]+)\.[0-9]+(-[0-9a-zA-Z\+\.]+)?$/\1\2/')" >> $GITHUB_ENV
 
       - uses: hashicorp/actions-docker-build@v1
         with:
@@ -365,8 +372,10 @@ jobs:
             docker.io/hashicorp/${{env.repo}}:${{env.version}}-ubi
             public.ecr.aws/hashicorp/${{env.repo}}:${{env.version}}-ubi
           dev_tags: |
-            docker.io/hashicorppreview/${{ env.repo }}:${{ env.dev_tag }}-ubi
-            docker.io/hashicorppreview/${{ env.repo }}:${{ env.dev_tag }}-ubi-${{ github.sha }}
+            docker.io/hashicorppreview/${{ env.repo }}:${{ env.full_dev_tag }}-ubi
+            docker.io/hashicorppreview/${{ env.repo }}:${{ env.full_dev_tag }}-ubi-${{ github.sha }}
+            docker.io/hashicorppreview/${{ env.repo }}:${{ env.minor_dev_tag }}-ubi
+            docker.io/hashicorppreview/${{ env.repo }}:${{ env.minor_dev_tag }}-ubi-${{ github.sha }}
           smoke_test: .github/scripts/verify_docker.sh v${{ env.version }}
 
   verify-linux:

.github/workflows/nightly-test-integrations-1.15.x.yml

@@ -68,7 +68,7 @@ jobs:
           # this is further going to multiplied in envoy-integration tests by the 
           # other dimensions in the matrix.  Currently TOTAL_RUNNERS would be
           # multiplied by 8 based on these values:
-          # envoy-version: ["1.22.11", "1.23.12", "1.24.10", "1.25.9"]
+          # envoy-version: ["1.22.11", "1.23.12", "1.24.12", "1.25.11"]
           # xds-target: ["server", "client"]
           TOTAL_RUNNERS: 4 
           JQ_SLICER: '[ inputs ] | [_nwise(length / $runnercount | floor)]'
@@ -102,7 +102,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        envoy-version: ["1.22.11", "1.23.12", "1.24.10", "1.25.9"]
+        envoy-version: ["1.22.11", "1.23.12", "1.24.12", "1.25.11"]
         xds-target: ["server", "client"]
         test-cases: ${{ fromJSON(needs.generate-envoy-job-matrices.outputs.envoy-matrix) }}
     env:

.github/workflows/nightly-test-integrations-1.16.x.yml

@@ -68,7 +68,7 @@ jobs:
           # this is further going to multiplied in envoy-integration tests by the 
           # other dimensions in the matrix.  Currently TOTAL_RUNNERS would be
           # multiplied by 8 based on these values:
-          # envoy-version: ["1.24.10", "1.25.9", "1.26.4", "1.27.0"]
+          # envoy-version: ["1.23.12", "1.24.12", "1.25.11", "1.26.6"]
           # xds-target: ["server", "client"]
           TOTAL_RUNNERS: 4 
           JQ_SLICER: '[ inputs ] | [_nwise(length / $runnercount | floor)]'
@@ -102,7 +102,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        envoy-version: ["1.23.12", "1.24.10", "1.25.9", "1.26.4"]
+        envoy-version: ["1.23.12", "1.24.12", "1.25.11", "1.26.6"]
         xds-target: ["server", "client"]
         test-cases: ${{ fromJSON(needs.generate-envoy-job-matrices.outputs.envoy-matrix) }}
     env:

.github/workflows/nightly-test-integrations.yml

@@ -65,7 +65,7 @@ jobs:
           # this is further going to multiplied in envoy-integration tests by the 
           # other dimensions in the matrix.  Currently TOTAL_RUNNERS would be
           # multiplied by 8 based on these values:
-          # envoy-version: ["1.24.10", "1.25.9", "1.26.4", "1.27.0"]
+          # envoy-version: ["1.24.12", "1.25.11", "1.26.6", "1.27.2"]
           # xds-target: ["server", "client"]
           TOTAL_RUNNERS: 4 
           JQ_SLICER: '[ inputs ] | [_nwise(length / $runnercount | floor)]'
@@ -99,7 +99,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        envoy-version: ["1.24.10", "1.25.9", "1.26.4", "1.27.0"]
+        envoy-version: ["1.24.12", "1.25.11", "1.26.6", "1.27.2"]
         xds-target: ["server", "client"]
         test-cases: ${{ fromJSON(needs.generate-envoy-job-matrices.outputs.envoy-matrix) }}
     env:

.github/workflows/test-integrations-windows.yml

@@ -54,7 +54,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        envoy-version: [ "1.27.0" ]
+        envoy-version: [ "1.27.2" ]
         xds-target: [ "server", "client" ]
     env:
       ENVOY_VERSION: ${{ matrix.envoy-version }}

.github/workflows/test-integrations.yml

@@ -80,7 +80,8 @@ jobs:
       contents: read
     strategy:
       matrix:
-        nomad-version: ['v1.6.1', 'v1.5.8', 'v1.4.12']
+        nomad-version: ['v1.6.2', 'v1.5.9', 'v1.4.13']
+
     steps:
       - name: Checkout Nomad
         uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
@@ -159,7 +160,7 @@ jobs:
       contents: read
     strategy:
       matrix:
-        vault-version: ["1.14.1", "1.13.5", "1.12.9", "1.11.12"]
+        vault-version: ["1.15.0", "1.14.4", "1.13.8", "1.12.11"]
     env:
       VAULT_BINARY_VERSION: ${{ matrix.vault-version }}
     steps:
@@ -259,8 +260,8 @@ jobs:
         env:
           # this is further going to multiplied in envoy-integration tests by the
           # other dimensions in the matrix.  Currently TOTAL_RUNNERS would be
-          # multiplied by 8 based on these values:
-          # envoy-version: ["1.24.10", "1.25.9", "1.26.4", "1.27.0"]
+          # multiplied by 2 based on these values:
+          # envoy-version: ["1.27.2"]
           # xds-target: ["server", "client"]
           TOTAL_RUNNERS: 4
           JQ_SLICER: '[ inputs ] | [_nwise(length / $runnercount | floor)]'
@@ -294,7 +295,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        envoy-version: ["1.27.0"]
+        envoy-version: ["1.27.2"]
         xds-target: ["server", "client"]
         test-cases: ${{ fromJSON(needs.generate-envoy-job-matrices.outputs.envoy-matrix) }}
     env:

.github/workflows/verify-envoy-version.yml

@@ -14,6 +14,9 @@ on:
       - main
       - release/**
 
+env:
+  SKIP_VERIFY_ENVOY_VERSION: ${{ vars.SKIP_VERIFY_ENVOY_VERSION }}
+
 jobs:
   verify-envoy-version:
     runs-on: ubuntu-latest

Makefile

@@ -438,6 +438,9 @@ codegen: codegen-tools ## Deep copy
 	@$(SHELL) $(CURDIR)/agent/consul/state/deep-copy.sh
 	@$(SHELL) $(CURDIR)/agent/config/deep-copy.sh
 	copywrite headers
+	# Special case for MPL headers in /api and /sdk
+	cd api && $(CURDIR)/build-support/scripts/copywrite-exceptions.sh
+	cd sdk && $(CURDIR)/build-support/scripts/copywrite-exceptions.sh
 
 print-%  : ; @echo $($*) ## utility to echo a makefile variable (i.e. 'make print-GOPATH')
 

acl/MockAuthorizer.go

@@ -224,6 +224,11 @@ func (m *MockAuthorizer) ServiceReadAll(ctx *AuthorizerContext) EnforcementDecis
 	return ret.Get(0).(EnforcementDecision)
 }
 
+func (m *MockAuthorizer) ServiceReadPrefix(prefix string, ctx *AuthorizerContext) EnforcementDecision {
+	ret := m.Called(ctx)
+	return ret.Get(0).(EnforcementDecision)
+}
+
 // ServiceWrite checks for permission to create or update a given
 // service
 func (m *MockAuthorizer) ServiceWrite(segment string, ctx *AuthorizerContext) EnforcementDecision {

acl/acl_test.go

@@ -300,6 +300,14 @@ func checkDenyServiceReadAll(t *testing.T, authz Authorizer, _ string, entCtx *A
 	require.Equal(t, Deny, authz.ServiceReadAll(entCtx))
 }
 
+func checkAllowServiceReadPrefix(t *testing.T, authz Authorizer, prefix string, entCtx *AuthorizerContext) {
+	require.Equal(t, Allow, authz.ServiceReadPrefix(prefix, entCtx))
+}
+
+func checkDenyServiceReadPrefix(t *testing.T, authz Authorizer, prefix string, entCtx *AuthorizerContext) {
+	require.Equal(t, Deny, authz.ServiceReadPrefix(prefix, entCtx))
+}
+
 func checkDenyServiceWrite(t *testing.T, authz Authorizer, prefix string, entCtx *AuthorizerContext) {
 	require.Equal(t, Deny, authz.ServiceWrite(prefix, entCtx))
 }
@@ -456,6 +464,10 @@ func checkDefaultServiceReadAll(t *testing.T, authz Authorizer, _ string, entCtx
 	require.Equal(t, Default, authz.ServiceReadAll(entCtx))
 }
 
+func checkDefaultServiceReadPrefix(t *testing.T, authz Authorizer, prefix string, entCtx *AuthorizerContext) {
+	require.Equal(t, Default, authz.ServiceReadPrefix(prefix, entCtx))
+}
+
 func checkDefaultServiceWrite(t *testing.T, authz Authorizer, prefix string, entCtx *AuthorizerContext) {
 	require.Equal(t, Default, authz.ServiceWrite(prefix, entCtx))
 }

acl/authorizer.go

@@ -171,6 +171,9 @@ type Authorizer interface {
 	// ServiceReadAll checks for permission to read all services
 	ServiceReadAll(*AuthorizerContext) EnforcementDecision
 
+	// ServiceReadPrefix checks for permission to read services within the given prefix.
+	ServiceReadPrefix(string, *AuthorizerContext) EnforcementDecision
+
 	// ServiceWrite checks for permission to create or update a given
 	// service
 	ServiceWrite(string, *AuthorizerContext) EnforcementDecision
@@ -507,6 +510,14 @@ func (a AllowAuthorizer) ServiceReadAllAllowed(ctx *AuthorizerContext) error {
 	return nil
 }
 
+// ServiceReadPrefixAllowed checks for permission to read services within the given prefix
+func (a AllowAuthorizer) ServiceReadPrefixAllowed(prefix string, ctx *AuthorizerContext) error {
+	if a.Authorizer.ServiceReadPrefix(prefix, ctx) != Allow {
+		return PermissionDeniedByACL(a, ctx, ResourceService, AccessRead, prefix) // read
+	}
+	return nil
+}
+
 // ServiceWriteAllowed checks for permission to create or update a given
 // service
 func (a AllowAuthorizer) ServiceWriteAllowed(name string, ctx *AuthorizerContext) error {

acl/chained_authorizer.go

@@ -275,6 +275,12 @@ func (c *ChainedAuthorizer) ServiceReadAll(entCtx *AuthorizerContext) Enforcemen
 	})
 }
 
+func (c *ChainedAuthorizer) ServiceReadPrefix(prefix string, entCtx *AuthorizerContext) EnforcementDecision {
+	return c.executeChain(func(authz Authorizer) EnforcementDecision {
+		return authz.ServiceReadPrefix(prefix, entCtx)
+	})
+}
+
 // ServiceWrite checks for permission to create or update a given
 // service
 func (c *ChainedAuthorizer) ServiceWrite(name string, entCtx *AuthorizerContext) EnforcementDecision {

acl/chained_authorizer_test.go

@@ -107,6 +107,9 @@ func (authz testAuthorizer) ServiceRead(string, *AuthorizerContext) EnforcementD
 func (authz testAuthorizer) ServiceReadAll(*AuthorizerContext) EnforcementDecision {
 	return EnforcementDecision(authz)
 }
+func (authz testAuthorizer) ServiceReadPrefix(string, *AuthorizerContext) EnforcementDecision {
+	return EnforcementDecision(authz)
+}
 func (authz testAuthorizer) ServiceWrite(string, *AuthorizerContext) EnforcementDecision {
 	return EnforcementDecision(authz)
 }