Merge branch 'main' into sso/gh-13120-oidc-login

.changelog/15455.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+scheduler: allow using device IDs in `affinity` and `constraint`
+```

.changelog/15469.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+cli: add a nomad operator client state command
+```

.changelog/15552.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+server: Fixed a bug where rejoin_after_leave config was not being respected
+```

.changelog/15558.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+api: improved error returned from AllocFS.Logs when response is not JSON
+```

.changelog/15701.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+consul: add client configuration for grpc_ca_file
+```

.changelog/15726.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+deps: Update github.com/containerd/containerd from 1.6.6 to 1.6.12
+```

.changelog/15732.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+docker: configure restart policy for bridge network pause container
+```

.changelog/15733.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+ui: Show events alongside logs in the Task sidebar
+```

.changelog/15735.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+ui: Add a button for expanding the Task sidebar to full width
+```

.changelog/15745.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+vault: configure Nomad User-Agent on vault clients
+```

.changelog/15749.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+connect: ingress http/2/grpc listeners may exclude hosts
+```

.changelog/15759.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+ui: Scale down logger height in the UI when the sidebar container also has task events
+```

.changelog/15761.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+consul/connect: Adds support for proxy upstream opaque config
+```

.changelog/15769.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+build: Update to go1.19.5
+```

.changelog/15770.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+env/ec2: update cpu metadata
+```

.changelog/15797.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+cli: fix nomad fmt -check flag not returning error code
+```

.circleci/config.yml

@@ -450,7 +450,7 @@ executors:
   go:
     working_directory: /go/src/github.com/hashicorp/nomad
     docker:
-      - image: docker.mirror.hashicorp.services/golang:1.19.3
+      - image: docker.mirror.hashicorp.services/golang:1.19.5
     resource_class: medium
     environment:
       <<: *common_envs
@@ -463,7 +463,7 @@ executors:
       resource_class: large
     environment: &machine_env
       <<: *common_envs
-      GOLANG_VERSION: 1.19.3
+      GOLANG_VERSION: 1.19.5
 
   go-macos:
     working_directory: ~/go/src/github.com/hashicorp/nomad
@@ -472,7 +472,7 @@ executors:
     environment:
       <<: *common_envs
       GOPATH: /Users/distiller/go
-      GOLANG_VERSION: 1.19.3
+      GOLANG_VERSION: 1.19.5
 
   go-windows:
     machine:
@@ -484,7 +484,7 @@ executors:
       GOPATH: c:\gopath
       GOBIN: c:\gopath\bin
       GOTESTSUM_PATH: c:\tmp\test-reports
-      GOLANG_VERSION: 1.19.3
+      GOLANG_VERSION: 1.19.5
       GOTESTSUM_VERSION: 1.7.0
       VAULT_VERSION: 1.4.1
 

.go-version

@@ -1 +1 @@
-1.19.3
+1.19.5

.semgrep/rpc_endpoint.yml

@@ -65,9 +65,11 @@ rules:
       # Pattern used by Authenticate method.
       # TODO: add authorization steps as well.
       - pattern-not-inside: |
+          authErr := $A.$B.Authenticate($A.ctx, args)
           ...
-          ... := $A.$B.Authenticate($A.ctx, args.AuthToken)
-          ...
+          if authErr != nil {
+            return authErr
+          }
       - metavariable-pattern:
           metavariable: $METHOD
           patterns:

api/consul.go

@@ -2,6 +2,8 @@ package api
 
 import (
 	"time"
+
+	"golang.org/x/exp/maps"
 )
 
 // Consul represents configuration related to consul.
@@ -203,6 +205,7 @@ type ConsulUpstream struct {
 	Datacenter           string             `mapstructure:"datacenter" hcl:"datacenter,optional"`
 	LocalBindAddress     string             `mapstructure:"local_bind_address" hcl:"local_bind_address,optional"`
 	MeshGateway          *ConsulMeshGateway `mapstructure:"mesh_gateway" hcl:"mesh_gateway,block"`
+	Config               map[string]any     `mapstructure:"config" hcl:"config,block"`
 }
 
 func (cu *ConsulUpstream) Copy() *ConsulUpstream {
@@ -216,6 +219,7 @@ func (cu *ConsulUpstream) Copy() *ConsulUpstream {
 		Datacenter:           cu.Datacenter,
 		LocalBindAddress:     cu.LocalBindAddress,
 		MeshGateway:          cu.MeshGateway.Copy(),
+		Config:               maps.Clone(cu.Config),
 	}
 }
 
@@ -224,6 +228,9 @@ func (cu *ConsulUpstream) Canonicalize() {
 		return
 	}
 	cu.MeshGateway.Canonicalize()
+	if len(cu.Config) == 0 {
+		cu.Config = nil
+	}
 }
 
 type ConsulExposeConfig struct {

api/consul_test.go

@@ -172,6 +172,7 @@ func TestConsulUpstream_Copy(t *testing.T) {
 			LocalBindPort:        2000,
 			LocalBindAddress:     "10.0.0.1",
 			MeshGateway:          &ConsulMeshGateway{Mode: "remote"},
+			Config:               map[string]any{"connect_timeout_ms": 5000},
 		}
 		result := cu.Copy()
 		must.Eq(t, cu, result)
@@ -195,6 +196,7 @@ func TestConsulUpstream_Canonicalize(t *testing.T) {
 			LocalBindPort:        2000,
 			LocalBindAddress:     "10.0.0.1",
 			MeshGateway:          &ConsulMeshGateway{Mode: ""},
+			Config:               make(map[string]any),
 		}
 		cu.Canonicalize()
 		must.Eq(t, &ConsulUpstream{
@@ -204,6 +206,7 @@ func TestConsulUpstream_Canonicalize(t *testing.T) {
 			LocalBindPort:        2000,
 			LocalBindAddress:     "10.0.0.1",
 			MeshGateway:          &ConsulMeshGateway{Mode: ""},
+			Config:               nil,
 		}, cu)
 	})
 }

api/fs.go

@@ -4,10 +4,13 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"net"
 	"strconv"
 	"sync"
 	"time"
+
+	"github.com/hashicorp/go-multierror"
 )
 
 const (
@@ -288,7 +291,12 @@ func (a *AllocFS) Logs(alloc *Allocation, follow bool, task, logType, origin str
 				if err == io.EOF || err == io.ErrClosedPipe {
 					close(frames)
 				} else {
-					errCh <- err
+					buf, err2 := ioutil.ReadAll(dec.Buffered())
+					if err2 != nil {
+						errCh <- fmt.Errorf("failed to decode and failed to read buffered data: %w", multierror.Append(err, err2))
+					} else {
+						errCh <- fmt.Errorf("failed to decode log endpoint response as JSON: %q", buf)
+					}
 				}
 				return
 			}

api/go.mod

@@ -7,14 +7,17 @@ require (
 	github.com/gorilla/websocket v1.5.0
 	github.com/hashicorp/cronexpr v1.1.1
 	github.com/hashicorp/go-cleanhttp v0.5.2
+	github.com/hashicorp/go-multierror v1.1.1
 	github.com/hashicorp/go-rootcerts v1.0.2
 	github.com/mitchellh/go-testing-interface v1.14.1
 	github.com/mitchellh/mapstructure v1.5.0
-	github.com/shoenig/test v0.5.2
+	github.com/shoenig/test v0.6.0
+	golang.org/x/exp v0.0.0-20230108222341-4b8118a2686a
 )
 
 require (
 	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/hashicorp/errwrap v1.0.0 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/stretchr/testify v1.8.1 // indirect
 )

api/go.sum

@@ -9,8 +9,12 @@ github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWm
 github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/hashicorp/cronexpr v1.1.1 h1:NJZDd87hGXjoZBdvyCF9mX4DCq5Wy7+A/w+A7q0wn6c=
 github.com/hashicorp/cronexpr v1.1.1/go.mod h1:P4wA0KBl9C5q2hABiMO7cp6jcIg96CDh1Efb3g1PWA4=
+github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
 github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
+github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
+github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
 github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc=
 github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
@@ -21,15 +25,17 @@ github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyua
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/shoenig/test v0.5.2 h1:ELZ7qZ/6CPrT71PXrSe2TFzLs4/cGCqqU5lZ5RhZ+B8=
-github.com/shoenig/test v0.5.2/go.mod h1:xYtyGBC5Q3kzCNyJg/SjgNpfAa2kvmgA0i5+lQso8x0=
+github.com/shoenig/test v0.6.0 h1:rU0ymLHmCRqz14gABce/DzYryKU+uaWqobCBvAY6DtU=
+github.com/shoenig/test v0.6.0/go.mod h1:xYtyGBC5Q3kzCNyJg/SjgNpfAa2kvmgA0i5+lQso8x0=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+golang.org/x/exp v0.0.0-20230108222341-4b8118a2686a h1:tlXy25amD5A7gOfbXdqCGN5k8ESEed/Ee1E5RcrYnqU=
+golang.org/x/exp v0.0.0-20230108222341-4b8118a2686a/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

api/internal/testutil/server.go

@@ -24,7 +24,6 @@ import (
 	"github.com/hashicorp/go-cleanhttp"
 	"github.com/hashicorp/nomad/api/internal/testutil/discover"
 	testing "github.com/mitchellh/go-testing-interface"
-	"github.com/shoenig/test"
 	"github.com/shoenig/test/must"
 	"github.com/shoenig/test/portal"
 	"github.com/shoenig/test/wait"
@@ -286,7 +285,7 @@ func (s *TestServer) waitForAPI() {
 		}
 		return nil
 	}
-	test.Wait(s.t,
+	must.Wait(s.t,
 		wait.InitialSuccess(
 			wait.ErrorFunc(f),
 			wait.Timeout(10*time.Second),
@@ -313,7 +312,7 @@ func (s *TestServer) waitForLeader() {
 		}
 		return nil
 	}
-	test.Wait(s.t,
+	must.Wait(s.t,
 		wait.InitialSuccess(
 			wait.ErrorFunc(f),
 			wait.Timeout(10*time.Second),
@@ -347,7 +346,7 @@ func (s *TestServer) waitForClient() {
 		}
 		return nil
 	}
-	test.Wait(s.t,
+	must.Wait(s.t,
 		wait.InitialSuccess(
 			wait.ErrorFunc(f),
 			wait.Timeout(10*time.Second),

client/allocrunner/consul_grpc_sock_hook.go

@@ -195,7 +195,6 @@ func (p *grpcSocketProxy) run(alloc *structs.Allocation) error {
 			return fmt.Errorf("error parsing Consul address %q: %v",
 				p.config.Addr, err)
 		}
-
 		destAddr = net.JoinHostPort(host, p.consulGRPCFallbackPort)
 	}
 

client/allocrunner/taskrunner/envoy_bootstrap_hook.go

@@ -44,27 +44,29 @@ const (
 )
 
 type consulTransportConfig struct {
-	HTTPAddr  string // required
-	Auth      string // optional, env CONSUL_HTTP_AUTH
-	SSL       string // optional, env CONSUL_HTTP_SSL
-	VerifySSL string // optional, env CONSUL_HTTP_SSL_VERIFY
-	CAFile    string // optional, arg -ca-file
-	CertFile  string // optional, arg -client-cert
-	KeyFile   string // optional, arg -client-key
-	Namespace string // optional, only consul Enterprise, env CONSUL_NAMESPACE
+	HTTPAddr   string // required
+	Auth       string // optional, env CONSUL_HTTP_AUTH
+	SSL        string // optional, env CONSUL_HTTP_SSL
+	VerifySSL  string // optional, env CONSUL_HTTP_SSL_VERIFY
+	GRPCCAFile string // optional, arg -grpc-ca-file
+	CAFile     string // optional, arg -ca-file
+	CertFile   string // optional, arg -client-cert
+	KeyFile    string // optional, arg -client-key
+	Namespace  string // optional, only consul Enterprise, env CONSUL_NAMESPACE
 	// CAPath (dir) not supported by Nomad's config object
 }
 
-func newConsulTransportConfig(consul *config.ConsulConfig) consulTransportConfig {
+func newConsulTransportConfig(cc *config.ConsulConfig) consulTransportConfig {
 	return consulTransportConfig{
-		HTTPAddr:  consul.Addr,
-		Auth:      consul.Auth,
-		SSL:       decodeTriState(consul.EnableSSL),
-		VerifySSL: decodeTriState(consul.VerifySSL),
-		CAFile:    consul.CAFile,
-		CertFile:  consul.CertFile,
-		KeyFile:   consul.KeyFile,
-		Namespace: consul.Namespace,
+		HTTPAddr:   cc.Addr,
+		Auth:       cc.Auth,
+		SSL:        decodeTriState(cc.EnableSSL),
+		VerifySSL:  decodeTriState(cc.VerifySSL),
+		GRPCCAFile: cc.GRPCCAFile,
+		CAFile:     cc.CAFile,
+		CertFile:   cc.CertFile,
+		KeyFile:    cc.KeyFile,
+		Namespace:  cc.Namespace,
 	}
 }
 
@@ -125,7 +127,7 @@ type envoyBootstrapHook struct {
 	// envoyBootstrapWaitTime is the total amount of time hook will wait for Consul
 	envoyBootstrapWaitTime time.Duration
 
-	// envoyBootstrapInitialGap is the initial wait gap when retyring
+	// envoyBootstrapInitialGap is the initial wait gap when retrying
 	envoyBoostrapInitialGap time.Duration
 
 	// envoyBootstrapMaxJitter is the maximum amount of jitter applied to retries
@@ -542,29 +544,19 @@ func (e envoyBootstrapArgs) args() []string {
 		"-bootstrap",
 	}
 
-	if v := e.gateway; v != "" {
-		arguments = append(arguments, "-gateway", v)
-	}
-
-	if v := e.siToken; v != "" {
-		arguments = append(arguments, "-token", v)
-	}
-
-	if v := e.consulConfig.CAFile; v != "" {
-		arguments = append(arguments, "-ca-file", v)
-	}
-
-	if v := e.consulConfig.CertFile; v != "" {
-		arguments = append(arguments, "-client-cert", v)
-	}
-
-	if v := e.consulConfig.KeyFile; v != "" {
-		arguments = append(arguments, "-client-key", v)
+	appendIfSet := func(param, value string) {
+		if value != "" {
+			arguments = append(arguments, param, value)
+		}
 	}
 
-	if v := e.namespace; v != "" {
-		arguments = append(arguments, "-namespace", v)
-	}
+	appendIfSet("-gateway", e.gateway)
+	appendIfSet("-token", e.siToken)
+	appendIfSet("-grpc-ca-file", e.consulConfig.GRPCCAFile)
+	appendIfSet("-ca-file", e.consulConfig.CAFile)
+	appendIfSet("-client-cert", e.consulConfig.CertFile)
+	appendIfSet("-client-key", e.consulConfig.KeyFile)
+	appendIfSet("-namespace", e.namespace)
 
 	return arguments
 }