Skip to content

Commit

Permalink
simplify and clean up tf generation
Browse files Browse the repository at this point in the history 
namely: don't pass arbitrary open-ended stuff (env, command) to the tf templates
  • Loading branch information
@nfi-hashicorp
nfi-hashicorp committed Aug 30, 2023
1 parent f13b60e commit 7324ad6
Show file tree
Hide file tree
Showing 8 changed files with 137 additions and 205 deletions.
8 changes: 4 additions & 4 deletions testing/deployer/sprawl/internal/tfgen/agent.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,14 @@ import (
"github.com/hashicorp/consul/testing/deployer/topology"
)

func (g *Generator) generateAgentHCL(node *topology.Node) (string, error) {
func (g *Generator) generateAgentHCL(node *topology.Node) string {
if !node.IsAgent() {
return "", fmt.Errorf("not an agent")
panic("generateAgentHCL only applies to agents")
}

cluster, ok := g.topology.Clusters[node.Cluster]
if !ok {
return "", fmt.Errorf("no such cluster: %s", node.Cluster)
panic(fmt.Sprintf("no such cluster: %s", node.Cluster))
}

var b HCLBuilder
Expand Down Expand Up @@ -167,7 +167,7 @@ func (g *Generator) generateAgentHCL(node *topology.Node) (string, error) {
}
}

return b.String(), nil
return b.String()
}

type HCLBuilder struct {
Expand Down
251 changes: 77 additions & 174 deletions testing/deployer/sprawl/internal/tfgen/nodes.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,6 @@ package tfgen

import (
"fmt"
"sort"
"strconv"

"github.com/hashicorp/consul/testing/deployer/topology"
)
Expand All @@ -21,40 +19,6 @@ type terraformPod struct {
DockerNetworkName string
}

type terraformConsulAgent struct {
terraformPod
ImageResource string
HCL string
EnterpriseLicense string
Env []string
}

type terraformMeshGatewayService struct {
terraformPod
EnvoyImageResource string
Service *topology.Service
Command []string
}

type terraformAgentfulService struct {
terraformPod
AppImageResource string
EnvoyImageResource string
Service *topology.Service
Env []string
Command []string
EnvoyCommand []string
}

type terraformAgentlessService struct {
terraformPod
AppImageResource string
DataplaneImageResource string // agentless
Service *topology.Service
Env []string
Command []string
}

func (g *Generator) generateNodeContainers(
step Step,
cluster *topology.Cluster,
Expand Down Expand Up @@ -89,161 +53,100 @@ func (g *Generator) generateNodeContainers(
}
pod.DockerNetworkName = net.DockerName

var (
containers []Resource
)
containers := []Resource{}

if node.IsAgent() {
agentHCL, err := g.generateAgentHCL(node)
if err != nil {
return nil, err
}

agent := terraformConsulAgent{
terraformPod: pod,
ImageResource: DockerImageResourceName(node.Images.Consul),
HCL: agentHCL,
EnterpriseLicense: g.license,
Env: node.AgentEnv,
}

switch {
case node.IsServer() && step.StartServers(),
!node.IsServer() && step.StartAgents():
containers = append(containers, Eval(tfConsulT, &agent))
containers = append(containers, Eval(tfConsulT, struct {
terraformPod
ImageResource string
HCL string
EnterpriseLicense string
}{
terraformPod: pod,
ImageResource: DockerImageResourceName(node.Images.Consul),
HCL: g.generateAgentHCL(node),
EnterpriseLicense: g.license,
}))
}
}

svcContainers := []Resource{}
for _, svc := range node.SortedServices() {
token := g.sec.ReadServiceToken(node.Cluster, svc.ID)
switch {

case svc.IsMeshGateway && node.IsDataplane():
panic("NOT READY YET")
case svc.IsMeshGateway && !node.IsDataplane():
gw := terraformMeshGatewayService{
terraformPod: pod,
EnvoyImageResource: DockerImageResourceName(node.Images.EnvoyConsulImage()),
Service: svc,
Command: []string{
"consul", "connect", "envoy",
"-register",
"-mesh-gateway",
},
}
if token := g.sec.ReadServiceToken(node.Cluster, svc.ID); token != "" {
gw.Command = append(gw.Command, "-token", token)
}
if cluster.Enterprise {
gw.Command = append(gw.Command,
"-partition",
svc.ID.Partition,
)
}
gw.Command = append(gw.Command,
"-address",
`{{ GetInterfaceIP \"eth0\" }}:`+strconv.Itoa(svc.Port),
"-wan-address",
`{{ GetInterfaceIP \"eth1\" }}:`+strconv.Itoa(svc.Port),
"-grpc-addr", "http://127.0.0.1:8502",
"-admin-bind",
// for demo purposes
"0.0.0.0:"+strconv.Itoa(svc.EnvoyAdminPort),
"--",
"-l",
"trace",
)
if step.StartServices() {
containers = append(containers, Eval(tfMeshGatewayT, &gw))
}

case !svc.IsMeshGateway && !node.IsDataplane():
tfsvc := terraformAgentfulService{
terraformPod: pod,
AppImageResource: DockerImageResourceName(svc.Image),
Service: svc,
Command: svc.Command,
}
tfsvc.Env = append(tfsvc.Env, svc.Env...)
if step.StartServices() {
containers = append(containers, Eval(tfAppT, &tfsvc))
}

if svc.DisableServiceMesh {
case svc.IsMeshGateway && !node.IsDataplane():
tfin := struct {
terraformPod
ImageResource string
Enterprise bool
Service *topology.Service
Token string
}{
terraformPod: pod,
Enterprise: cluster.Enterprise,
ImageResource: DockerImageResourceName(node.Images.EnvoyConsulImage()),
Service: svc,
Token: token,
}
svcContainers = append(svcContainers, Eval(tfMeshGatewayT, &tfin))

case !svc.IsMeshGateway:
svcContainers = append(svcContainers, Eval(tfAppT, struct {
terraformPod
ImageResource string
Service *topology.Service
}{
terraformPod: pod,
ImageResource: DockerImageResourceName(svc.Image),
Service: svc,
}))

if !svc.DisableServiceMesh {
break
}

tfsvc.EnvoyImageResource = DockerImageResourceName(node.Images.EnvoyConsulImage())
tfsvc.EnvoyCommand = []string{
"consul", "connect", "envoy",
"-sidecar-for", svc.ID.Name,
}
if cluster.Enterprise {
tfsvc.EnvoyCommand = append(tfsvc.EnvoyCommand,
"-partition",
svc.ID.Partition,
"-namespace",
svc.ID.Namespace,
)
}
if token := g.sec.ReadServiceToken(node.Cluster, svc.ID); token != "" {
tfsvc.EnvoyCommand = append(tfsvc.EnvoyCommand, "-token", token)
}
tfsvc.EnvoyCommand = append(tfsvc.EnvoyCommand,
"-grpc-addr", "http://127.0.0.1:8502",
"-admin-bind",
// for demo purposes
"0.0.0.0:"+strconv.Itoa(svc.EnvoyAdminPort),
"--",
"-l",
"trace",
)
if step.StartServices() {
sort.Strings(tfsvc.Env)
containers = append(containers, Eval(tfAppSidecarT, &tfsvc))
}

case !svc.IsMeshGateway && node.IsDataplane():
tfsvc := terraformAgentlessService{
terraformPod: pod,
AppImageResource: DockerImageResourceName(svc.Image),
Service: svc,
Command: svc.Command,
}
tfsvc.Env = append(tfsvc.Env, svc.Env...)
if step.StartServices() {
containers = append(containers, Eval(tfAppT, &tfsvc))
}

setenv := func(k, v string) {
tfsvc.Env = append(tfsvc.Env, k+"="+v)
}

if svc.DisableServiceMesh {
break
}
switch node.IsDataplane() {
case false:
svcContainers = append(svcContainers, Eval(tfAppSidecarT, struct {
terraformPod
ImageResource string
Service *topology.Service
Token string
Enterprise bool
// TODO: we used to use the env from the app container, doubt we need it, seems leaky
// Env []string
}{
terraformPod: pod,
ImageResource: DockerImageResourceName(node.Images.EnvoyConsulImage()),
Service: svc,
Token: token,
Enterprise: cluster.Enterprise,
}))

case true:
svcContainers = append(svcContainers, Eval(tfAppDataplaneT, &struct {
terraformPod
ImageResource string
Token string
}{
terraformPod: pod,
ImageResource: DockerImageResourceName(node.Images.LocalDataplaneImage()),
Token: token,
}))
}

default:
panic(fmt.Sprintf("unhandled node kind/dataplane type: %#v", svc))
}

tfsvc.DataplaneImageResource = DockerImageResourceName(node.Images.LocalDataplaneImage())
setenv("DP_CONSUL_ADDRESSES", "server."+node.Cluster+"-consulcluster.lan")
setenv("DP_SERVICE_NODE_NAME", node.PodName())
setenv("DP_PROXY_SERVICE_ID", svc.ID.Name+"-sidecar-proxy")
if cluster.Enterprise {
setenv("DP_SERVICE_NAMESPACE", svc.ID.Namespace)
setenv("DP_SERVICE_PARTITION", svc.ID.Partition)
}
if token := g.sec.ReadServiceToken(node.Cluster, svc.ID); token != "" {
setenv("DP_CREDENTIAL_TYPE", "static")
setenv("DP_CREDENTIAL_STATIC_TOKEN", token)
}
setenv("DP_ENVOY_ADMIN_BIND_ADDRESS", "0.0.0.0") // for demo purposes
setenv("DP_ENVOY_ADMIN_BIND_PORT", "19000")
setenv("DP_LOG_LEVEL", "trace")
setenv("DP_CA_CERTS", "/consul/config/certs/consul-agent-ca.pem")
setenv("DP_CONSUL_GRPC_PORT", "8503")
setenv("DP_TLS_SERVER_NAME", "server."+node.Datacenter+".consul")
if step.StartServices() {
sort.Strings(tfsvc.Env)
containers = append(containers, Eval(tfAppDataplaneT, &tfsvc))
}
if step.StartServices() {
containers = append(containers, svcContainers...)
}
}

Expand Down
25 changes: 20 additions & 5 deletions testing/deployer/sprawl/internal/tfgen/templates/container-app-dataplane.tf.tmpl
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
resource "docker_container" "{{.Node.DockerName}}-{{.Service.ID.TFString}}-sidecar" {
name = "{{.Node.DockerName}}-{{.Service.ID.TFString}}-sidecar"
network_mode = "container:${docker_container.{{.PodName}}.id}"
image = docker_image.{{.DataplaneImageResource}}.latest
restart = "on-failure"
image = docker_image.{{.ImageResource}}.latest
restart = "on-failure"

{{- range $k, $v := .Labels }}
labels {
Expand All @@ -18,9 +18,24 @@ resource "docker_container" "{{.Node.DockerName}}-{{.Service.ID.TFString}}-sidec
}

env = [
{{- range .Env }}
"{{.}}",
{{- end}}
"DP_CONSUL_ADDRESSES=server.{{.Node.Cluster}}-consulcluster.lan",
"DP_SERVICE_NODE_NAME", {{.Node.PodName}}",
"DP_PROXY_SERVICE_ID", svc.ID.Name+"-sidecar-proxy"),
{{ if .Enterprise }}
"DP_SERVICE_NAMESPACE={{.Service.ID.Namespace}}",
"DP_SERVICE_PARTITION={{.Service.ID.Partition}}",
{{ end }}
{{ if .Token }}
"DP_CREDENTIAL_TYPE", "static"),
"DP_CREDENTIAL_STATIC_TOKEN", token),
{{ end }}
// for demo purposes
"DP_ENVOY_ADMIN_BIND_ADDRESS=0.0.0.0",
"DP_ENVOY_ADMIN_BIND_PORT=19000",
"DP_LOG_LEVEL=trace",
"DP_CA_CERTS=/consul/config/certs/consul-agent-ca.pem",
"DP_CONSUL_GRPC_PORT=8503",
"DP_TLS_SERVER_NAME=server.{{.Node.Datacenter}}.consul",
]

command = [
Expand Down
24 changes: 15 additions & 9 deletions testing/deployer/sprawl/internal/tfgen/templates/container-app-sidecar.tf.tmpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,21 @@ resource "docker_container" "{{.Node.DockerName}}-{{.Service.ID.TFString}}-sidec
read_only = true
}

env = [
{{- range .Env }}
"{{.}}",
{{- end}}
]

command = [
{{- range .EnvoyCommand }}
"{{.}}",
{{- end }}
"consul", "connect", "envoy",
"-sidecar-for={{.Service.ID.Name}}",
"-grpc-addr=http://127.0.0.1:8502",
// for demo purposes (TODO: huh?)
"-admin-bind=0.0.0.0:{{ .Service.EnvoyAdminPort}},
{{if .Enterprise}}
"-partition", svc.ID.Partition,
"-namespace", svc.ID.Namespace,
{{end}}
{{if .Token }}
"-token={{.Token}}
{{end}}
"--",
"-l",
"trace",
]
}
Loading

0 comments on commit 7324ad6

Please sign in to comment.