Merge pull request #277 from ystia/bugfix/GH-274-backport

Backport of fix for issue deploying applications using a secured yorc/consul
ystia · Jan 29, 2019 · 4ce6e77 · 4ce6e77
2 parents ffa33a8 + d742d7b
commit 4ce6e77
Show file tree

Hide file tree

Showing 12 changed files with 163 additions and 178 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,7 @@
 
 ### BUG FIXES
 
+* Can't deploy applications using a secured yorc/consul ([GH-274](https://github.com/ystia/yorc/issues/274))
 * Unable to purge an application that appears in the list ([GH-238](https://github.com/ystia/yorc/issues/238))
 * K8S jobs namespace should not be removed if its provided ([GH-245](https://github.com/ystia/yorc/issues/245))
 

diff --git a/commands/server.go b/commands/server.go
@@ -85,6 +85,7 @@ var consulConfiguration = map[string]interface{}{
 	"consul.ssl":                    false,
 	"consul.ssl_verify":             true,
 	"consul.publisher_max_routines": config.DefaultConsulPubMaxRoutines,
+	"consul.tls_handshake_timeout":  config.DefaultConsulTLSHandshakeTimeout,
 }
 
 var terraformConfiguration = map[string]interface{}{
@@ -285,6 +286,7 @@ func setConfig() {
 	serverCmd.PersistentFlags().Bool("consul_ssl_verify", true, "Whether or not to disable certificate checking")
 
 	serverCmd.PersistentFlags().Int("consul_publisher_max_routines", config.DefaultConsulPubMaxRoutines, "Maximum number of parallelism used to store TOSCA definitions in Consul. If you increase the default value you may need to tweak the ulimit max open files. If set to 0 or less the default value will be used")
+	serverCmd.PersistentFlags().Duration("consul_tls_handshake_timeout", config.DefaultConsulTLSHandshakeTimeout, "Maximum duration to wait for a TLS handshake with Consul")
 
 	serverCmd.PersistentFlags().Bool("ansible_use_openssh", false, "Prefer OpenSSH over Paramiko a Python implementation of SSH (the default) to provision remote hosts")
 	serverCmd.PersistentFlags().Bool("ansible_debug", false, "Prints massive debug information from Ansible")

diff --git a/commands/server_test.go b/commands/server_test.go
@@ -172,16 +172,18 @@ func TestConfigFile(t *testing.T) {
 				JobsChecksPeriod:        15 * time.Second,
 			},
 			ConsulConfig: config.Consul{
-				Token:          "testToken",
-				Datacenter:     "testDC",
-				Address:        "http://127.0.0.1:8500",
-				Key:            "testKeyFile",
-				Cert:           "testCertFile",
-				CA:             "testCACert",
-				CAPath:         "testCAPath",
-				SSL:            true,
-				SSLVerify:      false,
-				PubMaxRoutines: 1234},
+				Token:               "testToken",
+				Datacenter:          "testDC",
+				Address:             "http://127.0.0.1:8500",
+				Key:                 "testKeyFile",
+				Cert:                "testCertFile",
+				CA:                  "testCACert",
+				CAPath:              "testCAPath",
+				SSL:                 true,
+				SSLVerify:           false,
+				PubMaxRoutines:      1234,
+				TLSHandshakeTimeout: 30 * time.Second,
+			},
 		},
 		{SubTestName: "config_structured",
 			FileName: "testdata/config_structured.yorc.json",
@@ -196,16 +198,17 @@ func TestConfigFile(t *testing.T) {
 				JobsChecksPeriod:        15 * time.Second,
 			},
 			ConsulConfig: config.Consul{
-				Token:          "testToken2",
-				Datacenter:     "testDC2",
-				Address:        "http://127.0.0.1:8502",
-				Key:            "testKeyFile2",
-				Cert:           "testCertFile2",
-				CA:             "testCACert2",
-				CAPath:         "testCAPath2",
-				SSL:            true,
-				SSLVerify:      false,
-				PubMaxRoutines: 4321,
+				Token:               "testToken2",
+				Datacenter:          "testDC2",
+				Address:             "http://127.0.0.1:8502",
+				Key:                 "testKeyFile2",
+				Cert:                "testCertFile2",
+				CA:                  "testCACert2",
+				CAPath:              "testCAPath2",
+				SSL:                 true,
+				SSLVerify:           false,
+				PubMaxRoutines:      4321,
+				TLSHandshakeTimeout: 51 * time.Second,
 			},
 		},
 	}
@@ -248,16 +251,17 @@ func TestAnsibleDefaultValues(t *testing.T) {
 func TestConsulDefaultValues(t *testing.T) {
 
 	expectedConsulConfig := config.Consul{
-		Token:          "anonymous",
-		Datacenter:     "dc1",
-		Address:        "",
-		Key:            "",
-		Cert:           "",
-		CA:             "",
-		CAPath:         "",
-		SSL:            false,
-		SSLVerify:      true,
-		PubMaxRoutines: config.DefaultConsulPubMaxRoutines,
+		Token:               "anonymous",
+		Datacenter:          "dc1",
+		Address:             "",
+		Key:                 "",
+		Cert:                "",
+		CA:                  "",
+		CAPath:              "",
+		SSL:                 false,
+		SSLVerify:           true,
+		PubMaxRoutines:      config.DefaultConsulPubMaxRoutines,
+		TLSHandshakeTimeout: config.DefaultConsulTLSHandshakeTimeout,
 	}
 
 	testResetConfig()
@@ -305,16 +309,17 @@ func TestAnsibleEnvVariables(t *testing.T) {
 func TestConsulEnvVariables(t *testing.T) {
 
 	expectedConsulConfig := config.Consul{
-		Token:          "testEnvToken",
-		Datacenter:     "testEnvDC",
-		Address:        "testEnvAddress",
-		Key:            "testEnvKey",
-		Cert:           "testEnvCert",
-		CA:             "testEnvCA",
-		CAPath:         "testEnvCAPath",
-		SSL:            true,
-		SSLVerify:      false,
-		PubMaxRoutines: 125,
+		Token:               "testEnvToken",
+		Datacenter:          "testEnvDC",
+		Address:             "testEnvAddress",
+		Key:                 "testEnvKey",
+		Cert:                "testEnvCert",
+		CA:                  "testEnvCA",
+		CAPath:              "testEnvCAPath",
+		SSL:                 true,
+		SSLVerify:           false,
+		PubMaxRoutines:      125,
+		TLSHandshakeTimeout: 11 * time.Second,
 	}
 
 	// Set Consul configuration environment variables
@@ -328,6 +333,7 @@ func TestConsulEnvVariables(t *testing.T) {
 	os.Setenv("YORC_CONSUL_SSL", strconv.FormatBool(expectedConsulConfig.SSL))
 	os.Setenv("YORC_CONSUL_SSL_VERIFY", strconv.FormatBool(expectedConsulConfig.SSLVerify))
 	os.Setenv("YORC_CONSUL_PUBLISHER_MAX_ROUTINES", strconv.Itoa(expectedConsulConfig.PubMaxRoutines))
+	os.Setenv("YORC_CONSUL_TLS_HANDSHAKE_TIMEOUT", expectedConsulConfig.TLSHandshakeTimeout.String())
 
 	testResetConfig()
 	setConfig()
@@ -347,6 +353,7 @@ func TestConsulEnvVariables(t *testing.T) {
 	os.Unsetenv("YORC_CONSUL_SSL")
 	os.Unsetenv("YORC_CONSUL_SSL_VERIFY")
 	os.Unsetenv("YORC_CONSUL_PUBLISHER_MAX_ROUTINES")
+	os.Unsetenv("YORC_CONSUL_TLS_HANDSHAKE_TIMEOUT")
 }
 
 // Tests Ansible configuration using persistent flags
@@ -389,16 +396,17 @@ func TestAnsiblePersistentFlags(t *testing.T) {
 func TestConsulPersistentFlags(t *testing.T) {
 
 	expectedConsulConfig := config.Consul{
-		Token:          "testPFlagToken",
-		Datacenter:     "testPFlagDC",
-		Address:        "testPFlagAddress",
-		Key:            "testPFlagKey",
-		Cert:           "testPFlagCert",
-		CA:             "testPFlagCA",
-		CAPath:         "testEnvCAPath",
-		SSL:            true,
-		SSLVerify:      false,
-		PubMaxRoutines: 123,
+		Token:               "testPFlagToken",
+		Datacenter:          "testPFlagDC",
+		Address:             "testPFlagAddress",
+		Key:                 "testPFlagKey",
+		Cert:                "testPFlagCert",
+		CA:                  "testPFlagCA",
+		CAPath:              "testEnvCAPath",
+		SSL:                 true,
+		SSLVerify:           false,
+		PubMaxRoutines:      123,
+		TLSHandshakeTimeout: 12 * time.Second,
 	}
 
 	consulPFlagConfiguration := map[string]string{
@@ -412,6 +420,7 @@ func TestConsulPersistentFlags(t *testing.T) {
 		"consul_ssl":                    strconv.FormatBool(expectedConsulConfig.SSL),
 		"consul_ssl_verify":             strconv.FormatBool(expectedConsulConfig.SSLVerify),
 		"consul_publisher_max_routines": strconv.Itoa(expectedConsulConfig.PubMaxRoutines),
+		"consul_tls_handshake_timeout":  expectedConsulConfig.TLSHandshakeTimeout.String(),
 	}
 
 	testResetConfig()

diff --git a/commands/testdata/config_flat.yorc.json b/commands/testdata/config_flat.yorc.json
@@ -24,6 +24,7 @@
   "consul_ca_path": "testCAPath",
   "consul_ssl": true,
   "consul_ssl_verify": false,
+  "consul_tls_handshake_timeout": "30s",
   "telemetry":{
     "statsd_address": "127.0.0.1:8125",
     "expose_prometheus_endpoint": true

diff --git a/commands/testdata/config_structured.yorc.json b/commands/testdata/config_structured.yorc.json
@@ -26,7 +26,8 @@
     "ca_cert": "testCACert2",
     "ca_path": "testCAPath2",
     "ssl": true,
-    "ssl_verify": false
+    "ssl_verify": false,
+    "tls_handshake_timeout": "51s"
   }, 
    "telemetry":{
     "statsd_address": "127.0.0.1:8125",

diff --git a/config/config.go b/config/config.go
@@ -32,6 +32,11 @@ import (
 // See consulutil package for more details
 const DefaultConsulPubMaxRoutines int = 500
 
+// DefaultConsulTLSHandshakeTimeout is the default maximum duration to wait for
+// a TLS handshake.
+// See consulutil package for more details
+const DefaultConsulTLSHandshakeTimeout = 50 * time.Second
+
 // DefaultWorkersNumber is the default number of workers in the Yorc server
 const DefaultWorkersNumber int = 30
 
@@ -128,16 +133,17 @@ type Ansible struct {
 
 // Consul configuration
 type Consul struct {
-	Token          string `yaml:"token,omitempty" mapstructure:"token"`
-	Datacenter     string `yaml:"datacenter,omitempty" mapstructure:"datacenter"`
-	Address        string `yaml:"address,omitempty" mapstructure:"address"`
-	Key            string `yaml:"key_file,omitempty" mapstructure:"key_file"`
-	Cert           string `yaml:"cert_file,omitempty" mapstructure:"cert_file"`
-	CA             string `yaml:"ca_cert,omitempty" mapstructure:"ca_cert"`
-	CAPath         string `yaml:"ca_path,omitempty" mapstructure:"ca_path"`
-	SSL            bool   `yaml:"ssl,omitempty" mapstructure:"ssl"`
-	SSLVerify      bool   `yaml:"ssl_verify,omitempty" mapstructure:"ssl_verify"`
-	PubMaxRoutines int    `yaml:"publisher_max_routines,omitempty" mapstructure:"publisher_max_routines"`
+	Token               string        `yaml:"token,omitempty" mapstructure:"token"`
+	Datacenter          string        `yaml:"datacenter,omitempty" mapstructure:"datacenter"`
+	Address             string        `yaml:"address,omitempty" mapstructure:"address"`
+	Key                 string        `yaml:"key_file,omitempty" mapstructure:"key_file"`
+	Cert                string        `yaml:"cert_file,omitempty" mapstructure:"cert_file"`
+	CA                  string        `yaml:"ca_cert,omitempty" mapstructure:"ca_cert"`
+	CAPath              string        `yaml:"ca_path,omitempty" mapstructure:"ca_path"`
+	SSL                 bool          `yaml:"ssl,omitempty" mapstructure:"ssl"`
+	SSLVerify           bool          `yaml:"ssl_verify,omitempty" mapstructure:"ssl_verify"`
+	PubMaxRoutines      int           `yaml:"publisher_max_routines,omitempty" mapstructure:"publisher_max_routines"`
+	TLSHandshakeTimeout time.Duration `yaml:"tls_handshake_timeout,omitempty" mapstructure:"tls_handshake_timeout"`
 }
 
 // Telemetry holds the configuration for the telemetry service

diff --git a/config/consul.go b/config/consul.go
@@ -15,11 +15,12 @@
 package config
 
 import (
+	"sync"
+	"time"
+
 	"github.com/hashicorp/consul/api"
 	"github.com/pkg/errors"
 	"github.com/ystia/yorc/log"
-	"sync"
-	"time"
 )
 
 var consulClient *api.Client
@@ -45,6 +46,7 @@ func (cfg Configuration) buildConsulClientInstance() (*api.Client, error) {
 	consulCustomConfig.Transport.MaxIdleConnsPerHost = cfg.Consul.PubMaxRoutines
 	consulCustomConfig.Transport.MaxIdleConns = cfg.Consul.PubMaxRoutines
 	consulCustomConfig.Transport.IdleConnTimeout = 10 * time.Second
+	consulCustomConfig.Transport.TLSHandshakeTimeout = cfg.Consul.TLSHandshakeTimeout
 	log.Debugf("consul http Transport config: %+v", consulCustomConfig.Transport)
 	if cfg.Consul.Address != "" {
 		consulCustomConfig.Address = cfg.Consul.Address

diff --git a/doc/configuration.rst b/doc/configuration.rst
@@ -98,6 +98,10 @@ Globals Command-line options
 
   * ``--consul_ssl_verify``: If set to false, disable Consul certificate checking (true by default is ssl enabled).
 
+.. _option_consul_tls_handshake_timeout_cmd:
+
+  * ``--consul_tls_handshake_timeout``: Maximum duration to wait for a TLS handshake with Consul, the default is ``50s``.
+
 .. _option_terraform_plugins_dir_cmd:
 
   * ``--terraform_plugins_dir``: Specify the directory where to find Terraform pre-installed providers plugins. If not specified, required plugins will be downloaded during deployment. See https://www.terraform.io/guides/running-terraform-in-automation.html#pre-installed-plugins for more information.
@@ -475,6 +479,10 @@ All available configuration options for Consul are:
 
   * ``ssl_verify``: Equivalent to :ref:`--consul_ssl_verify <option_consul_ssl_verify_cmd>` command-line flag.
 
+.. _option_consul_tls_handshake_timeout:
+
+  * ``tls_handshake_timeout``: Equivalent to :ref:`--consul_tls_handshake_timeout <option_consul_tls_handshake_timeout_cmd>` command-line flag.
+
 .. _option_pub_routines_cfg:
 
   * ``publisher_max_routines``: Equivalent to :ref:`--consul_publisher_max_routines <option_pub_routines_cmd>` command-line flag.
@@ -723,6 +731,11 @@ Environment variables
 
   * ``YORC_CONSUL_SSL_VERIFY``: Equivalent to :ref:`--consul_ssl_verify <option_consul_ssl_verify_cmd>` command-line flag.
 
+.. _option_consul_tls_handshake_timeout_env:
+
+  * ``YORC_CONSUL_TLS_HANDSHAKE_TIMEOUT``: Equivalent to :ref:`--consul_tls_handshake_timeout <option_consul_tls_handshake_timeout_cmd>` command-line flag.
+
+
 .. _option_pub_routines_env:
 
   * ``YORC_CONSUL_PUBLISHER_MAX_ROUTINES``: Equivalent to :ref:`--consul_publisher_max_routines <option_pub_routines_cmd>` command-line flag.

diff --git a/prov/terraform/aws/generator.go b/prov/terraform/aws/generator.go
@@ -48,35 +48,9 @@ func (g *awsGenerator) GenerateTerraformInfraForNode(ctx context.Context, cfg co
 
 	infrastructure := commons.Infrastructure{}
 
-	consulAddress := "127.0.0.1:8500"
-	if cfg.Consul.Address != "" {
-		consulAddress = cfg.Consul.Address
-	}
-	consulScheme := "http"
-	if cfg.Consul.SSL {
-		consulScheme = "https"
-	}
-	consulCA := ""
-	if cfg.Consul.CA != "" {
-		consulCA = cfg.Consul.CA
-	}
-	consulKey := ""
-	if cfg.Consul.Key != "" {
-		consulKey = cfg.Consul.Key
-	}
-	consulCert := ""
-	if cfg.Consul.Cert != "" {
-		consulCert = cfg.Consul.Cert
-	}
-
 	// Remote Configuration for Terraform State to store it in the Consul KV store
-	infrastructure.Terraform = map[string]interface{}{
-		"backend": map[string]interface{}{
-			"consul": map[string]interface{}{
-				"path": terraformStateKey,
-			},
-		},
-	}
+	infrastructure.Terraform = commons.GetBackendConfiguration(terraformStateKey, cfg)
+
 	cmdEnv := []string{
 		fmt.Sprintf("AWS_ACCESS_KEY_ID=%s", cfg.Infrastructures[infrastructureName].GetString("access_key")),
 		fmt.Sprintf("AWS_SECRET_ACCESS_KEY=%s", cfg.Infrastructures[infrastructureName].GetString("secret_key")),
@@ -87,14 +61,7 @@ func (g *awsGenerator) GenerateTerraformInfraForNode(ctx context.Context, cfg co
 			"region":  cfg.Infrastructures[infrastructureName].GetString("region"),
 			"version": cfg.Terraform.AWSPluginVersionConstraint,
 		},
-		"consul": map[string]interface{}{
-			"version":   cfg.Terraform.ConsulPluginVersionConstraint,
-			"address":   consulAddress,
-			"scheme":    consulScheme,
-			"ca_file":   consulCA,
-			"cert_file": consulCert,
-			"key_file":  consulKey,
-		},
+		"consul": commons.GetConsulProviderfiguration(cfg),
 		"null": map[string]interface{}{
 			"version": commons.NullPluginVersionConstraint,
 		},