Merge with develop

.travis.yml

@@ -20,7 +20,7 @@ env:
 
 install:
   - make tools
-  - pip install --user --upgrade sphinx==1.7.9 semantic-version requests urllib3[secure]
+  - pip install --user --upgrade sphinx==1.8.1 semantic-version requests urllib3[secure]==1.23
 
 before_script:
   - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64  > ./cc-test-reporter

CHANGELOG.md

@@ -6,6 +6,10 @@
 
 * Support GCE Block storages. ([GH-82](https://github.com/ystia/yorc/issues/81))
 
+### ENHANCEMENTS
+
+* Concurrent workflows and custom commands executions are now allowed except when a deployment/undeployment/scaling operation is in progress ([GH-182](https://github.com/ystia/yorc/issues/182))
+
 ## 3.1.0-M4 (October 08, 2018)
 
 ### DEPENDENCIES
@@ -39,6 +43,7 @@
 ### BUG FIXES
 
 * REQ_TARGET keyword into TOSCA functions was broken. This was introduced during the upgrade to Alien4Cloud 2.0 that changed how requirements definition on node templates ([GH-159](https://github.com/ystia/yorc/issues/159))
+* Parse Alien specific way of defining properties on relationships ([GH-155](https://github.com/ystia/yorc/issues/155))
 
 ## 3.1.0-M2 (August 24, 2018)
 

commands/server.go

@@ -63,6 +63,7 @@ var ansibleConfiguration = map[string]interface{}{
 	"ansible.keep_operation_remote_path": config.DefaultKeepOperationRemotePath,
 	"ansible.archive_artifacts":          config.DefaultArchiveArtifacts,
 	"ansible.cache_facts":                config.DefaultCacheFacts,
+	"ansible.keep_generated_recipes":     false,
 }
 
 var consulConfiguration = map[string]interface{}{
@@ -84,6 +85,7 @@ var terraformConfiguration = map[string]interface{}{
 	"terraform.aws_plugin_version_constraint":       tfAWSPluginVersionConstraint,
 	"terraform.google_plugin_version_constraint":    tfGooglePluginVersionConstraint,
 	"terraform.openstack_plugin_version_constraint": tfOpenStackPluginVersionConstraint,
+	"terraform.keep_generated_files":                false,
 }
 
 var cfgFile string
@@ -252,6 +254,10 @@ func setConfig() {
 	serverCmd.PersistentFlags().Bool("keep_operation_remote_path", config.DefaultKeepOperationRemotePath, "Define wether the path created to store artifacts on the nodes will be removed at the end of workflow executions.")
 	serverCmd.PersistentFlags().Bool("ansible_archive_artifacts", config.DefaultArchiveArtifacts, "Define wether artifacts should be ./archived before being copied on remote nodes (requires tar to be installed on remote nodes).")
 	serverCmd.PersistentFlags().Bool("ansible_cache_facts", config.DefaultCacheFacts, "Define wether Ansible facts (useful variables about remote hosts) should be cached.")
+	serverCmd.PersistentFlags().Bool("ansible_keep_generated_recipes", false, "Define if Yorc should not delete generated Ansible recipes")
+
+	//Flags definition for Terraform
+	serverCmd.PersistentFlags().Bool("terraform_keep_generated_files", false, "Define if Yorc should not delete generated Terraform infrastructures files")
 
 	//Flags definition for Terraform
 	serverCmd.PersistentFlags().StringP("terraform_plugins_dir", "", "", "The directory where to find Terraform plugins")

config/config.go

@@ -115,6 +115,7 @@ type Ansible struct {
 	ConnectionRetries       int              `mapstructure:"connection_retries"`
 	OperationRemoteBaseDir  string           `mapstructure:"operation_remote_base_dir"`
 	KeepOperationRemotePath bool             `mapstructure:"keep_operation_remote_path"`
+	KeepGeneratedRecipes    bool             `mapstructure:"keep_generated_recipes"`
 	ArchiveArtifacts        bool             `mapstructure:"archive_artifacts"`
 	CacheFacts              bool             `mapstructure:"cache_facts"`
 	HostedOperations        HostedOperations `mapstructure:"hosted_operations"`
@@ -151,6 +152,7 @@ type Terraform struct {
 	AWSPluginVersionConstraint       string `mapstructure:"aws_plugin_version_constraint"`
 	GooglePluginVersionConstraint    string `mapstructure:"google_plugin_version_constraint"`
 	OpenStackPluginVersionConstraint string `mapstructure:"openstack_plugin_version_constraint"`
+	KeepGeneratedFiles               bool   `mapstructure:"keep_generated_files"`
 }
 
 // DynamicMap allows to store configuration parameters that are not known in advance.

doc/configuration.rst

@@ -46,6 +46,10 @@ Globals Command-line options
 
   * ``--ansible_archive_artifacts``: If set to true, archives operation bash/python scripts locally, copies this archive and unarchives it on remote hosts (requires tar to be installed on remote hosts), to avoid multiple time consuming remote copy operations of individual scripts (false by default: no archive).
 
+.. _option_ansible_keep_generated_recipes_cmd:
+
+  * ``--ansible_keep_generated_recipes``: If set to true, generated Ansible recipes on Yorc server are not deleted. (false by default: generated recipes are deleted).
+
 .. _option_operation_remote_base_dir_cmd:
 
   * ``--operation_remote_base_dir``: Specify an alternative working directory for Ansible on provisioned Compute.
@@ -109,6 +113,9 @@ Globals Command-line options
 .. _option_terraform_openstack_plugin_version_constraint_cmd:
 
   * ``--terraform_openstack_plugin_version_constraint``: Specify the Terraform OpenStack plugin version constraint. Default one compatible with our source code is ``"~> 1.9"``. If you choose another, it's at your own risk. See https://www.terraform.io/docs/configuration/providers.html#provider-versions for more information.
+.. _option_terraform_keep_generated_files_cmd:
+
+  * ``--terraform_keep_generated_files``: If set to true, generated Terraform infrastructures files on Yorc server are not deleted. (false by default: generated files are deleted).
 
 .. _option_pub_routines_cmd:
 
@@ -336,6 +343,10 @@ All available configuration options for Ansible are:
 
   * ``keep_operation_remote_path``: Equivalent to :ref:`--keep_operation_remote_path <option_keep_remote_path_cmd>` command-line flag.
 
+.. _option_ansible_keep_generated_recipes_cfg:
+
+  * ``keep_generated_recipes``: Equivalent to :ref:`--ansible_keep_generated_recipes <option_ansible_keep_generated_recipes_cmd>` command-line flag.
+
 .. _option_ansible_sandbox_hosted_ops_cfg:
 
   * ``hosted_operations``: This is a complex structure that allow to define the behavior of a Yorc server when it executes an hosted operation.
@@ -478,7 +489,7 @@ Below is an example of configuration file with Terraform configuration options.
       }
     }
 
-All available configuration options for Consul are:
+All available configuration options for Terraform are:
 
 .. _option_plugins_dir_cfg:
 
@@ -500,6 +511,11 @@ All available configuration options for Consul are:
 
   * ``openstack_plugin_version_constraint``: Equivalent to :ref:`--terraform_openstack_plugin_version_constraint <option_terraform_openstack_plugin_version_constraint_cmd>` command-line flag.
 
+.. _option_terraform_keep_generated_files_cfg:
+
+  * ``keep_generated_files``: Equivalent to :ref:`--terraform_keep_generated_files <option_terraform_keep_generated_files_cmd>` command-line flag.
+
+
 .. _yorc_config_file_telemetry_section:
 
 Telemetry configuration
@@ -647,6 +663,10 @@ Environment variables
 
   * ``YORC_ANSIBLE_ARCHIVE_ARTIFACTS``: Equivalent to :ref:`--ansible_archive_artifacts <option_ansible_archive_artifacts_cmd>` command-line flag.
 
+.. _option_ansible_keep_generated_recipes_env:
+
+  * ``YORC_ANSIBLE_KEEP_GENERATED_RECIPES``: Equivalent to :ref:`--ansible_keep_generated_recipes <option_ansible_keep_generated_recipes_cmd>` command-line flag.
+
 .. _option_operation_remote_base_dir_env:
 
   * ``YORC_OPERATION_REMOTE_BASE_DIR``: Equivalent to :ref:`--operation_remote_base_dir <option_operation_remote_base_dir_cmd>` command-line flag.
@@ -751,14 +771,6 @@ Environment variables
 
   * ``YORC_LOG``: If set to ``1`` or ``DEBUG``, enables debug logging for Yorc.
 
-.. _option_aws_access_key:
-
-  * ``YORC_INFRA_AWS_ACCESS_KEY``: The AWS access key credential.
-
-.. _option_aws_secret_key:
-
-  * ``YORC_INFRA_AWS_SECRET_KEY``: The AWS secret key credential.
-
 .. _option_terraform_plugins_dir_env:
 
   * ``YORC_TERRAFORM_PLUGINS_DIR``: Equivalent to :ref:`--terraform_plugins_dir <option_terraform_plugins_dir_cmd>` command-line flag.
@@ -779,6 +791,10 @@ Environment variables
 
   * ``YORC_TERRAFORM_OPENSTACK_PLUGIN_VERSION_CONSTRAINT``: Equivalent to :ref:`--terraform_openstack_plugin_version_constraint <option_terraform_openstack_plugin_version_constraint_cmd>` command-line flag.
 
+.. _option_terraform_keep_generated_files_env:
+
+  * ``YORC_TERRAFORM_KEEP_GENERATED_FILES``: Equivalent to :ref:`--terraform_keep_generated_files <option_terraform_keep_generated_files_cmd>` command-line flag.
+
 
 Infrastructures configuration
 -----------------------------

prov/ansible/execution.go

@@ -621,7 +621,7 @@ func (e *executionCommon) resolveOperationOutputPath() error {
 				} else {
 					//If we are with an expression type {get_operation_output : [ SELF, ...]} in a relationship we store the result in the corresponding relationship instance
 					if oof.Operands[0].String() == "SELF" && e.operation.RelOp.IsRelationshipOperation {
-						relationShipPrefix := filepath.Join("relationship_instances", e.NodeName, e.operation.RelOp.RequirementIndex, instanceID)
+						relationShipPrefix := path.Join("relationship_instances", e.NodeName, e.operation.RelOp.RequirementIndex, instanceID)
 						e.Outputs[outputVariableName+"_"+fmt.Sprint(b)] = path.Join(relationShipPrefix, "outputs", interfaceName, operationName, outputVariableName)
 					} else if oof.Operands[0].String() == "HOST" {
 						// In this case we continue because the parsing has change this type on {get_operation_output : [ SELF, ...]}  on the host node
@@ -799,11 +799,11 @@ func (e *executionCommon) executeWithCurrentInstance(ctx context.Context, retry
 		return err
 	}
 
-	var ansibleRecipePath string
+	ansibleRecipePath := filepath.Join(ansiblePath, e.taskID, e.NodeName)
 	if e.operation.RelOp.IsRelationshipOperation {
-		ansibleRecipePath = filepath.Join(ansiblePath, e.NodeName, e.relationshipType, e.operation.RelOp.TargetRelationship, e.operation.Name, currentInstance)
+		ansibleRecipePath = filepath.Join(ansibleRecipePath, e.relationshipType, e.operation.RelOp.TargetRelationship, e.operation.Name, currentInstance)
 	} else {
-		ansibleRecipePath = filepath.Join(ansiblePath, e.NodeName, e.operation.Name, currentInstance)
+		ansibleRecipePath = filepath.Join(ansibleRecipePath, e.operation.Name, currentInstance)
 	}
 
 	if err = os.RemoveAll(ansibleRecipePath); err != nil {
@@ -812,6 +812,17 @@ func (e *executionCommon) executeWithCurrentInstance(ctx context.Context, retry
 		events.WithContextOptionalFields(ctx).NewLogEntry(events.LogLevelERROR, e.deploymentID).RegisterAsString(err.Error())
 		return err
 	}
+
+	defer func() {
+		if !e.cfg.Ansible.KeepGeneratedRecipes {
+			err := os.RemoveAll(ansibleRecipePath)
+			if err != nil {
+				err = errors.Wrapf(err, "Failed to remove ansible recipe directory %q for node %q operation %q", ansibleRecipePath, e.NodeName, e.operation.Name)
+				log.Debugf("%+v", err)
+				events.WithContextOptionalFields(ctx).NewLogEntry(events.LogLevelERROR, e.deploymentID).RegisterAsString(err.Error())
+			}
+		}
+	}()
 	ansibleHostVarsPath := filepath.Join(ansibleRecipePath, "host_vars")
 	if err = os.MkdirAll(ansibleHostVarsPath, 0775); err != nil {
 		events.WithContextOptionalFields(ctx).NewLogEntry(events.LogLevelERROR, e.deploymentID).RegisterAsString(err.Error())

prov/terraform/aws/generator.go

@@ -19,7 +19,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
-	"os"
 	"path"
 	"path/filepath"
 
@@ -37,7 +36,7 @@ const infrastructureName = "aws"
 type awsGenerator struct {
 }
 
-func (g *awsGenerator) GenerateTerraformInfraForNode(ctx context.Context, cfg config.Configuration, deploymentID, nodeName string) (bool, map[string]string, []string, error) {
+func (g *awsGenerator) GenerateTerraformInfraForNode(ctx context.Context, cfg config.Configuration, deploymentID, nodeName, infrastructurePath string) (bool, map[string]string, []string, error) {
 	log.Debugf("Generating infrastructure for deployment with id %s", deploymentID)
 	cClient, err := cfg.GetConsulClient()
 	if err != nil {
@@ -141,13 +140,9 @@ func (g *awsGenerator) GenerateTerraformInfraForNode(ctx context.Context, cfg co
 	if err != nil {
 		return false, nil, nil, errors.Wrap(err, "Failed to generate JSON of terraform Infrastructure description")
 	}
-	infraPath := filepath.Join(cfg.WorkingDirectory, "deployments", fmt.Sprint(deploymentID), "infra", nodeName)
-	if err = os.MkdirAll(infraPath, 0775); err != nil {
-		return false, nil, nil, errors.Wrapf(err, "Failed to create infrastructure working directory %q", infraPath)
-	}
 
-	if err = ioutil.WriteFile(filepath.Join(infraPath, "infra.tf.json"), jsonInfra, 0664); err != nil {
-		return false, nil, nil, errors.Wrapf(err, "Failed to write file %q", filepath.Join(infraPath, "infra.tf.json"))
+	if err = ioutil.WriteFile(filepath.Join(infrastructurePath, "infra.tf.json"), jsonInfra, 0664); err != nil {
+		return false, nil, nil, errors.Wrapf(err, "Failed to write file %q", filepath.Join(infrastructurePath, "infra.tf.json"))
 	}
 
 	log.Debugf("Infrastructure generated for deployment with id %s", deploymentID)

prov/terraform/commons/generator.go

@@ -37,15 +37,15 @@ type Generator interface {
 	// GenerateTerraformInfraForNode can also return a map of outputs names indexed by consul keys into which the outputs results should be stored.
 	// And a list of environment variables in form "key=value" to be added to the current process environment when running terraform commands.
 	// This is particularly useful for adding secrets that should not be in tf files.
-	GenerateTerraformInfraForNode(ctx context.Context, cfg config.Configuration, deploymentID, nodeName string) (bool, map[string]string, []string, error)
+	GenerateTerraformInfraForNode(ctx context.Context, cfg config.Configuration, deploymentID, nodeName, infrastructurePath string) (bool, map[string]string, []string, error)
 }
 
 // PreDestroyInfraCallback is a function that is call before destroying an infrastructure. If it returns false the node will not be destroyed.
-type PreDestroyInfraCallback func(ctx context.Context, kv *api.KV, cfg config.Configuration, deploymentID, nodeName string) (bool, error)
+type PreDestroyInfraCallback func(ctx context.Context, kv *api.KV, cfg config.Configuration, deploymentID, nodeName, infrastructurePath string) (bool, error)
 
 // PreDestroyStorageInfraCallback is a callback of type PreDestroyInfraCallback
 // checking if a block storage node is deletable on undeployment.
-func PreDestroyStorageInfraCallback(ctx context.Context, kv *api.KV, cfg config.Configuration, deploymentID, nodeName string) (bool, error) {
+func PreDestroyStorageInfraCallback(ctx context.Context, kv *api.KV, cfg config.Configuration, deploymentID, nodeName, infrastructurePath string) (bool, error) {
 	nodeType, err := deployments.GetNodeType(kv, deploymentID, nodeName)
 	if err != nil {
 		return false, err