From 4055b6357b502b717424829ea544ea578ff793f0 Mon Sep 17 00:00:00 2001 From: Laurent Ganne Date: Wed, 17 Oct 2018 16:01:49 +0000 Subject: [PATCH 1/7] Upgraded sphinx to 1.8.1 on travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3fb693731..8a80a8456 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,7 +20,7 @@ env: install: - make tools - - pip install --user --upgrade sphinx==1.7.9 semantic-version requests urllib3[secure] + - pip install --user --upgrade sphinx==1.8.1 semantic-version requests urllib3[secure] before_script: - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter From 3c74dcf0343c76407b90e3b5c8ff9e594ef8310b Mon Sep 17 00:00:00 2001 From: Laurent Ganne Date: Wed, 17 Oct 2018 16:21:53 +0000 Subject: [PATCH 2/7] Setting urllib3 version to 1.23 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8a80a8456..755115895 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,7 +20,7 @@ env: install: - make tools - - pip install --user --upgrade sphinx==1.8.1 semantic-version requests urllib3[secure] + - pip install --user --upgrade sphinx==1.8.1 semantic-version requests urllib3[secure]==1.23 before_script: - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter From 90b53b5a6f995c223bc70ce79ef1a5b844329ef9 Mon Sep 17 00:00:00 2001 From: Albertin Loic Date: Tue, 23 Oct 2018 15:26:06 +0200 Subject: [PATCH 3/7] Allow to run multiple workflows and custom commands in //Now terraform and ansible recipes are stored under a path containingthe execution task id.By default those files are deleted after the execution by config optionsare available to keep them --- commands/server.go | 6 +++ config/config.go | 2 + doc/configuration.rst | 34 ++++++++++++----- prov/ansible/execution.go | 19 ++++++++-- prov/terraform/aws/generator.go | 11 ++---- prov/terraform/commons/generator.go | 4 +- prov/terraform/executor.go | 53 +++++++++++++++++---------- prov/terraform/google/generator.go | 11 ++---- prov/terraform/openstack/generator.go | 11 ++---- prov/terraform/openstack/init.go | 2 +- tasks/tasks.go | 30 ++++++++------- tasks/tasks_test.go | 14 +++---- 12 files changed, 116 insertions(+), 81 deletions(-) diff --git a/commands/server.go b/commands/server.go index 59fb2bf16..a6502d426 100644 --- a/commands/server.go +++ b/commands/server.go @@ -63,6 +63,7 @@ var ansibleConfiguration = map[string]interface{}{ "ansible.keep_operation_remote_path": config.DefaultKeepOperationRemotePath, "ansible.archive_artifacts": config.DefaultArchiveArtifacts, "ansible.cache_facts": config.DefaultCacheFacts, + "ansible.keep_generated_recipes": false, } var consulConfiguration = map[string]interface{}{ @@ -84,6 +85,7 @@ var terraformConfiguration = map[string]interface{}{ "terraform.aws_plugin_version_constraint": tfAWSPluginVersionConstraint, "terraform.google_plugin_version_constraint": tfGooglePluginVersionConstraint, "terraform.openstack_plugin_version_constraint": tfOpenStackPluginVersionConstraint, + "terraform.keep_generated_files": false, } var cfgFile string @@ -252,6 +254,10 @@ func setConfig() { serverCmd.PersistentFlags().Bool("keep_operation_remote_path", config.DefaultKeepOperationRemotePath, "Define wether the path created to store artifacts on the nodes will be removed at the end of workflow executions.") serverCmd.PersistentFlags().Bool("ansible_archive_artifacts", config.DefaultArchiveArtifacts, "Define wether artifacts should be ./archived before being copied on remote nodes (requires tar to be installed on remote nodes).") serverCmd.PersistentFlags().Bool("ansible_cache_facts", config.DefaultCacheFacts, "Define wether Ansible facts (useful variables about remote hosts) should be cached.") + serverCmd.PersistentFlags().Bool("ansible_keep_generated_recipes", false, "Define if Yorc should not delete generated Ansible recipes") + + //Flags definition for Terraform + serverCmd.PersistentFlags().Bool("terraform_keep_generated_files", false, "Define if Yorc should not delete generated Terraform infrastructures files") //Flags definition for Terraform serverCmd.PersistentFlags().StringP("terraform_plugins_dir", "", "", "The directory where to find Terraform plugins") diff --git a/config/config.go b/config/config.go index ae8fd39ac..36bb12e64 100644 --- a/config/config.go +++ b/config/config.go @@ -115,6 +115,7 @@ type Ansible struct { ConnectionRetries int `mapstructure:"connection_retries"` OperationRemoteBaseDir string `mapstructure:"operation_remote_base_dir"` KeepOperationRemotePath bool `mapstructure:"keep_operation_remote_path"` + KeepGeneratedRecipes bool `mapstructure:"keep_generated_recipes"` ArchiveArtifacts bool `mapstructure:"archive_artifacts"` CacheFacts bool `mapstructure:"cache_facts"` HostedOperations HostedOperations `mapstructure:"hosted_operations"` @@ -151,6 +152,7 @@ type Terraform struct { AWSPluginVersionConstraint string `mapstructure:"aws_plugin_version_constraint"` GooglePluginVersionConstraint string `mapstructure:"google_plugin_version_constraint"` OpenStackPluginVersionConstraint string `mapstructure:"openstack_plugin_version_constraint"` + KeepGeneratedFiles bool `mapstructure:"keep_generated_files"` } // DynamicMap allows to store configuration parameters that are not known in advance. diff --git a/doc/configuration.rst b/doc/configuration.rst index 1346837d9..89948cc8e 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -46,6 +46,10 @@ Globals Command-line options * ``--ansible_archive_artifacts``: If set to true, archives operation bash/python scripts locally, copies this archive and unarchives it on remote hosts (requires tar to be installed on remote hosts), to avoid multiple time consuming remote copy operations of individual scripts (false by default: no archive). +.. _option_ansible_keep_generated_recipes_cmd: + + * ``--ansible_keep_generated_recipes``: If set to true, generated Ansible recipes on Yorc server are not delete. (false by default: generated recipes are deleted). + .. _option_operation_remote_base_dir_cmd: * ``--operation_remote_base_dir``: Specify an alternative working directory for Ansible on provisioned Compute. @@ -109,6 +113,9 @@ Globals Command-line options .. _option_terraform_openstack_plugin_version_constraint_cmd: * ``--terraform_openstack_plugin_version_constraint``: Specify the Terraform OpenStack plugin version constraint. Default one compatible with our source code is ``"~> 1.9"``. If you choose another, it's at your own risk. See https://www.terraform.io/docs/configuration/providers.html#provider-versions for more information. +.. _option_terraform_keep_generated_files_cmd: + + * ``--terraform_keep_generated_files``: If set to true, generated Terraform infrastructures files on Yorc server are not delete. (false by default: generated files are deleted). .. _option_pub_routines_cmd: @@ -336,6 +343,10 @@ All available configuration options for Ansible are: * ``keep_operation_remote_path``: Equivalent to :ref:`--keep_operation_remote_path ` command-line flag. +.. _option_ansible_keep_generated_recipes_cfg: + + * ``keep_generated_recipes``: Equivalent to :ref:`--ansible_keep_generated_recipes ` command-line flag. + .. _option_ansible_sandbox_hosted_ops_cfg: * ``hosted_operations``: This is a complex structure that allow to define the behavior of a Yorc server when it executes an hosted operation. @@ -478,7 +489,7 @@ Below is an example of configuration file with Terraform configuration options. } } -All available configuration options for Consul are: +All available configuration options for Terraform are: .. _option_plugins_dir_cfg: @@ -500,6 +511,11 @@ All available configuration options for Consul are: * ``openstack_plugin_version_constraint``: Equivalent to :ref:`--terraform_openstack_plugin_version_constraint ` command-line flag. +.. _option_terraform_keep_generated_files_cfg: + + * ``keep_generated_files``: Equivalent to :ref:`--terraform_keep_generated_files ` command-line flag. + + .. _yorc_config_file_telemetry_section: Telemetry configuration @@ -647,6 +663,10 @@ Environment variables * ``YORC_ANSIBLE_ARCHIVE_ARTIFACTS``: Equivalent to :ref:`--ansible_archive_artifacts ` command-line flag. +.. _option_ansible_keep_generated_recipes_env: + + * ``YORC_ANSIBLE_KEEP_GENERATED_RECIPES``: Equivalent to :ref:`--ansible_keep_generated_recipes ` command-line flag. + .. _option_operation_remote_base_dir_env: * ``YORC_OPERATION_REMOTE_BASE_DIR``: Equivalent to :ref:`--operation_remote_base_dir ` command-line flag. @@ -751,14 +771,6 @@ Environment variables * ``YORC_LOG``: If set to ``1`` or ``DEBUG``, enables debug logging for Yorc. -.. _option_aws_access_key: - - * ``YORC_INFRA_AWS_ACCESS_KEY``: The AWS access key credential. - -.. _option_aws_secret_key: - - * ``YORC_INFRA_AWS_SECRET_KEY``: The AWS secret key credential. - .. _option_terraform_plugins_dir_env: * ``YORC_TERRAFORM_PLUGINS_DIR``: Equivalent to :ref:`--terraform_plugins_dir ` command-line flag. @@ -779,6 +791,10 @@ Environment variables * ``YORC_TERRAFORM_OPENSTACK_PLUGIN_VERSION_CONSTRAINT``: Equivalent to :ref:`--terraform_openstack_plugin_version_constraint ` command-line flag. +.. _option_terraform_keep_generated_files_env: + + * ``YORC_TERRAFORM_KEEP_GENERATED_FILES``: Equivalent to :ref:`--terraform_keep_generated_files ` command-line flag. + Infrastructures configuration ----------------------------- diff --git a/prov/ansible/execution.go b/prov/ansible/execution.go index 4fe8d62a5..9f05df749 100644 --- a/prov/ansible/execution.go +++ b/prov/ansible/execution.go @@ -621,7 +621,7 @@ func (e *executionCommon) resolveOperationOutputPath() error { } else { //If we are with an expression type {get_operation_output : [ SELF, ...]} in a relationship we store the result in the corresponding relationship instance if oof.Operands[0].String() == "SELF" && e.operation.RelOp.IsRelationshipOperation { - relationShipPrefix := filepath.Join("relationship_instances", e.NodeName, e.operation.RelOp.RequirementIndex, instanceID) + relationShipPrefix := path.Join("relationship_instances", e.NodeName, e.operation.RelOp.RequirementIndex, instanceID) e.Outputs[outputVariableName+"_"+fmt.Sprint(b)] = path.Join(relationShipPrefix, "outputs", interfaceName, operationName, outputVariableName) } else if oof.Operands[0].String() == "HOST" { // In this case we continue because the parsing has change this type on {get_operation_output : [ SELF, ...]} on the host node @@ -799,11 +799,11 @@ func (e *executionCommon) executeWithCurrentInstance(ctx context.Context, retry return err } - var ansibleRecipePath string + ansibleRecipePath := filepath.Join(ansiblePath, e.taskID, e.NodeName) if e.operation.RelOp.IsRelationshipOperation { - ansibleRecipePath = filepath.Join(ansiblePath, e.NodeName, e.relationshipType, e.operation.RelOp.TargetRelationship, e.operation.Name, currentInstance) + ansibleRecipePath = filepath.Join(ansibleRecipePath, e.relationshipType, e.operation.RelOp.TargetRelationship, e.operation.Name, currentInstance) } else { - ansibleRecipePath = filepath.Join(ansiblePath, e.NodeName, e.operation.Name, currentInstance) + ansibleRecipePath = filepath.Join(ansibleRecipePath, e.operation.Name, currentInstance) } if err = os.RemoveAll(ansibleRecipePath); err != nil { @@ -812,6 +812,17 @@ func (e *executionCommon) executeWithCurrentInstance(ctx context.Context, retry events.WithContextOptionalFields(ctx).NewLogEntry(events.LogLevelERROR, e.deploymentID).RegisterAsString(err.Error()) return err } + + defer func() { + if !e.cfg.Ansible.KeepGeneratedRecipes { + err := os.RemoveAll(ansibleRecipePath) + if err != nil { + err = errors.Wrapf(err, "Failed to remove ansible recipe directory %q for node %q operation %q", ansibleRecipePath, e.NodeName, e.operation.Name) + log.Debugf("%+v", err) + events.WithContextOptionalFields(ctx).NewLogEntry(events.LogLevelERROR, e.deploymentID).RegisterAsString(err.Error()) + } + } + }() ansibleHostVarsPath := filepath.Join(ansibleRecipePath, "host_vars") if err = os.MkdirAll(ansibleHostVarsPath, 0775); err != nil { events.WithContextOptionalFields(ctx).NewLogEntry(events.LogLevelERROR, e.deploymentID).RegisterAsString(err.Error()) diff --git a/prov/terraform/aws/generator.go b/prov/terraform/aws/generator.go index 5c82b2ab2..03620c85d 100644 --- a/prov/terraform/aws/generator.go +++ b/prov/terraform/aws/generator.go @@ -19,7 +19,6 @@ import ( "encoding/json" "fmt" "io/ioutil" - "os" "path" "path/filepath" @@ -37,7 +36,7 @@ const infrastructureName = "aws" type awsGenerator struct { } -func (g *awsGenerator) GenerateTerraformInfraForNode(ctx context.Context, cfg config.Configuration, deploymentID, nodeName string) (bool, map[string]string, []string, error) { +func (g *awsGenerator) GenerateTerraformInfraForNode(ctx context.Context, cfg config.Configuration, deploymentID, nodeName, infrastructurePath string) (bool, map[string]string, []string, error) { log.Debugf("Generating infrastructure for deployment with id %s", deploymentID) cClient, err := cfg.GetConsulClient() if err != nil { @@ -141,13 +140,9 @@ func (g *awsGenerator) GenerateTerraformInfraForNode(ctx context.Context, cfg co if err != nil { return false, nil, nil, errors.Wrap(err, "Failed to generate JSON of terraform Infrastructure description") } - infraPath := filepath.Join(cfg.WorkingDirectory, "deployments", fmt.Sprint(deploymentID), "infra", nodeName) - if err = os.MkdirAll(infraPath, 0775); err != nil { - return false, nil, nil, errors.Wrapf(err, "Failed to create infrastructure working directory %q", infraPath) - } - if err = ioutil.WriteFile(filepath.Join(infraPath, "infra.tf.json"), jsonInfra, 0664); err != nil { - return false, nil, nil, errors.Wrapf(err, "Failed to write file %q", filepath.Join(infraPath, "infra.tf.json")) + if err = ioutil.WriteFile(filepath.Join(infrastructurePath, "infra.tf.json"), jsonInfra, 0664); err != nil { + return false, nil, nil, errors.Wrapf(err, "Failed to write file %q", filepath.Join(infrastructurePath, "infra.tf.json")) } log.Debugf("Infrastructure generated for deployment with id %s", deploymentID) diff --git a/prov/terraform/commons/generator.go b/prov/terraform/commons/generator.go index 9b5b80a24..00ae3343d 100644 --- a/prov/terraform/commons/generator.go +++ b/prov/terraform/commons/generator.go @@ -29,8 +29,8 @@ type Generator interface { // GenerateTerraformInfraForNode can also return a map of outputs names indexed by consul keys into which the outputs results should be stored. // And a list of environment variables in form "key=value" to be added to the current process environment when running terraform commands. // This is particularly useful for adding secrets that should not be in tf files. - GenerateTerraformInfraForNode(ctx context.Context, cfg config.Configuration, deploymentID, nodeName string) (bool, map[string]string, []string, error) + GenerateTerraformInfraForNode(ctx context.Context, cfg config.Configuration, deploymentID, nodeName, infrastructurePath string) (bool, map[string]string, []string, error) } // PreDestroyInfraCallback is a function that is call before destroying an infrastructure. If it returns false the node will not be destroyed. -type PreDestroyInfraCallback func(ctx context.Context, kv *api.KV, cfg config.Configuration, deploymentID, nodeName string) (bool, error) +type PreDestroyInfraCallback func(ctx context.Context, kv *api.KV, cfg config.Configuration, deploymentID, nodeName, infrastructurePath string) (bool, error) diff --git a/prov/terraform/executor.go b/prov/terraform/executor.go index def6a93b5..580837158 100644 --- a/prov/terraform/executor.go +++ b/prov/terraform/executor.go @@ -28,6 +28,7 @@ import ( "github.com/ystia/yorc/events" "github.com/ystia/yorc/helper/consulutil" "github.com/ystia/yorc/helper/executil" + "github.com/ystia/yorc/log" "github.com/ystia/yorc/prov" "github.com/ystia/yorc/prov/terraform/commons" "github.com/ystia/yorc/tasks" @@ -65,32 +66,46 @@ func (e *defaultExecutor) ExecDelegate(ctx context.Context, cfg config.Configura if err != nil { return err } - + infrastructurePath := filepath.Join(cfg.WorkingDirectory, "deployments", deploymentID, "terraform", taskID, nodeName) + if err = os.MkdirAll(infrastructurePath, 0775); err != nil { + return errors.Wrapf(err, "Failed to create infrastructure working directory %q", infrastructurePath) + } + defer func() { + if !cfg.Terraform.KeepGeneratedFiles { + err := os.RemoveAll(infrastructurePath) + if err != nil { + err = errors.Wrapf(err, "Failed to remove Terraform infrastructure directory %q for node %q operation %q", infrastructurePath, nodeName, delegateOperation) + log.Debugf("%+v", err) + events.WithContextOptionalFields(ctx).NewLogEntry(events.LogLevelERROR, deploymentID).RegisterAsString(err.Error()) + } + } + }() op := strings.ToLower(delegateOperation) switch { case op == "install": - err = e.installNode(ctx, kv, cfg, deploymentID, nodeName, instances) + err = e.installNode(ctx, kv, cfg, deploymentID, nodeName, infrastructurePath, instances) case op == "uninstall": - err = e.uninstallNode(ctx, kv, cfg, deploymentID, nodeName, instances) + err = e.uninstallNode(ctx, kv, cfg, deploymentID, nodeName, infrastructurePath, instances) default: return errors.Errorf("Unsupported operation %q", delegateOperation) } return err } -func (e *defaultExecutor) installNode(ctx context.Context, kv *api.KV, cfg config.Configuration, deploymentID, nodeName string, instances []string) error { +func (e *defaultExecutor) installNode(ctx context.Context, kv *api.KV, cfg config.Configuration, deploymentID, nodeName, infrastructurePath string, instances []string) error { for _, instance := range instances { err := deployments.SetInstanceStateWithContextualLogs(events.AddLogOptionalFields(ctx, events.LogOptionalFields{events.InstanceID: instance}), kv, deploymentID, nodeName, instance, tosca.NodeStateCreating) if err != nil { return err } } - infraGenerated, outputs, env, err := e.generator.GenerateTerraformInfraForNode(ctx, cfg, deploymentID, nodeName) + + infraGenerated, outputs, env, err := e.generator.GenerateTerraformInfraForNode(ctx, cfg, deploymentID, nodeName, infrastructurePath) if err != nil { return err } if infraGenerated { - if err = e.applyInfrastructure(ctx, kv, cfg, deploymentID, nodeName, outputs, env); err != nil { + if err = e.applyInfrastructure(ctx, kv, cfg, deploymentID, nodeName, infrastructurePath, outputs, env); err != nil { return err } } @@ -103,19 +118,19 @@ func (e *defaultExecutor) installNode(ctx context.Context, kv *api.KV, cfg confi return nil } -func (e *defaultExecutor) uninstallNode(ctx context.Context, kv *api.KV, cfg config.Configuration, deploymentID, nodeName string, instances []string) error { +func (e *defaultExecutor) uninstallNode(ctx context.Context, kv *api.KV, cfg config.Configuration, deploymentID, nodeName, infrastructurePath string, instances []string) error { for _, instance := range instances { err := deployments.SetInstanceStateWithContextualLogs(events.AddLogOptionalFields(ctx, events.LogOptionalFields{events.InstanceID: instance}), kv, deploymentID, nodeName, instance, tosca.NodeStateDeleting) if err != nil { return err } } - infraGenerated, outputs, env, err := e.generator.GenerateTerraformInfraForNode(ctx, cfg, deploymentID, nodeName) + infraGenerated, outputs, env, err := e.generator.GenerateTerraformInfraForNode(ctx, cfg, deploymentID, nodeName, infrastructurePath) if err != nil { return err } if infraGenerated { - if err = e.destroyInfrastructure(ctx, kv, cfg, deploymentID, nodeName, outputs, env); err != nil { + if err = e.destroyInfrastructure(ctx, kv, cfg, deploymentID, nodeName, infrastructurePath, outputs, env); err != nil { return err } } @@ -128,9 +143,8 @@ func (e *defaultExecutor) uninstallNode(ctx context.Context, kv *api.KV, cfg con return nil } -func (e *defaultExecutor) remoteConfigInfrastructure(ctx context.Context, kv *api.KV, cfg config.Configuration, deploymentID, nodeName string, env []string) error { +func (e *defaultExecutor) remoteConfigInfrastructure(ctx context.Context, kv *api.KV, cfg config.Configuration, deploymentID, nodeName, infrastructurePath string, env []string) error { events.WithContextOptionalFields(ctx).NewLogEntry(events.LogLevelINFO, deploymentID).RegisterAsString("Remote configuring the infrastructure") - infraPath := filepath.Join(cfg.WorkingDirectory, "deployments", deploymentID, "infra", nodeName) var cmd *executil.Cmd // Use pre-installed Terraform providers plugins if plugins directory exists // https://www.terraform.io/guides/running-terraform-in-automation.html#pre-installed-plugins @@ -140,7 +154,7 @@ func (e *defaultExecutor) remoteConfigInfrastructure(ctx context.Context, kv *ap cmd = executil.Command(ctx, "terraform", "init") } - cmd.Dir = infraPath + cmd.Dir = infrastructurePath cmd.Env = mergeEnvironments(env) errbuf := events.NewBufferedLogEntryWriter() out := events.NewBufferedLogEntryWriter() @@ -197,17 +211,16 @@ func (e *defaultExecutor) retrieveOutputs(ctx context.Context, kv *api.KV, infra return nil } -func (e *defaultExecutor) applyInfrastructure(ctx context.Context, kv *api.KV, cfg config.Configuration, deploymentID, nodeName string, outputs map[string]string, env []string) error { +func (e *defaultExecutor) applyInfrastructure(ctx context.Context, kv *api.KV, cfg config.Configuration, deploymentID, nodeName, infrastructurePath string, outputs map[string]string, env []string) error { // Remote Configuration for Terraform State to store it in the Consul KV store - if err := e.remoteConfigInfrastructure(ctx, kv, cfg, deploymentID, nodeName, env); err != nil { + if err := e.remoteConfigInfrastructure(ctx, kv, cfg, deploymentID, nodeName, infrastructurePath, env); err != nil { return err } events.WithContextOptionalFields(ctx).NewLogEntry(events.LogLevelINFO, deploymentID).RegisterAsString("Applying the infrastructure") - infraPath := filepath.Join(cfg.WorkingDirectory, "deployments", deploymentID, "infra", nodeName) cmd := executil.Command(ctx, "terraform", "apply", "-input=false", "-auto-approve") - cmd.Dir = infraPath + cmd.Dir = infrastructurePath cmd.Env = mergeEnvironments(env) errbuf := events.NewBufferedLogEntryWriter() out := events.NewBufferedLogEntryWriter() @@ -225,20 +238,20 @@ func (e *defaultExecutor) applyInfrastructure(ctx context.Context, kv *api.KV, c return errors.Wrap(err, "Failed to apply the infrastructure changes via terraform") } - return e.retrieveOutputs(ctx, kv, infraPath, outputs) + return e.retrieveOutputs(ctx, kv, infrastructurePath, outputs) } -func (e *defaultExecutor) destroyInfrastructure(ctx context.Context, kv *api.KV, cfg config.Configuration, deploymentID, nodeName string, outputs map[string]string, env []string) error { +func (e *defaultExecutor) destroyInfrastructure(ctx context.Context, kv *api.KV, cfg config.Configuration, deploymentID, nodeName, infrastructurePath string, outputs map[string]string, env []string) error { if e.preDestroyCheck != nil { - check, err := e.preDestroyCheck(ctx, kv, cfg, deploymentID, nodeName) + check, err := e.preDestroyCheck(ctx, kv, cfg, deploymentID, nodeName, infrastructurePath) if err != nil || !check { return err } } - return e.applyInfrastructure(ctx, kv, cfg, deploymentID, nodeName, outputs, env) + return e.applyInfrastructure(ctx, kv, cfg, deploymentID, nodeName, infrastructurePath, outputs, env) } // mergeEnvironments merges given env with current process env diff --git a/prov/terraform/google/generator.go b/prov/terraform/google/generator.go index 14949a7f1..5772217ea 100644 --- a/prov/terraform/google/generator.go +++ b/prov/terraform/google/generator.go @@ -19,7 +19,6 @@ import ( "encoding/json" "fmt" "io/ioutil" - "os" "path" "path/filepath" "strings" @@ -38,7 +37,7 @@ const infrastructureName = "google" type googleGenerator struct { } -func (g *googleGenerator) GenerateTerraformInfraForNode(ctx context.Context, cfg config.Configuration, deploymentID, nodeName string) (bool, map[string]string, []string, error) { +func (g *googleGenerator) GenerateTerraformInfraForNode(ctx context.Context, cfg config.Configuration, deploymentID, nodeName, infrastructurePath string) (bool, map[string]string, []string, error) { log.Debugf("Generating infrastructure for deployment with id %s", deploymentID) cClient, err := cfg.GetConsulClient() if err != nil { @@ -158,13 +157,9 @@ func (g *googleGenerator) GenerateTerraformInfraForNode(ctx context.Context, cfg if err != nil { return false, nil, nil, errors.Wrap(err, "Failed to generate JSON of terraform Infrastructure description") } - infraPath := filepath.Join(cfg.WorkingDirectory, "deployments", fmt.Sprint(deploymentID), "infra", nodeName) - if err = os.MkdirAll(infraPath, 0775); err != nil { - return false, nil, nil, errors.Wrapf(err, "Failed to create infrastructure working directory %q", infraPath) - } - if err = ioutil.WriteFile(filepath.Join(infraPath, "infra.tf.json"), jsonInfra, 0664); err != nil { - return false, nil, nil, errors.Wrapf(err, "Failed to write file %q", filepath.Join(infraPath, "infra.tf.json")) + if err = ioutil.WriteFile(filepath.Join(infrastructurePath, "infra.tf.json"), jsonInfra, 0664); err != nil { + return false, nil, nil, errors.Wrapf(err, "Failed to write file %q", filepath.Join(infrastructurePath, "infra.tf.json")) } log.Debugf("Infrastructure generated for deployment with id %s", deploymentID) diff --git a/prov/terraform/openstack/generator.go b/prov/terraform/openstack/generator.go index 6525cfb3a..3c9cf2c97 100644 --- a/prov/terraform/openstack/generator.go +++ b/prov/terraform/openstack/generator.go @@ -19,7 +19,6 @@ import ( "encoding/json" "fmt" "io/ioutil" - "os" "path" "path/filepath" "strconv" @@ -50,7 +49,7 @@ func (g *osGenerator) getStringFormConsul(kv *api.KV, baseURL, property string) return string(getResult.Value), nil } -func (g *osGenerator) GenerateTerraformInfraForNode(ctx context.Context, cfg config.Configuration, deploymentID, nodeName string) (bool, map[string]string, []string, error) { +func (g *osGenerator) GenerateTerraformInfraForNode(ctx context.Context, cfg config.Configuration, deploymentID, nodeName, infrastructurePath string) (bool, map[string]string, []string, error) { log.Debugf("Generating infrastructure for deployment with id %s", deploymentID) cClient, err := cfg.GetConsulClient() if err != nil { @@ -273,13 +272,9 @@ func (g *osGenerator) GenerateTerraformInfraForNode(ctx context.Context, cfg con if err != nil { return false, nil, nil, errors.Wrap(err, "Failed to generate JSON of terraform Infrastructure description") } - infraPath := filepath.Join(cfg.WorkingDirectory, "deployments", deploymentID, "infra", nodeName) - if err = os.MkdirAll(infraPath, 0775); err != nil { - return false, nil, nil, errors.Wrapf(err, "Failed to create infrastructure working directory %q", infraPath) - } - if err = ioutil.WriteFile(filepath.Join(infraPath, "infra.tf.json"), jsonInfra, 0664); err != nil { - return false, nil, nil, errors.Wrapf(err, "Failed to write file %q", filepath.Join(infraPath, "infra.tf.json")) + if err = ioutil.WriteFile(filepath.Join(infrastructurePath, "infra.tf.json"), jsonInfra, 0664); err != nil { + return false, nil, nil, errors.Wrapf(err, "Failed to write file %q", filepath.Join(infrastructurePath, "infra.tf.json")) } log.Debugf("Infrastructure generated for deployment with id %s", deploymentID) diff --git a/prov/terraform/openstack/init.go b/prov/terraform/openstack/init.go index 3f1780c48..69a974240 100644 --- a/prov/terraform/openstack/init.go +++ b/prov/terraform/openstack/init.go @@ -33,7 +33,7 @@ func init() { reg.RegisterDelegates([]string{`yorc\.nodes\.openstack\..*`}, terraform.NewExecutor(&osGenerator{}, preDestroyInfraCallback), registry.BuiltinOrigin) } -func preDestroyInfraCallback(ctx context.Context, kv *api.KV, cfg config.Configuration, deploymentID, nodeName string) (bool, error) { +func preDestroyInfraCallback(ctx context.Context, kv *api.KV, cfg config.Configuration, deploymentID, nodeName, infrastructurePath string) (bool, error) { nodeType, err := deployments.GetNodeType(kv, deploymentID, nodeName) if err != nil { return false, err diff --git a/tasks/tasks.go b/tasks/tasks.go index f57dbffd7..a5f3cb3ac 100644 --- a/tasks/tasks.go +++ b/tasks/tasks.go @@ -216,32 +216,34 @@ func DeleteTask(kv *api.KV, taskID string) error { } // TargetHasLivingTasks checks if a targetID has associated tasks in status INITIAL or RUNNING and returns the id and status of the first one found +// +// Only Deploy, UnDeploy, ScaleOut, ScaleIn and Purge task type are considered. func TargetHasLivingTasks(kv *api.KV, targetID string) (bool, string, string, error) { tasksKeys, _, err := kv.Keys(consulutil.TasksPrefix+"/", "/", nil) if err != nil { return false, "", "", errors.Wrap(err, consulutil.ConsulGenericErrMsg) } for _, taskKey := range tasksKeys { - kvp, _, err := kv.Get(path.Join(taskKey, "targetId"), nil) + taskID := path.Base(taskKey) + ttID, err := GetTaskTarget(kv, taskID) if err != nil { - return false, "", "", errors.Wrap(err, consulutil.ConsulGenericErrMsg) + return false, "", "", err } - if kvp != nil && len(kvp.Value) > 0 && string(kvp.Value) == targetID { - kvp, _, err := kv.Get(path.Join(taskKey, "status"), nil) - taskID := path.Base(taskKey) + if ttID == targetID { + tStatus, err := GetTaskStatus(kv, taskID) if err != nil { - return false, "", "", errors.Wrap(err, consulutil.ConsulGenericErrMsg) + return false, "", "", err } - if kvp == nil || len(kvp.Value) == 0 { - return false, "", "", errors.Errorf("Missing status for task with id %q", taskID) - } - statusInt, err := strconv.Atoi(string(kvp.Value)) + tType, err := GetTaskType(kv, taskID) if err != nil { - return false, "", "", errors.Wrap(err, "Invalid task status") + return false, "", "", err } - switch TaskStatus(statusInt) { - case TaskStatusINITIAL, TaskStatusRUNNING: - return true, taskID, TaskStatus(statusInt).String(), nil + + switch tType { + case TaskTypeDeploy, TaskTypeUnDeploy, TaskTypePurge, TaskTypeScaleIn, TaskTypeScaleOut: + if tStatus == TaskStatusINITIAL || tStatus == TaskStatusRUNNING { + return true, taskID, tStatus.String(), nil + } } } } diff --git a/tasks/tasks_test.go b/tasks/tasks_test.go index 411dd0149..558309a7e 100644 --- a/tasks/tasks_test.go +++ b/tasks/tasks_test.go @@ -15,17 +15,16 @@ package tasks import ( + "encoding/json" + "fmt" + "path" "reflect" "testing" - "github.com/ystia/yorc/helper/consulutil" - - "path" - - "encoding/json" - "fmt" "github.com/hashicorp/consul/api" "github.com/hashicorp/consul/testutil" + + "github.com/ystia/yorc/helper/consulutil" ) func populateKV(t *testing.T, srv *testutil.TestServer) { @@ -80,7 +79,8 @@ func populateKV(t *testing.T, srv *testutil.TestServer) { consulutil.TasksPrefix + "/t13/type": []byte("5"), consulutil.TasksPrefix + "/t13/status": []byte("3"), consulutil.TasksPrefix + "/t14/status": []byte("3"), - consulutil.TasksPrefix + "/t14/type": []byte("5"), + consulutil.TasksPrefix + "/t14/type": []byte("6"), + consulutil.TasksPrefix + "/t14/targetId": []byte("id"), consulutil.TasksPrefix + "/t15/targetId": []byte("xxx"), consulutil.TasksPrefix + "/t15/status": []byte("2"), From bb9174f98a91951922d4af2d3f20baa3b07e7ade Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Albertin?= Date: Tue, 23 Oct 2018 17:44:04 +0200 Subject: [PATCH 4/7] Update CHANGELOG.md --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e0cc0ca2..70e326b80 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ ## 3.1.0-M4 (October 08, 2018) +### ENHANCEMENTS + +* Concurrent workflows and custom commands executions are now allowed except when a deployment/undeployment/scaling operation is in progress ([GH-182](https://github.com/ystia/yorc/issues/182)) + ### DEPENDENCIES * The orchestrator requires now at least Terraform 0.11.8 and following Terraform plugins (with corresponding version constraints): `Consul (~> 2.1)`, `AWS (~> 1.36)`, `OpenStack (~> 1.9)`, `Google (~ 1.18)` and `null provider (~ 1.0)`. (Terraform upgrade from 0.9.11 introduced in [GH-82](https://github.com/ystia/yorc/issues/82)) From 1f9dd5b2c129e4893cd05b7238ed0118bdc1b86b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Albertin?= Date: Tue, 23 Oct 2018 17:44:47 +0200 Subject: [PATCH 5/7] Update CHANGELOG.md Oups wrong place :) --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70e326b80..ef6a3bda1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,12 +2,13 @@ ## UNRELEASED -## 3.1.0-M4 (October 08, 2018) - ### ENHANCEMENTS * Concurrent workflows and custom commands executions are now allowed except when a deployment/undeployment/scaling operation is in progress ([GH-182](https://github.com/ystia/yorc/issues/182)) + +## 3.1.0-M4 (October 08, 2018) + ### DEPENDENCIES * The orchestrator requires now at least Terraform 0.11.8 and following Terraform plugins (with corresponding version constraints): `Consul (~> 2.1)`, `AWS (~> 1.36)`, `OpenStack (~> 1.9)`, `Google (~ 1.18)` and `null provider (~ 1.0)`. (Terraform upgrade from 0.9.11 introduced in [GH-82](https://github.com/ystia/yorc/issues/82)) From edecb828e944df58ad15842233d6f3d180c76e78 Mon Sep 17 00:00:00 2001 From: Laurent Ganne <33217305+laurentganne@users.noreply.github.com> Date: Tue, 23 Oct 2018 22:37:06 +0200 Subject: [PATCH 6/7] Fixed typo --- doc/configuration.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index 89948cc8e..3c8e970de 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -48,7 +48,7 @@ Globals Command-line options .. _option_ansible_keep_generated_recipes_cmd: - * ``--ansible_keep_generated_recipes``: If set to true, generated Ansible recipes on Yorc server are not delete. (false by default: generated recipes are deleted). + * ``--ansible_keep_generated_recipes``: If set to true, generated Ansible recipes on Yorc server are not deleted. (false by default: generated recipes are deleted). .. _option_operation_remote_base_dir_cmd: @@ -115,7 +115,7 @@ Globals Command-line options * ``--terraform_openstack_plugin_version_constraint``: Specify the Terraform OpenStack plugin version constraint. Default one compatible with our source code is ``"~> 1.9"``. If you choose another, it's at your own risk. See https://www.terraform.io/docs/configuration/providers.html#provider-versions for more information. .. _option_terraform_keep_generated_files_cmd: - * ``--terraform_keep_generated_files``: If set to true, generated Terraform infrastructures files on Yorc server are not delete. (false by default: generated files are deleted). + * ``--terraform_keep_generated_files``: If set to true, generated Terraform infrastructures files on Yorc server are not deleted. (false by default: generated files are deleted). .. _option_pub_routines_cmd: From 039a10ab8a93baddd672265329945ae041bf6bfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Albertin?= Date: Wed, 24 Oct 2018 16:34:46 +0200 Subject: [PATCH 7/7] Update CHANGELOG.md Added missing entry in 3.1.0-M3 version --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef6a3bda1..e892fc3b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ ### BUG FIXES * REQ_TARGET keyword into TOSCA functions was broken. This was introduced during the upgrade to Alien4Cloud 2.0 that changed how requirements definition on node templates ([GH-159](https://github.com/ystia/yorc/issues/159)) +* Parse Alien specific way of defining properties on relationships ([GH-155](https://github.com/ystia/yorc/issues/155)) ## 3.1.0-M2 (August 24, 2018)