Merge branch 'develop' into feature/GH-131-bootstrap-yorc

CHANGELOG.md

@@ -2,6 +2,14 @@
 
 ## UNRELEASED
 
+### FEATURES
+
+* Support GCE Block storages. ([GH-82](https://github.com/ystia/yorc/issues/81))
+
+### ENHANCEMENTS
+
+* Concurrent workflows and custom commands executions are now allowed except when a deployment/undeployment/scaling operation is in progress ([GH-182](https://github.com/ystia/yorc/issues/182))
+
 ## 3.1.0-M4 (October 08, 2018)
 
 ### DEPENDENCIES
@@ -35,6 +43,7 @@
 ### BUG FIXES
 
 * REQ_TARGET keyword into TOSCA functions was broken. This was introduced during the upgrade to Alien4Cloud 2.0 that changed how requirements definition on node templates ([GH-159](https://github.com/ystia/yorc/issues/159))
+* Parse Alien specific way of defining properties on relationships ([GH-155](https://github.com/ystia/yorc/issues/155))
 
 ## 3.1.0-M2 (August 24, 2018)
 

commands/server.go

@@ -63,6 +63,7 @@ var ansibleConfiguration = map[string]interface{}{
 	"ansible.keep_operation_remote_path": config.DefaultKeepOperationRemotePath,
 	"ansible.archive_artifacts":          config.DefaultArchiveArtifacts,
 	"ansible.cache_facts":                config.DefaultCacheFacts,
+	"ansible.keep_generated_recipes":     false,
 }
 
 var consulConfiguration = map[string]interface{}{
@@ -84,6 +85,7 @@ var terraformConfiguration = map[string]interface{}{
 	"terraform.aws_plugin_version_constraint":       tfAWSPluginVersionConstraint,
 	"terraform.google_plugin_version_constraint":    tfGooglePluginVersionConstraint,
 	"terraform.openstack_plugin_version_constraint": tfOpenStackPluginVersionConstraint,
+	"terraform.keep_generated_files":                false,
 }
 
 var cfgFile string
@@ -257,6 +259,10 @@ func setConfig() {
 	serverCmd.PersistentFlags().Bool("keep_operation_remote_path", config.DefaultKeepOperationRemotePath, "Define wether the path created to store artifacts on the nodes will be removed at the end of workflow executions.")
 	serverCmd.PersistentFlags().Bool("ansible_archive_artifacts", config.DefaultArchiveArtifacts, "Define wether artifacts should be ./archived before being copied on remote nodes (requires tar to be installed on remote nodes).")
 	serverCmd.PersistentFlags().Bool("ansible_cache_facts", config.DefaultCacheFacts, "Define wether Ansible facts (useful variables about remote hosts) should be cached.")
+	serverCmd.PersistentFlags().Bool("ansible_keep_generated_recipes", false, "Define if Yorc should not delete generated Ansible recipes")
+
+	//Flags definition for Terraform
+	serverCmd.PersistentFlags().Bool("terraform_keep_generated_files", false, "Define if Yorc should not delete generated Terraform infrastructures files")
 
 	//Flags definition for Terraform
 	serverCmd.PersistentFlags().StringP("terraform_plugins_dir", "", "", "The directory where to find Terraform plugins")

config/config.go

@@ -115,6 +115,7 @@ type Ansible struct {
 	ConnectionRetries       int              `mapstructure:"connection_retries"`
 	OperationRemoteBaseDir  string           `mapstructure:"operation_remote_base_dir"`
 	KeepOperationRemotePath bool             `mapstructure:"keep_operation_remote_path"`
+	KeepGeneratedRecipes    bool             `mapstructure:"keep_generated_recipes"`
 	ArchiveArtifacts        bool             `mapstructure:"archive_artifacts"`
 	CacheFacts              bool             `mapstructure:"cache_facts"`
 	HostedOperations        HostedOperations `mapstructure:"hosted_operations"`
@@ -151,6 +152,7 @@ type Terraform struct {
 	AWSPluginVersionConstraint       string `mapstructure:"aws_plugin_version_constraint"`
 	GooglePluginVersionConstraint    string `mapstructure:"google_plugin_version_constraint"`
 	OpenStackPluginVersionConstraint string `mapstructure:"openstack_plugin_version_constraint"`
+	KeepGeneratedFiles               bool   `mapstructure:"keep_generated_files"`
 }
 
 // DynamicMap allows to store configuration parameters that are not known in advance.

data/tosca/yorc-google-types.yml

@@ -12,6 +12,47 @@ artifact_types:
   yorc.artifacts.google.Deployment:
     derived_from: tosca.artifacts.Deployment
 
+data_types:
+  yorc.datatypes.google.EncryptionKey:
+    derived_from: tosca.datatypes.Root
+    properties:
+      raw_key:
+        type: string
+        required: false
+        description: Specifies a 256-bit customer-supplied encryption key, encoded in RFC 4648 base64 to either encrypt or decrypt this resource.
+      sha256:
+        type: string
+        required: false
+        description: The RFC 4648 base64 encoded SHA-256 hash of the customer-supplied encryption key that protects this resource.
+
+  yorc.datatypes.google.ScratchDisk:
+    derived_from: tosca.datatypes.Root
+    properties:
+      interface:
+        type: string
+        required: false
+        description: The disk interface to use for attaching the scratch disks; either SCSI or NVME. Defaults to SCSI.
+
+relationship_types:
+  yorc.relationships.google.AttachesTo:
+    derived_from: tosca.relationships.AttachTo
+    description: >
+      This type represents an attachment relationship between two nodes.
+      For example, an AttachesTo relationship type would be used for attaching a storage node to a Compute node.
+    valid_target_types: [ tosca.capabilities.Attachment ]
+    properties:
+      mode:
+        type: string
+        description: >
+          The mode in which to attach this disk, either READ_WRITE or READ_ONLY. If not specified, the default is to attach the disk in READ_WRITE mode.
+        required: false
+    attributes:
+      device:
+        type: string
+        description: >
+          The logical name of the device as exposed to the instance.
+          Note: A runtime property that gets set when the model gets instantiated by the orchestrator.
+
 node_types:
   yorc.nodes.google.Compute:
     derived_from: yorc.nodes.Compute
@@ -103,6 +144,15 @@ node_types:
           Comma-separated list of tags to apply to the instances for identifying
           the instances to which network firewall rules will apply.
         required: false
+      scratch_disks:
+        type: list
+        description: Additional scratch disks to attach to the instance. Maximum allowed is 8.
+        required: false
+        entry_schema:
+          type: yorc.datatypes.google.ScratchDisk
+          constraints:
+            - greater_or_equal: 0
+            - max_length: 8
     requirements:
       - assignment:
           capability: yorc.capabilities.Assignable
@@ -150,8 +200,6 @@ node_types:
         description: >
           Comma-separated list of label KEY=VALUE pairs to assign to the Compute Address.
         required: false
-        entry_schema:
-          type: string
       region:
         type: string
         description: >
@@ -180,3 +228,66 @@ node_types:
         type: string
         description: The compute IP address.
 
+  yorc.nodes.google.PersistentDisk:
+    derived_from: tosca.nodes.BlockStorage
+    properties:
+    # See https://www.terraform.io/docs/providers/google/r/compute_disk.html
+      description:
+        type: string
+        description: >
+          An optional description of this resource.
+        required: false
+      type:
+        type: string
+        description: >
+          URL of the disk type resource describing which disk type to use to create the disk.
+          If this field is not specified, it is assumed to be pd-standard for Standard Persistent Disk HDD.
+          pd-ssd is for solid-state drives (SSD).
+        required: false
+      labels:
+        type: string
+        description: >
+          Comma-separated list of label KEY=VALUE pairs to assign to the Compute Disk.
+        required: false
+      zone:
+        type: string
+        description: >
+          A reference to the zone where the disk resides. In any case the disk must be on the same zone as the associated Compute instances.
+        required: true
+      deletable:
+        type: boolean
+        description: should this volume be deleted at undeployment
+        required: false
+        default: false
+      disk_encryption_key:
+        type: yorc.datatypes.google.EncryptionKey
+        description: >
+          Encrypts the disk using a customer-supplied encryption key. After you encrypt a disk with a customer-supplied key, you must provide the same key if you use the disk later
+          (e.g. to create a disk snapshot or an image, or to attach the disk to a virtual machine).
+          Customer-supplied encryption keys do not protect access to metadata of the disk.
+          If you do not provide an encryption key when creating the disk, then the disk will be encrypted using an automatically generated key and you do not need to provide a key to use the disk later.
+        required: false
+      image_id:
+        type: string
+        description: >
+          The image from which to initialize this disk. This can be one of: the image's self_link, projects/{project}/global/images/{image}, projects/{project}/global/images/family/{family}, global/images/{image}, global/images/family/{family}, family/{family}, {project}/{family}, {project}/{image}, {family}, or {image}.
+          If referred by family, the images names must include the family name. If they don't, use the google_compute_image data source. For instance, the image centos-6-v20180104 includes its family name centos-6. These images can be referred by family name here.
+        required: false
+      image_encryption_key:
+        type: yorc.datatypes.google.EncryptionKey
+        description: >
+          The customer-supplied encryption key of the source image. Required if the source image is protected by a customer-supplied encryption key.
+        required: false
+      snapshot_id:
+        type: string
+        description: >
+          The source snapshot used to create this disk. You can provide this as a partial or full URL to the resource.
+          For example, https://www.googleapis.com/compute/v1/projects/project/global/snapshots/snapshot, projects/project/global/snapshots/snapshot, global/snapshots/snapshot, snapshot are valid values
+        required: false
+      snapshot_encryption_key:
+        type: yorc.datatypes.google.EncryptionKey
+        description: >
+          The customer-supplied encryption key of the source snapshot. Required if the source snapshot is protected by a customer-supplied encryption key.
+        required: false
+
+

deployments/consul_test.go

@@ -96,5 +96,8 @@ func TestRunConsulDeploymentsPackageTests(t *testing.T) {
 		t.Run("TestOperationHost", func(t *testing.T) {
 			testOperationHost(t, kv)
 		})
+		t.Run("testIssueGetEmptyPropOnRelationship", func(t *testing.T) {
+			testIssueGetEmptyPropOnRelationship(t, kv)
+		})
 	})
 }

deployments/definition_store.go

@@ -1091,9 +1091,11 @@ func fixAlienBlockStorages(ctx context.Context, kv *api.KV, deploymentID, nodeNa
 			if err != nil {
 				return errors.Wrapf(err, "Failed to fix Alien-specific BlockStorage %q", nodeName)
 			}
+
+			req.RelationshipProps = make(map[string]*tosca.ValueAssignment)
+
 			if device != nil {
 				va := &tosca.ValueAssignment{}
-				req.RelationshipProps = make(map[string]*tosca.ValueAssignment)
 				if device.RawString() != "" {
 					err = yaml.Unmarshal([]byte(device.RawString()), &va)
 					if err != nil {
@@ -1103,6 +1105,19 @@ func fixAlienBlockStorages(ctx context.Context, kv *api.KV, deploymentID, nodeNa
 				req.RelationshipProps["device"] = va
 			}
 
+			// Get all requirement properties
+			kvps, _, err := kv.List(path.Join(attachReq, "properties"), nil)
+			if err != nil {
+				return errors.Wrapf(err, "Failed to fix Alien-specific BlockStorage %q", nodeName)
+			}
+			for _, kvp := range kvps {
+				va := &tosca.ValueAssignment{}
+				err := yaml.Unmarshal(kvp.Value, va)
+				if err != nil {
+					return errors.Wrapf(err, "Failed to fix Alien-specific BlockStorage %q", nodeName)
+				}
+				req.RelationshipProps[path.Base(kvp.Key)] = va
+			}
 			newReqID, err := GetNbRequirementsForNode(kv, deploymentID, computeNodeName)
 			if err != nil {
 				return err
@@ -1179,7 +1194,7 @@ func createMissingBlockStorageForNode(consulStore consulutil.ConsulStore, kv *ap
 }
 
 /**
-This function check if a nodes need a floating IP, and return the name of Floating IP node.
+This function check if a nodes need a block storage, and return the name of BlockStorage node.
 */
 func checkBlockStorage(kv *api.KV, deploymentID, nodeName string) (bool, []string, error) {
 	requirementsKey, err := GetRequirementsKeysByTypeForNode(kv, deploymentID, nodeName, "local_storage")

deployments/definition_store_test.go

@@ -765,6 +765,27 @@ func testIssueGetEmptyPropRel(t *testing.T, kv *api.KV) {
 	require.Equal(t, "", results[0].Value)
 }
 
+func testIssueGetEmptyPropOnRelationship(t *testing.T, kv *api.KV) {
+	// t.Parallel()
+	deploymentID := strings.Replace(t.Name(), "/", "_", -1)
+	err := StoreDeploymentDefinition(context.Background(), kv, deploymentID, "testdata/issue_get_empty_prop_rel.yaml")
+	require.Nil(t, err)
+	// First test operation outputs detection
+
+	results, err := GetOperationInput(kv, deploymentID, "ValueAssignmentNode2", prov.Operation{
+		Name:                   "configure.pre_configure_source",
+		ImplementedInType:      "yorc.tests.relationships.ValueAssignmentConnectsTo",
+		ImplementationArtifact: "",
+		RelOp: prov.RelationshipOperation{
+			IsRelationshipOperation: true,
+			RequirementIndex:        "1",
+			TargetNodeName:          "ValueAssignmentNode1",
+		}}, "input_empty_prop")
+	require.Nil(t, err)
+	require.Len(t, results, 1)
+	require.Equal(t, "", results[0].Value)
+}
+
 func testRelationshipWorkflow(t *testing.T, kv *api.KV) {
 	// t.Parallel()
 	deploymentID := strings.Replace(t.Name(), "/", "_", -1)

deployments/testdata/issue_get_empty_prop_rel.yaml

@@ -166,6 +166,11 @@ node_types:
 relationship_types:
   yorc.tests.relationships.ValueAssignmentConnectsTo:
     derived_from: tosca.relationships.ConnectsTo
+    properties:
+      empty_prop:
+        type: string
+        required: false
+        default: ""
     interfaces:
       Configure:
         pre_configure_source:
@@ -174,6 +179,9 @@ relationship_types:
             input_list: ["l1", "l2"]
             input_list_ex:
               - "le1"
+
+
+
               - "le2"
             input_map: {"key1": "value1", "key2": "value2"}
             input_map_ex:
@@ -188,6 +196,7 @@ relationship_types:
             input_propList_all: { get_property: [SOURCE, list] }
             input_propList_0_alien: { get_property: [SOURCE, "list[0]"] }
             input_propList_0_tosca: { get_property: [SOURCE, list, 0] }
+            input_empty_prop: { get_property: [SELF, empty_prop] }
           implementation: scripts/show_inputs.sh
         post_configure_source:
           inputs: