Merge branch 'develop' into feature/GH-131-bootstrap-yorc

ystia · Nov 16, 2018 · 030199b · 030199b
2 parents b21eaff + aabb3b9
commit 030199b
Show file tree

Hide file tree

Showing 15 changed files with 984 additions and 6 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,8 +2,11 @@
 
 ## UNRELEASED
 
+## 3.1.0-M6 (November 16, 2018)
+
 ### FEATURES
 
+* Support GCE virtual private networks (VPC) ([GH-80](https://github.com/ystia/yorc/issues/80))
 * Support Kubernetes Jobs. ([GH-86](https://github.com/ystia/yorc/issues/86))
 
 ### ENHANCEMENTS

diff --git a/data/tosca/yorc-google-types.yml b/data/tosca/yorc-google-types.yml
@@ -25,6 +25,67 @@ data_types:
         required: false
         description: The RFC 4648 base64 encoded SHA-256 hash of the customer-supplied encryption key that protects this resource.
 
+  yorc.datatypes.google.IPRange:
+    derived_from: tosca.datatypes.Root
+    properties:
+      name:
+        type: string
+        required: true
+        description: >
+          The name of this IP range, used when adding an alias IP range to a VM instance.
+          The name must be 1-63 characters long, and comply with RFC1035. The name must be unique within a subnetwork.
+      ip_cidr_range:
+        type: string
+        required: true
+        description: >
+          The range of IP addresses belonging to this range.
+          Ranges must be unique and non-overlapping with all primary and secondary IP ranges within a network.
+          Only IPv4 is supported.
+
+  yorc.datatypes.google.Subnetwork:
+    derived_from: tosca.datatypes.Root
+    properties:
+      name:
+        type: string
+        required: true
+        description: >
+          The name of the resource, provided by the client when initially creating the resource. The name must be 1-63 characters long, and comply with RFC1035.
+          Specifically, the name must be 1-63 characters long and match the regular expression [a-z]([-a-z0-9]*[a-z0-9])?
+          which means the first character must be a lowercase letter, and all following characters must be a dash, lowercase letter, or digit, except the last character, which cannot be a dash.
+      ip_cidr_range:
+        type: string
+        required: true
+        description: >
+          The range of internal addresses that are owned by this sub-network.
+          For example, 10.0.0.0/8 or 192.168.0.0/16. Ranges must be unique and non-overlapping within a network. Only IPv4 is supported.
+      project:
+        type: string
+        description: >
+          The ID of the project in which the resource belongs. If it is not provided, the infrastructure location project is used.
+        required: false
+      region:
+        type: string
+        description: >
+          The Region in which this subnet should reside.
+        required: true
+      enable_flow_logs:
+        type: boolean
+        description: Whether to enable flow logging for this subnetwork.
+        required: false
+      private_ip_google_access:
+        type: boolean
+        description: Whether the VMs in this subnet can access Google services without assigned external IP addresses.
+        required: false
+      secondary_ip_ranges:
+        type: list
+        description: >
+          A list of configurations for secondary IP ranges for VM instances contained in this sub-network.
+          The primary IP of such VM must belong to the primary ip_cidr_range of the sub-network.
+          The alias IPs may belong to either primary or secondary ranges.
+        entry_schema:
+          type: yorc.datatypes.google.IPRange
+        required: false
+
   yorc.datatypes.google.ScratchDisk:
     derived_from: tosca.datatypes.Root
     properties:
@@ -52,6 +113,16 @@ relationship_types:
         description: >
           The logical name of the device as exposed to the instance.
           Note: A runtime property that gets set when the model gets instantiated by the orchestrator.
+  yorc.relationships.google.Network:
+    derived_from: tosca.relationships.Network
+    valid_target_types: [ tosca.capabilities.Connectivity ]
+    properties:
+      subnet:
+        type: string
+        description: >
+          Specify the sub-network to use for this relationship if network contains multiple sub-networks.
+          If none is defined, the first matching subnet with the target Google region will be associated to it.
+        required: false
 
 node_types:
   yorc.nodes.google.Compute:
@@ -160,8 +231,122 @@ node_types:
           relationship: yorc.relationships.AssignsTo
           occurrences: [0, UNBOUNDED]
 
-  yorc.nodes.google.Network:
+  yorc.nodes.google.Subnetwork:
     derived_from: tosca.nodes.Network
+    # See https://www.terraform.io/docs/providers/google/r/compute_subnetwork.html
+    properties:
+      name:
+        type: string
+        required: true
+        description: >
+          The name of the resource, provided by the client when initially creating the resource. The name must be 1-63 characters long, and comply with RFC1035.
+          Specifically, the name must be 1-63 characters long and match the regular expression [a-z]([-a-z0-9]*[a-z0-9])?
+          which means the first character must be a lowercase letter, and all following characters must be a dash, lowercase letter, or digit, except the last character, which cannot be a dash.
+      network:
+        type: string
+        required: true
+        description: >
+          The network this subnet belongs to.
+      ip_cidr_range:
+        type: string
+        required: true
+        description: >
+          The range of internal addresses that are owned by this sub-network.
+          For example, 10.0.0.0/8 or 192.168.0.0/16. Ranges must be unique and non-overlapping within a network. Only IPv4 is supported.
+      project:
+        type: string
+        description: >
+          The ID of the project in which the resource belongs. If it is not provided, the infrastructure location project is used.
+        required: false
+      region:
+        type: string
+        description: >
+          The Region in which this subnet should reside.
+        required: true
+      enable_flow_logs:
+        type: boolean
+        description: Whether to enable flow logging for this subnetwork.
+        required: false
+      private_ip_google_access:
+        type: boolean
+        description: Whether the VMs in this subnet can access Google services without assigned external IP addresses.
+        required: false
+      secondary_ip_ranges:
+        type: list
+        description: >
+          A list of configurations for secondary IP ranges for VM instances contained in this sub-network.
+          The primary IP of such VM must belong to the primary ip_cidr_range of the sub-network.
+          The alias IPs may belong to either primary or secondary ranges.
+        entry_schema:
+          type: yorc.datatypes.google.IPRange
+        required: false
+    attributes:
+      gateway_ip:
+        type: string
+        description: The IPv4 address of the gateway.
+      network_name:
+        type: string
+        description: The related name of the network of this subnet.
+      subnetwork_name:
+        type: string
+        description: The related name of this subnet.
+
+  yorc.nodes.google.PrivateNetwork:
+    derived_from: tosca.nodes.Network
+    properties:
+    # See https://www.terraform.io/docs/providers/google/r/compute_network.html
+      auto_create_subnetworks:
+        type: boolean
+        description: >
+          If set to true, this network will be created in auto subnet mode, and Google will create a subnet for each region automatically.
+          If set to false, a custom subnetted network must be created. Defaults to true.
+        required: false
+        default: false
+      routing_mode:
+        type: string
+        description: >
+          Sets the network-wide routing mode for Cloud Routers to use. Accepted values are "GLOBAL" or "REGIONAL". Defaults to "REGIONAL".
+          Refer to the Cloud Router (https://cloud.google.com/router/docs/concepts/overview#dynamic-routing-mode) documentation for more details.
+        required: false
+      description:
+        type: string
+        description: >
+          An optional description of this resource.
+        required: false
+      network_name:
+        type: string
+        description: >
+          The existing network to use. You can create default or custom subnets with existing network.
+        required: false
+      cidr:
+        type: string
+        description: >
+          CIDR range for default subnet creation. If this field is specified, you must fill the cidr_region too.
+        required: false
+      cidr_region:
+        type: string
+        description: >
+          The Google region to used with specified cidr for default subnet creation. This field is mandatory is cidr is set.
+        required: false
+      project:
+        type: string
+        description: >
+          The ID of the project in which the resource belongs. If it is not provided, the infrastructure location project is used.
+        required: false
+      custom_subnetworks:
+        type: list
+        description: >
+          if auto_create_subnetworks is set to false, you must create at least one custom sub-network for this network.
+          Each VPC network is subdivided into subnets, and each subnet is contained within a single region.
+          You can have more than one subnet in a region for a given VPC network. Each subnet has a contiguous private RFC1918 IP space.
+          When you create an instance in a subnet, the instance draws its internal IP address from that subnet.
+        entry_schema:
+          type: yorc.datatypes.google.Subnetwork
+        required: false
+    attributes:
+      network_name:
+        type: string
+        description: The unique name of the network.
 
   yorc.nodes.google.Address:
     derived_from: tosca.nodes.Root

diff --git a/deployments/requirements.go b/deployments/requirements.go
@@ -241,7 +241,38 @@ func HasAnyRequirementCapability(kv *api.KV, deploymentID, nodeName, requirement
 			if capability == capabilityType {
 				return true, relatedNodeName, nil
 			}
-			is, err := IsNodeDerivedFrom(kv, deploymentID, relatedNodeName, capabilityType)
+			is, err := IsTypeDerivedFrom(kv, deploymentID, capability, capabilityType)
+			if err != nil {
+				return false, "", err
+			}
+			return is, relatedNodeName, nil
+		}
+	}
+
+	return false, "", nil
+}
+
+// HasAnyRequirementFromNodeType returns true and the the related node name addressing the capability
+// if node with name nodeName has the requirement with the node type equal or derived from the provided type
+// otherwise it returns false and empty string
+func HasAnyRequirementFromNodeType(kv *api.KV, deploymentID, nodeName, requirement, nodeType string) (bool, string, error) {
+	reqkKeys, err := GetRequirementsKeysByTypeForNode(kv, deploymentID, nodeName, requirement)
+	if err != nil {
+		return false, "", err
+	}
+	for _, reqPrefix := range reqkKeys {
+		requirementIndex := GetRequirementIndexFromRequirementKey(reqPrefix)
+		capability, err := GetCapabilityForRequirement(kv, deploymentID, nodeName, requirementIndex)
+		if err != nil {
+			return false, "", err
+		}
+		relatedNodeName, err := GetTargetNodeForRequirement(kv, deploymentID, nodeName, requirementIndex)
+		if err != nil {
+			return false, "", err
+		}
+
+		if capability != "" {
+			is, err := IsNodeDerivedFrom(kv, deploymentID, relatedNodeName, nodeType)
 			if err != nil {
 				return false, "", err
 			} else if is {

diff --git a/prov/terraform/aws/aws_instance.go b/prov/terraform/aws/aws_instance.go
@@ -162,7 +162,7 @@ func (g *awsGenerator) generateAWSInstance(ctx context.Context, kv *api.KV, cfg
 	// Check existing network requirement otherwise
 	var isElasticIP = len(instance.ElasticIps) > 0
 	if !isElasticIP {
-		isElasticIP, _, err = deployments.HasAnyRequirementCapability(kv, deploymentID, nodeName, "network", "yorc.nodes.aws.PublicNetwork")
+		isElasticIP, _, err = deployments.HasAnyRequirementFromNodeType(kv, deploymentID, nodeName, "network", "yorc.nodes.aws.PublicNetwork")
 		if err != nil {
 			return err
 		}

diff --git a/prov/terraform/google/compute_instance.go b/prov/terraform/google/compute_instance.go
@@ -109,7 +109,29 @@ func (g *googleGenerator) generateComputeInstance(ctx context.Context, kv *api.K
 		return err
 	}
 
-	networkInterface := NetworkInterface{Network: "default"}
+	// Define if a private network access is required
+	var netInterfaces []NetworkInterface
+	reqPrivateNetwork, _, err := deployments.HasAnyRequirementFromNodeType(kv, deploymentID, nodeName, "network", "yorc.nodes.google.PrivateNetwork")
+	if err != nil {
+		return err
+	}
+	// Check for subnet otherwise
+	if !reqPrivateNetwork {
+		reqPrivateNetwork, _, err = deployments.HasAnyRequirementFromNodeType(kv, deploymentID, nodeName, "network", "yorc.nodes.google.Subnetwork")
+		if err != nil {
+			return err
+		}
+	}
+	if reqPrivateNetwork {
+		netInterfaces, err = addPrivateNetworkInterfaces(ctx, kv, deploymentID, nodeName)
+		if err != nil {
+			return err
+		}
+	} else {
+		// Create a default private network interface
+		netInterfaces = append(netInterfaces, NetworkInterface{Network: "default"})
+	}
+
 	// Define an external access if there will be an external IP address
 	if !noAddress {
 		hasStaticAddressReq, addressNode, err := deployments.HasAnyRequirementCapability(kv, deploymentID, nodeName, "assignment", "yorc.capabilities.Assignable")
@@ -128,9 +150,9 @@ func (g *googleGenerator) generateComputeInstance(ctx context.Context, kv *api.K
 		// else externalAddress is empty, which means an ephemeral external IP
 		// address will be assigned to the instance
 		accessConfig := AccessConfig{NatIP: externalAddress}
-		networkInterface.AccessConfigs = []AccessConfig{accessConfig}
+		netInterfaces[0].AccessConfigs = []AccessConfig{accessConfig}
 	}
-	instance.NetworkInterfaces = []NetworkInterface{networkInterface}
+	instance.NetworkInterfaces = netInterfaces
 
 	// Scheduling definition
 	var preemptible bool
@@ -422,3 +444,56 @@ func addAttachedDisks(ctx context.Context, cfg config.Configuration, kv *api.KV,
 	}
 	return devices, nil
 }
+
+func addPrivateNetworkInterfaces(ctx context.Context, kv *api.KV, deploymentID, nodeName string) ([]NetworkInterface, error) {
+	var netInterfaces []NetworkInterface
+
+	// Check if subnets have been specified by user into network relationship
+	storageKeys, err := deployments.GetRequirementsKeysByTypeForNode(kv, deploymentID, nodeName, "network")
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to add network interfaces for deploymentID:%q, nodeName:%q", deploymentID, nodeName)
+	}
+	for _, storagePrefix := range storageKeys {
+		requirementIndex := deployments.GetRequirementIndexFromRequirementKey(storagePrefix)
+
+		networkNodeName, err := deployments.GetTargetNodeForRequirement(kv, deploymentID, nodeName, requirementIndex)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to add network interfaces for deploymentID:%q, nodeName:%q", deploymentID, nodeName)
+		}
+
+		// Check if node is network or subnet
+		netType, err := deployments.GetNodeType(kv, deploymentID, networkNodeName)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to add network interfaces for deploymentID:%q, nodeName:%q", deploymentID, nodeName)
+		}
+		switch netType {
+		case "yorc.nodes.google.Subnetwork":
+			subnet, err := attributeLookup(ctx, kv, deploymentID, "0", networkNodeName, "subnetwork_name")
+			if err != nil {
+				return nil, errors.Wrapf(err, "failed to add network interfaces for deploymentID:%q, nodeName:%q, networkName:%q", deploymentID, nodeName, networkNodeName)
+			}
+			log.Debugf("add network interface with sub-network property:%s", subnet)
+			netInterfaces = append(netInterfaces, NetworkInterface{Subnetwork: subnet})
+		case "yorc.nodes.google.PrivateNetwork":
+			// We mention subnet if provided by network relationship property
+			subRaw, err := deployments.GetRelationshipPropertyValueFromRequirement(kv, deploymentID, nodeName, requirementIndex, "subnet")
+			if err != nil {
+				return nil, err
+			}
+			if subRaw != nil && subRaw.RawString() != "" {
+				log.Debugf("add network interface with user-specified sub-network property:%s", subRaw.RawString())
+				netInterfaces = append(netInterfaces, NetworkInterface{Subnetwork: subRaw.RawString()})
+			} else { // we mention the network
+				network, err := attributeLookup(ctx, kv, deploymentID, "0", networkNodeName, "network_name")
+				if err != nil {
+					return nil, errors.Wrapf(err, "failed to add network interfaces for deploymentID:%q, nodeName:%q, networkName:%q", deploymentID, nodeName, networkNodeName)
+				}
+				log.Debugf("add network interface with network property:%s", network)
+				netInterfaces = append(netInterfaces, NetworkInterface{Network: network})
+			}
+		default:
+			return nil, errors.Errorf("type:%q is not handled for compute network interface addition", netType)
+		}
+	}
+	return netInterfaces, nil
+}