[DPM/ACA Schema] added message_type and updated router and security g…

…roup configuration - not for 830 release (#362)

* added message_type and updated route state

* adding PushNetworkResourceStatesStream in GoalStateProvisioner

* update comment for VpcConfiguration

* adding router_e2e_workflow.puml

* complete routing scenario A

* moved neighborType inside FixedIp list

* add pm->dpm data exchange format

* added scenario B for router creation after port

* Upload a router e2e workflow

* added routing rule proposed workflow

* update routing rule workflow based on discussion

Co-authored-by: albert <[email protected]>
Co-authored-by: Liguang Xie <[email protected]>

docs/modules/ROOT/workflows/router_e2e_workflow.puml

@@ -0,0 +1,103 @@
+' Copyright 2019 The Alcor Authors.
+
+' Licensed under the Apache License, Version 2.0 (the "License");
+'        you may not use this file except in compliance with the License.
+'        You may obtain a copy of the License at
+
+'        http://www.apache.org/licenses/LICENSE-2.0
+
+'        Unless required by applicable law or agreed to in writing, software
+'        distributed under the License is distributed on an "AS IS" BASIS,
+'        WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+'        See the License for the specific language governing permissions and
+'        limitations under the License.
+
+@startuml
+
+skinparam monochrome true
+
+actor "Customer"
+collections "Route Manager"
+collections "Node Manager"
+collections "Subnet Manager"
+collections "Port Manager"
+collections "Data Plane Manager"
+
+box "Compute Host 1" #LightBlue
+participant "ACA on \n Host 1"
+end box
+
+box "Compute Host 2" #LightGreen
+participant "ACA on \n Host 2"
+end box
+
+===Scenarios A: Create GREEN subnet P1 on Host 1, then RED subnet P2 on Host 2, both subnets connected to same router==
+
+==Scenario A1: Setup GREEN subnet P1 on Host 1 ==
+autonumber 10
+"Port Manager" -> "Route Manager": Query routing info and connected subnets {GREEN subnet id, vpc id}
+"Route Manager" -> "Port Manager": Response {routerinfo(connected to GREEN subnet)\n Subnetids(the list of connected subnets including GREEN subnet)}
+"Port Manager" -> "Node Manager": Get node info input: {host_id}, new port\n output: {host_ip, local_host_dvr_mac (new)}
+"Port Manager" -> "Node Manager": Get node info input: {host_id}, L3 neighbor\n output {neighbor host_ip, neighbor host DVR mac(new)}
+"Port Manager" -> "Data Plane Manager": POST /port sync with RequestBody NetworkConfiguration=\n {[P1 port w/ L2_neighbor_Id=NULL \n L3_neighbor_Id=NULL],[GREEN Subnet],[VPC],[SG]}
+"Data Plane Manager" -> "ACA on \n Host 1": Port: CREATE (FULL) P1\n DHCP: CREATE (FULL) P1
+
+==Scenario A2: Setup RED subnet P2 on Host 2 ==
+autonumber 20
+"Port Manager" -> "Route Manager": Query routing info and connected subnets {RED subnet id, vpc id}
+"Route Manager" -> "Port Manager": Response {routerinfo(connected to RED subnet)\n Subnetids(the list of connected subnets including RED subnet)}
+"Port Manager" -> "Node Manager": Get node info input: {host_id}, new port\n output: {host_ip, local_host_dvr_mac (new)}
+"Port Manager" -> "Node Manager": Get node info input: {host_id}, L3 neighbor\n output {neighbor host_ip, neighbor host DVR mac(new)}
+"Port Manager" -> "Data Plane Manager": POST /port sync with RequestBody NetworkConfiguration=\n {[P2 port w/ L2_neighbor_Id=NULL \n L3_neighbor_Id=[P1]],[Router INFO][GREEN+RED Subnet],[VPC],[SG]}
+"Data Plane Manager" -> "ACA on \n Host 2": Port: CREATE (FULL) P2\n DHCP: CREATE (FULL) P2\n L3 neighbors CREATE (DELTA) P1 + Router: INFO \n (w/ host 2 local DVR mac), Subnet: INFO (GREEN+RED)
+
+==Scenario A3: Setup L3 neighbor about RED subnet P2 on Host 1 ==
+"Data Plane Manager" -> "ACA on \n Host 1": L3 neighbors CREATE (DELTA) P2 + Router: INFO\n (w/ host 1 local DVR mac), Subnet: INFO (RED+GREEN needed for router)
+
+
+===Scenarios B: (Neutron only) Create GREEN subnet P1 on Host 1, then RED subnet P2 on Host 2, GREEN and RED subnets NOT connected until after P1 and P2 is created==
+
+==Scenario B1: Setup GREEN subnet P1 on Host 1 ==
+autonumber 30
+"Port Manager" -> "Route Manager": Query routing info and connected subnets {GREEN subnet id, vpc id}
+"Route Manager" -> "Port Manager": Response {routerinfo(connected to GREEN subnet),\n Subnetids(the list of connected subnets including GREEN subnet)}\n Should return none
+"Port Manager" -> "Node Manager": Get node info input: {host_id}, new port \n output: {host_ip, local_host_dvr_mac (new)}
+"Port Manager" -> "Node Manager": Get node info input: {host_id}, L3 neighbor \n output {neighbor host_ip, neighbor host DVR mac(new)}\n May merge with call above
+"Port Manager" -> "Data Plane Manager": POST /port sync with RequestBody NetworkConfiguration=\n {[P1 port w/ L2_neighbor_Id=NULL \n L3_neighbor_Id=NULL],[GREEN Subnet],[VPC],[SG]}
+"Data Plane Manager" -> "ACA on \n Host 1": Port: CREATE (FULL) P1, \n DHCP: CREATE (FULL) P1\n Subnet: INFO (GREEN only)
+
+==Scenario B2: Setup RED subnet P2 on Host 2 ==
+autonumber 40
+"Port Manager" -> "Route Manager": Query routing info and connected subnets {RED subnet id, vpc id}
+"Route Manager" -> "Port Manager": Response {routerinfo(connected to RED subnet)\n Subnetids(the list of connected subnets including RED subnet)}\n Should return none
+"Port Manager" -> "Node Manager": Get node info input: {host_id}, new port\n output: {host_ip, local_host_dvr_mac (new)}
+"Port Manager" -> "Node Manager": Get node info input: {host_id}, L3 neighbor\n output {neighbor host_ip, neighbor host DVR mac(new)}\n May merge with call above
+"Port Manager" -> "Data Plane Manager": POST /port sync with RequestBody NetworkConfiguration=\n {[P2 port w/ L2_neighbor_Id=NULL \n L3_neighbor_Id=NULL],[VPC],[SG]}
+"Data Plane Manager" -> "ACA on \n Host 2": Port: CREATE (FULL) P2\n DHCP: CREATE (FULL) P2\n Subnet: INFO (RED only)
+
+==Scenario B3: Router created and connected GREEN + RED subnet together ==
+autonumber 50
+"Customer" -> "Route Manager": Create new router
+"Customer" -> "Route Manager": Connect GREEN subnet\n to new router
+"Route Manager" -> "Subnet Manager": Update router ID for GREEN subnet
+"Customer" -> "Route Manager": Connect RED subnet\n to new router
+"Route Manager" -> "Subnet Manager": Update router ID for RED subnet
+"Route Manager" -> "Port Manager": Update L3 Neighbor
+"Port Manager" -> "Node Manager": Get node info input: {host_id}, new port\n output: {host_ip, local_host_dvr_mac (new)}
+"Port Manager" -> "Node Manager": Get node info input: {host_id}, L3 neighbor\n output {neighbor host_ip, neighbor host DVR mac(new)}\n May merge with call above
+"Port Manager" -> "Data Plane Manager": POST /port sync with RequestBody NetworkConfiguration=\n {[Neighbor port Id=[P1,P2]],[Router INFO][GREEN+RED Subnet],[VPC],[SG]} 
+"Data Plane Manager" -> "ACA on \n Host 1": L3 neighbors CREATE (DELTA) P2 + Router: INFO\n (w/ host 1 local DVR mac), Subnet: INFO (RED+GREEN needed for router)
+"Data Plane Manager" -> "ACA on \n Host 2": L3 neighbors CREATE (DELTA) P1 + Router: INFO\n (w/ host 2 local DVR mac), Subnet: INFO (RED+GREEN needed for router)
+
+
+===Scenario C: (VPC only) Routing rule added for GREEN subnet, GREEN subnet P1 on Host 1, RED subnet P2 on Host 2  ==
+autonumber 60
+"Customer" -> "Route Manager": Existing router created with GREEN and RED subnet connected
+"Customer" -> "Route Manager": Add routing rule on GREEN subnet\n customer provide subnet ID
+"Route Manager" -> "Subnet Manager": For new routing rule like new external routing rule\n (new) send GREEN subnet level routing rule to subnet manager
+"Subnet Manager" -> "Subnet Manager": Look at all the ports in GREEN subnet\n Look up cache for their {host_ip,\n local_host_dvr_mac (not needed)}
+"Subnet Manager" -> "Data Plane Manager": For all the involved host which has GREEN subnet port, Update router info\n which includes the new routing rule on GREEN subnet
+"Data Plane Manager" -> "ACA on \n Host 1": Router: INFO (add new routing rule on GREEN subnet)\n Subnet: INFO (GREEN needed for router)
+
+@enduml
+

schema/proto3/common.proto

@@ -68,4 +68,9 @@ enum NetworkType {
     GRE = 2;
     GENEVE = 3;
     VXLAN_GPE = 4;
+}
+
+enum MessageType { 
+    DELTA = 0; // the default type
+    FULL = 1;
 }

schema/proto3/dhcp.proto

@@ -28,11 +28,12 @@ message DHCPConfiguration {
     uint32 revision_number = 2;
 
     string request_id = 3;
-    string subnet_id = 4;
-    string mac_address = 5;
-    string ipv4_address = 6;
-    string ipv6_address = 7;
-    string port_host_name = 8; // for local DNS response
+    MessageType message_type = 4; // DELTA (default) or FULL
+    string subnet_id = 5;
+    string mac_address = 6;
+    string ipv4_address = 7;
+    string ipv6_address = 8;
+    string port_host_name = 9; // for local DNS response
 
     message ExtraDhcpOption {
         string name = 1;
@@ -43,8 +44,8 @@ message DHCPConfiguration {
         string entry = 1;
     }
 
-    repeated ExtraDhcpOption extra_dhcp_options = 9;
-    repeated DnsEntry dns_entry_list = 10;
+    repeated ExtraDhcpOption extra_dhcp_options = 10;
+    repeated DnsEntry dns_entry_list = 11;
 }
 
 message DHCPState {

schema/proto3/goalstateprovisioner.proto

@@ -35,6 +35,11 @@ service GoalStateProvisioner {
     rpc PushNetworkResourceStates (GoalState) returns (GoalStateOperationReply) {
     }
 
+    // same as PushNetworkResourceStates with streaming GoalState and streaming GoalStateOperationReply
+    rpc PushNetworkResourceStatesStream (stream GoalState) returns (stream GoalStateOperationReply) {
+    }
+
+
     // Retrieve a group of network resource states (stored as a steam of GoalState objects)
     rpc RetrieveNetworkResourceStates (GoalStateRequest) returns (stream GoalState) {
     }

schema/proto3/neighbor.proto

@@ -34,7 +34,7 @@ message NeighborConfiguration {
 
     string request_id = 3;
     string id = 4;
-    NeighborType neighbor_type = 5; // L2 (default) or L3
+    MessageType message_type = 5; // DELTA (default) or FULL
     string project_id = 6;
     string vpc_id = 7;
     string name = 8;
@@ -43,8 +43,9 @@ message NeighborConfiguration {
     string neighbor_host_dvr_mac = 11;
 
     message FixedIp {
-        string subnet_id = 1;
-        string ip_address = 2;
+        NeighborType neighbor_type = 1;
+        string subnet_id = 2;
+        string ip_address = 3;
     }
 
     message AllowAddressPair {

schema/proto3/port.proto

@@ -23,18 +23,13 @@ option java_outer_classname = "Port";
 
 import "common.proto";
 
-enum MessageType { 
-    DELTA = 0; // the default type
-    FULL = 1;
-}
-
 message PortConfiguration {
     uint32 format_version = 1;
     uint32 revision_number = 2;
 
     string request_id = 3;
     string id = 4;
-    MessageType message_type = 5;
+    MessageType message_type = 5; // DELTA (default) or FULL
     NetworkType network_type = 6; // to be removed, will use the one in subnet
     string project_id = 7;
     string vpc_id = 8;

schema/proto3/router.proto

@@ -29,8 +29,24 @@ message RouterConfiguration {
 
     string request_id = 3;
     string id = 4;
-    string host_dvr_mac_address = 5;
-    repeated string subnet_ids = 6;
+    MessageType message_type = 5; // DELTA (default) or FULL
+    string host_dvr_mac_address = 6;
+
+    message RoutingRule {
+        OperationType operation_type = 1;
+        string id = 2;
+        string name = 3;
+        string destination = 4;
+        string nextHop = 5;
+        uint32 priority = 6;
+    }
+
+    message SubnetRoutingTable {
+        string subnet_id = 1;
+        repeated RoutingRule routing_rules = 2;
+    }
+
+    repeated SubnetRoutingTable subnet_routing_tables = 7;
 }
 
 message RouterState {

schema/proto3/securitygroup.proto

@@ -29,28 +29,30 @@ message SecurityGroupConfiguration {
 
     string request_id = 3;
     string id = 4;
-    string project_id = 5;
-    string vpc_id = 6;
-    string name = 7;
+    MessageType message_type = 5; // DELTA (default) or FULL
+    string project_id = 6;
+    string vpc_id = 7;
+    string name = 8;
 
     enum Direction {
         EGRESS = 0;
         INGRESS = 1;
     }
 
     message SecurityGroupRule {
-        string security_group_id = 1;
-        string id = 2;
-        Direction direction = 3;
-        EtherType ethertype = 4;
-        Protocol protocol = 5;
-        uint32 port_range_min = 6;
-        uint32 port_range_max = 7;
-        string remote_ip_prefix = 8;
-        string remote_group_id = 9;
+        OperationType operation_type = 1;
+        string security_group_id = 2;
+        string id = 3;
+        Direction direction = 4;
+        EtherType ethertype = 5;
+        Protocol protocol = 6;
+        uint32 port_range_min = 7;
+        uint32 port_range_max = 8;
+        string remote_ip_prefix = 9;
+        string remote_group_id = 10;
     }
 
-    repeated SecurityGroupRule security_group_rules = 8;
+    repeated SecurityGroupRule security_group_rules = 9;
 }
 
 message SecurityGroupState {

schema/proto3/subnet.proto

@@ -29,25 +29,26 @@ message SubnetConfiguration {
 
     string request_id = 3;
     string id = 4;
-    NetworkType network_type = 5;
-    string project_id = 6;
-    string vpc_id = 7;
-    string name = 8;
-    string cidr = 9;
+    MessageType message_type = 5; // DELTA (default) or FULL
+    NetworkType network_type = 6;
+    string project_id = 7;
+    string vpc_id = 8;
+    string name = 9;
+    string cidr = 10;
 
     // TODO: change to uint32 but that would require change in DPM
-    uint64 tunnel_id = 10;
+    uint64 tunnel_id = 11;
 
     message Gateway {
         string ip_address = 3;
         string mac_address = 4;
     }
 
-    Gateway gateway = 11;
-    bool dhcp_enable = 12;
-    string availability_zone = 13;
-    string primary_dns = 14;
-    string secondary_dns = 15;
+    Gateway gateway = 12;
+    bool dhcp_enable = 13;
+    string availability_zone = 14;
+    string primary_dns = 15;
+    string secondary_dns = 16;
 }
 
 message SubnetState {

schema/proto3/vpc.proto

@@ -23,16 +23,19 @@ option java_outer_classname = "Vpc";
 
 import "common.proto";
 
-message VpcConfiguration {
+// Not used based on the current design including router E2E
+// To be removed once we complete the design for all major features
+message VpcConfiguration {  
     uint32 format_version = 1;
     uint32 revision_number = 2;
 
     string request_id = 3;
     string id = 4;
-    string project_id = 5;
-    string name = 6;
-    string cidr = 7;
-    int64 tunnel_id = 8;
+    MessageType message_type = 5; // DELTA (default) or FULL
+    string project_id = 6;
+    string name = 7;
+    string cidr = 8;
+    int64 tunnel_id = 9;
 
     message SubnetId {
         string id = 1;
@@ -43,8 +46,8 @@ message VpcConfiguration {
         string next_hop = 2;
     }
 
-    repeated SubnetId subnet_ids = 9;
-    repeated Route routes = 10;
+    repeated SubnetId subnet_ids = 10;
+    repeated Route routes = 11;
 }
 
 message VpcState {

services/data_plane_manager/src/main/java/com/futurewei/alcor/dataplane/utils/GoalStateManager.java

@@ -244,7 +244,7 @@ public Map<String, Goalstate.GoalState> transformNorthToSouth(
                                     .setNetworkTypeValue(Common.NetworkType.VXLAN_VALUE)
                                     .setId(pid)
                                     .setHostInfo(h)
-                                    .setMessageTypeValue(Port.MessageType.DELTA_VALUE)
+                                    .setMessageTypeValue(Common.MessageType.DELTA_VALUE)
                                     .build();
                             final PortState portStateSB =
                                 PortState.newBuilder()
@@ -259,7 +259,7 @@ public Map<String, Goalstate.GoalState> transformNorthToSouth(
                                 .toBuilder()
                                 .setId(portStateWithEverythingFilledNB.getId())
                                 .setNetworkTypeValue(Common.NetworkType.VXLAN_VALUE)
-                                .setMessageTypeValue(Port.MessageType.FULL_VALUE)
+                                .setMessageTypeValue(Common.MessageType.FULL_VALUE)
                                 .build();
 
                         final PortState portStateSB =