Use allowed_ssh_sources for workspace to simplify the firewall config…

…urations (oap-project#562)

README.md

@@ -109,17 +109,12 @@ workspace_name: example-workspace
 provider:
     type: aws
     region: us-west-2
-    security_group:
-        # Use IpPermissions to allow SSH access from your working node
-        # Remember to restrict IpRanges for security 
-        IpPermissions:
-        - FromPort: 22
-          ToPort: 22
-          IpProtocol: TCP
-          IpRanges:
-          - CidrIp: 0.0.0.0/0
-```
-*NOTE:* Remember to change `CidrIp` from `0.0.0.0/0` to restricted IpRanges for TCP port 22 security.
+    # Use allowed_ssh_sources to allow SSH access from your client machine
+    allowed_ssh_sources:
+      - 0.0.0.0/0
+```
+*NOTE:* `0.0.0.0/0` in `allowed_ssh_sources` will allow any IP addresses to connect to your cluster as long as it has the cluster private key.
+For more security, make sure to change from `0.0.0.0/0` to restricted CIDR ranges for your case.
 
 Use the following command to create and provision a Workspace:
 

docs/source/GettingStarted/quick-start.md

@@ -95,17 +95,12 @@ workspace_name: example-workspace
 provider:
     type: aws
     region: us-west-2
-    security_group:
-        # Use IpPermissions to allow SSH access from your working node
-        # Remember to restrict IpRanges for security 
-        IpPermissions:
-        - FromPort: 22
-          ToPort: 22
-          IpProtocol: TCP
-          IpRanges:
-          - CidrIp: 0.0.0.0/0
-```
-*NOTE:* Remember to change `CidrIp` from `0.0.0.0/0` to restricted IpRanges for TCP port 22 security.
+    # Use allowed_ssh_sources to allow SSH access from your client machine
+    allowed_ssh_sources:
+      - 0.0.0.0/0
+```
+*NOTE:* `0.0.0.0/0` in `allowed_ssh_sources` will allow any IP addresses to connect to your cluster as long as it has the cluster private key.
+For more security, make sure to change from `0.0.0.0/0` to restricted CIDR ranges for your case.
 
 Use the following command to create and provision a Workspace:
 

example/cluster/aws/example-workspace.yaml

@@ -5,11 +5,6 @@ workspace_name: example-workspace
 provider:
     type: aws
     region: us-west-2
-    security_group:
-        # Use IpPermissions to allow SSH access from your working node
-        IpPermissions:
-          - FromPort: 22
-            ToPort: 22
-            IpProtocol: TCP
-            IpRanges:
-              - CidrIp: 0.0.0.0/0
+    # Use allowed_ssh_sources to allow SSH access from your client machine
+    allowed_ssh_sources:
+      - 0.0.0.0/0

example/cluster/azure/example-workspace.yaml

@@ -6,14 +6,6 @@ provider:
     type: azure
     location: westus
     subscription_id: your_subscription_id
-    # Use securityRules to allow SSH access from your working node
-    securityRules:
-      - priority: 1000
-        protocol: Tcp
-        access: Allow
-        direction: Inbound
-        source_address_prefixes:
-          - 0.0.0.0/0
-        source_port_range: "*"
-        destination_address_prefix: "*"
-        destination_port_range: 22
+    # Use allowed_ssh_sources to allow SSH access from your client machine
+    allowed_ssh_sources:
+      - 0.0.0.0/0

example/cluster/gcp/example-workspace.yaml

@@ -7,12 +7,6 @@ provider:
     region: us-central1
     availability_zone: us-central1-a
     project_id: your_project_id
-    firewalls:
-        # Use firewall_rules to allow SSH access from your working node
-        firewall_rules:
-          - allowed:
-              - IPProtocol: tcp
-                ports:
-                  - 22
-            sourceRanges:
-              - 0.0.0.0/0
+    # Use allowed_ssh_sources to allow SSH access from your client machine
+    allowed_ssh_sources:
+      - 0.0.0.0/0

python/cloudtik/core/workspace-schema.json

@@ -112,6 +112,13 @@
                     "type": "boolean",
                     "description": "Whether to create managed cloud storage of workspace.",
                     "default": true
+                },
+                "allowed_ssh_sources": {
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    },
+                    "description": "The list of CIDR definitions for hosts allowing ssh connection. For example, 0.0.0.0/0 for all hosts."
                 }
             }
         }

python/cloudtik/core/workspace_provider.py

@@ -53,6 +53,6 @@ def validate_config(provider_config: Dict[str, Any]):
         pass
 
     @staticmethod
-    def bootstrap_workspace_config(cluster_config: Dict[str, Any]) -> Dict[str, Any]:
-        """Bootstraps the cluster config by adding env defaults if needed."""
-        return cluster_config
+    def bootstrap_workspace_config(config: Dict[str, Any]) -> Dict[str, Any]:
+        """Bootstraps the workspace config by adding env defaults if needed."""
+        return config

python/cloudtik/providers/_private/_azure/config.py

@@ -1551,6 +1551,39 @@ def bootstrap_azure_from_workspace(config):
     return config
 
 
+def bootstrap_azure_workspace(config):
+    # create a copy of the input config to modify
+    config = copy.deepcopy(config)
+    _configure_allowed_ssh_sources(config)
+    return config
+
+
+def _configure_allowed_ssh_sources(config):
+    provider_config = config["provider"]
+    if "allowed_ssh_sources" not in provider_config:
+        return
+
+    allowed_ssh_sources = provider_config["allowed_ssh_sources"]
+    if len(allowed_ssh_sources) == 0:
+        return
+
+    if "securityRules" not in provider_config:
+        provider_config["securityRules"] = []
+    security_rules = provider_config["securityRules"]
+
+    security_rule = {
+        "priority": 1000,
+        "protocol": "Tcp",
+        "access": "Allow",
+        "direction": "Inbound",
+        "source_address_prefixes": [allowed_ssh_source for allowed_ssh_source in allowed_ssh_sources],
+        "source_port_range": "*",
+        "destination_address_prefix": "*",
+        "destination_port_range": 22
+    }
+    security_rules.append(security_rule)
+
+
 def _configure_workspace_resource(config):
     config = _configure_resource_group_from_workspace(config)
     config = _configure_virtual_network_from_workspace(config)

python/cloudtik/providers/_private/_azure/workspace_provider.py

@@ -4,7 +4,7 @@
 from cloudtik.core._private.utils import get_running_head_node
 from cloudtik.providers._private._azure.config import create_azure_workspace, \
     delete_workspace_azure, check_azure_workspace_resource, update_azure_workspace_firewalls, \
-    get_workspace_head_nodes, list_azure_clusters
+    get_workspace_head_nodes, list_azure_clusters, bootstrap_azure_workspace
 from cloudtik.core._private.providers import _get_node_provider
 from cloudtik.core.tags import CLOUDTIK_GLOBAL_VARIABLE_KEY_PREFIX, CLOUDTIK_GLOBAL_VARIABLE_KEY
 from cloudtik.core.workspace_provider import WorkspaceProvider
@@ -62,6 +62,6 @@ def validate_config(
         pass
 
     @staticmethod
-    def bootstrap_workspace_config(cluster_config):
-        return cluster_config
+    def bootstrap_workspace_config(config):
+        return bootstrap_azure_workspace(config)
 

python/cloudtik/providers/_private/_kubernetes/config.py

@@ -55,9 +55,9 @@ def created_msg(resource_type, name):
 def not_provided_msg(resource_type):
     return "no {} config provided, must already exist".format(resource_type)
 
-def bootstrap_workspace_kubernetes(cluster_config):
-    # TODO:
-    return cluster_config
+
+def bootstrap_kubernetes_workspace(config):
+    return config
 
 
 def bootstrap_kubernetes(config):

python/cloudtik/providers/_private/_kubernetes/workspace_provider.py

@@ -1,7 +1,7 @@
 import logging
 from typing import Any, Dict
 
-from cloudtik.providers._private._kubernetes.config import bootstrap_workspace_kubernetes
+from cloudtik.providers._private._kubernetes.config import bootstrap_kubernetes_workspace
 from cloudtik.core.workspace_provider import WorkspaceProvider
 
 logger = logging.getLogger(__name__)
@@ -17,5 +17,5 @@ def validate_config(
         pass
 
     @staticmethod
-    def bootstrap_workspace_config(cluster_config):
-        return bootstrap_workspace_kubernetes(cluster_config)
+    def bootstrap_workspace_config(config):
+        return bootstrap_kubernetes_workspace(config)

python/cloudtik/providers/_private/aws/config.py

@@ -718,6 +718,38 @@ def bootstrap_aws_from_workspace(config):
     return config
 
 
+def bootstrap_aws_workspace(config):
+    # create a copy of the input config to modify
+    config = copy.deepcopy(config)
+    _configure_allowed_ssh_sources(config)
+    return config
+
+
+def _configure_allowed_ssh_sources(config):
+    provider_config = config["provider"]
+    if "allowed_ssh_sources" not in provider_config:
+        return
+
+    allowed_ssh_sources = provider_config["allowed_ssh_sources"]
+    if len(allowed_ssh_sources) == 0:
+        return
+
+    if "security_group" not in provider_config:
+        provider_config["security_group"] = {}
+    security_group_config = provider_config["security_group"]
+
+    if "IpPermissions" not in security_group_config:
+        security_group_config["IpPermissions"] = []
+    ip_permissions = security_group_config["IpPermissions"]
+    ip_permission = {
+        "IpProtocol": "tcp",
+        "FromPort": 22,
+        "ToPort": 22,
+        "IpRanges": [{"CidrIp": allowed_ssh_source} for allowed_ssh_source in allowed_ssh_sources]
+    }
+    ip_permissions.append(ip_permission)
+
+
 def get_workspace_head_nodes(config):
     return _get_workspace_head_nodes(
         config["provider"], config["workspace_name"])

python/cloudtik/providers/_private/aws/workspace_provider.py

@@ -4,7 +4,7 @@
 from cloudtik.core._private.utils import get_running_head_node
 from cloudtik.providers._private.aws.config import create_aws_workspace, \
     delete_workspace_aws, check_aws_workspace_resource, update_aws_workspace_firewalls, \
-    list_aws_clusters, _get_workspace_head_nodes
+    list_aws_clusters, _get_workspace_head_nodes, bootstrap_aws_workspace
 from cloudtik.core._private.providers import _get_node_provider
 from cloudtik.core.tags import CLOUDTIK_GLOBAL_VARIABLE_KEY_PREFIX, CLOUDTIK_GLOBAL_VARIABLE_KEY
 from cloudtik.core.workspace_provider import WorkspaceProvider
@@ -63,5 +63,5 @@ def validate_config(
         pass
 
     @staticmethod
-    def bootstrap_workspace_config(cluster_config):
-        return cluster_config
+    def bootstrap_workspace_config(config):
+        return bootstrap_aws_workspace(config)

python/cloudtik/providers/_private/gcp/config.py

@@ -1405,6 +1405,44 @@ def bootstrap_gcp_from_workspace(config):
     return config
 
 
+def bootstrap_gcp_workspace(config):
+    # create a copy of the input config to modify
+    config = copy.deepcopy(config)
+    _configure_allowed_ssh_sources(config)
+    return config
+
+
+def _configure_allowed_ssh_sources(config):
+    provider_config = config["provider"]
+    if "allowed_ssh_sources" not in provider_config:
+        return
+
+    allowed_ssh_sources = provider_config["allowed_ssh_sources"]
+    if len(allowed_ssh_sources) == 0:
+        return
+
+    if "firewalls" not in provider_config:
+        provider_config["firewalls"] = {}
+    fire_walls = provider_config["firewalls"]
+
+    if "firewall_rules" not in fire_walls:
+        fire_walls["firewall_rules"] = []
+    firewall_rules = fire_walls["firewall_rules"]
+
+    firewall_rule = {
+        "allowed": [
+            {
+              "IPProtocol": "tcp",
+              "ports": [
+                "22"
+              ]
+            }
+        ],
+        "sourceRanges": [allowed_ssh_source for allowed_ssh_source in allowed_ssh_sources]
+    }
+    firewall_rules.append(firewall_rule)
+
+
 def _configure_project(config, crm):
     """Setup a Google Cloud Platform Project.
 

python/cloudtik/providers/_private/gcp/workspace_provider.py

@@ -3,7 +3,7 @@
 
 from cloudtik.providers._private.gcp.config import create_gcp_workspace, \
     delete_workspace_gcp, check_gcp_workspace_resource, update_gcp_workspace_firewalls, \
-    get_workspace_head_nodes, list_gcp_clusters
+    get_workspace_head_nodes, list_gcp_clusters, bootstrap_gcp_workspace
 from cloudtik.core._private.providers import _get_node_provider
 from cloudtik.core._private.utils import binary_to_hex, hex_to_binary, get_running_head_node
 from cloudtik.core.tags import CLOUDTIK_GLOBAL_VARIABLE_KEY_PREFIX, CLOUDTIK_GLOBAL_VARIABLE_KEY
@@ -66,5 +66,5 @@ def validate_config(
         pass
 
     @staticmethod
-    def bootstrap_workspace_config(cluster_config):
-        return cluster_config
+    def bootstrap_workspace_config(config):
+        return bootstrap_gcp_workspace(config)