Merge pull request #944 from seriva/fixes/init-fixes

Fixed addition of infrastructure documents when using --ful flag (#940)
Switched from PyYAML to ruamel.yaml to preserve comments and commented out configs in the defaults. (#611)
Fixed running epicli init -p any --full generating cloud sample configuration (#736)
Checked if additional security rules for NSGs are applied properly for Azure (#942)
Exclusion of SET_BY_AUTOMATION fields on generated configs
Updated gen-licenses.py to use requirements.txt to generate licenses.py file instead of PipEnv

core/src/epicli/.devcontainer/requirements.txt

@@ -1,10 +1,12 @@
 adal==1.2.2
 ansible==2.8.6
-antlr4-python3-runtime==4.7.2 ; python_version >= '3.0'
+antlr4-python3-runtime==4.7.2
 applicationinsights==0.11.7
 argcomplete==1.10.0
+astroid==2.3.3
 attrs==19.3.0
 azure-batch==6.0.0
+azure-cli==2.0.67
 azure-cli-acr==2.2.9
 azure-cli-acs==2.4.4
 azure-cli-advisor==2.0.1
@@ -64,7 +66,6 @@ azure-cli-sqlvm==0.2.0
 azure-cli-storage==2.4.3
 azure-cli-telemetry==1.0.2
 azure-cli-vm==2.2.23
-azure-cli==2.0.67
 azure-common==1.1.23
 azure-cosmos==3.1.2
 azure-datalake-store==0.0.39
@@ -136,6 +137,7 @@ azure-storage-blob==1.3.1
 azure-storage-common==1.4.2
 azure-storage-nspkg==3.1.0
 bcrypt==3.1.7
+bleach==3.1.1
 boto3==1.10.9
 botocore==1.13.9
 certifi==2019.9.11
@@ -150,45 +152,63 @@ idna==2.8
 importlib-metadata==0.23
 invoke==1.3.0
 isodate==0.6.0
-jinja2==2.10.3
+isort==4.3.21
+jeepney==0.4.2
+Jinja2==2.10.3
 jmespath==0.9.4
 jsonschema==3.1.1
+keyring==21.1.0
 knack==0.6.3
-markupsafe==1.1.1
+lazy-object-proxy==1.4.3
+MarkupSafe==1.1.1
+mccabe==0.6.1
 mock==3.0.5
 more-itertools==7.2.0
 msrest==0.6.10
 msrestazure==0.6.2
 oauthlib==3.1.0
+packaging==20.1
 paramiko==2.6.0
+pkginfo==1.5.0.1
+pluggy==0.13.1
 portalocker==1.2.1
 prompt-toolkit==1.0.18
 psutil==5.6.4
+py==1.8.1
 pycparser==2.19
-pygments==2.4.2
-pyjwt==1.7.1
-pynacl==1.3.0
-pyopenssl==19.0.0
+Pygments==2.4.2
+PyJWT==1.7.1
+PyNaCl==1.3.0
+pyOpenSSL==19.0.0
+pyparsing==2.4.6
 pyperclip==1.7.0
 pyrsistent==0.15.5
-python-dateutil==2.8.0 ; python_version >= '2.7'
+python-dateutil==2.8.0
 python-json-logger==0.1.11
 pytz==2019.3
-pyyaml==5.1.2
-requests-oauthlib==1.2.0
+PyYAML==5.3
+readme-renderer==24.0
 requests==2.22.0
+requests-oauthlib==1.2.0
+requests-toolbelt==0.9.1
+ruamel.yaml==0.16.10
+ruamel.yaml.clib==0.2.0
 s3transfer==0.2.1
 scp==0.13.2
+SecretStorage==3.1.2
 six==1.12.0
 skopeo-bin==1.0.3
 sshtunnel==0.1.5
 tabulate==0.8.5
 terraform-bin==1.0.1
-urllib3[secure]==1.25.6 ; python_version >= '3.4'
-vsts-cd-manager==1.0.2
+tqdm==4.43.0
+typed-ast==1.4.1
+urllib3==1.25.6
 vsts==0.1.25
+vsts-cd-manager==1.0.2
 wcwidth==0.1.7
+webencodings==0.5.1
 websocket-client==0.56.0
-wheel==0.30.0
+wrapt==1.11.2
 xmltodict==0.12.0
 zipp==0.6.0

core/src/epicli/cli/engine/InitEngine.py

@@ -1,11 +1,14 @@
 
 import os
+import sys
 
 from cli.helpers.Step import Step
 from cli.helpers.build_saver import save_manifest, get_build_path
 from cli.helpers.data_loader import load_all_yaml_objs, types, load_all_documents_from_folder
 from cli.engine.BuildEngine import BuildEngine
+from cli.helpers.objdict_helpers import remove_value
 from cli.version import VERSION
+from cli.helpers.doc_list_helpers import select_all, select_single
 
 
 class InitEngine(Step):
@@ -24,32 +27,64 @@ def __exit__(self, exc_type, exc_value, traceback):
         super().__exit__(exc_type, exc_value, traceback)  
 
     def init(self):
-        defaults = load_all_yaml_objs(types.DEFAULT, self.provider, 'configuration/minimal-cluster-config')
-        defaults[0].specification.name = self.name
-
-        for i in range(len(defaults)): 
-            defaults[i]['version'] = VERSION       
+        input = load_all_yaml_objs(types.DEFAULT, self.provider, 'configuration/minimal-cluster-config')
+        input[0].specification.name = self.name     
 
         if self.is_full_config:
-            defaults = self.get_full_config(defaults)
+            config = self.get_config_docs(input)
+            config_only = select_all(config, lambda x: not(x.kind.startswith('epiphany-cluster')))
+            if self.provider == 'any':
+                # for any provider we want to use the default config from minimal-cluster-config
+                cluster_model = select_single(input, lambda x: x.kind == 'epiphany-cluster')
+            else:
+                # for azure|aws provider we want to use the extended defaults cluster-config after dry run.
+                # TODO: We probably wants this comming from seperate documents since Azure and AWS overlap now...
+                cluster_model = select_single(config, lambda x: x.kind == 'epiphany-cluster')
+            infra  = self.get_infra_docs(input)
+            docs = [cluster_model, *config_only, *infra]
+        else:
+            docs = [*input]
+
+        # set the provider and version for all docs
+        for doc in docs:
+            doc['provider'] = self.provider
+            doc['version'] = VERSION  
+
+        # remove SET_BY_AUTOMATION fields
+        remove_value(docs, 'SET_BY_AUTOMATION')  
 
-        save_manifest(defaults, self.name, self.name+'.yml')
+        # save document
+        save_manifest(docs, self.name, self.name+'.yml')
 
-        self.logger.info('Initialized user configuration and saved it to "' + os.path.join(get_build_path(self.name), self.name + '.yml') + '"')
+        self.logger.info('Initialized new configuration and saved it to "' + os.path.join(get_build_path(self.name), self.name + '.yml') + '"')
         return 0
 
-    def get_full_config(self, config_docs):
-        cluster_config_path = save_manifest(config_docs, self.name, self.name + '.yml')
+    def get_config_docs(self, input_docs):
+        cluster_config_path = save_manifest(input_docs, self.name, self.name + '.yml')
         args = type('obj', (object,), {'file': cluster_config_path})()
-        with BuildEngine(args) as build:
-            docs = build.dry_run()
 
-        # set the provider for all docs
-        for doc in docs:
-            if 'provider' not in doc.keys():
-                doc['provider'] = self.provider
+        # generate the config documents
+        with BuildEngine(args) as build:
+            config = build.dry_run() 
+
+        return config
+
+    def get_infra_docs(self, input_docs):
+        if self.provider == 'any':
+            # For any we can include the machine documents from the minimal-cluster-config
+            infra = select_all(input_docs, lambda x: x.kind.startswith('infrastructure/machine'))
+        else:
+            # VMs are curently the infrastructure documents the user might interact with for:
+            # - type/size
+            # - distro
+            # - network security rules
+            # ...
+            # So we add the defaults here.
+            # TODO: Check if we want to include possible other infrastructure documents.           
+            infra = load_all_yaml_objs(types.DEFAULT, self.provider, 'infrastructure/virtual-machine')
+
+        return infra
 
-        return docs
 
 
 

core/src/epicli/cli/helpers/objdict_helpers.py

@@ -37,3 +37,20 @@ def merge_objdict(to_merge, extend_by):
                 to_merge[key] = val
         else:
             to_merge[key] = val
+
+
+def remove_value(d, value):
+    if isinstance(d, str):
+        return
+    elif isinstance(d, list):
+        for dd in d:     
+            remove_value(dd, value)
+    else:
+        for k in list(d):
+            v = d[k]
+            if isinstance(v, list) or isinstance(v, dict):
+                remove_value(v, value)
+            else:
+                if value == v:
+                    del d[k]
+

core/src/epicli/cli/helpers/yaml_helpers.py

@@ -1,28 +1,36 @@
-import yaml
+from ruamel.yaml import YAML
 from cli.helpers.objdict_helpers import objdict_to_dict, dict_to_objdict
 
 
 def safe_load_all(stream):
-    docs = list(yaml.safe_load_all(stream))
+    yaml = YAML()
+    yaml.default_flow_style = False     
+    docs = list(yaml.load_all(stream))
     conv_docs = []
     for doc in docs:
         conv_docs.append(dict_to_objdict(doc))
     return conv_docs
 
 
 def safe_load(stream):
-    doc = yaml.safe_load(stream)
+    yaml = YAML()
+    yaml.default_flow_style = False
+    doc = yaml.load(stream)
     return dict_to_objdict(doc)
 
 
 def dump_all(docs, stream):
+    yaml = YAML()
+    yaml.default_flow_style = False    
     doc2 = docs
     conv_docs = []
     for doc in doc2:
         conv_docs.append(objdict_to_dict(doc))
-    yaml.dump_all(conv_docs, stream, default_flow_style=False)
+    yaml.dump_all(conv_docs, stream)
 
 
 def dump(doc, stream):
-    yaml.dump(objdict_to_dict(doc), stream, default_flow_style=False)
+    yaml = YAML()
+    yaml.default_flow_style = False
+    yaml.dump(objdict_to_dict(doc), stream)