Merge remote-tracking branch 'upstream/feature/upgrade-postgresql-13'…

… into feature/upgrade-pgbouncer

.github/ISSUE_TEMPLATE/feature-request.md

@@ -25,6 +25,7 @@ Add any other context or screenshots about the feature request here.
 
 * [ ] Changelog updated
 * [ ] COMPONENTS.md updated / doesn't need to be updated
+* [ ] Schema updated / doesn't need to be updated
 * [ ] Feature has automated tests
 * [ ] Automated tests passed (QA pipelines)
   * [ ] apply

CHANGELOG-1.2.md

@@ -7,12 +7,16 @@
 - [#126](https://github.com/epiphany-platform/epiphany/issues/126) - Added default Kibana dashboards
 - [#2127](https://github.com/epiphany-platform/epiphany/issues/2127) - Allow to specify configuration to be used in upgrade mode
 - [#2397](https://github.com/epiphany-platform/epiphany/issues/2397) - Restart CoreDNS pods conditionally
+- [#195](https://github.com/epiphany-platform/epiphany/issues/195) - Basic configuration type and schema validation
+- [#2434](https://github.com/epiphany-platform/epiphany/issues/2434) - Python 3 installation
 
 ### Fixed
 
 - [#2406](https://github.com/epiphany-platform/epiphany/issues/2406) - [Upgrade] [Filebeat] All settings for multiline feature are lost after upgrade
 - [#2380](https://github.com/epiphany-platform/epiphany/issues/2380) - Unable to drain nodes with Istio application enabled due to PodDisruptionBudgets
 - [#2332](https://github.com/epiphany-platform/epiphany/issues/2332) - [Elasticsearch] Error when having multiple VMs and non-clustered mode
+- [#2425](https://github.com/epiphany-platform/epiphany/issues/2425) - Feature-mapping - 'enabled: no' do nothing
+- [#2449](https://github.com/epiphany-platform/epiphany/issues/2449) - [Grafana] Unable to add Grafana repository
 
 ### Updated
 

core/src/epicli/cli/engine/ApplyEngine.py

@@ -59,8 +59,9 @@ def process_input_docs(self):
         if self.cluster_model is None:
             raise Exception('No cluster model defined in input YAML file')
 
-        # Validate input documents
-        with SchemaValidator(self.cluster_model.provider, self.input_docs) as schema_validator:
+        # Validate cluster input document.
+        # Other documents might need more processing (SET_BY_AUTOMATION) so will be validated at a later stage.
+        with SchemaValidator(self.cluster_model.provider, [self.cluster_model]) as schema_validator:
             schema_validator.run()
 
     def process_infrastructure_docs(self):

core/src/epicli/cli/engine/providers/aws/InfrastructureBuilder.py

@@ -237,7 +237,7 @@ def add_security_rules_inbound_efs(self, infrastructure, security_group):
                     rule.specification.description = 'NFS inbound for '+subnet.specification.name
                     rule.specification.direction = 'ingress'
                     rule.specification.protocol = 'tcp'
-                    rule.specification.destination_port_range = 2049
+                    rule.specification.destination_port_range = "2049"
                     rule.specification.source_address_prefix = subnet.specification.cidr_block
                     rule.specification.destination_address_prefix = '*'
                     security_group.specification.rules.append(rule.specification)

core/src/epicli/cli/engine/schema/SchemaValidator.py

@@ -1,9 +1,8 @@
-from jsonschema import validate
+from jsonschema import validate, Draft7Validator
 from cli.helpers.data_loader import load_yaml_obj, types
-from cli.helpers.objdict_helpers import objdict_to_dict, dict_to_objdict
+from cli.helpers.objdict_helpers import objdict_to_dict, dict_to_objdict, replace_yesno_with_booleans
 from cli.helpers.Step import Step
 from copy import deepcopy
-from cli.helpers.doc_list_helpers import select_single
 
 
 class SchemaValidator(Step):
@@ -32,6 +31,16 @@ def get_base_schema(self, kind):
         schema.properties.kind.pattern = '^(' + kind + ')$'
         return schema
 
+    def validate_document(self, doc, schema):
+        try:
+            replace_yesno_with_booleans(doc)
+            Draft7Validator.check_schema(schema)
+            validate(instance=objdict_to_dict(doc), schema=schema)
+        except Exception as e:
+            self.logger.error(f'Failed validating: {doc.kind}')
+            self.logger.error(e)
+            raise Exception('Schema validation error, see the error above.')        
+
     def run_for_individual_documents(self):
         for doc in self.validation_docs:
             # Load document schema
@@ -46,12 +55,8 @@ def run_for_individual_documents(self):
                     self.logger.warn('No specification validation for ' + doc.kind)
 
             # Assert the schema
-            try:
-                validate(instance=objdict_to_dict(doc), schema=objdict_to_dict(schema))
-            except Exception as e:
-                self.logger.error(f'Failed validating: {doc.kind}')
-                self.logger.error(e)
-                raise Exception('Schema validation error, see the error above.')
+            schema_dict = objdict_to_dict(schema)
+            self.validate_document(doc, schema_dict)
 
     def run(self):
         for doc in self.validation_docs:
@@ -61,9 +66,5 @@ def run(self):
             if hasattr(schema['properties']["specification"], '$ref'):
                 if schema['properties']["specification"]['$ref'] == '#/definitions/unvalidated_specification':
                     self.logger.warn('No specification validation for ' + doc.kind)
-            try:
-                validate(instance=objdict_to_dict(doc), schema=objdict_to_dict(schema))
-            except Exception as e:
-                self.logger.error(f'Failed validating: {doc.kind}')
-                self.logger.error(e)
-                raise Exception('Schema validation error, see the error above.')
+            schema_dict = objdict_to_dict(schema)
+            self.validate_document(doc, schema_dict)

core/src/epicli/cli/helpers/objdict_helpers.py

@@ -52,3 +52,18 @@ def remove_value(d, value):
             else:
                 if value == v:
                     del d[k]
+
+
+def replace_yesno_with_booleans(d):
+    if isinstance(d, list):
+        for dd in d:
+            replace_yesno_with_booleans(dd)
+    elif isinstance(d, ObjDict):
+        for key, val in d.items():
+            if isinstance(d[key], str):
+                if val == 'yes':        
+                    d[key] = True
+                elif val == 'no':             
+                    d[key] = False
+            else:      
+                replace_yesno_with_booleans(d[key])

core/src/epicli/data/any/validation/infrastructure/machine.yml

@@ -1 +1,10 @@
-$ref: '#/definitions/unvalidated_specification'
+"$id": "#/specification"
+title: "Machine specification schema"
+description: "Machine specification schema"
+specification:
+  type: object
+  properties:
+    hostname:
+      type: string
+    ip:
+      type: string

core/src/epicli/data/aws/defaults/infrastructure/launch-configuration.yml

@@ -12,6 +12,3 @@ specification:
   associate_public_ip: SET_BY_AUTOMATION
   security_groups: [] # SET_BY_AUTOMATION
   enable_monitoring: true
-
-
-

core/src/epicli/data/aws/validation/infrastructure/default-security-group.yml

@@ -1 +1,28 @@
-$ref: '#/definitions/unvalidated_specification'
+"$id": "#/specification"
+title: "Default-security-group specification schema"
+description: "Default-security-group specification schema"
+type: object
+properties:
+  name:
+    type: string
+  vpc_name:
+    type: string
+  rules:
+    type: array
+    items:
+      type: object
+      properties:
+        name:
+          type: string
+        description:
+          type: string
+        direction:
+          type: string
+        protocol:
+          type: string
+        destination_port_range:
+          type: string
+        source_address_prefix:
+          type: string
+        destination_address_prefix:
+          type: string

core/src/epicli/data/aws/validation/infrastructure/efs-storage.yml

@@ -1 +1,29 @@
-$ref: '#/definitions/unvalidated_specification'
+"$id": "#/specification"
+title: "Efs-storage specification schema"
+description: "Efs-storage specification schema"
+type: object
+properties:
+  name:
+    type: string
+  token:
+    type: string
+  encrypted:
+    type: boolean
+  performance_mode:
+    type: string
+  throughput_mode:
+    type: string
+  mount_targets:
+    type: array
+    items:
+      type: object
+      properties:
+        name:
+          type: string
+        subnet_name:
+          type: string
+  security:
+    type: object
+    properties:
+      populate_sg_rules:
+        type: boolean

core/src/epicli/data/aws/validation/infrastructure/internet-gateway.yml

@@ -1 +1,15 @@
-$ref: '#/definitions/unvalidated_specification'
+"$id": "#/specification"
+title: "Internet-gateway specification schema"
+description: "Internet-gateway specification schema"
+type: object
+properties:
+  name:
+    type: string
+  vpc_name:
+    type: string
+  cluster_name:
+    type: string
+  tag:
+    type: array
+    items:
+      type: string

core/src/epicli/data/aws/validation/infrastructure/launch-configuration.yml

@@ -1 +1,52 @@
-$ref: '#/definitions/unvalidated_specification'
+"$id": "#/specification"
+title: "Launch-configuration specification schema"
+description: "Launch-configuration specification schema"
+type: object
+properties:
+  name:
+    type: string
+  image_id:
+    type: string
+  size:
+    type: string
+  key_name:
+    type: string
+  disks:
+    type: object
+    properties:
+      root:
+        type: object
+        properties:
+          volume_type:
+            type: string
+          volume_size:
+            type: integer
+          delete_on_termination:
+            type: boolean
+          encrypted:
+            type: boolean
+      additional_disks:
+        type: array
+        items:
+          type: object
+          properties:
+            device_name:
+              type: string
+            volume_type:
+              type: string
+            volume_size:
+              type: integer
+            delete_on_termination:
+              type: boolean
+            encrypted:
+              type: boolean
+  ebs_optimized:
+    type: boolean
+  associate_public_ip:
+    type: boolean
+  security_groups:
+    type: array
+    items:
+    - type: string
+  enable_monitoring:
+    type: boolean

core/src/epicli/data/aws/validation/infrastructure/public-key.yml

@@ -1 +1,9 @@
-$ref: '#/definitions/unvalidated_specification'
+"$id": "#/specification"
+title: "Public-key specification schema"
+description: "Public-key specification schema"
+type: object
+properties:
+  name:
+    type: string
+  public_key:
+    type: string

core/src/epicli/data/aws/validation/infrastructure/resource-group.yml

@@ -1 +1,9 @@
-$ref: '#/definitions/unvalidated_specification'
+"$id": "#/specification"
+title: "Resource-group specification schema"
+description: "Resource-group specification schema"
+type: object
+properties:
+  name:
+    type: string
+  cluster_name:
+    type: string

core/src/epicli/data/aws/validation/infrastructure/route-table-association.yml

@@ -1 +1,11 @@
-$ref: '#/definitions/unvalidated_specification'
+"$id": "#/specification"
+title: "Route-table-association specification schema"
+description: "Route-table-association specification schema"
+type: object
+properties:
+  name:
+    type: string
+  subnet_name:
+    type: string
+  route_table_name:
+    type: string

core/src/epicli/data/aws/validation/infrastructure/route-table.yml

@@ -1 +1,22 @@
-$ref: '#/definitions/unvalidated_specification'
+"$id": "#/specification"
+title: "Route-table specification schema"
+description: "Route-table specification schema"
+type: object
+properties:
+  name:
+    type: string
+  vpc_name:
+    type: string
+  route:
+    type: object
+    properties:
+      cidr_block:
+        type: string
+      gateway_name:
+        type: string
+  tag:
+    type: array
+    items:
+      type: string
+  cluster_name:
+    type: string

core/src/epicli/data/aws/validation/infrastructure/security-group-rule.yml

@@ -1 +1,19 @@
-$ref: '#/definitions/unvalidated_specification'
+"$id": "#/specification"
+title: "Security-group-rule specification schema"
+description: "Security-group-rule specification schema"
+type: object
+properties:
+  name:
+    type: string
+  protocol:
+    type: string
+  description:
+    type: string
+  direction:
+    type: string
+  destination_port_range:
+    type: string
+  source_address_prefix:
+    type: string
+  destination_address_prefix:
+    type: string

core/src/epicli/data/aws/validation/infrastructure/security-group.yml

@@ -1 +1,32 @@
-$ref: '#/definitions/unvalidated_specification'
+"$id": "#/specification"
+title: "Security-group specification schema"
+description: "Security-group specification schema"
+type: object
+properties:
+  vpc_name:
+    type: string
+  name:
+    type: string
+  cidr_block:
+    type: string
+  rules:
+    type: array
+    items:
+      type: object
+      properties:
+        name:
+          type: string
+        description:
+          type: string
+        direction:
+          type: string
+        protocol:
+          type: string
+        destination_port_range:
+          type: string
+        source_address_prefix:
+          type: string
+        destination_address_prefix:
+          type: string
+  cluster_name:
+    type: string