Back merge of fixes from 0.6.0 to 0.4.4

CHANGELOG-0.4.md

@@ -1,5 +1,17 @@
 # Changelog 0.4
 
+## [0.4.4] 2020-04-xx
+
+### Fixed
+
+- [#966](https://github.com/epiphany-platform/epiphany/issues/966) - Ubuntu builds get stuck on 'Create epirepo repository' task waiting for user input in offline mode
+- [#1043](https://github.com/epiphany-platform/epiphany/issues/1043) - For vm template on Azure disk_size_gb is missing in storage_os_disk
+- [#1049](https://github.com/epiphany-platform/epiphany/issues/1049) - Azure/RedHat specified disk size 30 GB is smaller than the size of the disk in the VM image
+- [#1063](https://github.com/epiphany-platform/epiphany/issues/1063) - Issues with single_machine install
+- [#1108](https://github.com/epiphany-platform/epiphany/issues/1108) - [Azure RedHat] Create epirepo: package httpd-2.4.6-93 requires httpd-tools = 2.4.6-93 but latest available is 2.4.6-90
+- [#1110](https://github.com/epiphany-platform/epiphany/issues/1110) - Install fixed version of httpd when latest fails (RHEL)
+- [#1016](https://github.com/epiphany-platform/epiphany/issues/1016) - Disable verify, backup and recovery as they are not fully implemented
+
 ## [0.4.3] 2020-03-16
 
 ### Fixed

CHANGELOG.md

@@ -1,4 +1,4 @@
-# Development release
+# Changelog
 
 All notable changes to this project will be documented in linked files.
 
@@ -7,30 +7,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 Reference for actual cluster component versions can be found [here](docs/home/COMPONENTS.md)
 
-## Current release
-
-### 0.4.x
-
-- [CHANGELOG-0.4.3](./CHANGELOG-0.4.md#043-2020-03-16)
-- [CHANGELOG-0.4.2](./CHANGELOG-0.4.md#042-2019-11-20)
-- [CHANGELOG-0.4.1](./CHANGELOG-0.4.md#041-2019-10-17)
-- [CHANGELOG-0.4.0](./CHANGELOG-0.4.md#040-2019-09-30)
-
-## Older releases
-
-### 0.3.x
-
-- [CHANGELOG-0.3.0](./CHANGELOG-0.3.md#030-2019-07-31)
-
-### 0.2.x
-
-- [CHANGELOG-0.2.3](./CHANGELOG-0.2.md#023-2019-05-20)
-- [CHANGELOG-0.2.2](./CHANGELOG-0.2.md#022-2019-03-29)
-- [CHANGELOG-0.2.1](./CHANGELOG-0.2.md#021-2019-03-07)
-- [CHANGELOG-0.2.0](./CHANGELOG-0.2.md#020-2019-02-19)
-
-### 0.1.x
-
-- [CHANGELOG-0.1.2](./CHANGELOG-0.1.md#012-2018-12-05)
-- [CHANGELOG-0.1.1](./CHANGELOG-0.1.md#011-2018-11-21)
-- [CHANGELOG-0.1.0](./CHANGELOG-0.1.md#010-2018-10-19)
+- [CHANGELOG-0.4.x](./CHANGELOG-0.4.md)
+- [CHANGELOG-0.3.x](./CHANGELOG-0.3.md)
+- [CHANGELOG-0.2.x](./CHANGELOG-0.2.md)
+- [CHANGELOG-0.1.x](./CHANGELOG-0.1.md)

core/src/epicli/cli/epicli.py

@@ -55,15 +55,18 @@ def main():
 
     # setup subparsers
     subparsers = parser.add_subparsers()
-    apply_parser(subparsers)
-    validate_parser(subparsers)
+    prepare_parser(subparsers)
     init_parser(subparsers)
+    apply_parser(subparsers)
     upgrade_parser(subparsers)
-    backup_parser(subparsers)
-    recovery_parser(subparsers)
     delete_parser(subparsers)
-    prepare_parser(subparsers)
 
+    '''
+    validate_parser(subparsers)
+    backup_parser(subparsers)
+    recovery_parser(subparsers)
+    '''
+
     # check if there were any variables and display full help
     if len(sys.argv) < 2:
         parser.print_help()
@@ -96,6 +99,19 @@ def main():
         return 1
 
 
+def prepare_parser(subparsers):
+    sub_parser = subparsers.add_parser('prepare', description='Creates a folder with all prerequisites to setup the offline requirements to install a cluster offline.')
+    sub_parser.add_argument('--os', type=str, required=True, dest='os',
+                            help='The OS to prepare the offline requirements for.')
+
+    def run_prepare(args):
+        adjust_paths_from_output_dir()
+        with PrepareEngine(args) as engine:
+            return engine.prepare()
+
+    sub_parser.set_defaults(func=run_prepare) 
+
+
 def init_parser(subparsers):
     sub_parser = subparsers.add_parser('init', description='Creates configuration file in working directory.')
     sub_parser.add_argument('-p', '--provider', dest='provider', choices=['aws', 'azure', 'any'], default='any',
@@ -131,20 +147,22 @@ def run_apply(args):
 
     sub_parser.set_defaults(func=run_apply)
 
+def upgrade_parser(subparsers):
+    sub_parser = subparsers.add_parser('upgrade',
+                                       description='Upgrades common and K8s components of an existing Epiphany Platform cluster.')
+    sub_parser.add_argument('-b', '--build', dest='build_directory', type=str, required=True,
+                            help='Absolute path to directory with build artifacts.')
+    sub_parser.add_argument('--wait-for-pods', dest='wait_for_pods', action="store_true",
+                            help="Waits for all pods to be in the 'Ready' state before proceeding to the next step of the K8s upgrade.")
+    sub_parser.add_argument('--offline-requirements', dest='offline_requirements', type=str, required=False,
+                            help='Path to the folder with pre-prepared offline requirements.')
 
-def validate_parser(subparsers):
-    sub_parser = subparsers.add_parser('verify', description='Validates the configuration from file by executing a dry '
-                                                             'run without changing the physical '
-                                                             'infrastructure/configuration')
-    sub_parser.add_argument('-f', '--file', dest='file', type=str,
-                            help='File with infrastructure/configuration definitions to use.')
-
-    def run_validate(args):
-        adjust_paths_from_file(args)
-        with BuildEngine(args) as engine:
-            return engine.validate()
+    def run_upgrade(args):
+        adjust_paths_from_build(args)
+        with UpgradeEngine(args) as engine:
+            return engine.upgrade()
 
-    sub_parser.set_defaults(func=run_validate)
+    sub_parser.set_defaults(func=run_upgrade)
 
 
 def delete_parser(subparsers):
@@ -160,25 +178,23 @@ def run_delete(args):
         with DeleteEngine(args) as engine:
             return engine.delete()
 
-    sub_parser.set_defaults(func=run_delete)
+    sub_parser.set_defaults(func=run_delete)    
 
 
-def upgrade_parser(subparsers):
-    sub_parser = subparsers.add_parser('upgrade',
-                                       description='Upgrades common and K8s components of an existing Epiphany Platform cluster.')
-    sub_parser.add_argument('-b', '--build', dest='build_directory', type=str, required=True,
-                            help='Absolute path to directory with build artifacts.')
-    sub_parser.add_argument('--wait-for-pods', dest='wait_for_pods', action="store_true",
-                            help="Waits for all pods to be in the 'Ready' state before proceeding to the next step of the K8s upgrade.")
-    sub_parser.add_argument('--offline-requirements', dest='offline_requirements', type=str, required=False,
-                            help='Path to the folder with pre-prepared offline requirements.')
+'''
+def validate_parser(subparsers):
+    sub_parser = subparsers.add_parser('verify', description='Validates the configuration from file by executing a dry '
+                                                             'run without changing the physical '
+                                                             'infrastructure/configuration')
+    sub_parser.add_argument('-f', '--file', dest='file', type=str,
+                            help='File with infrastructure/configuration definitions to use.')
 
-    def run_upgrade(args):
-        adjust_paths_from_build(args)
-        with UpgradeEngine(args) as engine:
-            return engine.upgrade()
+    def run_validate(args):
+        adjust_paths_from_file(args)
+        with BuildEngine(args) as engine:
+            return engine.validate()
 
-    sub_parser.set_defaults(func=run_upgrade)
+    sub_parser.set_defaults(func=run_validate)
 
 
 def backup_parser(subparsers):
@@ -208,19 +224,7 @@ def run_recovery(args):
             return engine.recovery()
 
     sub_parser.set_defaults(func=run_recovery)
-
-
-def prepare_parser(subparsers):
-    sub_parser = subparsers.add_parser('prepare', description='Creates a folder with all prerequisites to setup the offline requirements to install a cluster offline.')
-    sub_parser.add_argument('--os', type=str, required=True, dest='os',
-                            help='The OS to prepare the offline requirements for.')
-
-    def run_prepare(args):
-        adjust_paths_from_output_dir()
-        with PrepareEngine(args) as engine:
-            return engine.prepare()
-
-    sub_parser.set_defaults(func=run_prepare)   
+'''
 
 
 def experimental_query():

core/src/epicli/data/aws/defaults/infrastructure/virtual-machine.yml

@@ -739,3 +739,56 @@ specification:
        destination_port_range: "0"
        source_address_prefix: "0.0.0.0/0"
        destination_address_prefix: "0.0.0.0/0"
+---
+kind: infrastructure/virtual-machine
+title: "Virtual Machine Infra"
+provider: aws
+name: single-machine
+specification:
+  size: t3.xlarge
+  authorized_to_efs: true
+  mount_efs: true
+  security:
+    rules:
+     - name: ssh
+       description: Allow ssh traffic
+       priority: 101
+       direction: Inbound
+       access: Allow
+       protocol: Tcp
+       source_port_range: "*"
+       destination_port_range: "22"
+       source_address_prefix: "0.0.0.0/0"
+       destination_address_prefix: "0.0.0.0/0"
+     - name: repository
+       description: Allow repository traffic
+       priority: 302
+       direction: Inbound
+       access: Allow
+       protocol: Tcp
+       source_port_range: "*"
+       destination_port_range: "80"
+       source_address_prefix: "10.1.0.0/20"
+       destination_address_prefix: "0.0.0.0/0"
+     - name: subnet-traffic
+       description: Allow subnet traffic
+       priority: 102
+       direction: Inbound
+       access: Allow
+       protocol: ALL
+       source_port_range: "*"
+       destination_from_port: 0
+       destination_to_port: 65536
+       destination_port_range: "0"
+       source_address_prefix: "10.1.1.0/24"
+       destination_address_prefix: "0.0.0.0/0"
+     - name: out
+       description: Allow out
+       priority: 101
+       direction: Egress
+       access: Allow
+       protocol: "all"
+       source_port_range: "*"
+       destination_port_range: "0"
+       source_address_prefix: "0.0.0.0/0"
+       destination_address_prefix: "0.0.0.0/0"       

core/src/epicli/data/azure/defaults/infrastructure/virtual-machine.yml

@@ -21,7 +21,7 @@ specification:
     managed: false
     caching: ReadWrite
     create_option: FromImage
-    disk_size_gb: 30
+    disk_size_gb: 32
     managed_disk_type: Premium_LRS
   network_interface:
     enable_accelerated_networking: false
@@ -657,4 +657,77 @@ specification:
         source_port_range: "*"
         destination_port_range: "9101"
         source_address_prefix: "10.1.0.0/20"
-        destination_address_prefix: "0.0.0.0/0"
+        destination_address_prefix: "0.0.0.0/0"
+---
+kind: infrastructure/virtual-machine
+title: "Virtual Machine Infra"
+provider: azure
+name: single-machine
+specification:
+  size: Standard_DS3_v2
+  security:
+    rules:
+      - name: ssh
+        description: Allow SSH
+        priority: 100
+        direction: Inbound
+        access: Allow
+        protocol: Tcp
+        source_port_range: "*"
+        destination_port_range: "22"
+        source_address_prefix: "0.0.0.0/0"
+        destination_address_prefix: "0.0.0.0/0"
+      - name: out
+        description: Allow out
+        priority: 101
+        direction: Outbound
+        access: Allow
+        protocol: "*"
+        source_port_range: "*"
+        destination_port_range: "0"
+        source_address_prefix: "0.0.0.0/0"
+        destination_address_prefix: "0.0.0.0/0"
+      - name: subnet-traffic
+        description: Allow subnet traffic
+        priority: 201
+        direction: Inbound
+        access: Allow
+        protocol: "*"
+        source_port_range: "*"
+        destination_from_port: 0
+        destination_to_port: 65536
+        destination_port_range: "0"
+        source_address_prefix: "10.1.1.0/24"
+        destination_address_prefix: "0.0.0.0/0"
+      - name: package_repository
+        description: Allow package repository traffic
+        priority: 205
+        direction: Inbound
+        access: Allow
+        protocol: Tcp
+        source_port_range: "*"
+        destination_port_range: "80"
+        source_address_prefix: "10.1.0.0/20"
+        destination_address_prefix: "0.0.0.0/0"
+      - name: image_repository
+        description: Allow image repository traffic
+        priority: 206
+        direction: Inbound
+        access: Allow
+        protocol: Tcp
+        source_port_range: "*"
+        destination_port_range: "5000"
+        source_address_prefix: "10.1.0.0/20"
+        destination_address_prefix: "0.0.0.0/0"
+  #    - name: node2-subnet-traffic
+  #      description: Allow node subnet traffic
+  #      priority: 102
+  #      direction: Inbound
+  #      access: Allow
+  #      protocol: "*"
+  #      source_port_range: "*"
+  #      destination_from_port: 0
+  #      destination_to_port: 65536
+  #      destination_port_range: "0"
+  #      source_address_prefix: "10.1.4.0/24"
+  #      destination_address_prefix: "0.0.0.0/0"        

core/src/epicli/data/azure/terraform/infrastructure/virtual-machine.j2

@@ -51,6 +51,7 @@ resource "azurerm_virtual_machine" "{{ specification.name  }}" {
     name          = "{{ specification.name }}-os-disk"
     caching       = "{{ specification.storage_os_disk.caching }}"
     create_option = "{{ specification.storage_os_disk.create_option }}"
+    disk_size_gb  = "{{ specification.storage_os_disk.disk_size_gb }}"
     {%- if specification.storage_os_disk.managed != true %}
     managed_disk_type = "{{ specification.storage_os_disk.managed_disk_type | lower }}"
     {%- else %}

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.txt

@@ -37,6 +37,7 @@ filebeat-6.5.4 # actually it's filebeat-oss
 firewalld
 fontconfig # for grafana
 fping
+gnutls # for cifs-utils
 grafana-6.2.5
 gssproxy # for nfs-utils
 htop
@@ -69,6 +70,7 @@ perl-Pod-Perldoc # for vim
 perl-Pod-Simple # for vim
 perl-Pod-Usage # for vim
 policycoreutils-python # for container-selinux
+pyldb # for cifs-utils
 python-firewall # for firewalld
 python-kitchen # for yum-utils
 python-lxml # for java-1.8.0-openjdk-headless
@@ -91,6 +93,7 @@ rh-postgresql10-postgresql-syspaths
 samba-client
 samba-client-libs # for samba-client
 samba-common
+samba-libs # for cifs-utils
 socat
 sysstat
 tar