Back merge of fixes from 0.6.0 to 0.5.4

CHANGELOG-0.5.md

@@ -1,5 +1,22 @@
 # Changelog 0.5
 
+## [0.5.4] 2020-04-xx
+
+### Added
+
+- [#986](https://github.com/epiphany-platform/epiphany/issues/986) - Add vim to Epicli container and devcontainer
+- [#1044](https://github.com/epiphany-platform/epiphany/issues/1044) - Add ability to add subscriptionId to sp.yml on Azure
+
+### Fixed
+
+- [#624](https://github.com/epiphany-platform/epiphany/issues/624) - Don't run epicli as root in container
+- [#966](https://github.com/epiphany-platform/epiphany/issues/966) - Ubuntu builds get stuck on 'Create epirepo repository' task waiting for user input in offline mode
+- [#1043](https://github.com/epiphany-platform/epiphany/issues/1043) - For vm template on Azure disk_size_gb is missing in storage_os_disk
+- [#1049](https://github.com/epiphany-platform/epiphany/issues/1049) - Azure/RedHat specified disk size 30 GB is smaller than the size of the disk in the VM image
+- [#1063](https://github.com/epiphany-platform/epiphany/issues/1063) - Issues with single_machine install
+- [#1108](https://github.com/epiphany-platform/epiphany/issues/1108) - [Azure RedHat] Create epirepo: package httpd-2.4.6-93 requires httpd-tools = 2.4.6-93 but latest available is 2.4.6-90
+- [#1110](https://github.com/epiphany-platform/epiphany/issues/1110) - Install fixed version of httpd when latest fails (RHEL)
+
 ## [0.5.3] 2020-03-09
 
 ### Added
@@ -33,6 +50,7 @@
 - [#906](https://github.com/epiphany-platform/epiphany/issues/906) - Open Distro for Elasticsearch: Task 'Install Elasticsearch package' fails
 - [#909](https://github.com/epiphany-platform/epiphany/issues/909) - Upgrade: Missing property 'name' when running epicli upgrade
 - [#869](https://github.com/epiphany-platform/epiphany/issues/869) - Common: Issue installing Debian packages
+- [#1016](https://github.com/epiphany-platform/epiphany/issues/1016) - Disable verify, backup and recovery as they are not fully implemented
 
 ### Known issues
 

CHANGELOG.md

@@ -1,4 +1,4 @@
-# Development release
+# Changelog
 
 All notable changes to this project will be documented in linked files.
 
@@ -7,37 +7,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 Reference for actual cluster component versions can be found [here](docs/home/COMPONENTS.md)
 
-## Current release
-
-### 0.5.x
-
-- [CHANGELOG-0.5.3](./CHANGELOG-0.5.md#051-2020-03-09)
-- [CHANGELOG-0.5.2](./CHANGELOG-0.5.md#051-2020-02-17)
-- [CHANGELOG-0.5.1](./CHANGELOG-0.5.md#051-2020-01-23)
-- [CHANGELOG-0.5.0](./CHANGELOG-0.5.md#050-2020-01-17)
-
-## Older releases
-
-### 0.4.x
-
-- [CHANGELOG-0.4.3](./CHANGELOG-0.4.md#043-2020-03-16)
-- [CHANGELOG-0.4.2](./CHANGELOG-0.4.md#042-2019-11-20)
-- [CHANGELOG-0.4.1](./CHANGELOG-0.4.md#041-2019-10-17)
-- [CHANGELOG-0.4.0](./CHANGELOG-0.4.md#040-2019-10-11)
-
-### 0.3.x
-
-- [CHANGELOG-0.3.0](./CHANGELOG-0.3.md#030-2019-07-31)
-
-### 0.2.x
-
-- [CHANGELOG-0.2.3](./CHANGELOG-0.2.md#023-2019-05-20)
-- [CHANGELOG-0.2.2](./CHANGELOG-0.2.md#022-2019-03-29)
-- [CHANGELOG-0.2.1](./CHANGELOG-0.2.md#021-2019-03-07)
-- [CHANGELOG-0.2.0](./CHANGELOG-0.2.md#020-2019-02-19)
-
-### 0.1.x
-
-- [CHANGELOG-0.1.2](./CHANGELOG-0.1.md#012-2018-12-05)
-- [CHANGELOG-0.1.1](./CHANGELOG-0.1.md#011-2018-11-21)
-- [CHANGELOG-0.1.0](./CHANGELOG-0.1.md#010-2018-10-19)
+- [CHANGELOG-0.5.x](./CHANGELOG-0.5.md)
+- [CHANGELOG-0.4.x](./CHANGELOG-0.4.md)
+- [CHANGELOG-0.3.x](./CHANGELOG-0.3.md)
+- [CHANGELOG-0.2.x](./CHANGELOG-0.2.md)
+- [CHANGELOG-0.1.x](./CHANGELOG-0.1.md)

core/src/epicli/.devcontainer/Dockerfile

@@ -17,7 +17,7 @@ RUN chmod +x /config-pre.sh \
     && apt-get update \
     && apt-get -y install --no-install-recommends apt-utils dialog 2>&1 \
 
-    && apt-get -y install git procps lsb-release gcc make musl-dev libffi-dev tar unzip \
+    && apt-get -y install git procps lsb-release gcc make musl-dev libffi-dev tar unzip vim \
 
     && apt-get -y install ruby-full \
 

core/src/epicli/Dockerfile-debian

@@ -2,17 +2,34 @@ FROM python:3.7-slim
 
 ARG EPICLI_VERSION
 
+ARG USERNAME=epiuser
+ARG USER_UID=1000
+ARG USER_GID=$USER_UID
+
 ENV DOCKER_CLI Yes
 
 COPY /dist/ /epicli
 WORKDIR /epicli
 
 RUN apt-get update \
-    && apt-get -y install gcc make musl-dev libffi-dev tar unzip openssh-client
+    && apt-get -y install gcc make musl-dev libffi-dev tar unzip openssh-client vim  
 
 RUN pip install epicli-${EPICLI_VERSION}-py3-none-any.whl
 
+RUN groupadd --gid $USER_GID $USERNAME \
+    && useradd -s /bin/bash --uid $USER_UID --gid $USER_GID -m $USERNAME \
+    && apt-get install -y sudo \
+
+    && echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \
+    && chmod 0440 /etc/sudoers.d/$USERNAME \   
+
+    && apt-get autoremove -y \
+    && apt-get clean -y \
+    && rm -rf /var/lib/apt/lists/*   
+
 RUN mkdir /shared/
 WORKDIR /shared
 
+USER $USERNAME
+
 ENTRYPOINT ["/bin/bash"]

core/src/epicli/cli/engine/providers/azure/APIProxy.py

@@ -34,7 +34,7 @@ def login_sp(self, sp_data):
         name = sp_data['name']
         password = sp_data['password']
         tenant = sp_data['tenant']
-        return self.run(self, f'az login --service-principal -u {name} -p {password} --tenant {tenant}')      
+        return self.run(self, f'az login --service-principal -u {name} -p {password} --tenant {tenant}', False)      
 
     def set_active_subscribtion(self, subscription_id):
         self.run(self, f'az account set --subscription {subscription_id}')  
@@ -77,8 +77,9 @@ def wait(self, seconds):
             time.sleep(1)
 
     @staticmethod
-    def run(self, cmd):
-        self.logger.info('Running: "' + cmd + '"')
+    def run(self, cmd, log_cmd=True):
+        if log_cmd:
+            self.logger.info('Running: "' + cmd + '"')
 
         logpipe = LogPipe(__name__)
         with Popen(cmd, stdout=PIPE, stderr=logpipe, shell=True) as sp:
@@ -92,7 +93,11 @@ def run(self, cmd):
                 output = {}
 
         if sp.returncode != 0:
-            raise Exception(f'Error running: "{cmd}"')
+            if log_cmd:
+                raise Exception(f'Error running: "{cmd}"')
+            else:
+                raise Exception(f'Error running Azure APIProxy cmd')
         else:
-            self.logger.info(f'Done running "{cmd}"')
+            if log_cmd:
+                self.logger.info(f'Done running "{cmd}"')
             return output            

core/src/epicli/cli/engine/terraform/TerraformRunner.py

@@ -42,29 +42,37 @@ def azure_login(self):
             subscription = apiproxy.login_account()
             apiproxy.set_active_subscribtion(subscription['id'])
         else:
-            # Service principle
+            # Service principal
             sp_file = os.path.join(get_terraform_path(self.cluster_model.specification.name), SP_FILE_NAME)
             if not os.path.exists(sp_file):
-                # If no service principle exists or is defined we created one and for that we need to login using an account
+                # If no service principal exists or is defined we created one and for that we need to login using an account
                 subscription = apiproxy.login_account()
                 apiproxy.set_active_subscribtion(subscription['id'])
 
-                # Create the service principle
+                # Create the service principal, for now we use the default subscription
                 self.logger.info('Creating service principal')
                 cluster_name = self.cluster_model.specification.name.lower()
                 cluster_prefix = self.cluster_model.specification.prefix.lower()               
                 resource_group_name = resource_name(cluster_prefix, cluster_name, 'rg')
                 sp = apiproxy.create_sp(resource_group_name, subscription['id'])
+                sp['subscriptionId'] = subscription['id']
                 save_sp(sp, self.cluster_model.specification.name)
             else:
                 self.logger.info('Using service principal from file')
                 sp = load_yaml_file(sp_file)
 
-            # Login as SP.
+            # Login as SP and get the default subscription.
             subscription = apiproxy.login_sp(sp)
 
-            # Setup environment variables for Terraform when working with Azure and service principal.
-            self.new_env['ARM_SUBSCRIPTION_ID'] = subscription[0]['id']
+            if 'subscriptionId' in sp:
+                # Set active subscription if sp contains it. 
+                apiproxy.set_active_subscribtion(sp['subscriptionId'])
+                self.new_env['ARM_SUBSCRIPTION_ID'] = sp['subscriptionId']
+            else:
+                # No subscriptionId in sp.yml so use the default one from Azure SP login.
+                self.new_env['ARM_SUBSCRIPTION_ID'] = subscription[0]['id']
+
+             # Set other environment variables for Terraform when working with Azure and service principal.
             self.new_env['ARM_TENANT_ID'] = sp['tenant']
             self.new_env['ARM_CLIENT_ID'] = sp['appId']
             self.new_env['ARM_CLIENT_SECRET'] = sp['password']

core/src/epicli/cli/epicli.py

@@ -58,14 +58,17 @@ def main():
 
     # setup subparsers
     subparsers = parser.add_subparsers()
-    apply_parser(subparsers)
-    validate_parser(subparsers)
+    prepare_parser(subparsers)
     init_parser(subparsers)
+    apply_parser(subparsers)
     upgrade_parser(subparsers)
-    backup_parser(subparsers)
-    recovery_parser(subparsers)
     delete_parser(subparsers)
-    prepare_parser(subparsers)
+
+    '''
+    validate_parser(subparsers)
+    backup_parser(subparsers)
+    recovery_parser(subparsers)  
+    '''  
 
     # check if there were any variables and display full help
     if len(sys.argv) < 2:
@@ -99,6 +102,19 @@ def main():
         return 1
 
 
+def prepare_parser(subparsers):
+    sub_parser = subparsers.add_parser('prepare', description='Creates a folder with all prerequisites to setup the offline requirements to install a cluster offline.')
+    sub_parser.add_argument('--os', type=str, required=True, dest='os',
+                            help='The OS to prepare the offline requirements for.')
+
+    def run_prepare(args):
+        adjust_paths_from_output_dir()
+        with PrepareEngine(args) as engine:
+            return engine.prepare()
+
+    sub_parser.set_defaults(func=run_prepare)         
+
+
 def init_parser(subparsers):
     sub_parser = subparsers.add_parser('init', description='Creates configuration file in working directory.')
     sub_parser.add_argument('-p', '--provider', dest='provider', choices=['aws', 'azure', 'any'], default='any',
@@ -144,19 +160,22 @@ def run_apply(args):
     sub_parser.set_defaults(func=run_apply)
 
 
-def validate_parser(subparsers):
-    sub_parser = subparsers.add_parser('verify', description='Validates the configuration from file by executing a dry '
-                                                             'run without changing the physical '
-                                                             'infrastructure/configuration')
-    sub_parser.add_argument('-f', '--file', dest='file', type=str,
-                            help='File with infrastructure/configuration definitions to use.')
+def upgrade_parser(subparsers):
+    sub_parser = subparsers.add_parser('upgrade',
+                                       description='Upgrades common and K8s components of an existing Epiphany Platform cluster.')
+    sub_parser.add_argument('-b', '--build', dest='build_directory', type=str, required=True,
+                            help='Absolute path to directory with build artifacts.')
+    sub_parser.add_argument('--wait-for-pods', dest='wait_for_pods', action="store_true",
+                            help="Waits for all pods to be in the 'Ready' state before proceeding to the next step of the K8s upgrade.")
+    sub_parser.add_argument('--offline-requirements', dest='offline_requirements', type=str, required=False,
+                            help='Path to the folder with pre-prepared offline requirements.')
 
-    def run_validate(args):
-        adjust_paths_from_file(args)
-        with BuildEngine(args) as engine:
-            return engine.validate()
+    def run_upgrade(args):
+        adjust_paths_from_build(args)
+        with UpgradeEngine(args) as engine:
+            return engine.upgrade()
 
-    sub_parser.set_defaults(func=run_validate)
+    sub_parser.set_defaults(func=run_upgrade)
 
 
 def delete_parser(subparsers):
@@ -174,22 +193,20 @@ def run_delete(args):
     sub_parser.set_defaults(func=run_delete)
 
 
-def upgrade_parser(subparsers):
-    sub_parser = subparsers.add_parser('upgrade',
-                                       description='Upgrades common and K8s components of an existing Epiphany Platform cluster.')
-    sub_parser.add_argument('-b', '--build', dest='build_directory', type=str, required=True,
-                            help='Absolute path to directory with build artifacts.')
-    sub_parser.add_argument('--wait-for-pods', dest='wait_for_pods', action="store_true",
-                            help="Waits for all pods to be in the 'Ready' state before proceeding to the next step of the K8s upgrade.")
-    sub_parser.add_argument('--offline-requirements', dest='offline_requirements', type=str, required=False,
-                            help='Path to the folder with pre-prepared offline requirements.')
+'''
+def validate_parser(subparsers):
+    sub_parser = subparsers.add_parser('verify', description='Validates the configuration from file by executing a dry '
+                                                             'run without changing the physical '
+                                                             'infrastructure/configuration')
+    sub_parser.add_argument('-f', '--file', dest='file', type=str,
+                            help='File with infrastructure/configuration definitions to use.')
 
-    def run_upgrade(args):
-        adjust_paths_from_build(args)
-        with UpgradeEngine(args) as engine:
-            return engine.upgrade()
+    def run_validate(args):
+        adjust_paths_from_file(args)
+        with BuildEngine(args) as engine:
+            return engine.validate()
 
-    sub_parser.set_defaults(func=run_upgrade)
+    sub_parser.set_defaults(func=run_validate)    
 
 
 def backup_parser(subparsers):
@@ -219,20 +236,7 @@ def run_recovery(args):
             return engine.recovery()
 
     sub_parser.set_defaults(func=run_recovery)
-
-
-def prepare_parser(subparsers):
-    sub_parser = subparsers.add_parser('prepare', description='Creates a folder with all prerequisites to setup the offline requirements to install a cluster offline.')
-    sub_parser.add_argument('--os', type=str, required=True, dest='os',
-                            help='The OS to prepare the offline requirements for.')
-
-    def run_prepare(args):
-        adjust_paths_from_output_dir()
-        with PrepareEngine(args) as engine:
-            return engine.prepare()
-
-    sub_parser.set_defaults(func=run_prepare)   
-
+'''
 
 def experimental_query():
     if not query_yes_no('This is an experimental feature and could change at any time. Do you want to continue?'):

core/src/epicli/cli/version.txt.py

@@ -1 +1 @@
-0.5.3
+0.5.4