diff --git a/CHANGELOG-0.4.md b/CHANGELOG-0.4.md index 6bfb41f44d..2c0e1559cf 100644 --- a/CHANGELOG-0.4.md +++ b/CHANGELOG-0.4.md @@ -1,5 +1,16 @@ # Changelog 0.4 +## [0.4.1] 2019-10-17 + +### Fixed + + +- [#612](https://github.com/epiphany-platform/epiphany/issues/612) - 'epicli delete' - cannot delete a partially built infrastructure +- [#613](https://github.com/epiphany-platform/epiphany/pull/613) - Hotfixes for Ubuntu offline installation in air-gap mode +- [#614](https://github.com/epiphany-platform/epiphany/pull/614) - Fixed RotatingFileHandler permission error (for Docker Toolbox on Windows) +- [#615](https://github.com/epiphany-platform/epiphany/issues/615) - Minor Azure bugs for 0.4.0 release +- [#620](https://github.com/epiphany-platform/epiphany/issues/620) - Incorrect Ansible metadata (prerequisite) for Kubernetes Node + ## [0.4.0] 2019-10-11 ### Added diff --git a/core/src/epicli/Dockerfile b/core/src/epicli/Dockerfile index 7e9b476691..35870a9cf4 100644 --- a/core/src/epicli/Dockerfile +++ b/core/src/epicli/Dockerfile @@ -1,7 +1,7 @@ FROM python:3.7-alpine ENV DOCKER_CLI Yes -ENV EPICLI_VERSION 0.4.0 +ENV EPICLI_VERSION 0.4.1 COPY /dist/ /epicli WORKDIR /epicli diff --git a/core/src/epicli/cli/engine/BuildEngine.py b/core/src/epicli/cli/engine/BuildEngine.py index 33f5992217..60c572da04 100644 --- a/core/src/epicli/cli/engine/BuildEngine.py +++ b/core/src/epicli/cli/engine/BuildEngine.py @@ -93,6 +93,8 @@ def apply(self): self.process_infrastructure_docs() + save_manifest([*self.input_docs, *self.infrastructure_docs], self.cluster_model.specification.name) + if not (self.skip_infrastructure or self.is_provider_any(self.cluster_model)): # Generate terraform templates with TerraformTemplateGenerator(self.cluster_model, self.infrastructure_docs) as template_generator: @@ -124,7 +126,7 @@ def dry_run(self): self.process_configuration_docs() - return [*self.input_docs, *self.configuration_docs] + return [*self.configuration_docs, *self.infrastructure_docs] @staticmethod def is_provider_any(cluster_model): diff --git a/core/src/epicli/cli/engine/InitEngine.py b/core/src/epicli/cli/engine/InitEngine.py index a7a94323c2..091021320b 100644 --- a/core/src/epicli/cli/engine/InitEngine.py +++ b/core/src/epicli/cli/engine/InitEngine.py @@ -38,17 +38,14 @@ def get_full_config(self, config_docs): cluster_config_path = save_manifest(config_docs, self.name, self.name + '.yml') args = type('obj', (object,), {'file': cluster_config_path})() with BuildEngine(args) as build: - config_docs = build.dry_run() - - infra_docs = load_all_documents_from_folder(self.provider, 'defaults/infrastructure') - merged_docs = [*config_docs, *infra_docs] + docs = build.dry_run() # set the provider for all docs - for doc in merged_docs: + for doc in docs: if 'provider' not in doc.keys(): doc['provider'] = self.provider - return merged_docs + return docs diff --git a/core/src/epicli/cli/engine/providers/azure/InfrastructureBuilder.py b/core/src/epicli/cli/engine/providers/azure/InfrastructureBuilder.py index 3afdc4c2bd..e5b703d2fc 100644 --- a/core/src/epicli/cli/engine/providers/azure/InfrastructureBuilder.py +++ b/core/src/epicli/cli/engine/providers/azure/InfrastructureBuilder.py @@ -45,6 +45,10 @@ def run(self): if (len(component_value.subnets) > 1): self.logger.warning(f'On Azure only one subnet per component is supported for now. Taking first and ignoring others.') + # Add message for ignoring availabiltity zones if present. + if 'availability_zone' in component_value.subnets[0]: + self.logger.warning(f'On Azure availability_zones are not supported jet. Ignoring definition.') + subnet_definition = component_value.subnets[0] subnet = select_first(infrastructure, lambda item: item.kind == 'infrastructure/subnet' and item.specification.address_prefix == subnet_definition['address_pool']) diff --git a/core/src/epicli/cli/helpers/Log.py b/core/src/epicli/cli/helpers/Log.py index fd3bb008f7..89440cb1d4 100644 --- a/core/src/epicli/cli/helpers/Log.py +++ b/core/src/epicli/cli/helpers/Log.py @@ -17,14 +17,14 @@ def __init__(self): log_path = os.path.join(get_output_path(), config.log_file) logging.basicConfig(level=logging.INFO, format=config.log_format, datefmt=config.log_date_format) formatter = jsonlogger.JsonFormatter(config.log_format, datefmt=config.log_date_format) - self.json_file_handler = logging.FileHandler(filename=log_path) - self.json_file_handler.setFormatter(formatter) - self.json_stream_handler = logging.StreamHandler() - self.json_stream_handler.setFormatter(formatter) should_roll_over = os.path.isfile(log_path) - handler = logging.handlers.RotatingFileHandler(log_path, mode='w', backupCount=config.log_count) + handler = logging.handlers.RotatingFileHandler(log_path, backupCount=config.log_count) if should_roll_over: handler.doRollover() + self.json_file_handler = handler + self.json_file_handler.setFormatter(formatter) + self.json_stream_handler = logging.StreamHandler() + self.json_stream_handler.setFormatter(formatter) instance = None diff --git a/core/src/epicli/cli/helpers/naming_helpers.py b/core/src/epicli/cli/helpers/naming_helpers.py index 688afda2df..b0d35d45a2 100644 --- a/core/src/epicli/cli/helpers/naming_helpers.py +++ b/core/src/epicli/cli/helpers/naming_helpers.py @@ -46,11 +46,12 @@ def storage_account_name(prefix, cluster_name, storage_use): sto = storage_use.lower() clu = '' + cn = cluster_name.replace('-', '') length = 24 - (len(pre)+len(sto)) - if len(cluster_name) > length: - clu = cluster_name[:length].lower() + if len(cn) > length: + clu = cn[:length].lower() else: - clu = cluster_name.lower() + clu = cn.lower() return f'{pre}{clu}{sto}' diff --git a/core/src/epicli/cli/version.py b/core/src/epicli/cli/version.py index 698a3f5800..0d4089c181 100644 --- a/core/src/epicli/cli/version.py +++ b/core/src/epicli/cli/version.py @@ -1 +1 @@ -VERSION = '0.4.0' \ No newline at end of file +VERSION = '0.4.1' \ No newline at end of file diff --git a/core/src/epicli/data/any/defaults/configuration/minimal-cluster-config.yml b/core/src/epicli/data/any/defaults/configuration/minimal-cluster-config.yml index dfe7ad50f4..08629fd071 100644 --- a/core/src/epicli/data/any/defaults/configuration/minimal-cluster-config.yml +++ b/core/src/epicli/data/any/defaults/configuration/minimal-cluster-config.yml @@ -1,5 +1,5 @@ kind: epiphany-cluster -version: 0.4.0 +version: 0.4.1 title: "Epiphany cluster Config" provider: any name: "default" diff --git a/core/src/epicli/data/aws/defaults/configuration/minimal-cluster-config.yml b/core/src/epicli/data/aws/defaults/configuration/minimal-cluster-config.yml index 1a06ed15ed..d1d1585c4d 100644 --- a/core/src/epicli/data/aws/defaults/configuration/minimal-cluster-config.yml +++ b/core/src/epicli/data/aws/defaults/configuration/minimal-cluster-config.yml @@ -1,5 +1,5 @@ kind: epiphany-cluster -version: 0.4.0 +version: 0.4.1 title: "Epiphany cluster Config" provider: aws name: "default" diff --git a/core/src/epicli/data/aws/defaults/infrastructure/default-security-group.yml b/core/src/epicli/data/aws/defaults/infrastructure/default-security-group.yml index d9d2e1281e..cf164c4b4c 100644 --- a/core/src/epicli/data/aws/defaults/infrastructure/default-security-group.yml +++ b/core/src/epicli/data/aws/defaults/infrastructure/default-security-group.yml @@ -1,5 +1,5 @@ kind: infrastructure/default-security-group -version: 0.4.0 +version: 0.4.1 title: "Default Security Group Config" provider: aws name: default-security-group diff --git a/core/src/epicli/data/aws/defaults/infrastructure/efs-storage.yml b/core/src/epicli/data/aws/defaults/infrastructure/efs-storage.yml index e9b6189c2d..5a220a03b0 100644 --- a/core/src/epicli/data/aws/defaults/infrastructure/efs-storage.yml +++ b/core/src/epicli/data/aws/defaults/infrastructure/efs-storage.yml @@ -1,5 +1,5 @@ kind: infrastructure/efs-storage -version: 0.4.0 +version: 0.4.1 title: "Elastic File System Config" provider: aws name: default diff --git a/core/src/epicli/data/aws/defaults/infrastructure/internet-gateway.yml b/core/src/epicli/data/aws/defaults/infrastructure/internet-gateway.yml index f7f07752bc..3030940c73 100644 --- a/core/src/epicli/data/aws/defaults/infrastructure/internet-gateway.yml +++ b/core/src/epicli/data/aws/defaults/infrastructure/internet-gateway.yml @@ -1,5 +1,5 @@ kind: infrastructure/internet-gateway -version: 0.4.0 +version: 0.4.1 title: "Internet Gateway Config" provider: aws name: default diff --git a/core/src/epicli/data/aws/defaults/infrastructure/launch-configuration.yml b/core/src/epicli/data/aws/defaults/infrastructure/launch-configuration.yml index 983334c25c..62ade13a08 100644 --- a/core/src/epicli/data/aws/defaults/infrastructure/launch-configuration.yml +++ b/core/src/epicli/data/aws/defaults/infrastructure/launch-configuration.yml @@ -1,5 +1,5 @@ kind: infrastructure/launch-configuration -version: 0.4.0 +version: 0.4.1 title: "Launch configuration" provider: aws name: default diff --git a/core/src/epicli/data/aws/defaults/infrastructure/public-key.yml b/core/src/epicli/data/aws/defaults/infrastructure/public-key.yml index ab420ee069..de78a9c448 100644 --- a/core/src/epicli/data/aws/defaults/infrastructure/public-key.yml +++ b/core/src/epicli/data/aws/defaults/infrastructure/public-key.yml @@ -1,5 +1,5 @@ kind: infrastructure/public-key -version: 0.4.0 +version: 0.4.1 title: "Public Key" provider: aws name: default diff --git a/core/src/epicli/data/aws/defaults/infrastructure/resource-group.yml b/core/src/epicli/data/aws/defaults/infrastructure/resource-group.yml index 92fa7b24f5..0ea15a4822 100644 --- a/core/src/epicli/data/aws/defaults/infrastructure/resource-group.yml +++ b/core/src/epicli/data/aws/defaults/infrastructure/resource-group.yml @@ -1,5 +1,5 @@ kind: infrastructure/resource-group -version: 0.4.0 +version: 0.4.1 title: "Resource Group" provider: aws name: default diff --git a/core/src/epicli/data/aws/defaults/infrastructure/route-table-association.yml b/core/src/epicli/data/aws/defaults/infrastructure/route-table-association.yml index 8595e68034..2e1725fa09 100644 --- a/core/src/epicli/data/aws/defaults/infrastructure/route-table-association.yml +++ b/core/src/epicli/data/aws/defaults/infrastructure/route-table-association.yml @@ -1,5 +1,5 @@ kind: infrastructure/route-table-association -version: 0.4.0 +version: 0.4.1 title: Route Table Association Config" provider: aws name: default diff --git a/core/src/epicli/data/aws/defaults/infrastructure/route-table.yml b/core/src/epicli/data/aws/defaults/infrastructure/route-table.yml index 46deb87ccf..4b0dfd55c4 100644 --- a/core/src/epicli/data/aws/defaults/infrastructure/route-table.yml +++ b/core/src/epicli/data/aws/defaults/infrastructure/route-table.yml @@ -1,5 +1,5 @@ kind: infrastructure/route-table -version: 0.4.0 +version: 0.4.1 title: "Route Table Config" provider: aws name: default diff --git a/core/src/epicli/data/aws/defaults/infrastructure/security-group-rule.yml b/core/src/epicli/data/aws/defaults/infrastructure/security-group-rule.yml index d37a024ff5..3feadc55aa 100644 --- a/core/src/epicli/data/aws/defaults/infrastructure/security-group-rule.yml +++ b/core/src/epicli/data/aws/defaults/infrastructure/security-group-rule.yml @@ -1,5 +1,5 @@ kind: infrastructure/security-group-rule -version: 0.4.0 +version: 0.4.1 title: "Default Group Rule" provider: aws name: default diff --git a/core/src/epicli/data/aws/defaults/infrastructure/security-group.yml b/core/src/epicli/data/aws/defaults/infrastructure/security-group.yml index 71f0f7eabb..a35819c1f3 100644 --- a/core/src/epicli/data/aws/defaults/infrastructure/security-group.yml +++ b/core/src/epicli/data/aws/defaults/infrastructure/security-group.yml @@ -1,5 +1,5 @@ kind: infrastructure/security-group -version: 0.4.0 +version: 0.4.1 title: "Security Group Config" provider: aws name: default diff --git a/core/src/epicli/data/aws/defaults/infrastructure/subnet.yml b/core/src/epicli/data/aws/defaults/infrastructure/subnet.yml index cd9dd7e7b7..96f00f48ff 100644 --- a/core/src/epicli/data/aws/defaults/infrastructure/subnet.yml +++ b/core/src/epicli/data/aws/defaults/infrastructure/subnet.yml @@ -1,5 +1,5 @@ kind: infrastructure/subnet -version: 0.4.0 +version: 0.4.1 title: "Subnet Config" provider: aws name: default diff --git a/core/src/epicli/data/aws/defaults/infrastructure/virtual-machine.yml b/core/src/epicli/data/aws/defaults/infrastructure/virtual-machine.yml index 66b490d0cf..d48273e568 100644 --- a/core/src/epicli/data/aws/defaults/infrastructure/virtual-machine.yml +++ b/core/src/epicli/data/aws/defaults/infrastructure/virtual-machine.yml @@ -1,5 +1,5 @@ kind: infrastructure/virtual-machine -version: 0.4.0 +version: 0.4.1 title: "Virtual Machine Infra" provider: aws name: default @@ -13,7 +13,7 @@ specification: authorized_to_efs: false mount_efs: false tags: - - version: 0.4.0 + - version: 0.4.1 size: t2.micro os_full_name: "ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20190212.1" os_type: linux @@ -84,13 +84,13 @@ specification: --- kind: infrastructure/virtual-machine -version: 0.4.0 +version: 0.4.1 title: "Virtual Machine Infra" provider: aws name: default-size-t3 specification: tags: - - version: 0.4.0 + - version: 0.4.1 size: t3.micro os_type: linux security: @@ -117,13 +117,13 @@ specification: destination_address_prefix: "0.0.0.0/0" --- kind: infrastructure/virtual-machine -version: 0.4.0 +version: 0.4.1 title: "Virtual Machine Infra" provider: aws name: rabbitmq-machine specification: tags: - - version: 0.4.0 + - version: 0.4.1 size: t3.micro os_type: linux security: @@ -190,13 +190,13 @@ specification: destination_address_prefix: "0.0.0.0/0" --- kind: infrastructure/virtual-machine -version: 0.4.0 +version: 0.4.1 title: "Virtual Machine Infra" provider: aws name: load-balancer-machine specification: tags: - - version: 0.4.0 + - version: 0.4.1 size: t3.micro os_type: linux security: @@ -243,7 +243,7 @@ specification: destination_address_prefix: "0.0.0.0/0" --- kind: infrastructure/virtual-machine -version: 0.4.0 +version: 0.4.1 title: "Virtual Machine Infra" provider: aws name: kubernetes-master-machine @@ -343,7 +343,7 @@ specification: destination_address_prefix: "0.0.0.0/0" --- kind: infrastructure/virtual-machine -version: 0.4.0 +version: 0.4.1 title: "Virtual Machine Infra" provider: aws name: kubernetes-node-machine @@ -421,7 +421,7 @@ specification: destination_address_prefix: "0.0.0.0/0" --- kind: infrastructure/virtual-machine -version: 0.4.0 +version: 0.4.1 title: "Virtual Machine Infra" provider: aws name: kafka-machine @@ -547,7 +547,7 @@ specification: destination_address_prefix: "0.0.0.0/0" --- kind: infrastructure/virtual-machine -version: 0.4.0 +version: 0.4.1 title: "Virtual Machine Infra" provider: aws name: monitoring-machine @@ -607,7 +607,7 @@ specification: destination_address_prefix: "0.0.0.0/0" --- kind: infrastructure/virtual-machine -version: 0.4.0 +version: 0.4.1 title: "Virtual Machine Infra" provider: aws name: postgresql-machine @@ -681,7 +681,7 @@ specification: destination_address_prefix: "0.0.0.0/0" --- kind: infrastructure/virtual-machine -version: 0.4.0 +version: 0.4.1 title: "Virtual Machine Infra" provider: aws name: logging-machine diff --git a/core/src/epicli/data/aws/defaults/infrastructure/vpc.yml b/core/src/epicli/data/aws/defaults/infrastructure/vpc.yml index b8ab68df82..20b06f8fc4 100644 --- a/core/src/epicli/data/aws/defaults/infrastructure/vpc.yml +++ b/core/src/epicli/data/aws/defaults/infrastructure/vpc.yml @@ -1,5 +1,5 @@ kind: infrastructure/vpc -version: 0.4.0 +version: 0.4.1 title: "VPC Config" provider: aws name: default diff --git a/core/src/epicli/data/azure/defaults/configuration/minimal-cluster-config.yml b/core/src/epicli/data/azure/defaults/configuration/minimal-cluster-config.yml index fc09003f52..5e3bc341a7 100644 --- a/core/src/epicli/data/azure/defaults/configuration/minimal-cluster-config.yml +++ b/core/src/epicli/data/azure/defaults/configuration/minimal-cluster-config.yml @@ -1,5 +1,5 @@ kind: epiphany-cluster -version: 0.4.0 +version: 0.4.1 title: "Epiphany cluster Config" provider: azure name: "default" diff --git a/core/src/epicli/data/azure/defaults/infrastructure/network-interface.yml b/core/src/epicli/data/azure/defaults/infrastructure/network-interface.yml index bb9972c982..62b0b5d14b 100644 --- a/core/src/epicli/data/azure/defaults/infrastructure/network-interface.yml +++ b/core/src/epicli/data/azure/defaults/infrastructure/network-interface.yml @@ -1,5 +1,5 @@ kind: infrastructure/network-interface -version: 0.4.0 +version: 0.4.1 title: "Network Interface Config" provider: azure name: default diff --git a/core/src/epicli/data/azure/defaults/infrastructure/network-security-group.yml b/core/src/epicli/data/azure/defaults/infrastructure/network-security-group.yml index 7093dd1716..4a42da5ebb 100644 --- a/core/src/epicli/data/azure/defaults/infrastructure/network-security-group.yml +++ b/core/src/epicli/data/azure/defaults/infrastructure/network-security-group.yml @@ -1,5 +1,5 @@ kind: infrastructure/network-security-group -version: 0.4.0 +version: 0.4.1 title: "Security Group Config" provider: azure name: default diff --git a/core/src/epicli/data/azure/defaults/infrastructure/public-ip.yml b/core/src/epicli/data/azure/defaults/infrastructure/public-ip.yml index 476131eff8..5652c03887 100644 --- a/core/src/epicli/data/azure/defaults/infrastructure/public-ip.yml +++ b/core/src/epicli/data/azure/defaults/infrastructure/public-ip.yml @@ -1,5 +1,5 @@ kind: infrastructure/public-ip -version: 0.4.0 +version: 0.4.1 title: "Public IP Config" provider: azure name: default diff --git a/core/src/epicli/data/azure/defaults/infrastructure/resource-group.yml b/core/src/epicli/data/azure/defaults/infrastructure/resource-group.yml index 30e7e80ef4..1bdcd0878d 100644 --- a/core/src/epicli/data/azure/defaults/infrastructure/resource-group.yml +++ b/core/src/epicli/data/azure/defaults/infrastructure/resource-group.yml @@ -1,5 +1,5 @@ kind: infrastructure/resource-group -version: 0.4.0 +version: 0.4.1 title: "Resource Group" provider: azure name: default diff --git a/core/src/epicli/data/azure/defaults/infrastructure/storage-share.yml b/core/src/epicli/data/azure/defaults/infrastructure/storage-share.yml index a376f7caed..949d1344ec 100644 --- a/core/src/epicli/data/azure/defaults/infrastructure/storage-share.yml +++ b/core/src/epicli/data/azure/defaults/infrastructure/storage-share.yml @@ -1,5 +1,5 @@ kind: infrastructure/storage-share -version: 0.4.0 +version: 0.4.1 title: "Azure shared storage" provider: azure name: default diff --git a/core/src/epicli/data/azure/defaults/infrastructure/subnet-network-security-group-association.yml b/core/src/epicli/data/azure/defaults/infrastructure/subnet-network-security-group-association.yml index d949997924..8308e272c1 100644 --- a/core/src/epicli/data/azure/defaults/infrastructure/subnet-network-security-group-association.yml +++ b/core/src/epicli/data/azure/defaults/infrastructure/subnet-network-security-group-association.yml @@ -1,5 +1,5 @@ kind: infrastructure/subnet-network-security-group-association -version: 0.4.0 +version: 0.4.1 title: "Subnet Network Security Group Association" provider: azure name: default diff --git a/core/src/epicli/data/azure/defaults/infrastructure/subnet.yml b/core/src/epicli/data/azure/defaults/infrastructure/subnet.yml index d6a680316d..a8717516a2 100644 --- a/core/src/epicli/data/azure/defaults/infrastructure/subnet.yml +++ b/core/src/epicli/data/azure/defaults/infrastructure/subnet.yml @@ -1,5 +1,5 @@ kind: infrastructure/subnet -version: 0.4.0 +version: 0.4.1 title: "Subnet Config" provider: azure name: default diff --git a/core/src/epicli/data/azure/defaults/infrastructure/virtual-machine.yml b/core/src/epicli/data/azure/defaults/infrastructure/virtual-machine.yml index dca2fce330..fbc77b362b 100644 --- a/core/src/epicli/data/azure/defaults/infrastructure/virtual-machine.yml +++ b/core/src/epicli/data/azure/defaults/infrastructure/virtual-machine.yml @@ -1,5 +1,5 @@ kind: infrastructure/virtual-machine -version: 0.4.0 +version: 0.4.1 title: "Virtual Machine Infra" provider: azure name: default @@ -65,7 +65,7 @@ specification: destination_address_prefix: "0.0.0.0/0" --- kind: infrastructure/virtual-machine -version: 0.4.0 +version: 0.4.1 title: "Virtual Machine Infra" provider: azure name: kubernetes-master-machine @@ -173,7 +173,7 @@ specification: # destination_address_prefix: "0.0.0.0/0" --- kind: infrastructure/virtual-machine -version: 0.4.0 +version: 0.4.1 title: "Virtual Machine Infra" provider: azure name: kubernetes-node-machine @@ -249,7 +249,7 @@ specification: destination_address_prefix: "0.0.0.0/0" --- kind: infrastructure/virtual-machine -version: 0.4.0 +version: 0.4.1 title: "Virtual Machine Infra" provider: azure name: kafka-machine @@ -375,7 +375,7 @@ specification: # destination_address_prefix: "0.0.0.0/0" --- kind: infrastructure/virtual-machine -version: 0.4.0 +version: 0.4.1 title: "Virtual Machine Infra" provider: azure name: rabbitmq-machine @@ -425,7 +425,7 @@ specification: destination_address_prefix: "0.0.0.0/0" --- kind: infrastructure/virtual-machine -version: 0.4.0 +version: 0.4.1 title: "Virtual Machine Infra" provider: azure name: monitoring-machine @@ -485,7 +485,7 @@ specification: destination_address_prefix: "0.0.0.0/0" --- kind: infrastructure/virtual-machine -version: 0.4.0 +version: 0.4.1 title: "Virtual Machine Infra" provider: azure name: postgresql-machine @@ -549,7 +549,7 @@ specification: # destination_address_prefix: "0.0.0.0/0" --- kind: infrastructure/virtual-machine -version: 0.4.0 +version: 0.4.1 title: "Virtual Machine Infra" provider: azure name: logging-machine @@ -619,7 +619,7 @@ specification: destination_address_prefix: "0.0.0.0/0" --- kind: infrastructure/virtual-machine -version: 0.4.0 +version: 0.4.1 title: "Virtual Machine Infra" provider: azure name: load-balancer-machine diff --git a/core/src/epicli/data/azure/defaults/infrastructure/vnet.yml b/core/src/epicli/data/azure/defaults/infrastructure/vnet.yml index b524337231..3cbf27b6d2 100644 --- a/core/src/epicli/data/azure/defaults/infrastructure/vnet.yml +++ b/core/src/epicli/data/azure/defaults/infrastructure/vnet.yml @@ -1,5 +1,5 @@ kind: infrastructure/vnet -version: 0.4.0 +version: 0.4.1 title: "VNET Config" provider: azure name: default diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_node/meta/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_node/meta/main.yml index b3e1010012..3591c1731f 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_node/meta/main.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_node/meta/main.yml @@ -1,4 +1,3 @@ --- dependencies: - - role: kubernetes_master - role: kubernetes_common diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/common.sh b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/common.sh index f115e98eae..6089107000 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/common.sh +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/common.sh @@ -46,7 +46,7 @@ download_image() { else local tmp_file=$(mktemp) echo "Downloading image: $1" - echo "Skopeo command is: ./skopeo_linux --insecure-policy copy docker://{$image_name} docker-archive:${dst_image}:${repository}:${tag}" + echo "Skopeo command is: ./skopeo_linux --insecure-policy copy docker://${image_name} docker-archive:${dst_image}:${repository}:${tag}" # use temporary file for downloading to be safe from sudden interruptions (network, ctrl+c) ./skopeo_linux --insecure-policy copy docker://${image_name} docker-archive:${tmp_file}:${repository}:${tag} && chmod 644 ${tmp_file} && mv ${tmp_file} ${dst_image} fi @@ -74,8 +74,5 @@ download_file() { # --no-use-server-timestamps - we don't use --timestamping and we need to expire files somehow # --continue - don't download the same file multiple times, gracefully skip if file is fully downloaded - wget --no-use-server-timestamps --continue --show-progress --directory-prefix="${dest_dir}" "${file_url}" - - #wget --no-verbose --directory-prefix="$dest_dir" "$file_url" || - #exit_with_error "Command failed: wget --no-verbose --directory-prefix=\"$dest_dir\" \"$file_url\"" + wget --no-use-server-timestamps --continue --show-progress --prefer-family=IPv4 --directory-prefix="${dest_dir}" "${file_url}" } diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/server/Debian/create-repository.sh b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/server/Debian/create-repository.sh index ea5db3debf..1d4c874627 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/server/Debian/create-repository.sh +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/server/Debian/create-repository.sh @@ -2,26 +2,38 @@ EPI_REPO_SERVER_PATH=$1 # /var/www/html/epirepo is the default IS_OFFLINE_MODE=$2 +script_path="$( cd "$(dirname "$0")" ; pwd -P )" if $IS_OFFLINE_MODE = true; then # bootstrap apache and dpkg-dev installation in air-gap mode + if [[ -f /etc/apt/sources.list ]]; then + echo "disabling default repositories..." + mv /etc/apt/sources.list /etc/apt/sources.list.bak + fi if ! dpkg -l | grep -q libdpkg-perl; then echo libdpkg-perl not found, installing... - dpkg -i ${EPI_REPO_SERVER_PATH}/packages/libdpkg-perl*.deb + dpkg -i "${EPI_REPO_SERVER_PATH}"/packages/libdpkg-perl*.deb fi - cd ${EPI_REPO_SERVER_PATH}/packages && /tmp/epi-repository-setup-scripts/dpkg-scanpackages -m . | gzip -9c > Packages.gz && cd - + echo "generating repository metadata..." + cd "${EPI_REPO_SERVER_PATH}"/packages && /tmp/epi-repository-setup-scripts/dpkg-scanpackages -m . | gzip -9c > Packages.gz && cd "${script_path}" echo "deb [trusted=yes] file:${EPI_REPO_SERVER_PATH}/packages ./" > /etc/apt/sources.list.d/epilocal.list - apt update --assume-no # workaround for botched docker repository https://github.com/docker/for-linux/issues/812 + #apt update --assume-no # workaround for botched docker repository https://github.com/docker/for-linux/issues/812 + echo "updating apt and installing apache..." + apt -y update apt -y install apache2 dpkg-dev + echo "removing temporary repo definition: /etc/apt/sources.list.d/epilocal.list..." rm -f /etc/apt/sources.list.d/epilocal.list - rm -f ${EPI_REPO_SERVER_PATH}/packages/Packages.gz - apt update --assume-no + #rm -f ${EPI_REPO_SERVER_PATH}/packages/Packages.gz + echo "updating apt..." + apt -y update else + # for online mode just install apache apt -y install apache2 dpkg-dev + + # -m is important because it allow same packages with different versions + # 'cd' is needed here becuase 'dpkg-scanpackages' prepends path to "Filename" field in Packages.gz + # otherwise it would break package URL for apt + cd /var/www/html/epirepo/packages && dpkg-scanpackages -m . | gzip -9c > Packages.gz && cd "${script_path}" fi systemctl start apache2 - -# -m is important because it allow same packages with different versions -# 'cd' is needed here becuase 'dpkg-scanpackages' prepends path to "Filename" field in Packages.gz, otherwise it would break package URL for apt -cd /var/www/html/epirepo/packages && dpkg-scanpackages -m . | gzip -9c > Packages.gz diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/Debian/setup.yml b/core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/Debian/setup.yml index 121ca9a49f..e57afe6b4e 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/Debian/setup.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/Debian/setup.yml @@ -1,7 +1,7 @@ --- - name: Create epirepo repository shell: >- - /tmp/epi-repository-setup-scripts/create-repository.sh /var/www/html/epirepo {{ offline_mode | lower }} |& + set -o pipefail && /tmp/epi-repository-setup-scripts/create-repository.sh /var/www/html/epirepo {{ offline_mode | lower }} |& tee /tmp/epi-repository-setup-scripts/create-repository.log args: executable: /bin/bash @@ -9,6 +9,14 @@ - not custom_repository_url - inventory_hostname in groups['repository'] +- name: Ensure apache is running + service: + name: apache2 + state: started + when: + - not custom_repository_url + - inventory_hostname in groups['repository'] + - name: Disable system repositories and set up epirepo block: - name: Create list of enabled repositories diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/RedHat/setup.yml b/core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/RedHat/setup.yml index 99596e6591..7ec97b7764 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/RedHat/setup.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/RedHat/setup.yml @@ -17,7 +17,6 @@ name: httpd state: started when: - - ansible_os_family == "RedHat" - not custom_repository_url - inventory_hostname in groups['repository'] diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/teardown.yml b/core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/teardown.yml index ee00250ec5..a2544bf63e 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/teardown.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/teardown.yml @@ -8,4 +8,5 @@ loop: - /tmp/epi-download-requirements - /tmp/epi-repository-setup-scripts - - /var/tmp/enabled-system-repos.txt \ No newline at end of file + - /var/tmp/enabled-system-repos.txt + - /var/tmp/enabled-system-repos.tar \ No newline at end of file diff --git a/core/src/epicli/data/common/defaults/configuration/applications.yml b/core/src/epicli/data/common/defaults/configuration/applications.yml index 5ee34f7259..df50688323 100644 --- a/core/src/epicli/data/common/defaults/configuration/applications.yml +++ b/core/src/epicli/data/common/defaults/configuration/applications.yml @@ -1,5 +1,5 @@ kind: configuration/applications -version: 0.4.0 +version: 0.4.1 title: "Kubernetes Applications Config" name: default specification: @@ -8,7 +8,7 @@ specification: # Abstract these configs to seperate default files and add # the ability to add custom application roles. -# - name: rabbitmq +# - name: rabbitmq 2 # image_path: rabbitmq:3.7.10 # #image_pull_secret_name: regcred # optional # service: diff --git a/core/src/epicli/data/common/defaults/configuration/elasticsearch-curator.yml b/core/src/epicli/data/common/defaults/configuration/elasticsearch-curator.yml index c7061d5c80..66dc8f1f63 100644 --- a/core/src/epicli/data/common/defaults/configuration/elasticsearch-curator.yml +++ b/core/src/epicli/data/common/defaults/configuration/elasticsearch-curator.yml @@ -1,5 +1,5 @@ kind: configuration/elasticsearch-curator -version: 0.4.0 +version: 0.4.1 title: "ElasticSearch curator" name: default specification: diff --git a/core/src/epicli/data/common/defaults/configuration/elasticsearch.yml b/core/src/epicli/data/common/defaults/configuration/elasticsearch.yml index 50bfd0fa91..37b3f8d75b 100644 --- a/core/src/epicli/data/common/defaults/configuration/elasticsearch.yml +++ b/core/src/epicli/data/common/defaults/configuration/elasticsearch.yml @@ -1,5 +1,5 @@ kind: configuration/elasticsearch -version: 0.4.0 +version: 0.4.1 title: "ElasticSearch Config" name: default specification: diff --git a/core/src/epicli/data/common/defaults/configuration/feature-mapping.yml b/core/src/epicli/data/common/defaults/configuration/feature-mapping.yml index 2f2d5d720c..137502dac8 100644 --- a/core/src/epicli/data/common/defaults/configuration/feature-mapping.yml +++ b/core/src/epicli/data/common/defaults/configuration/feature-mapping.yml @@ -1,5 +1,5 @@ kind: configuration/feature-mapping -version: 0.4.0 +version: 0.4.1 title: "Feature mapping to roles" name: default specification: diff --git a/core/src/epicli/data/common/defaults/configuration/filebeat.yml b/core/src/epicli/data/common/defaults/configuration/filebeat.yml index ba8712a03b..05d23cd162 100644 --- a/core/src/epicli/data/common/defaults/configuration/filebeat.yml +++ b/core/src/epicli/data/common/defaults/configuration/filebeat.yml @@ -1,5 +1,5 @@ kind: configuration/filebeat -version: 0.4.0 +version: 0.4.1 title: "Filebeat" name: default specification: diff --git a/core/src/epicli/data/common/defaults/configuration/grafana.yml b/core/src/epicli/data/common/defaults/configuration/grafana.yml index ed50556f63..c30c055740 100644 --- a/core/src/epicli/data/common/defaults/configuration/grafana.yml +++ b/core/src/epicli/data/common/defaults/configuration/grafana.yml @@ -1,5 +1,5 @@ kind: configuration/grafana -version: 0.4.0 +version: 0.4.1 title: "Grafana" name: default specification: diff --git a/core/src/epicli/data/common/defaults/configuration/haproxy-exporter.yml b/core/src/epicli/data/common/defaults/configuration/haproxy-exporter.yml index 15b59b5e16..910d4e522a 100644 --- a/core/src/epicli/data/common/defaults/configuration/haproxy-exporter.yml +++ b/core/src/epicli/data/common/defaults/configuration/haproxy-exporter.yml @@ -1,5 +1,5 @@ kind: configuration/haproxy-exporter -version: 0.4.0 +version: 0.4.1 title: "HAProxy exporter" name: default specification: diff --git a/core/src/epicli/data/common/defaults/configuration/haproxy.yml b/core/src/epicli/data/common/defaults/configuration/haproxy.yml index d44a7dd6e4..67571b6b35 100644 --- a/core/src/epicli/data/common/defaults/configuration/haproxy.yml +++ b/core/src/epicli/data/common/defaults/configuration/haproxy.yml @@ -1,5 +1,5 @@ kind: configuration/haproxy -version: 0.4.0 +version: 0.4.1 title: "HAProxy" name: default specification: diff --git a/core/src/epicli/data/common/defaults/configuration/image-registry.yml b/core/src/epicli/data/common/defaults/configuration/image-registry.yml index bb912a77ce..b77d301f3c 100644 --- a/core/src/epicli/data/common/defaults/configuration/image-registry.yml +++ b/core/src/epicli/data/common/defaults/configuration/image-registry.yml @@ -1,5 +1,5 @@ kind: configuration/image-registry -version: 0.4.0 +version: 0.4.1 title: "Epiphany image registry" name: default specification: diff --git a/core/src/epicli/data/common/defaults/configuration/jmx-exporter.yml b/core/src/epicli/data/common/defaults/configuration/jmx-exporter.yml index eb98933ad8..9f05751271 100644 --- a/core/src/epicli/data/common/defaults/configuration/jmx-exporter.yml +++ b/core/src/epicli/data/common/defaults/configuration/jmx-exporter.yml @@ -1,5 +1,5 @@ kind: configuration/jmx-exporter -version: 0.4.0 +version: 0.4.1 title: "JMX exporter" name: default specification: diff --git a/core/src/epicli/data/common/defaults/configuration/kafka-exporter.yml b/core/src/epicli/data/common/defaults/configuration/kafka-exporter.yml index f8463e0976..f9eba15a4c 100644 --- a/core/src/epicli/data/common/defaults/configuration/kafka-exporter.yml +++ b/core/src/epicli/data/common/defaults/configuration/kafka-exporter.yml @@ -1,5 +1,5 @@ kind: configuration/kafka-exporter -version: 0.4.0 +version: 0.4.1 title: "Kafka exporter" name: default specification: diff --git a/core/src/epicli/data/common/defaults/configuration/kafka.yml b/core/src/epicli/data/common/defaults/configuration/kafka.yml index 975cf24019..87d92d6ed3 100644 --- a/core/src/epicli/data/common/defaults/configuration/kafka.yml +++ b/core/src/epicli/data/common/defaults/configuration/kafka.yml @@ -1,5 +1,5 @@ kind: configuration/kafka -version: 0.4.0 +version: 0.4.1 title: "Kafka" name: default specification: diff --git a/core/src/epicli/data/common/defaults/configuration/kibana.yml b/core/src/epicli/data/common/defaults/configuration/kibana.yml index 54c5388061..5dde8842ee 100644 --- a/core/src/epicli/data/common/defaults/configuration/kibana.yml +++ b/core/src/epicli/data/common/defaults/configuration/kibana.yml @@ -1,5 +1,5 @@ kind: configuration/kibana -version: 0.4.0 +version: 0.4.1 title: "Kibana" name: default specification: diff --git a/core/src/epicli/data/common/defaults/configuration/kubernetes-master.yml b/core/src/epicli/data/common/defaults/configuration/kubernetes-master.yml index 1e2af0e394..a176003827 100644 --- a/core/src/epicli/data/common/defaults/configuration/kubernetes-master.yml +++ b/core/src/epicli/data/common/defaults/configuration/kubernetes-master.yml @@ -1,5 +1,5 @@ kind: configuration/kubernetes-master -version: 0.4.0 +version: 0.4.1 title: "Kubernetes Master Config" name: default specification: diff --git a/core/src/epicli/data/common/defaults/configuration/kubernetes-node.yml b/core/src/epicli/data/common/defaults/configuration/kubernetes-node.yml index 89226f8ec9..ca36cf8cf5 100644 --- a/core/src/epicli/data/common/defaults/configuration/kubernetes-node.yml +++ b/core/src/epicli/data/common/defaults/configuration/kubernetes-node.yml @@ -1,12 +1,12 @@ kind: configuration/kubernetes-node -version: 0.4.0 +version: 0.4.1 title: "Kubernetes Node Config" name: default specification: version: 1.14.6 images_to_load: - name: "jboss/keycloak:4.8.3.Final" - file_name: keycloak:4.8.3.Final.tar + file_name: keycloak-4.8.3.Final.tar - name: "rabbitmq:3.7.10" - file_name: rabbitmq:3.7.10.tar + file_name: rabbitmq-3.7.10.tar node_labels: "node-type=epiphany" diff --git a/core/src/epicli/data/common/defaults/configuration/node-exporter.yml b/core/src/epicli/data/common/defaults/configuration/node-exporter.yml index 1883daed59..e4eb3c66e9 100644 --- a/core/src/epicli/data/common/defaults/configuration/node-exporter.yml +++ b/core/src/epicli/data/common/defaults/configuration/node-exporter.yml @@ -1,5 +1,5 @@ kind: configuration/node-exporter -version: 0.4.0 +version: 0.4.1 title: "Node exporter" name: default specification: diff --git a/core/src/epicli/data/common/defaults/configuration/postgresql.yml b/core/src/epicli/data/common/defaults/configuration/postgresql.yml index c6764ee21f..c63066c5ff 100644 --- a/core/src/epicli/data/common/defaults/configuration/postgresql.yml +++ b/core/src/epicli/data/common/defaults/configuration/postgresql.yml @@ -1,5 +1,5 @@ kind: configuration/postgresql -version: 0.4.0 +version: 0.4.1 title: "Postgresql" name: default specification: diff --git a/core/src/epicli/data/common/defaults/configuration/prometheus.yml b/core/src/epicli/data/common/defaults/configuration/prometheus.yml index 02b5615406..4236eaa5e5 100644 --- a/core/src/epicli/data/common/defaults/configuration/prometheus.yml +++ b/core/src/epicli/data/common/defaults/configuration/prometheus.yml @@ -1,5 +1,5 @@ kind: configuration/prometheus -version: 0.4.0 +version: 0.4.1 title: "Prometheus" name: default specification: diff --git a/core/src/epicli/data/common/defaults/configuration/rabbitmq.yml b/core/src/epicli/data/common/defaults/configuration/rabbitmq.yml index 3ae15ad69a..3990a900c6 100644 --- a/core/src/epicli/data/common/defaults/configuration/rabbitmq.yml +++ b/core/src/epicli/data/common/defaults/configuration/rabbitmq.yml @@ -1,5 +1,5 @@ kind: configuration/rabbitmq -version: 0.4.0 +version: 0.4.1 title: "RabbitMQ" name: default specification: diff --git a/core/src/epicli/data/common/defaults/configuration/repository.yml b/core/src/epicli/data/common/defaults/configuration/repository.yml index 2cbd8b8a91..1607e3baca 100644 --- a/core/src/epicli/data/common/defaults/configuration/repository.yml +++ b/core/src/epicli/data/common/defaults/configuration/repository.yml @@ -1,5 +1,5 @@ kind: configuration/repository -version: 0.4.0 +version: 0.4.1 title: "Epiphany requirements repository" name: default specification: diff --git a/core/src/epicli/data/common/defaults/configuration/shared-config.yml b/core/src/epicli/data/common/defaults/configuration/shared-config.yml index d865fa78df..18ad12bc8e 100644 --- a/core/src/epicli/data/common/defaults/configuration/shared-config.yml +++ b/core/src/epicli/data/common/defaults/configuration/shared-config.yml @@ -1,5 +1,5 @@ kind: configuration/shared-config -version: 0.4.0 +version: 0.4.1 title: "Shared configuration that will be visible to all roles" name: default specification: diff --git a/core/src/epicli/data/common/defaults/configuration/zookeeper.yml b/core/src/epicli/data/common/defaults/configuration/zookeeper.yml index fcf8c3cb98..63135499ac 100644 --- a/core/src/epicli/data/common/defaults/configuration/zookeeper.yml +++ b/core/src/epicli/data/common/defaults/configuration/zookeeper.yml @@ -1,5 +1,5 @@ kind: configuration/zookeeper -version: 0.4.0 +version: 0.4.1 title: "Zookeeper" name: default specification: diff --git a/core/src/epicli/data/common/defaults/epiphany-cluster.yml b/core/src/epicli/data/common/defaults/epiphany-cluster.yml index fb0d4bb217..8334a318b4 100644 --- a/core/src/epicli/data/common/defaults/epiphany-cluster.yml +++ b/core/src/epicli/data/common/defaults/epiphany-cluster.yml @@ -1,81 +1,81 @@ -kind: epiphany-cluster -version: 0.4.0 -title: "Epiphany cluster Config" -provider: aws -name: "default" -specification: - prefix: default - name: epiphanycluster - admin_user: - name: operations # YOUR-ADMIN-USERNAME - key_path: /root/.ssh/epiphany-operations/id_rsa # YOUR-SSH-KEY-PATH - cloud: - subscription_name: YOUR-SUB-NAME - vnet_address_pool: 10.1.0.0/20 - use_public_ips: False # When not using public IPs you have to provide connectivity via private IPs (VPN) - use_service_principal: False - region: eu-west-2 - credentials: # todo change it to get credentials from vault - key: 3124-4124-4124 - secret: DADFAFHCJHCAUYEAk - components: - kubernetes_master: - count: 1 - machine: kubernetes-master-machine - configuration: default - subnets: - - availability_zone: eu-west-2a - address_pool: 10.1.1.0/24 - - availability_zone: eu-west-2b - address_pool: 10.1.2.0/24 - kubernetes_node: - count: 2 - machine: kubernetes-node-machine - configuration: default - subnets: - - availability_zone: eu-west-2a - address_pool: 10.1.1.0/24 - - availability_zone: eu-west-2b - address_pool: 10.1.2.0/24 - logging: - count: 1 - machine: logging-machine - configuration: default - subnets: - - availability_zone: eu-west-2a - address_pool: 10.1.3.0/24 - monitoring: - count: 1 - machine: monitoring-machine - configuration: default - subnets: - - availability_zone: eu-west-2a - address_pool: 10.1.4.0/24 - kafka: - count: 2 - machine: kafka-machine - configuration: default - subnets: - - availability_zone: eu-west-2a - address_pool: 10.1.5.0/24 - postgresql: - count: 0 - machine: postgresql-machine - configuration: default - subnets: - - availability_zone: eu-west-2a - address_pool: 10.1.6.0/24 - load_balancer: - count: 1 - machine: load-balancer-machine - configuration: default - subnets: - - availability_zone: eu-west-2a - address_pool: 10.1.7.0/24 - rabbitmq: - count: 0 - machine: rabbitmq-machine - configuration: default - subnets: - - availability_zone: eu-west-2a - address_pool: 10.1.8.0/24 +kind: epiphany-cluster +version: 0.4.1 +title: "Epiphany cluster Config" +provider: aws +name: "default" +specification: + prefix: default + name: epiphanycluster + admin_user: + name: operations # YOUR-ADMIN-USERNAME + key_path: /root/.ssh/epiphany-operations/id_rsa # YOUR-SSH-KEY-PATH + cloud: + subscription_name: YOUR-SUB-NAME + vnet_address_pool: 10.1.0.0/20 + use_public_ips: False # When not using public IPs you have to provide connectivity via private IPs (VPN) + use_service_principal: False + region: eu-west-2 + credentials: # todo change it to get credentials from vault + key: 3124-4124-4124 + secret: DADFAFHCJHCAUYEAk + components: + kubernetes_master: + count: 1 + machine: kubernetes-master-machine + configuration: default + subnets: + - availability_zone: eu-west-2a + address_pool: 10.1.1.0/24 + - availability_zone: eu-west-2b + address_pool: 10.1.2.0/24 + kubernetes_node: + count: 2 + machine: kubernetes-node-machine + configuration: default + subnets: + - availability_zone: eu-west-2a + address_pool: 10.1.1.0/24 + - availability_zone: eu-west-2b + address_pool: 10.1.2.0/24 + logging: + count: 1 + machine: logging-machine + configuration: default + subnets: + - availability_zone: eu-west-2a + address_pool: 10.1.3.0/24 + monitoring: + count: 1 + machine: monitoring-machine + configuration: default + subnets: + - availability_zone: eu-west-2a + address_pool: 10.1.4.0/24 + kafka: + count: 2 + machine: kafka-machine + configuration: default + subnets: + - availability_zone: eu-west-2a + address_pool: 10.1.5.0/24 + postgresql: + count: 0 + machine: postgresql-machine + configuration: default + subnets: + - availability_zone: eu-west-2a + address_pool: 10.1.6.0/24 + load_balancer: + count: 1 + machine: load-balancer-machine + configuration: default + subnets: + - availability_zone: eu-west-2a + address_pool: 10.1.7.0/24 + rabbitmq: + count: 0 + machine: rabbitmq-machine + configuration: default + subnets: + - availability_zone: eu-west-2a + address_pool: 10.1.8.0/24 diff --git a/core/src/epicli/data/common/defaults/infrastructure/machine.yml b/core/src/epicli/data/common/defaults/infrastructure/machine.yml index bcc968ef14..0e5614074d 100644 --- a/core/src/epicli/data/common/defaults/infrastructure/machine.yml +++ b/core/src/epicli/data/common/defaults/infrastructure/machine.yml @@ -1,5 +1,5 @@ kind: infrastructure/virtual-machine -version: 0.4.0 +version: 0.4.1 title: "Virtual Machine Infra" provider: any name: default diff --git a/core/src/epicli/data/common/validation/core/definitions.yml b/core/src/epicli/data/common/validation/core/definitions.yml index 08bc0ed4fd..8b0716f7fb 100644 --- a/core/src/epicli/data/common/validation/core/definitions.yml +++ b/core/src/epicli/data/common/validation/core/definitions.yml @@ -24,9 +24,9 @@ provider: version: type: string title: The Version Schema - default: '0.4.0' + default: '0.4.1' examples: - - 0.4.0 + - 0.4.1 pattern: ^((\d+\.)(\d+\.)(\d))$ unvalidated_specification: type: diff --git a/core/src/epicli/data/common/validation/epiphany-cluster.yml b/core/src/epicli/data/common/validation/epiphany-cluster.yml index 25537a58d5..6f1e6b649a 100644 --- a/core/src/epicli/data/common/validation/epiphany-cluster.yml +++ b/core/src/epicli/data/common/validation/epiphany-cluster.yml @@ -1,43 +1,43 @@ -"$id": "#/epiphany-cluster/specification" -title: "Cluster specification schema" -description: "The main cluster specification" -type: object -required: - - name - - admin_user - - components -properties: - prefix: - "$id": "#/epiphany-cluster/properties/prefix" - title: "Cluster prefix" - description: "A prefix the can be prepended to the cluster name" - examples: - - prod01 - - dev02 - - test03 - - cust04 - type: string - pattern: "^[a-z0-9]{2,8}$" - name: - "$id": "#/epiphany-cluster/properties/name" - title: "Cluster name" - description: "The name of the cluster" - examples: - - clustername01 - type: string - pattern: "^[a-z0-9]{3,20}$" - admin_user: - "$id": "#/epiphany-cluster/properties/admin_user" - title: "The admin_user Schema" - description: "Settings needed for the SSH connection to the cluster machines or VM's" - type: object - cloud: - "$id": "#/epiphany-cluster/properties/cloud" - title: "Cloud Schema" - description: "Settings specific to cloud providers (AWS, Azure)" - type: object - components: - "$id": "#/epiphany-cluster/properties/components" - title: "Components schema" - description: "Cluster component layout specification" +"$id": "#/epiphany-cluster/specification" +title: "Cluster specification schema" +description: "The main cluster specification" +type: object +required: + - name + - admin_user + - components +properties: + prefix: + "$id": "#/epiphany-cluster/properties/prefix" + title: "Cluster prefix" + description: "A prefix the can be prepended to the cluster name" + examples: + - prod01 + - dev02 + - test03 + - cust04 + type: string + pattern: "^[a-z0-9]{2,8}$" + name: + "$id": "#/epiphany-cluster/properties/name" + title: "Cluster name" + description: "The name of the cluster" + examples: + - clustername01 + type: string + pattern: "^[a-z0-9\\-]{3,20}$" + admin_user: + "$id": "#/epiphany-cluster/properties/admin_user" + title: "The admin_user Schema" + description: "Settings needed for the SSH connection to the cluster machines or VM's" + type: object + cloud: + "$id": "#/epiphany-cluster/properties/cloud" + title: "Cloud Schema" + description: "Settings specific to cloud providers (AWS, Azure)" + type: object + components: + "$id": "#/epiphany-cluster/properties/components" + title: "Components schema" + description: "Cluster component layout specification" type: object \ No newline at end of file diff --git a/docs/design-docs/backup/backup_component.png b/docs/design-docs/backup/backup_component.png new file mode 100644 index 0000000000..33eb36f0cb --- /dev/null +++ b/docs/design-docs/backup/backup_component.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82183c16a801f0e7be0f2b4052d6c052639013c1ee0cddcbceb22987fa2dc8d7 +size 102655 diff --git a/docs/design-docs/backup/backups.md b/docs/design-docs/backup/backups.md new file mode 100644 index 0000000000..ad0c5bfdd0 --- /dev/null +++ b/docs/design-docs/backup/backups.md @@ -0,0 +1,148 @@ +# Epiphany Platform backup design document + +Affected version: 0.4.x + +## Goals + +Provide backup functionality for Epiphany Platform - cluster created using epicli tool. + +Backup will cover following areas: + +1. [Kubernetes cluster backup](#1.-Kubernetes-cluster-backup) + + 1.1 etcd database + + 1.2 kubeadm config + + 1.3 certificates + + 1.4 persistent volumes + + 1.5 applications deployed on the cluster + +2. [Kafka backup](#2.-Kafka-backup) + + 2.1 Kafka topic data + + 2.2 Kafka index + + 2.3 Zookeeper settings and data + +3. [Elastic stack backup](#3.-Elastic-stack-backup) + + 3.1 Elasticsearch data + + 3.2 Kibana settings + +4. [Monitoring backup](#4.-Monitoring-backup) + + 4.1 Prometheus data + + 4.2 Prometheus settings (properties, targets) + + 4.3 Alertmanager settings + + 4.4 Grafana settings (datasources, dashboards) + +5. [PostgreSQL backup](#5.-PostgreSQL-backup) + + 5.1 All databases from DB + +6. [RabbitMQ settings and user data](#6.-RabbitMQ-settings-and-user-data) + +7. [HAProxy settings backup](#7.-HAProxy-settings-backup) + +## Use cases + +User/background service/job is able to backup whole cluster or backup selected parts and store files in desired location. +There are few options possible to use for storing backup: +- S3 +- Azure file storage +- local file +- NFS + +Application/tool will create metadata file that will be definition of the backup - information that can be useful for restore tool. This metadata file will be stored within backup file. + +Backup is packed to zip/gz/tar.gz file that has timestamp in the name. If name collision occurred `name+'_1'` will be used. + +## Example use + +```bash +epibackup -b /path/to/build/dir -t /target/location/for/backup +``` + +Where `-b` is path to build folder that contains Ansible inventory and `-t` contains target path to store backup. + +## Backup Component View + +![Epiphany backup component](backup_component.png) + +User/background service/job executes `epibackup` (code name) application. Application takes parameters: +- `-b`: build directory of existing cluster. Most important is ansible inventory existing in this directory - so it can be assumed that this should be folder of Ansible inventory file. +- `-t`: target location of zip/tar.gz file that will contain backup files and metadata file. + +Tool when executed looks for the inventory file in `-b` location and executes backup playbooks. All playbooks are optional, in MVP version it can try to backup all components (it they exists in the inventory). After that, some components can be skipped (by providing additional flag, or parameter to cli). + +Tool also produces metadata file that describes backup with time, backed up components and their versions. + +## 1. Kubernetes cluster backup + +There are few ways of doing backups of existing Kuberntes cluster. Going to take into further research two approaches. + +**First**: Backup etcd database and kubeadm config of single master node. Instruction can be found [here](https://elastisys.com/2018/12/10/backup-kubernetes-how-and-why/). Simple solution for that will backup etcd which contains all workload definitions and settings. + +**Second**: Use 3rd party software to create a backup like [Heptio Velero](https://velero.io/docs/v1.1.0/support-matrix/) - Apache 2.0 license, [Velero GitHub](https://github.com/vmware-tanzu/velero) + +## 2. Kafka backup + +Possible options for backing up Kafka broker data and indexes: +1. Mirror using [Kafka Mirror Maker](https://kafka.apache.org/documentation/). It requires second Kafka cluster running independently that will replicate all data (including current offset and consumer groups). It is used mostly for multi-cloud replication. +2. Kafka-connect – use Kafka connect to get all topic and offset data from Kafka an save to it filesystem (NFS, local, S3, ...) called Sink connector. + + 2.1 [Confluent Kafka connector](https://github.com/confluentinc/kafka-connect-storage-common) – that use Confluent Kafka Community License Agreement + 2.2 Use another Open Source connector like [kafka-connect-s3](https://github.com/spredfast/kafka-connect-s3) (BSD) or [kafka-backup](https://github.com/itadventurer/kafka-backup) (Apache 2.0) + +3. File system copy: take Kafka broker and ZooKeeper data stored in files and copy it to backup location. It requires Kafka Broker to be stopped. Solution described in Digital Ocean [post](https://www.digitalocean.com/community/tutorials/how-to-back-up-import-and-migrate-your-apache-kafka-data-on-ubuntu-18-04). + +## 3. Elastic stack backup + +Use built-in features of Elasticsearch to create backup like: + +```REST +PUT /_snapshot/my_unverified_backup?verify=false +{ + "type": "fs", + "settings": { + "location": "my_unverified_backup_location" + } +} +``` + +More information can be found [here](https://www.elastic.co/guide/en/elasticsearch/reference/6.5/modules-snapshots.html). + +OpenDistro uses similar way of doing backups - it should be compatible. [OpenDistro backups link](https://opendistro.github.io/for-elasticsearch-docs/docs/elasticsearch/snapshot-restore/). + +## 4. Monitoring backup + +Prometheus from version 2.1 is able to create data snapshot by doing HTTP request: + +```bash +curl -XPOST http://localhost:9090/api/v1/admin/tsdb/snapshot +``` +Snapshot will be created in `/snapshots/SNAPSHOT-NAME-RETURNED-IN-RESPONSE` + +[More info](https://prometheus.io/docs/prometheus/2.1/querying/api/#snapshot) + +Files like targets and Prometheus/AlertManager settings should be also copied to backup location. + +## 5. PostgreSQL backup + +Relational DB backup mechanisms are the most mature ones. Simplest solution is to use [standard PostgreSQL backup funtions](https://www.postgresql.org/docs/10/backup.html). Valid option is also to use [pg_dump](https://www.postgresql.org/docs/current/app-pgdump.html). + +## 6. RabbitMQ settings and user data + +RabbitMQ has [standard way of creating backup](https://www.rabbitmq.com/backup.html). + +## 7. HAProxy settings backup + +Copy HAProxy configuration files to backup location. diff --git a/docs/design-docs/cache-storage/cache-storage.md b/docs/design-docs/cache-storage/cache-storage.md new file mode 100644 index 0000000000..2b60dde294 --- /dev/null +++ b/docs/design-docs/cache-storage/cache-storage.md @@ -0,0 +1,43 @@ +# Epiphany Platform cache storage design document + +Affected version: 0.4.x + +## Goals + +Provide in-memory cache storage that will be capable of store large amount of data with hight performance. + +## Use cases + +Platform should provide cache storage for key-value stores, latest value taken from queue (Kafka). + +## Architectural decision + +Considered options are: +- Apache Ignite +- Redis + +Description | Apache Ignite | Redis | +--- | ---| --- | +License | Apache 2.0 | three clause BSD license +Partition method | Sharding | Sharding +Replication | Yes | Master-slave - yes, Master - Master - only enterprise version +Transaction concept | ACID | Optimistic lock | +Data Grid | Yes | N/A | +In-memory DB | Distributed key-value store, in-memory distributed SQL database | key-value store +Integration with RDBMS | Can integrate with any relational DB that supports JDBC driver (Oracle, PostgreSQL, Microsoft SQL Server, and MySQL) | Possible using 3rd party software +Integration with Kafka | Using `Streamer` (Kafka Streamer, MQTT Streamer, ...) possible to insert to cache | Required 3rd party service +Machine learning | Apache Ignite Machine Learning - tools for building predictive ML models | N/A + +Based on above - Apache Ignite is not just scalable in-memory cache/database but cache and processing platform which can run transactional, analytical and streaming workloads. While Redis is simpler, Apache Ignite offers lot more features with Apache 2.0 licence. + +Choice: **Apache Ignite** + +## Design proposal + +[MVP] Add Ansible role to `epicli` that installs Apache Ignite and sets up cluster if there is more than one instance. Ansible playbook is also responsible for adding more nodes to existing cluster (scaling). + +Possible problems while implementing Ignite clustering: +- Ignite uses multicast for node discovery which is not supported on AWS. Ignite distribution comes with `TcpDiscoveryS3IpFinder` so S3-based discovery can be used. + +To consider: +- Deploy Apache Ignite cluster in Kubernetes diff --git a/docs/design-docs/offline-upgrade/epiphany-offline-upgrade.png b/docs/design-docs/offline-upgrade/epiphany-offline-upgrade.png new file mode 100644 index 0000000000..bee880cf01 --- /dev/null +++ b/docs/design-docs/offline-upgrade/epiphany-offline-upgrade.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56d8de6d1edb76730bca2aa47bb14708ab74a79c313a3370acf71dc29e6e0401 +size 83512 diff --git a/docs/design-docs/offline-upgrade/offline-upgrade.md b/docs/design-docs/offline-upgrade/offline-upgrade.md new file mode 100644 index 0000000000..03b4f64bb5 --- /dev/null +++ b/docs/design-docs/offline-upgrade/offline-upgrade.md @@ -0,0 +1,34 @@ +# Epiphany Platform offline upgrade design document + +Affected version: 0.4.x + +## Goals + +Provide upgrade functionality for Epiphany Platform so Kubernetes and other components can be upgraded when working offline. + +## Use cases + +Platform should be upgradeable when there is no internet connection. It requires all packages and dependencies to be downloaded on machine that has internet connection and then moved to air-gap server. + +## Example use + +```bash +epiupgrade -b /path/to/build/dir +``` + +Where `-b` is path to build folder that contains Ansible inventory. + +## Design proposal + +MVP for upgrade function will contain Kubernetes upgrade procedure to the latest supported version of Kubernetes. Later it will be extended to all other Epiphany Platform components. + +![Epiphany offline upgrade app](epiphany-offline-upgrade.png) + +`epiupgrade` application or module takes build path location (directory path that contains Ansible inventory file). + +First part of upgrade execution is to download/upload packages to repository so new packages will exist and be ready for upgrade process. +When repository module will finish its work then upgrade Ansible playbooks will be executed. + +Upgrade application/module shall implement following functions: +- [MVP] `apply` it will execute upgrade +- `--plan` where there will be no changes made to the cluster - it will return list of changes that will be made during upgrade execution.