diff --git a/core/src/epicli/cli/engine/ApplyEngine.py b/core/src/epicli/cli/engine/ApplyEngine.py index 8cab9a9bef..3e56cb48e2 100644 --- a/core/src/epicli/cli/engine/ApplyEngine.py +++ b/core/src/epicli/cli/engine/ApplyEngine.py @@ -5,7 +5,8 @@ from cli.helpers.Step import Step from cli.helpers.doc_list_helpers import select_single, select_all -from cli.helpers.build_saver import save_manifest, get_inventory_path +from cli.helpers.build_saver import save_manifest, get_inventory_path, get_manifest_path, get_build_path +from cli.helpers.data_loader import load_manifest_docs from cli.helpers.yaml_helpers import safe_load_all from cli.helpers.Log import Log from cli.helpers.os_images import get_os_distro_normalized @@ -17,6 +18,7 @@ from cli.engine.terraform.TerraformFileCopier import TerraformFileCopier from cli.engine.terraform.TerraformRunner import TerraformRunner from cli.engine.ansible.AnsibleRunner import AnsibleRunner +from cli.version import VERSION class ApplyEngine(Step): @@ -31,6 +33,7 @@ def __init__(self, input_data): self.input_docs = [] self.configuration_docs = [] self.infrastructure_docs = [] + self.manifest_docs = [] def __enter__(self): return self @@ -61,9 +64,12 @@ def process_input_docs(self): schema_validator.run() def process_infrastructure_docs(self): + # Load any posible existing manifest docs + self.load_manifest_docs() + # Build the infrastructure docs with provider_class_loader(self.cluster_model.provider, 'InfrastructureBuilder')( - self.input_docs) as infrastructure_builder: + self.input_docs, self.manifest_docs) as infrastructure_builder: self.infrastructure_docs = infrastructure_builder.run() # Validate infrastructure documents @@ -84,16 +90,10 @@ def collect_infrastructure_config(self): [*self.configuration_docs, *self.infrastructure_docs]) as config_collector: config_collector.run() - def validate(self): - self.process_input_docs() - - self.process_configuration_docs() - - self.process_infrastructure_docs() - - save_manifest([*self.input_docs, *self.configuration_docs, *self.infrastructure_docs], self.cluster_model.specification.name) - - return 0 + def load_manifest_docs(self): + path_to_manifest = get_manifest_path(self.cluster_model.specification.name) + if os.path.isfile(path_to_manifest): + self.manifest_docs = load_manifest_docs(get_build_path(self.cluster_model.specification.name)) def assert_no_master_downscale(self): components = self.cluster_model.specification.components diff --git a/core/src/epicli/cli/engine/providers/any/InfrastructureBuilder.py b/core/src/epicli/cli/engine/providers/any/InfrastructureBuilder.py index 1602157a7a..3894c8e7e8 100644 --- a/core/src/epicli/cli/engine/providers/any/InfrastructureBuilder.py +++ b/core/src/epicli/cli/engine/providers/any/InfrastructureBuilder.py @@ -3,10 +3,11 @@ from cli.version import VERSION class InfrastructureBuilder(Step): - def __init__(self, docs): + def __init__(self, docs, manifest_docs=[]): super().__init__(__name__) self.cluster_model = select_single(docs, lambda x: x.kind == 'epiphany-cluster') self.docs = docs + self.manifest_docs = manifest_docs def run(self): infrastructure_docs = select_all(self.docs, lambda x: x.kind.startswith('infrastructure/')) diff --git a/core/src/epicli/cli/engine/providers/aws/InfrastructureBuilder.py b/core/src/epicli/cli/engine/providers/aws/InfrastructureBuilder.py index cfee23af68..7978ac5d82 100644 --- a/core/src/epicli/cli/engine/providers/aws/InfrastructureBuilder.py +++ b/core/src/epicli/cli/engine/providers/aws/InfrastructureBuilder.py @@ -13,16 +13,18 @@ from cli.helpers.naming_helpers import resource_name from cli.helpers.objdict_helpers import objdict_to_dict, dict_to_objdict from cli.version import VERSION +from cli.helpers.query_yes_no import query_yes_no class InfrastructureBuilder(Step): - def __init__(self, docs): + def __init__(self, docs, manifest_docs=[]): super().__init__(__name__) self.cluster_model = select_single(docs, lambda x: x.kind == 'epiphany-cluster') self.cluster_name = self.cluster_model.specification.name.lower() self.cluster_prefix = self.cluster_model.specification.prefix.lower() self.use_network_security_groups = self.cluster_model.specification.cloud.network.use_network_security_groups self.docs = docs + self.manifest_docs = manifest_docs def run(self): infrastructure = [] @@ -136,7 +138,7 @@ def get_efs_config(self): return efs_config def get_autoscaling_group(self, component_key, component_value, subnets_to_create, index): - autoscaling_group = dict_to_objdict(deepcopy(self.get_virtual_machine(component_value, self.cluster_model, self.docs))) + autoscaling_group = dict_to_objdict(deepcopy(self.get_virtual_machine(component_value))) autoscaling_group.specification.cluster_name = self.cluster_name autoscaling_group.specification.name = resource_name(self.cluster_prefix, self.cluster_name, 'asg' + '-' + str(index), component_key) autoscaling_group.specification.count = component_value.count @@ -245,6 +247,45 @@ def add_security_rules_inbound_efs(self, infrastructure, security_group): rules.append(objdict_to_dict(rule)) security_group.specification.rules = rules + def get_virtual_machine(self, component_value): + machine_selector = component_value.machine + model_with_defaults = select_first(self.docs, lambda x: x.kind == 'infrastructure/virtual-machine' and + x.name == machine_selector) + + # Merge with defaults + if model_with_defaults is None: + model_with_defaults = merge_with_defaults(self.cluster_model.provider, 'infrastructure/virtual-machine', + machine_selector, self.docs) + + # Check if we have a cluster-config OS image defined that we want to apply cluster wide. + cloud_os_image_defaults = self.get_config_or_default(self.docs, 'infrastructure/cloud-os-image-defaults') + cloud_image = self.cluster_model.specification.cloud.default_os_image + if cloud_image != 'default': + if not hasattr(cloud_os_image_defaults.specification, cloud_image): + raise NotImplementedError(f'default_os_image "{cloud_image}" is unsupported for "{self.cluster_model.provider}" provider.') + model_with_defaults.specification.os_full_name = cloud_os_image_defaults.specification[cloud_image] + + # finally check if we are trying to re-apply a configuration. + if self.manifest_docs: + manifest_vm_config = select_first(self.manifest_docs, lambda x: x.name == machine_selector and x.kind == 'infrastructure/virtual-machine') + manifest_firstvm_config = select_first(self.manifest_docs, lambda x: x.kind == 'infrastructure/virtual-machine') + + if manifest_vm_config is not None and model_with_defaults.specification.os_full_name == manifest_vm_config.specification.os_full_name: + return model_with_defaults + + if model_with_defaults.specification.os_full_name == manifest_firstvm_config.specification.os_full_name: + return model_with_defaults + + self.logger.warning(f"Re-applying a different OS image might lead to data loss and/or other issues. Preserving the existing OS image used for VM definition '{machine_selector}'.") + + if manifest_vm_config is not None: + model_with_defaults.specification.os_full_name = manifest_vm_config.specification.os_full_name + else: + model_with_defaults.specification.os_full_name = manifest_firstvm_config.specification.os_full_name + + return model_with_defaults + + @staticmethod def efs_add_mount_target_config(efs_config, subnet): target = select_first(efs_config.specification.mount_targets, @@ -275,17 +316,6 @@ def get_config_or_default(docs, kind): config['version'] = VERSION return config - @staticmethod - def get_virtual_machine(component_value, cluster_model, docs): - machine_selector = component_value.machine - model_with_defaults = select_first(docs, lambda x: x.kind == 'infrastructure/virtual-machine' and - x.name == machine_selector) - if model_with_defaults is None: - model_with_defaults = merge_with_defaults(cluster_model.provider, 'infrastructure/virtual-machine', - machine_selector, docs) - - return model_with_defaults - @staticmethod def rule_exists_in_list(rule_list, rule_to_check): for rule in rule_list: diff --git a/core/src/epicli/cli/engine/providers/azure/InfrastructureBuilder.py b/core/src/epicli/cli/engine/providers/azure/InfrastructureBuilder.py index 8cb2069ef8..6e84e30df5 100644 --- a/core/src/epicli/cli/engine/providers/azure/InfrastructureBuilder.py +++ b/core/src/epicli/cli/engine/providers/azure/InfrastructureBuilder.py @@ -10,9 +10,10 @@ from cli.helpers.objdict_helpers import objdict_to_dict, dict_to_objdict from cli.helpers.os_images import get_os_distro_normalized from cli.version import VERSION +from cli.helpers.query_yes_no import query_yes_no class InfrastructureBuilder(Step): - def __init__(self, docs): + def __init__(self, docs, manifest_docs=[]): super().__init__(__name__) self.cluster_model = select_single(docs, lambda x: x.kind == 'epiphany-cluster') self.cluster_name = self.cluster_model.specification.name.lower() @@ -22,6 +23,7 @@ def __init__(self, docs): self.use_network_security_groups = self.cluster_model.specification.cloud.network.use_network_security_groups self.use_public_ips = self.cluster_model.specification.cloud.use_public_ips self.docs = docs + self.manifest_docs = manifest_docs def run(self): infrastructure = [] @@ -44,7 +46,7 @@ def run(self): # The vm config also contains some other stuff we use for network and security config. # So get it here and pass it allong. - vm_config = self.get_virtual_machine(component_value, self.cluster_model, self.docs) + vm_config = self.get_virtual_machine(component_value) # Set property that controls cloud-init. vm_config.specification['use_cloud_init_custom_data'] = cloud_init_custom_data.specification.enabled @@ -221,6 +223,44 @@ def get_cloud_init_custom_data(self): cloud_init_custom_data.specification.file_name = 'cloud-config.yml' return cloud_init_custom_data + def get_virtual_machine(self, component_value): + machine_selector = component_value.machine + model_with_defaults = select_first(self.docs, lambda x: x.kind == 'infrastructure/virtual-machine' and + x.name == machine_selector) + + # Merge with defaults + if model_with_defaults is None: + model_with_defaults = merge_with_defaults(self.cluster_model.provider, 'infrastructure/virtual-machine', + machine_selector, self.docs) + + # Check if we have a cluster-config OS image defined that we want to apply cluster wide. + cloud_os_image_defaults = self.get_config_or_default(self.docs, 'infrastructure/cloud-os-image-defaults') + cloud_image = self.cluster_model.specification.cloud.default_os_image + if cloud_image != 'default': + if not hasattr(cloud_os_image_defaults.specification, cloud_image): + raise NotImplementedError(f'default_os_image "{cloud_image}" is unsupported for "{self.cluster_model.provider}" provider.') + model_with_defaults.specification.storage_image_reference = dict_to_objdict(deepcopy(cloud_os_image_defaults.specification[cloud_image])) + + # finally check if we are trying to re-apply a configuration. + if self.manifest_docs: + manifest_vm_config = select_first(self.manifest_docs, lambda x: x.name == machine_selector and x.kind == 'infrastructure/virtual-machine') + manifest_firstvm_config = select_first(self.manifest_docs, lambda x: x.kind == 'infrastructure/virtual-machine') + + if manifest_vm_config is not None and model_with_defaults.specification.storage_image_reference == manifest_vm_config.specification.storage_image_reference: + return model_with_defaults + + if model_with_defaults.specification.storage_image_reference == manifest_firstvm_config.specification.storage_image_reference: + return model_with_defaults + + self.logger.warning(f"Re-applying a different OS image might lead to data loss and/or other issues. Preserving the existing OS image used for VM definition '{machine_selector}'.") + + if manifest_vm_config is not None: + model_with_defaults.specification.storage_image_reference = dict_to_objdict(deepcopy(manifest_vm_config.specification.storage_image_reference)) + else: + model_with_defaults.specification.storage_image_reference = dict_to_objdict(deepcopy(manifest_firstvm_config.specification.storage_image_reference)) + + return model_with_defaults + @staticmethod def get_config_or_default(docs, kind): config = select_first(docs, lambda x: x.kind == kind) @@ -228,14 +268,3 @@ def get_config_or_default(docs, kind): config = load_yaml_obj(types.DEFAULT, 'azure', kind) config['version'] = VERSION return config - - @staticmethod - def get_virtual_machine(component_value, cluster_model, docs): - machine_selector = component_value.machine - model_with_defaults = select_first(docs, lambda x: x.kind == 'infrastructure/virtual-machine' and - x.name == machine_selector) - if model_with_defaults is None: - model_with_defaults = merge_with_defaults(cluster_model.provider, 'infrastructure/virtual-machine', - machine_selector, docs) - - return model_with_defaults diff --git a/core/src/epicli/cli/epicli.py b/core/src/epicli/cli/epicli.py index 8abdbe641e..b298ccbf7b 100644 --- a/core/src/epicli/cli/epicli.py +++ b/core/src/epicli/cli/epicli.py @@ -93,9 +93,6 @@ def debug_level(x): upgrade_parser(subparsers) delete_parser(subparsers) test_parser(subparsers) - ''' - validate_parser(subparsers) - ''' backup_parser(subparsers) recovery_parser(subparsers) @@ -318,23 +315,6 @@ def run_test(args): sub_parser.set_defaults(func=run_test) -''' -def validate_parser(subparsers): - sub_parser = subparsers.add_parser('verify', description='Validates the configuration from file by executing a dry ' - 'run without changing the physical ' - 'infrastructure/configuration') - sub_parser.add_argument('-f', '--file', dest='file', type=str, - help='File with infrastructure/configuration definitions to use.') - - def run_validate(args): - adjust_paths_from_file(args) - with ApplyEngine(args) as engine: - return engine.validate() - - sub_parser.set_defaults(func=run_validate) -''' - - def backup_parser(subparsers): """Configure and execute backup of cluster components.""" diff --git a/core/src/epicli/cli/helpers/build_saver.py b/core/src/epicli/cli/helpers/build_saver.py index 3f61587135..e8874a4148 100644 --- a/core/src/epicli/cli/helpers/build_saver.py +++ b/core/src/epicli/cli/helpers/build_saver.py @@ -85,6 +85,10 @@ def get_inventory_path(cluster_name): return os.path.join(get_build_path(cluster_name), INVENTORY_FILE_NAME) +def get_manifest_path(cluster_name): + return os.path.join(get_build_path(cluster_name), MANIFEST_FILE_NAME) + + def get_inventory_path_for_build(build_directory): build_version = check_build_output_version(build_directory) inventory = os.path.join(build_directory, INVENTORY_FILE_NAME) diff --git a/core/src/epicli/data/aws/defaults/configuration/minimal-cluster-config.yml b/core/src/epicli/data/aws/defaults/configuration/minimal-cluster-config.yml index 9005e443ab..049c019715 100644 --- a/core/src/epicli/data/aws/defaults/configuration/minimal-cluster-config.yml +++ b/core/src/epicli/data/aws/defaults/configuration/minimal-cluster-config.yml @@ -14,6 +14,7 @@ specification: credentials: key: XXXX-XXXX-XXXX secret: XXXXXXXXXXXXXXXX + default_os_image: default components: repository: count: 1 diff --git a/core/src/epicli/data/aws/defaults/infrastructure/cloud-os-image-defaults.yml b/core/src/epicli/data/aws/defaults/infrastructure/cloud-os-image-defaults.yml new file mode 100644 index 0000000000..7d85f23776 --- /dev/null +++ b/core/src/epicli/data/aws/defaults/infrastructure/cloud-os-image-defaults.yml @@ -0,0 +1,9 @@ +kind: infrastructure/cloud-os-image-defaults +title: "Cloud OS Image Defaults" +name: default +specification: + ubuntu-18.04-x86_64: ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20210323 + redhat-7-x86_64: RHEL-7.9_HVM-20210208-x86_64-0-Hourly2-GP2 + centos-7-x86_64: CentOS 7.9.2009 x86_64 + centos-7-arm64: CentOS 7.9.2009 aarch64 + diff --git a/core/src/epicli/data/azure/defaults/configuration/minimal-cluster-config.yml b/core/src/epicli/data/azure/defaults/configuration/minimal-cluster-config.yml index 41af4a2155..050e2bf5b8 100644 --- a/core/src/epicli/data/azure/defaults/configuration/minimal-cluster-config.yml +++ b/core/src/epicli/data/azure/defaults/configuration/minimal-cluster-config.yml @@ -11,6 +11,7 @@ specification: cloud: k8s_as_cloud_service: False use_public_ips: False # When not using public IPs you have to provide connectivity via private IPs (VPN) + default_os_image: default components: repository: count: 1 diff --git a/core/src/epicli/data/azure/defaults/infrastructure/cloud-os-image-defaults.yml b/core/src/epicli/data/azure/defaults/infrastructure/cloud-os-image-defaults.yml new file mode 100644 index 0000000000..432f1affca --- /dev/null +++ b/core/src/epicli/data/azure/defaults/infrastructure/cloud-os-image-defaults.yml @@ -0,0 +1,19 @@ +kind: infrastructure/cloud-os-image-defaults +title: "Cloud OS Image Defaults" +name: default +specification: + ubuntu-18.04-x86_64: + publisher: Canonical + offer: UbuntuServer + sku: 18.04-LTS + version: "18.04.202103151" + redhat-7-x86_64: + publisher: RedHat + offer: RHEL + sku: 7-LVM + version: "7.9.2020111202" + centos-7-x86_64: + publisher: OpenLogic + offer: CentOS + sku: "7_9" + version: "7.9.2021020400" diff --git a/core/src/epicli/data/common/defaults/epiphany-cluster.yml b/core/src/epicli/data/common/defaults/epiphany-cluster.yml index 27ed2a581c..2f7ba91bd4 100644 --- a/core/src/epicli/data/common/defaults/epiphany-cluster.yml +++ b/core/src/epicli/data/common/defaults/epiphany-cluster.yml @@ -20,6 +20,7 @@ specification: secret: DADFAFHCJHCAUYEAk network: use_network_security_groups: True + default_os_image: default components: kubernetes_master: count: 1 diff --git a/core/src/epicli/data/common/validation/epiphany-cluster.yml b/core/src/epicli/data/common/validation/epiphany-cluster.yml index 914516dee2..0451c69c4f 100644 --- a/core/src/epicli/data/common/validation/epiphany-cluster.yml +++ b/core/src/epicli/data/common/validation/epiphany-cluster.yml @@ -104,6 +104,17 @@ properties: default: false examples: - true + default_os_image: + type: string + title: Set the latest cloud OS image verified for use by the Epiphany team for this Epiphany version. + default: 'default' + examples: + - default + - ubuntu-18.04-x86_64 + - redhat-7-x86_64 + - centos-7-x86_64 + - centos-7-arm64 + pattern: ^(default|ubuntu-18.04-x86_64|redhat-7-x86_64|centos-7-x86_64|centos-7-arm64)$ components: "$id": "#/properties/components" type: object diff --git a/core/src/epicli/tests/engine/providers/azure/test_AzureConfigBuilder.py b/core/src/epicli/tests/engine/providers/azure/test_AzureConfigBuilder.py index 3b307956bb..e24a15acf1 100644 --- a/core/src/epicli/tests/engine/providers/azure/test_AzureConfigBuilder.py +++ b/core/src/epicli/tests/engine/providers/azure/test_AzureConfigBuilder.py @@ -68,7 +68,7 @@ def test_get_public_ip_should_set_proper_values_to_model(): component_value = dict_to_objdict({ 'machine': 'kubernetes-master-machine' }) - vm_config = builder.get_virtual_machine(component_value, cluster_model, []) + vm_config = builder.get_virtual_machine(component_value) actual = builder.get_public_ip('kubernetes_master', component_value, vm_config, 1) @@ -84,7 +84,7 @@ def test_get_network_interface_should_set_proper_values_to_model(): component_value = dict_to_objdict({ 'machine': 'kubernetes-master-machine' }) - vm_config = builder.get_virtual_machine(component_value, cluster_model, []) + vm_config = builder.get_virtual_machine(component_value) actual = builder.get_network_interface( 'kubernetes_master', @@ -119,6 +119,7 @@ def get_cluster_model(address_pool='10.22.0.0/22', cluster_name='EpiphanyTestClu cluster_model = dict_to_objdict({ 'kind': 'epiphany-cluster', 'provider': 'azure', + 'name': 'default', 'specification': { 'name': cluster_name, 'prefix': 'prefix', @@ -126,6 +127,7 @@ def get_cluster_model(address_pool='10.22.0.0/22', cluster_name='EpiphanyTestClu 'region': 'West Europe', 'vnet_address_pool': address_pool, 'use_public_ips': True, + 'default_os_image': 'default', 'network': { 'use_network_security_groups': True } diff --git a/docs/home/howto/CLUSTER.md b/docs/home/howto/CLUSTER.md index 9c1baa468d..4aa3e4f8a1 100644 --- a/docs/home/howto/CLUSTER.md +++ b/docs/home/howto/CLUSTER.md @@ -427,6 +427,7 @@ To setup the cluster do the following steps from the provisioning machine: key: aws_key secret: aws_secret use_public_ips: false + default_os_image: default ``` The [region](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.RegionsAndAvailabilityZones.html) lets you chose the most optimal place to deploy your cluster. The `key` and `secret` are needed by Terraform and can be generated in the AWS console. More information about that [here](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) @@ -439,6 +440,7 @@ To setup the cluster do the following steps from the provisioning machine: subscription_name: Subscribtion_name use_service_principal: false use_public_ips: false + default_os_image: default ``` The [region](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.RegionsAndAvailabilityZones.html) lets you chose the most optimal place to deploy your cluster. The `subscription_name` is the Azure subscribtion under which you want to deploy the cluster. @@ -464,7 +466,16 @@ To setup the cluster do the following steps from the provisioning machine: Epicli will read this file and automaticly use it for authentication for resource creation and management. - For both `aws`and `azure` there is a `use_public_ips` tag. When this is true the VM's will also have a direct inferface to the internet. While this is easy for setting up a cluster for testing it should not be used in production. A VPN setup should be used which we will document in a different section (TODO). + For both `aws`and `azure` the following cloud attributes overlap: + - `use_public_ips`: When `true`, the VMs will also have a direct interface to the internet. While this is easy for setting up a cluster for testing, it should not be used in production. A VPN setup should be used which we will document in a different section (TODO). + - `default_os_image`: Lets you more easily select Epiphany team validated and tested OS images. When one is selected, it will be applied to **every** `infrastructure/virtual-machine` document in the cluster regardless of user defined ones. + The following values are accepted: + - `default`: Applies user defined `infrastructure/virtual-machine` documents when generating a new configuration. + - `ubuntu-18.04-x86_64`: Applies the latest validated and tested Ubuntu 18.04 image to all `infrastructure/virtual-machine` documents on `x86_64` on Azure and AWS. + - `redhat-7-x86_64`: Applies the latest validated and tested RedHat 7.x image to all `infrastructure/virtual-machine` documents on `x86_64` on Azure and AWS. + - `centos-7-x86_64`: Applies the latest validated and tested CentOS 7.x image to all `infrastructure/virtual-machine` documents on `x86_64` on Azure and AWS. + - `centos-7-arm64`: Applies the latest validated and tested CentOS 7.x image to all `infrastructure/virtual-machine` documents on `arm64` on AWS. Azure currently doesn't support `arm64`. + The images which will be used for these values will be updated and tested on regular basis. 4. Define the components you want to install: diff --git a/docs/home/howto/UPGRADE.md b/docs/home/howto/UPGRADE.md index a6fa9833d4..c6da3134e1 100644 --- a/docs/home/howto/UPGRADE.md +++ b/docs/home/howto/UPGRADE.md @@ -124,6 +124,7 @@ The `epicli upgrade` command has additional flags: ### Run *apply* after *upgrade* Currently Epiphany does not fully support apply after upgrade. There is a possibility to re-apply configuration from newer version of Epicli but this needs some manual work from Administrator. Re-apply on already upgraded cluster needs to be called with `--no-infra` option to skip Terraform part of configuration. +If `apply` after `upgrade` is run with `--no-infra`, the used system images from the older Epiphany version are preserved to prevent the destruction of the VMs. If you plan modify any infrastructure unit (eg. add Kubernetes Node) you need to create machine by yourself and attach it into configuration yaml. While running `epicli apply...` on already upgraded cluster you should use config yamls generated in newer version of Epiphany and apply changes you had in older one. If the cluster is upgraded to version 0.8 or newer you need also add additional feature mapping for repository role as shown on example below: