From 4e4b0e2daabe6842476f83604eeb2d7b9d6f5b50 Mon Sep 17 00:00:00 2001 From: Anatoli Tsikhamirau Date: Fri, 14 Jan 2022 15:50:14 +0100 Subject: [PATCH 1/2] Remove Ignite (#2867) Co-authored-by: atsikham --- .vscode/launch.json | 2 +- ansible/playbooks/ignite.yml | 12 -- .../applications/ignite-stateless/main.yml | 5 - .../ignite-stateless/00-namespace.yml.j2 | 5 - .../templates/ignite-stateless/01-rbac.yml.j2 | 37 ---- .../ignite-stateless/02-configmap.yml.j2 | 32 ---- .../ignite-stateless/03-service.yml.j2 | 29 --- .../ignite-stateless/04-deployment.yml.j2 | 50 ------ .../playbooks/roles/ignite/defaults/main.yml | 36 ---- .../playbooks/roles/ignite/handlers/main.yml | 15 -- ansible/playbooks/roles/ignite/meta/main.yml | 3 - .../roles/ignite/tasks/configure.yml | 84 --------- .../roles/ignite/tasks/install-ignite.yml | 87 --------- .../roles/ignite/tasks/install-java.yml | 33 ---- ansible/playbooks/roles/ignite/tasks/main.yml | 6 - .../ignite/templates/default-config.xml.j2 | 1 - .../roles/ignite/templates/ignite.service.j2 | 18 -- .../ignite/templates/ip-discovery.xml.j2 | 5 - .../roles/preflight/defaults/main.yml | 2 - .../centos-7/requirements.aarch64.txt | 2 - .../centos-7/requirements.x86_64.txt | 2 - .../redhat-7/requirements.x86_64.txt | 2 - .../ubuntu-20.04/requirements.x86_64.txt | 2 - .../repository/tasks/clean-up-epirepo.yml | 1 - .../playbooks/roles/upgrade/tasks/ignite.yml | 48 ----- ansible/playbooks/upgrade.yml | 9 - cli/engine/ansible/AnsibleVarsGenerator.py | 2 +- cli/epicli.py | 1 - .../current-c4/c2-arch-1-execution.puml | 1 - .../c2-arch-2-human-interactions.puml | 1 - .../c2-arch-3-runtime-dependencies.puml | 1 - .../c2-arch-4-metrics-and-logs.puml | 1 - docs/changelogs/CHANGELOG-2.0.md | 1 + docs/design-docs/arm/centos-arm-analysis.md | 2 - docs/design-docs/arm/redhat-arm-analysis.md | 1 - docs/design-docs/arm/ubuntu-arm-analysis.md | 2 - .../cache-storage/cache-storage.md | 43 ----- .../infrastructure/naming-convention.md | 2 +- docs/home/ARM.md | 52 ------ docs/home/COMPONENTS.md | 1 - docs/home/HOWTO.md | 2 - docs/home/SECURITY.md | 19 +- docs/home/howto/CLUSTER.md | 4 - docs/home/howto/DATABASES.md | 124 ------------- docs/home/howto/KUBERNETES.md | 1 - docs/home/howto/MODULES.md | 13 -- docs/home/howto/SECURITY_GROUPS.md | 2 - docs/home/howto/UPGRADE.md | 2 - schema/any/defaults/epiphany-cluster.yml | 4 - schema/aws/defaults/epiphany-cluster.yml | 7 - .../infrastructure/virtual-machine.yml | 17 -- schema/azure/defaults/epiphany-cluster.yml | 6 - .../infrastructure/virtual-machine.yml | 20 --- .../defaults/configuration/applications.yml | 16 -- .../configuration/feature-mapping.yml | 7 - .../defaults/configuration/firewall.yml | 9 - .../common/defaults/configuration/ignite.yml | 96 ---------- .../defaults/configuration/image-registry.yml | 4 - .../configuration/feature-mapping.yml | 4 - .../validation/configuration/firewall.yml | 9 - .../validation/configuration/ignite.yml | 11 -- .../spec/applications/applications_spec.rb | 12 +- .../ignite-stateless/ignite-stateless.rb | 122 ------------- tests/spec/spec/ignite/ignite_spec.rb | 165 ------------------ 64 files changed, 8 insertions(+), 1307 deletions(-) delete mode 100644 ansible/playbooks/ignite.yml delete mode 100644 ansible/playbooks/roles/applications/tasks/applications/ignite-stateless/main.yml delete mode 100644 ansible/playbooks/roles/applications/templates/ignite-stateless/00-namespace.yml.j2 delete mode 100644 ansible/playbooks/roles/applications/templates/ignite-stateless/01-rbac.yml.j2 delete mode 100644 ansible/playbooks/roles/applications/templates/ignite-stateless/02-configmap.yml.j2 delete mode 100644 ansible/playbooks/roles/applications/templates/ignite-stateless/03-service.yml.j2 delete mode 100644 ansible/playbooks/roles/applications/templates/ignite-stateless/04-deployment.yml.j2 delete mode 100644 ansible/playbooks/roles/ignite/defaults/main.yml delete mode 100644 ansible/playbooks/roles/ignite/handlers/main.yml delete mode 100644 ansible/playbooks/roles/ignite/meta/main.yml delete mode 100644 ansible/playbooks/roles/ignite/tasks/configure.yml delete mode 100644 ansible/playbooks/roles/ignite/tasks/install-ignite.yml delete mode 100644 ansible/playbooks/roles/ignite/tasks/install-java.yml delete mode 100644 ansible/playbooks/roles/ignite/tasks/main.yml delete mode 100644 ansible/playbooks/roles/ignite/templates/default-config.xml.j2 delete mode 100644 ansible/playbooks/roles/ignite/templates/ignite.service.j2 delete mode 100644 ansible/playbooks/roles/ignite/templates/ip-discovery.xml.j2 delete mode 100644 ansible/playbooks/roles/upgrade/tasks/ignite.yml delete mode 100644 docs/design-docs/cache-storage/cache-storage.md delete mode 100644 schema/common/defaults/configuration/ignite.yml delete mode 100644 schema/common/validation/configuration/ignite.yml delete mode 100644 tests/spec/spec/applications/ignite-stateless/ignite-stateless.rb delete mode 100644 tests/spec/spec/ignite/ignite_spec.rb diff --git a/.vscode/launch.json b/.vscode/launch.json index 36742f567b..604e3df3f3 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -24,7 +24,7 @@ // "args": ["test", "-b", "${workspaceFolder}/clusters/build/"] // "args": ["test", "-b", "${workspaceFolder}/clusters/build/", "-g", ""] // "args": ["upgrade", "-b", "${workspaceFolder}/clusters/build/"] - // "args": ["upgrade", "-b", "${workspaceFolder}/clusters/build/","--upgrade-components","kafka,ignite"] + // "args": ["upgrade", "-b", "${workspaceFolder}/clusters/build/","--upgrade-components","kafka"] } ] } diff --git a/ansible/playbooks/ignite.yml b/ansible/playbooks/ignite.yml deleted file mode 100644 index 837c13ba07..0000000000 --- a/ansible/playbooks/ignite.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -# Ansible playbook for installing Apache Ignite - -- hosts: repository - gather_facts: true - tasks: [] - -- hosts: ignite - become: true - become_method: sudo - roles: - - ignite diff --git a/ansible/playbooks/roles/applications/tasks/applications/ignite-stateless/main.yml b/ansible/playbooks/roles/applications/tasks/applications/ignite-stateless/main.yml deleted file mode 100644 index 0357b900b3..0000000000 --- a/ansible/playbooks/roles/applications/tasks/applications/ignite-stateless/main.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -- name: Deploy Ignite - include_tasks: deploy-k8s-app.yml - vars: - app_dir_name: "{{ data.name }}" diff --git a/ansible/playbooks/roles/applications/templates/ignite-stateless/00-namespace.yml.j2 b/ansible/playbooks/roles/applications/templates/ignite-stateless/00-namespace.yml.j2 deleted file mode 100644 index e83d849586..0000000000 --- a/ansible/playbooks/roles/applications/templates/ignite-stateless/00-namespace.yml.j2 +++ /dev/null @@ -1,5 +0,0 @@ ---- -apiVersion: v1 -kind: Namespace -metadata: - name: {{ data.namespace }} diff --git a/ansible/playbooks/roles/applications/templates/ignite-stateless/01-rbac.yml.j2 b/ansible/playbooks/roles/applications/templates/ignite-stateless/01-rbac.yml.j2 deleted file mode 100644 index 6c08204c00..0000000000 --- a/ansible/playbooks/roles/applications/templates/ignite-stateless/01-rbac.yml.j2 +++ /dev/null @@ -1,37 +0,0 @@ ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: ignite - namespace: {{ data.namespace }} - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: ignite - namespace: {{ data.namespace }} -rules: -- apiGroups: - - "" - resources: # Here are resources you can access - - pods - - endpoints - verbs: # That is what you can do with them - - get - - list - - watch - ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: ignite -roleRef: - kind: ClusterRole - name: ignite - apiGroup: rbac.authorization.k8s.io -subjects: -- kind: ServiceAccount - name: ignite - namespace: {{ data.namespace }} diff --git a/ansible/playbooks/roles/applications/templates/ignite-stateless/02-configmap.yml.j2 b/ansible/playbooks/roles/applications/templates/ignite-stateless/02-configmap.yml.j2 deleted file mode 100644 index 53d8b2ff64..0000000000 --- a/ansible/playbooks/roles/applications/templates/ignite-stateless/02-configmap.yml.j2 +++ /dev/null @@ -1,32 +0,0 @@ ---- -apiVersion: v1 -kind: ConfigMap -metadata: - labels: - app: ignite - name: ignite - namespace: {{ data.namespace }} -data: - ignite-config.xml: | - - - - - - - - - - - - - - - diff --git a/ansible/playbooks/roles/applications/templates/ignite-stateless/03-service.yml.j2 b/ansible/playbooks/roles/applications/templates/ignite-stateless/03-service.yml.j2 deleted file mode 100644 index d96785b0c2..0000000000 --- a/ansible/playbooks/roles/applications/templates/ignite-stateless/03-service.yml.j2 +++ /dev/null @@ -1,29 +0,0 @@ ---- -apiVersion: v1 -kind: Service -metadata: - # The name must be equal to TcpDiscoveryKubernetesIpFinder.serviceName - name: ignite - # The name must be equal to TcpDiscoveryKubernetesIpFinder.namespaceName - namespace: {{ data.namespace }} -spec: - type: LoadBalancer - ports: - - name: rest - port: 8080 - targetPort: 8080 - nodePort: {{ data.service.rest_nodeport }} - - name: sql - port: 10800 - targetPort: 10800 - nodePort: {{ data.service.sql_nodeport }} - - name: thinclients - port: 10900 - targetPort: 10900 - nodePort: {{ data.service.thinclients_nodeport }} - # Optional - remove 'sessionAffinity' property if the Ignite cluster - # and applications deployed within Kubernetes - sessionAffinity: ClientIP - selector: - # Must be equal to the label set for Ignite pods. - app: ignite diff --git a/ansible/playbooks/roles/applications/templates/ignite-stateless/04-deployment.yml.j2 b/ansible/playbooks/roles/applications/templates/ignite-stateless/04-deployment.yml.j2 deleted file mode 100644 index 478cfeac6e..0000000000 --- a/ansible/playbooks/roles/applications/templates/ignite-stateless/04-deployment.yml.j2 +++ /dev/null @@ -1,50 +0,0 @@ -#jinja2: trim_blocks:False ---- -# An example of a Kubernetes configuration for Ignite pods deployment. -apiVersion: apps/v1 -kind: Deployment -metadata: - # Custom Ignite cluster's name. - name: ignite-cluster - namespace: {{ data.namespace }} -spec: - selector: - matchLabels: - app: ignite - # A number of Ignite pods to be started by Kubernetes initially. - replicas: {{ data.replicas }} - template: - metadata: - labels: - app: ignite - spec: - serviceAccountName: ignite - containers: - # Custom Ignite pod name. - - name: ignite-node -{% if use_epiphany_image_registry %} - image: {{ image_registry_address }}/{{ data.image_path }} -{% else %} - image: {{ data.image_path }} -{% endif %} - env: - - name: OPTION_LIBS - value: {% for plugin in data.enabled_plugins %}{{ plugin }}{{ "," if not loop.last else "" }}{% endfor %} - - name: CONFIG_URI - value: /ignite-config/ignite-config.xml - ports: - # Ports to open. - # Might be optional depending on your Kubernetes environment. - - containerPort: 11211 # REST port number. - - containerPort: 47100 # communication SPI port number. - - containerPort: 47500 # discovery SPI port number. - - containerPort: 49112 # JMX port number. - - containerPort: 10800 # SQL port number. - - containerPort: 10900 # Thin clients port number. - volumeMounts: - - name: ignite-config - mountPath: /ignite-config - volumes: - - configMap: - name: ignite - name: ignite-config diff --git a/ansible/playbooks/roles/ignite/defaults/main.yml b/ansible/playbooks/roles/ignite/defaults/main.yml deleted file mode 100644 index 91720b8fe8..0000000000 --- a/ansible/playbooks/roles/ignite/defaults/main.yml +++ /dev/null @@ -1,36 +0,0 @@ ---- -ignite_version: 2.9.1 -ignite_file_name: apache-ignite-{{ ignite_version }}-bin.zip - -ignite_ports_from_ephemeral_range: # strings are required - - '47100-47109' - - '47500-47509' - - '49112' - -# The following block is appended to $IGNITE_HOME/bin/include/functions.sh -block_to_append_to_ignite_functions_script: | - # - # The function exports JMX_MON variable with Java JMX options. - # Overrides original version in order to: - # 1) Use fixed ports for JMX (https://github.com/epiphany-platform/epiphany/issues/1181) - # 2) Disable direct remote access to unsecured JMX (remote access possible through SSH tunnel) - # - findAvailableJmxPort() { - export IGNITE_JMX_PORT=49112 - - JMX_PORT=`"$JAVA" -cp "${IGNITE_LIBS}" org.apache.ignite.internal.util.portscanner.GridJmxPortFinder` - - # - # This variable defines parameters for JMX monitoring and management. - # - if [ -n "$JMX_PORT" ]; then - # java.rmi.server.hostname=127.0.0.1 is used to make JMX accessible through SSH tunnel - JMX_MON="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=${JMX_PORT} \ - -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false \ - -Dcom.sun.management.jmxremote.rmi.port=${JMX_PORT} -Djava.rmi.server.hostname=127.0.0.1" - else - # If JMX port wasn't found do not initialize JMX. - echo "$0, WARN: Failed to resolve JMX host (JMX will be disabled): $HOSTNAME" - JMX_MON="" - fi - } diff --git a/ansible/playbooks/roles/ignite/handlers/main.yml b/ansible/playbooks/roles/ignite/handlers/main.yml deleted file mode 100644 index c7d21eddbf..0000000000 --- a/ansible/playbooks/roles/ignite/handlers/main.yml +++ /dev/null @@ -1,15 +0,0 @@ ---- -- name: Reload SystemD daemon - systemd: - daemon_reload: true - -- name: Start Ignite service - systemd: - name: ignite - state: started - enabled: true - -- name: Restart Ignite service - systemd: - name: ignite - state: restarted diff --git a/ansible/playbooks/roles/ignite/meta/main.yml b/ansible/playbooks/roles/ignite/meta/main.yml deleted file mode 100644 index 745ba4d956..0000000000 --- a/ansible/playbooks/roles/ignite/meta/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -dependencies: - - role: preflight_facts diff --git a/ansible/playbooks/roles/ignite/tasks/configure.yml b/ansible/playbooks/roles/ignite/tasks/configure.yml deleted file mode 100644 index 2e79c0895f..0000000000 --- a/ansible/playbooks/roles/ignite/tasks/configure.yml +++ /dev/null @@ -1,84 +0,0 @@ ---- -- name: Get effective value of net.ipv4.ip_local_reserved_ports - slurp: - path: /proc/sys/net/ipv4/ip_local_reserved_ports - register: slurp_ip_local_reserved_ports - -- name: Reserve ports from ephemeral range - when: _computed_ports != _reserved_ports - sysctl: - name: net.ipv4.ip_local_reserved_ports - state: present - sysctl_set: true - reload: true - value: "{{ _computed_ports | join(_separator) }}" - vars: - _computed_ports: >- - {{ (_reserved_ports + ignite_ports_from_ephemeral_range) | select - | unique - | sort }} - _reserved_ports: >- - {{ (slurp_ip_local_reserved_ports.content | b64decode | trim).split(_separator) | select - | unique - | sort }} - _separator: "," - -- name: Append block to /opt/ignite/bin/include/functions.sh - blockinfile: - path: /opt/ignite/bin/include/functions.sh - marker: "# {mark} ANSIBLE MANAGED BLOCK" - insertafter: EOF - backup: true - block: "{{ block_to_append_to_ignite_functions_script }}" - owner: ignite - group: ignite - mode: ug=rwx,o= - -- name: Prepare Ignite configuration - set_fact: - ignite_configuration: >- - {{ specification.config | replace('IP_LIST_PLACEHOLDER', _ip_discovery_list) }} - vars: - _ip_discovery_list: >- - {{ lookup('template', 'ip-discovery.xml.j2') }} - -- name: Ensure /var/lib/ignite/ directory exists - file: - path: /var/lib/ignite/ - state: directory - owner: ignite - group: ignite - mode: ug=rwx,o= - -- name: Create systemd service - template: - dest: /etc/systemd/system/ignite.service - src: ignite.service.j2 - owner: ignite - group: ignite - mode: ug=rw,o= - notify: - - Reload SystemD daemon - - Start Ignite service - -- name: Create config file - template: - dest: /opt/ignite/config/default-config.xml - src: default-config.xml.j2 - owner: ignite - group: ignite - mode: ug=rw,o= - notify: - - Restart Ignite service - -- name: Copy Ignite enabled plugins - copy: - dest: /opt/ignite/libs - src: /opt/ignite/libs/optional/{{ item }} - remote_src: true - owner: ignite - group: ignite - mode: preserve - loop: "{{ specification.enabled_plugins | default([]) }}" - notify: - - Restart Ignite service diff --git a/ansible/playbooks/roles/ignite/tasks/install-ignite.yml b/ansible/playbooks/roles/ignite/tasks/install-ignite.yml deleted file mode 100644 index 2e059c1e99..0000000000 --- a/ansible/playbooks/roles/ignite/tasks/install-ignite.yml +++ /dev/null @@ -1,87 +0,0 @@ ---- -- name: Add ignite group - group: - name: ignite - state: present - -- name: Add ignite user - user: - name: ignite - group: ignite - state: present - -- name: Check if Ignite in current version exists - stat: - path: /opt/ignite # this should be a symlink - get_attributes: false - get_checksum: false - get_mime: false - follow: false - register: stat_ignite_dir - -- name: Install Ignite - when: stat_ignite_dir.stat.lnk_source is undefined - or stat_ignite_dir.stat.lnk_source != _ignite_dest_path - vars: - _ignite_dest_path: /opt/ignite_{{ ignite_version }} - block: - - name: Download Ignite archive - include_role: - name: download - tasks_from: download_file - vars: - file_name: "{{ ignite_file_name }}" - - - name: Get file list in the archive - unarchive: - dest: /tmp/ - src: "{{ download_directory }}/{{ ignite_file_name }}" - remote_src: true - list_files: true - register: unarchive_list_files - changed_when: false - check_mode: true # check mode works only for zip files (to be extracted via unzip) - - - name: Decompress Ignite - vars: - _archive_root_dir: >- - {{ (unarchive_list_files.files | sort | first).rstrip('/') }} - always: - - &cleanup - name: Delete temporary destination - file: - dest: /tmp/{{ _archive_root_dir }} - state: absent - block: - - name: Create destination directory - file: - dest: "{{ _ignite_dest_path }}" - state: directory - owner: ignite - group: ignite - mode: ug=rwx,o= - - - *cleanup - - # This is here to aid unarchive module in achieving idempotence. - - name: Create temporary destination - file: - dest: /tmp/{{ _archive_root_dir }} - src: "{{ _ignite_dest_path }}" - state: link - - # Files are extracted via symlink to destination directory without copy/move operation. - - name: Decompress {{ ignite_file_name }} - unarchive: - dest: /tmp/ - src: "{{ download_directory }}/{{ ignite_file_name }}" - remote_src: true - owner: ignite - group: ignite - creates: "{{ _ignite_dest_path }}/bin/ignite.sh" - - - name: Link /opt/ignite to {{ _ignite_dest_path }}/ - file: - dest: /opt/ignite - src: "{{ _ignite_dest_path }}" - state: link diff --git a/ansible/playbooks/roles/ignite/tasks/install-java.yml b/ansible/playbooks/roles/ignite/tasks/install-java.yml deleted file mode 100644 index 0a98d19f1d..0000000000 --- a/ansible/playbooks/roles/ignite/tasks/install-java.yml +++ /dev/null @@ -1,33 +0,0 @@ ---- -- name: Install Java package - package: - name: "{{ _packages[ansible_os_family] }}" - state: present - vars: - _packages: - Debian: - - openjdk-8-jre-headless - RedHat: - - java-1.8.0-openjdk-headless - -- name: Find default java binary - command: which java - register: command_which_java - changed_when: false - -- name: Get absolute path of the default java binary - stat: - path: "{{ command_which_java.stdout_lines[0] }}" - get_attributes: false - get_checksum: false - get_mime: false - follow: false - register: stat_java - -- name: Compute ignite_java_home fact - set_fact: - ignite_java_home: >- - {{ stat_java.stat.islnk | default(false) - | ternary(stat_java.stat.lnk_source, stat_java.stat.path) - | dirname - | dirname }} diff --git a/ansible/playbooks/roles/ignite/tasks/main.yml b/ansible/playbooks/roles/ignite/tasks/main.yml deleted file mode 100644 index dc5fed5f0f..0000000000 --- a/ansible/playbooks/roles/ignite/tasks/main.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -- include_tasks: install-java.yml - -- include_tasks: install-ignite.yml - -- include_tasks: configure.yml diff --git a/ansible/playbooks/roles/ignite/templates/default-config.xml.j2 b/ansible/playbooks/roles/ignite/templates/default-config.xml.j2 deleted file mode 100644 index 22db209f5f..0000000000 --- a/ansible/playbooks/roles/ignite/templates/default-config.xml.j2 +++ /dev/null @@ -1 +0,0 @@ -{{ ignite_configuration }} diff --git a/ansible/playbooks/roles/ignite/templates/ignite.service.j2 b/ansible/playbooks/roles/ignite/templates/ignite.service.j2 deleted file mode 100644 index f0ad60874e..0000000000 --- a/ansible/playbooks/roles/ignite/templates/ignite.service.j2 +++ /dev/null @@ -1,18 +0,0 @@ -# {{ ansible_managed }} -[Unit] -Description=Apache Ignite Server -After=network.target - -[Service] -Type=simple -User=ignite -Group=ignite -DynamicUser=false -StateDirectory=ignite -WorkingDirectory=/opt/ignite -Environment="JAVA_HOME={{ ignite_java_home }}" -ExecStart=/opt/ignite/bin/ignite.sh -Restart=on-failure - -[Install] -WantedBy=multi-user.target diff --git a/ansible/playbooks/roles/ignite/templates/ip-discovery.xml.j2 b/ansible/playbooks/roles/ignite/templates/ip-discovery.xml.j2 deleted file mode 100644 index 3fe99bb04c..0000000000 --- a/ansible/playbooks/roles/ignite/templates/ip-discovery.xml.j2 +++ /dev/null @@ -1,5 +0,0 @@ - - {% for host in groups.ignite | default({}) %} - {{ hostvars[host].ansible_default_ipv4.address }} - {% endfor %} - diff --git a/ansible/playbooks/roles/preflight/defaults/main.yml b/ansible/playbooks/roles/preflight/defaults/main.yml index a4a151c3a3..129df20b77 100644 --- a/ansible/playbooks/roles/preflight/defaults/main.yml +++ b/ansible/playbooks/roles/preflight/defaults/main.yml @@ -53,7 +53,6 @@ unsupported_roles: - postgresql - postgres_exporter - applications - - ignite - distro: CentOS arch: x86_64 roles: [] # all supported @@ -92,7 +91,6 @@ unsupported_roles: - postgresql - postgres_exporter - applications - - ignite unsupported_postgres_extensions: x86_64: [] diff --git a/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.aarch64.txt b/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.aarch64.txt index ff0c03ebfa..2b9692dbcf 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.aarch64.txt +++ b/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.aarch64.txt @@ -163,7 +163,6 @@ https://archive.apache.org/dist/kafka/2.6.0/kafka_2.12-2.6.0.tgz https://github.com/prometheus/prometheus/releases/download/v2.31.1/prometheus-2.31.1.linux-arm64.tar.gz https://github.com/prometheus/alertmanager/releases/download/v0.23.0/alertmanager-0.23.0.linux-arm64.tar.gz https://archive.apache.org/dist/zookeeper/zookeeper-3.5.8/apache-zookeeper-3.5.8-bin.tar.gz -https://archive.apache.org/dist/ignite/2.9.1/apache-ignite-2.9.1-bin.zip https://get.helm.sh/helm-v3.2.0-linux-arm64.tar.gz https://archive.apache.org/dist/logging/log4j/2.17.1/apache-log4j-2.17.1-bin.tar.gz # --- Helm charts --- @@ -203,7 +202,6 @@ kubernetesui/dashboard:v2.3.1 kubernetesui/metrics-scraper:v1.0.7 registry:2 # applications -epiphanyplatform/ignite:2.9.1 epiphanyplatform/keycloak:14.0.0 rabbitmq:3.8.9 # K8s diff --git a/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.x86_64.txt b/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.x86_64.txt index 8e1cd804da..ec19ff95d2 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.x86_64.txt +++ b/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.x86_64.txt @@ -164,7 +164,6 @@ https://archive.apache.org/dist/kafka/2.6.0/kafka_2.12-2.6.0.tgz https://github.com/prometheus/prometheus/releases/download/v2.31.1/prometheus-2.31.1.linux-amd64.tar.gz https://github.com/prometheus/alertmanager/releases/download/v0.23.0/alertmanager-0.23.0.linux-amd64.tar.gz https://archive.apache.org/dist/zookeeper/zookeeper-3.5.8/apache-zookeeper-3.5.8-bin.tar.gz -https://archive.apache.org/dist/ignite/2.9.1/apache-ignite-2.9.1-bin.zip https://get.helm.sh/helm-v3.2.0-linux-amd64.tar.gz https://archive.apache.org/dist/logging/log4j/2.17.1/apache-log4j-2.17.1-bin.tar.gz # --- Helm charts --- @@ -204,7 +203,6 @@ kubernetesui/dashboard:v2.3.1 kubernetesui/metrics-scraper:v1.0.7 registry:2 # applications -epiphanyplatform/ignite:2.9.1 bitnami/pgpool:4.2.4 bitnami/pgbouncer:1.16.0 # istio diff --git a/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.x86_64.txt b/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.x86_64.txt index d3e5b5fae1..930cb9b87b 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.x86_64.txt +++ b/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.x86_64.txt @@ -162,7 +162,6 @@ https://archive.apache.org/dist/kafka/2.6.0/kafka_2.12-2.6.0.tgz https://github.com/prometheus/prometheus/releases/download/v2.31.1/prometheus-2.31.1.linux-amd64.tar.gz https://github.com/prometheus/alertmanager/releases/download/v0.23.0/alertmanager-0.23.0.linux-amd64.tar.gz https://archive.apache.org/dist/zookeeper/zookeeper-3.5.8/apache-zookeeper-3.5.8-bin.tar.gz -https://archive.apache.org/dist/ignite/2.9.1/apache-ignite-2.9.1-bin.zip https://get.helm.sh/helm-v3.2.0-linux-amd64.tar.gz https://archive.apache.org/dist/logging/log4j/2.17.1/apache-log4j-2.17.1-bin.tar.gz # --- Helm charts --- @@ -202,7 +201,6 @@ kubernetesui/dashboard:v2.3.1 kubernetesui/metrics-scraper:v1.0.7 registry:2 # applications -epiphanyplatform/ignite:2.9.1 bitnami/pgpool:4.2.4 bitnami/pgbouncer:1.16.0 # istio diff --git a/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-20.04/requirements.x86_64.txt b/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-20.04/requirements.x86_64.txt index 574d59a7d8..12b1da7a6f 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-20.04/requirements.x86_64.txt +++ b/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-20.04/requirements.x86_64.txt @@ -199,7 +199,6 @@ https://archive.apache.org/dist/kafka/2.6.0/kafka_2.12-2.6.0.tgz https://archive.apache.org/dist/zookeeper/zookeeper-3.5.8/apache-zookeeper-3.5.8-bin.tar.gz https://github.com/prometheus/alertmanager/releases/download/v0.23.0/alertmanager-0.23.0.linux-amd64.tar.gz https://github.com/prometheus/prometheus/releases/download/v2.31.1/prometheus-2.31.1.linux-amd64.tar.gz -https://archive.apache.org/dist/ignite/2.9.1/apache-ignite-2.9.1-bin.zip https://get.helm.sh/helm-v3.2.0-linux-amd64.tar.gz https://archive.apache.org/dist/logging/log4j/2.17.1/apache-log4j-2.17.1-bin.tar.gz # --- Helm charts --- @@ -239,7 +238,6 @@ kubernetesui/dashboard:v2.3.1 kubernetesui/metrics-scraper:v1.0.7 registry:2 # applications -epiphanyplatform/ignite:2.9.1 bitnami/pgpool:4.2.4 bitnami/pgbouncer:1.16.0 # istio diff --git a/ansible/playbooks/roles/repository/tasks/clean-up-epirepo.yml b/ansible/playbooks/roles/repository/tasks/clean-up-epirepo.yml index 0d536bc218..f294a00d33 100644 --- a/ansible/playbooks/roles/repository/tasks/clean-up-epirepo.yml +++ b/ansible/playbooks/roles/repository/tasks/clean-up-epirepo.yml @@ -74,7 +74,6 @@ paths: /tmp patterns: - keycloak*.tar - - ignite*.tar - etcd*.tar - node*.tar - dashboard*.tar diff --git a/ansible/playbooks/roles/upgrade/tasks/ignite.yml b/ansible/playbooks/roles/upgrade/tasks/ignite.yml deleted file mode 100644 index af1acee05c..0000000000 --- a/ansible/playbooks/roles/upgrade/tasks/ignite.yml +++ /dev/null @@ -1,48 +0,0 @@ ---- -- name: Ignite | Populate service facts - service_facts: - when: ansible_facts.service is undefined - no_log: true - -- name: Ignite | Stat /opt/ignite - stat: - path: /opt/ignite - get_attributes: false - get_checksum: false - get_mime: false - follow: false - register: stat_opt_ignite - -- name: Ignite | Compare versions and upgrade - when: - - ansible_facts.services["ignite.service"] is defined - - stat_opt_ignite.stat.exists - - stat_opt_ignite.stat.islnk | default(false) - vars: - _ignite_defaults: - ignite_version: 2.9.1 - _ignite_version: >- - {{ stat_opt_ignite.stat.lnk_source.split('_') | last }} - block: - - name: Ignite | Print versions - debug: - msg: - - "Installed version: {{ _ignite_version }}" - - "Target version: {{ _ignite_defaults.ignite_version }}" - - # NOTE: non-commercial Ignite does not support rolling upgrades (downtime is always expected). - - name: Ignite | Stop and upgrade - when: - - _ignite_defaults.ignite_version is version(_ignite_version, '>') - block: - - name: Ignite | Stop service - systemd: - name: ignite - state: stopped - - - name: Ignite | Upgrade - import_role: - name: ignite - vars_from: manifest - - - meta: flush_handlers diff --git a/ansible/playbooks/upgrade.yml b/ansible/playbooks/upgrade.yml index 8a5171a288..1b8fb40e20 100644 --- a/ansible/playbooks/upgrade.yml +++ b/ansible/playbooks/upgrade.yml @@ -267,15 +267,6 @@ vars: { lock_file: /var/tmp/rabbitmq-upgrade-in-progress.flag } when: "'rabbitmq' in upgrade_components or upgrade_components|length == 0" -- hosts: ignite - become: true - become_method: sudo - tasks: - - import_role: - name: upgrade - tasks_from: ignite - when: "'ignite' in upgrade_components or upgrade_components|length == 0" - - hosts: haproxy become: true become_method: sudo diff --git a/cli/engine/ansible/AnsibleVarsGenerator.py b/cli/engine/ansible/AnsibleVarsGenerator.py index 20d937442a..f24ef9d39a 100644 --- a/cli/engine/ansible/AnsibleVarsGenerator.py +++ b/cli/engine/ansible/AnsibleVarsGenerator.py @@ -76,7 +76,7 @@ def generate(self): # In special cases (like haproxy), where user specifies majority of the config, it's easier (and less # awkward) to re-render config templates instead of modifying (for example with regular expressions) # no-longer-compatible config files. - roles_with_manifest = ['filebeat', 'ignite', 'postgresql', 'repository'] + roles_with_manifest = ['filebeat', 'postgresql', 'repository'] else: roles_with_defaults = self.inventory_creator.get_enabled_roles() roles_with_manifest = [] # applies only to upgrades diff --git a/cli/epicli.py b/cli/epicli.py index 42192131f6..60537b6c0b 100644 --- a/cli/epicli.py +++ b/cli/epicli.py @@ -252,7 +252,6 @@ def upgrade_parser(subparsers): 'elasticsearch_curator', 'filebeat', 'grafana', - 'ignite', 'image_registry', 'jmx_exporter', 'kafka', diff --git a/docs/architecture/current-c4/c2-arch-1-execution.puml b/docs/architecture/current-c4/c2-arch-1-execution.puml index de742a943e..d14d27453c 100644 --- a/docs/architecture/current-c4/c2-arch-1-execution.puml +++ b/docs/architecture/current-c4/c2-arch-1-execution.puml @@ -14,7 +14,6 @@ Container_Boundary(epiphany, "Epiphany managed environment"){ Container(rdb, "Relational Database", "PostrgreSQL") Container(lb, "Load Balancer", "HAProxy") Container(messages, "Message Broker", "RabbitMQ") - Container(memdb, "In-Memory Database", "Apache Ignite") Container(repository, "Repository", "Apache") Container(auth, "Authentication and Authorization", "Keycloak") diff --git a/docs/architecture/current-c4/c2-arch-2-human-interactions.puml b/docs/architecture/current-c4/c2-arch-2-human-interactions.puml index b687ebb422..3754b12b04 100644 --- a/docs/architecture/current-c4/c2-arch-2-human-interactions.puml +++ b/docs/architecture/current-c4/c2-arch-2-human-interactions.puml @@ -19,7 +19,6 @@ Container_Boundary(epiphany, "Epiphany managed environment"){ Container(rdb, "Relational Database", "PostrgreSQL") Container(lb, "Load Balancer", "HAProxy") Container(messages, "Message Broker", "RabbitMQ") - Container(memdb, "In-Memory Database", "Apache Ignite") Container(repository, "Repository", "Apache") Container(auth, "Authentication and Authorization", "Keycloak") diff --git a/docs/architecture/current-c4/c2-arch-3-runtime-dependencies.puml b/docs/architecture/current-c4/c2-arch-3-runtime-dependencies.puml index d5991a6800..ae1df0bbff 100644 --- a/docs/architecture/current-c4/c2-arch-3-runtime-dependencies.puml +++ b/docs/architecture/current-c4/c2-arch-3-runtime-dependencies.puml @@ -16,7 +16,6 @@ Container_Boundary(epiphany, "Epiphany managed environment"){ Container(rdb, "Relational Database", "PostrgreSQL") Container(lb, "Load Balancer", "HAProxy") Container(messages, "Message Broker", "RabbitMQ") - Container(memdb, "In-Memory Database", "Apache Ignite") Container(repository, "Repository", "Apache") Container(auth, "Authentication and Authorization", "Keycloak") diff --git a/docs/architecture/current-c4/c2-arch-4-metrics-and-logs.puml b/docs/architecture/current-c4/c2-arch-4-metrics-and-logs.puml index 15b2d1de9e..e4801be369 100644 --- a/docs/architecture/current-c4/c2-arch-4-metrics-and-logs.puml +++ b/docs/architecture/current-c4/c2-arch-4-metrics-and-logs.puml @@ -14,7 +14,6 @@ Container_Boundary(epiphany, "Epiphany managed environment"){ Container(rdb, "Relational Database", "PostrgreSQL") Container(lb, "Load Balancer", "HAProxy") Container(messages, "Message Broker", "RabbitMQ") - Container(memdb, "In-Memory Database", "Apache Ignite") Container(repository, "Repository", "Apache") Container(auth, "Authentication and Authorization", "Keycloak") diff --git a/docs/changelogs/CHANGELOG-2.0.md b/docs/changelogs/CHANGELOG-2.0.md index 16ffdc8e11..ef1e32dd23 100644 --- a/docs/changelogs/CHANGELOG-2.0.md +++ b/docs/changelogs/CHANGELOG-2.0.md @@ -11,6 +11,7 @@ ### Removed - [#2834](https://github.com/epiphany-platform/epiphany/issues/2834) - Removal of Hashicorp Vault component +- [#2837](https://github.com/epiphany-platform/epiphany/issues/2837) - Removal of Apache Ignite component ### Deprecated diff --git a/docs/design-docs/arm/centos-arm-analysis.md b/docs/design-docs/arm/centos-arm-analysis.md index 2fca781b86..b7211606dc 100644 --- a/docs/design-docs/arm/centos-arm-analysis.md +++ b/docs/design-docs/arm/centos-arm-analysis.md @@ -128,7 +128,6 @@ | https://github.com/prometheus/prometheus/releases/download/v2.10.0/prometheus-2.10.0.linux-arm64.tar.gz | + | dedicated package | + | | https://github.com/prometheus/alertmanager/releases/download/v0.17.0/alertmanager-0.17.0.linux-arm64.tar.gz | + | dedicated package | + | | https://archive.apache.org/dist/zookeeper/zookeeper-3.5.8/apache-zookeeper-3.5.8-bin.tar.gz | + | shell scripts + jar libraries | --- | -| https://archive.apache.org/dist/ignite/2.9.1/apache-ignite-2.9.1-bin.zip | + | shell scripts + jar libraries | --- | | https://releases.hashicorp.com/vault/1.7.0/vault_1.7.0_linux_arm64.zip | + | dedicated package | --- | | https://get.helm.sh/helm-v3.2.0-linux-arm64.tar.gz | + | dedicated package | --- | | https://github.com/hashicorp/vault-helm/archive/v0.9.0.tar.gz | + | yaml files | --- | @@ -148,7 +147,6 @@ | registry:2 | + | | | | hashicorp/vault-k8s:0.7.0 | --- | https://hub.docker.com/r/moikot/vault-k8s / custom build | --- | | vault:1.7.0 | + | | --- | -| apacheignite/ignite:2.9.1 | --- | https://github.com/apache/ignite/tree/master/docker/apache-ignite / custom build | --- | | bitnami/pgpool:4.1.1-debian-10-r29 | --- | | --- | | brainsam/pgbouncer:1.12 | --- | | --- | | istio/pilot:1.8.1 | --- | https://github.com/istio/istio/issues/21094 / custom build | --- | diff --git a/docs/design-docs/arm/redhat-arm-analysis.md b/docs/design-docs/arm/redhat-arm-analysis.md index 552082f663..17c13a5f1f 100644 --- a/docs/design-docs/arm/redhat-arm-analysis.md +++ b/docs/design-docs/arm/redhat-arm-analysis.md @@ -132,7 +132,6 @@ | https://github.com/prometheus/prometheus/releases/download/v2.10.0/prometheus-2.10.0.linux-arm64.tar.gz | + | dedicated package | + | | https://github.com/prometheus/alertmanager/releases/download/v0.17.0/alertmanager-0.17.0.linux-arm64.tar.gz | + | dedicated package | + | | https://archive.apache.org/dist/zookeeper/zookeeper-3.5.8/apache-zookeeper-3.5.8-bin.tar.gz | + | shell scripts + jar libraries | --- | -| https://archive.apache.org/dist/ignite/2.9.1/apache-ignite-2.9.1-bin.zip | + | shell scripts + jar libraries | --- | | https://releases.hashicorp.com/vault/1.7.0/vault_1.7.0_linux_arm64.zip | + | dedicated package | --- | | https://get.helm.sh/helm-v3.2.0-linux-arm64.tar.gz | + | dedicated package | --- | | https://github.com/hashicorp/vault-helm/archive/v0.9.0.tar.gz | + | yaml files | --- | diff --git a/docs/design-docs/arm/ubuntu-arm-analysis.md b/docs/design-docs/arm/ubuntu-arm-analysis.md index 07895439b1..0a5c635ead 100644 --- a/docs/design-docs/arm/ubuntu-arm-analysis.md +++ b/docs/design-docs/arm/ubuntu-arm-analysis.md @@ -156,7 +156,6 @@ | https://github.com/prometheus/prometheus/releases/download/v2.10.0/prometheus-2.10.0.linux-arm64.tar.gz | + | dedicated package | + | | https://github.com/prometheus/alertmanager/releases/download/v0.17.0/alertmanager-0.17.0.linux-arm64.tar.gz | + | dedicated package | + | | https://archive.apache.org/dist/zookeeper/zookeeper-3.5.8/apache-zookeeper-3.5.8-bin.tar.gz | + | shell scripts + jar libraries | --- | -| https://archive.apache.org/dist/ignite/2.9.1/apache-ignite-2.9.1-bin.zip | + | shell scripts + jar libraries | --- | | https://releases.hashicorp.com/vault/1.7.0/vault_1.7.0_linux_arm64.zip | + | dedicated package | --- | | https://get.helm.sh/helm-v3.2.0-linux-arm64.tar.gz | + | dedicated package | --- | | https://github.com/hashicorp/vault-helm/archive/v0.9.0.tar.gz | + | yaml files | --- | @@ -176,7 +175,6 @@ | registry:2 | + | | | | hashicorp/vault-k8s:0.7.0 | --- | https://hub.docker.com/r/moikot/vault-k8s / custom build | --- | | vault:1.7.0 | + | | --- | -| apacheignite/ignite:2.9.1 | --- | https://github.com/apache/ignite/tree/master/docker/apache-ignite / custom build | --- | | bitnami/pgpool:4.1.1-debian-10-r29 | --- | | --- | | brainsam/pgbouncer:1.12 | --- | | --- | | istio/pilot:1.8.1 | --- | https://github.com/istio/istio/issues/21094 / custom build | --- | diff --git a/docs/design-docs/cache-storage/cache-storage.md b/docs/design-docs/cache-storage/cache-storage.md deleted file mode 100644 index 2b60dde294..0000000000 --- a/docs/design-docs/cache-storage/cache-storage.md +++ /dev/null @@ -1,43 +0,0 @@ -# Epiphany Platform cache storage design document - -Affected version: 0.4.x - -## Goals - -Provide in-memory cache storage that will be capable of store large amount of data with hight performance. - -## Use cases - -Platform should provide cache storage for key-value stores, latest value taken from queue (Kafka). - -## Architectural decision - -Considered options are: -- Apache Ignite -- Redis - -Description | Apache Ignite | Redis | ---- | ---| --- | -License | Apache 2.0 | three clause BSD license -Partition method | Sharding | Sharding -Replication | Yes | Master-slave - yes, Master - Master - only enterprise version -Transaction concept | ACID | Optimistic lock | -Data Grid | Yes | N/A | -In-memory DB | Distributed key-value store, in-memory distributed SQL database | key-value store -Integration with RDBMS | Can integrate with any relational DB that supports JDBC driver (Oracle, PostgreSQL, Microsoft SQL Server, and MySQL) | Possible using 3rd party software -Integration with Kafka | Using `Streamer` (Kafka Streamer, MQTT Streamer, ...) possible to insert to cache | Required 3rd party service -Machine learning | Apache Ignite Machine Learning - tools for building predictive ML models | N/A - -Based on above - Apache Ignite is not just scalable in-memory cache/database but cache and processing platform which can run transactional, analytical and streaming workloads. While Redis is simpler, Apache Ignite offers lot more features with Apache 2.0 licence. - -Choice: **Apache Ignite** - -## Design proposal - -[MVP] Add Ansible role to `epicli` that installs Apache Ignite and sets up cluster if there is more than one instance. Ansible playbook is also responsible for adding more nodes to existing cluster (scaling). - -Possible problems while implementing Ignite clustering: -- Ignite uses multicast for node discovery which is not supported on AWS. Ignite distribution comes with `TcpDiscoveryS3IpFinder` so S3-based discovery can be used. - -To consider: -- Deploy Apache Ignite cluster in Kubernetes diff --git a/docs/design-docs/infrastructure/naming-convention.md b/docs/design-docs/infrastructure/naming-convention.md index 407152acb0..dc8c713a35 100644 --- a/docs/design-docs/infrastructure/naming-convention.md +++ b/docs/design-docs/infrastructure/naming-convention.md @@ -20,7 +20,7 @@ with a dash `-` as a delimiter. | Type | Required | Description | Examples | | ---- | -------- | ----------- | -------- | | Owner | yes | The name of the person or team which resource belongs to | epiphany | -| Application or service name | no | Name of the application, workload, or service that the resource is a part of | kafka, ignite, opendistro | +| Application or service name | no | Name of the application, workload, or service that the resource is a part of | kafka, opendistro | | Environment | no | The stage of the development lifecycle for the workload that the resource supports | prod, dev, qa | | VM group | no | The name of VM group that resource is created for | group-0 | diff --git a/docs/home/ARM.md b/docs/home/ARM.md index 59fda6a113..6c26cba273 100644 --- a/docs/home/ARM.md +++ b/docs/home/ARM.md @@ -29,7 +29,6 @@ Besides making sure that the selected providers, operating systems, components a | monitoring | :heavy_check_mark: | :x: | :x: | | load_balancer | :heavy_check_mark: | :x: | :x: | | postgresql | :heavy_check_mark: | :x: | :x: | -| ignite | :heavy_check_mark: | :x: | :x: | | opendistro_for_elasticsearch | :heavy_check_mark: | :x: | :x: | | single_machine | :heavy_check_mark: | :x: | :x: | @@ -43,7 +42,6 @@ Besides making sure that the selected providers, operating systems, components a | Application | Supported | | - | - | -| ignite-stateless | :heavy_check_mark: | | rabbitmq | :heavy_check_mark: | | auth-service | :heavy_check_mark: | | pgpool | :x: | @@ -95,9 +93,6 @@ specification: rabbitmq: count: 2 machine: rabbitmq-machine-arm - ignite: - count: 2 - machine: ignite-machine-arm opendistro_for_elasticsearch: count: 1 machine: opendistro-machine-arm @@ -170,14 +165,6 @@ specification: ip: x.x.x.x --- kind: infrastructure/virtual-machine -name: ignite-machine-arm -provider: any -based_on: ignite-machine -specification: - hostname: hostname - ip: x.x.x.x ---- -kind: infrastructure/virtual-machine name: opendistro-machine-arm provider: any based_on: logging-machine @@ -263,19 +250,6 @@ specification: #cluster: #is_clustered: true #redundant in in-Kubernetes installation, it will always be clustered #cookie: "cookieSetFromDataYaml" #optional - default value will be random generated string - - name: ignite-stateless - enabled: yes - image_path: "epiphanyplatform/ignite:2.9.1" # it will be part of the image path: {{local_repository}}/{{image_path}} - use_local_image_registry: true - namespace: ignite - service: - rest_nodeport: 32300 - sql_nodeport: 32301 - thinclients_nodeport: 32302 - replicas: 2 - enabled_plugins: - - ignite-kubernetes # required to work on K8s - - ignite-rest-http ``` ### ```AWS``` provider @@ -354,12 +328,6 @@ specification: subnets: - availability_zone: eu-west-1a address_pool: 10.1.8.0/24 - ignite: - count: 2 - machine: ignite-machine-arm - subnets: - - availability_zone: eu-west-1a - address_pool: 10.1.9.0/24 opendistro_for_elasticsearch: count: 1 machine: opendistro-machine-arm @@ -437,13 +405,6 @@ specification: size: a1.medium --- kind: infrastructure/virtual-machine -name: ignite-machine-arm -provider: aws -based_on: ignite-machine -specification: - size: a1.large ---- -kind: infrastructure/virtual-machine name: opendistro-machine-arm provider: aws based_on: logging-machine @@ -531,19 +492,6 @@ specification: #cluster: #is_clustered: true #redundant in in-Kubernetes installation, it will always be clustered #cookie: "cookieSetFromDataYaml" #optional - default value will be random generated string - - name: ignite-stateless - enabled: yes - image_path: "epiphanyplatform/ignite:2.9.1" # it will be part of the image path: {{local_repository}}/{{image_path}} - use_local_image_registry: true - namespace: ignite - service: - rest_nodeport: 32300 - sql_nodeport: 32301 - thinclients_nodeport: 32302 - replicas: 2 - enabled_plugins: - - ignite-kubernetes # required to work on K8s - - ignite-rest-http ``` ### ```Azure``` provider diff --git a/docs/home/COMPONENTS.md b/docs/home/COMPONENTS.md index fd1cb811a4..68520484cc 100644 --- a/docs/home/COMPONENTS.md +++ b/docs/home/COMPONENTS.md @@ -38,7 +38,6 @@ Note that versions are default versions and can be changed in certain cases thro | repmgr | 5.2.1 | https://github.com/EnterpriseDB/repmgr | [GNU General Public License 3.0](https://github.com/EnterpriseDB/repmgr/blob/master/LICENSE) | | Pgpool | 4.2.4 | https://www.pgpool.net/ | [License](https://www.pgpool.net/mediawiki/index.php/pgpool-II_License) | | Alertmanager | 0.23.0 | https://github.com/prometheus/alertmanager | [Apache License 2.0](https://github.com/prometheus/alertmanager/blob/master/LICENSE) | -| Apache Ignite | 2.9.1 | https://github.com/apache/ignite | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | | Apache2 | 2.4.29 | https://httpd.apache.org/ | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | | Istio | 1.8.1 | https://github.com/istio/istio | [Apache License 2.0](https://github.com/istio/istio/blob/master/LICENSE) | diff --git a/docs/home/HOWTO.md b/docs/home/HOWTO.md index 66bac44a5a..81bb4659bd 100644 --- a/docs/home/HOWTO.md +++ b/docs/home/HOWTO.md @@ -92,8 +92,6 @@ - [How to start working with OpenDistro for Elasticsearch](./howto/DATABASES.md#how-to-start-working-with-opendistro-for-elasticsearch) - [How to manage Opendistro for Elasticsearch data](./howto/LOGGING.md#how-to-manage-opendistro-for-elasticsearch-data) - [How to export Elasticsearch data to csv format](./howto/LOGGING.md#how-to-export-elasticsearch-data-to-csv-format) - - [How to start working with Apache Ignite Stateful setup](./howto/DATABASES.md#how-to-start-working-with-apache-ignite-stateful-setup) - - [How to start working with Apache Ignite Stateless setup](./howto/DATABASES.md#how-to-start-working-with-apache-ignite-stateless-setup) - [Backup and Recovery](./howto/BACKUP.md) - [Epiphany backup and restore](./howto/BACKUP.md#epiphany-backup-and-restore) diff --git a/docs/home/SECURITY.md b/docs/home/SECURITY.md index 50978aece9..66ec421967 100644 --- a/docs/home/SECURITY.md +++ b/docs/home/SECURITY.md @@ -114,7 +114,6 @@ different values. The list does not include ports that are bound to the loopback 13. Kubernetes apps: - 30104 - auth-service (Keycloak) - - 32300-32302 - Ignite (REST API, SQL port, Thin clients) - 30672,31672 - RabbitMQ (AMQP, management) 14. HAProxy: @@ -126,25 +125,11 @@ different values. The list does not include ports that are bound to the loopback **NOTE:** Not applicable for Ubuntu where UNIX socket is used (deb package's default). -15. Ignite: - - - 8080 - REST API - - 10800-10809* - JDBC Thin Driver - - 11211-11220* - JDBC Client Driver - - 47100-47109* - communication SPI - - 47500-47509* - discovery SPI - - 49112 - JMX (remote access), limited by Epiphany to be accessible only through SSH tunnel ( - java.rmi.server.hostname=127.0.0.1) - - unconfigurable random port from ephemeral range - JMX (for local access only), see note [[1]](#notes) - - **NOTE:** By default, only the first port from the range is used (port ranges are handy when starting multiple grid - nodes on the same machine) - -16. Repository: +15. Repository: - 80 - deb/rpm package repository (httpd is stopped at the end of installation) - 5000 - Docker image registry - + ### Connection protocols and ciphers used by components in Epiphany Below you can find list of cipersuites and protocols used for communication set in Epiphany on per component basis. This diff --git a/docs/home/howto/CLUSTER.md b/docs/home/howto/CLUSTER.md index 1e77191d3a..c2f6914fd7 100644 --- a/docs/home/howto/CLUSTER.md +++ b/docs/home/howto/CLUSTER.md @@ -638,8 +638,6 @@ specification: count: 0 rabbitmq: count: 0 - ignite: - count: 0 opendistro_for_elasticsearch: count: 0 single_machine: @@ -820,7 +818,6 @@ Monitoring | :heavy_check_mark: | :heavy_check_mark: | :x: | :x: | --- Logging | :heavy_check_mark: | :heavy_check_mark: | :x: | :x: | --- Kubernetes master | :heavy_check_mark: | :x: | :heavy_check_mark: | :heavy_check_mark: | [#1579](https://github.com/epiphany-platform/epiphany/issues/1579) Kubernetes node | :heavy_check_mark: | :x: | :heavy_check_mark: | :heavy_check_mark: | [#1580](https://github.com/epiphany-platform/epiphany/issues/1580) -Ignite | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | --- Kafka | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | --- Load Balancer | :heavy_check_mark: | :heavy_check_mark: | :x: | :x: | --- Opendistro for elasticsearch | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | --- @@ -830,7 +827,6 @@ RabbitMQ K8s | :heavy_check_mark: | :heavy_check_mark: | :x: | :heavy_check_mark Keycloak K8s | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | --- Pgpool K8s | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | --- Pgbouncer K8s | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | --- -Ignite K8s | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | --- Additional notes: diff --git a/docs/home/howto/DATABASES.md b/docs/home/howto/DATABASES.md index f3ed8e9f5e..cdb5e8a67f 100644 --- a/docs/home/howto/DATABASES.md +++ b/docs/home/howto/DATABASES.md @@ -507,127 +507,3 @@ specification: ``` Filebeat running on `opendistro_for_elasticsearch` hosts will always point to centralized logging hosts (./LOGGING.md). - -## How to start working with Apache Ignite Stateful setup - -Apache Ignite can be installed in Epiphany if `count` property for `ignite` feature is greater than 0. Example: - -```yaml -kind: epiphany-cluster -specification: - components: - load_balancer: - count: 1 - ignite: - count: 2 - rabbitmq: - count: 0 - ... -``` - -Configuration like in this example will create Virtual Machines with Apache Ignite cluster installed. There is possible -to modify configuration for Apache Ignite and plugins used. - -```yaml -kind: configuration/ignite -title: "Apache Ignite stateful installation" -name: default -specification: - version: 2.7.6 - file_name: apache-ignite-2.7.6-bin.zip - enabled_plugins: - - ignite-rest-http - config: | - - - - - - - - - - - - - - - - - - - - - - - - - - - - IP_LIST_PLACEHOLDER - - - - - - - -``` - -Property `enabled_plugins` contains list with plugin names that will be enabled. Property `config` contains xml -configuration for Apache Ignite. Important placeholder variable is `IP_LIST_PLACEHOLDER` which will be replaced by -automation with list of Apache Ignite nodes for self discovery. - -## How to start working with Apache Ignite Stateless setup - -Stateless setup of Apache Ignite is done using Kubernetes deployments. This setup uses standard `applications` -Epiphany's feature (similar to `auth-service`, `rabbitmq`). To enable stateless Ignite deployment use following -document: - -```yaml -kind: configuration/applications -title: "Kubernetes Applications Config" -name: default -specification: - applications: - - name: ignite-stateless - image_path: "epiphanyplatform/ignite:2.9.1" # it will be part of the image path: {{local_repository}}/{{image_path}} - namespace: ignite - service: - rest_nodeport: 32300 - sql_nodeport: 32301 - thinclients_nodeport: 32302 - replicas: 1 - enabled_plugins: - - ignite-kubernetes # required to work on K8s - - ignite-rest-http -``` - -Adjust this config to your requirements with number of replicas and plugins that should be enabled. diff --git a/docs/home/howto/KUBERNETES.md b/docs/home/howto/KUBERNETES.md index b7e002736f..7868d9e34f 100644 --- a/docs/home/howto/KUBERNETES.md +++ b/docs/home/howto/KUBERNETES.md @@ -33,7 +33,6 @@ specification: Currently, Epiphany provides the following predefined applications which may be deployed with epicli: -- ignite - rabbitmq - auth-service (Keycloak) - pgpool diff --git a/docs/home/howto/MODULES.md b/docs/home/howto/MODULES.md index dab15dcb22..05e7df4727 100644 --- a/docs/home/howto/MODULES.md +++ b/docs/home/howto/MODULES.md @@ -263,19 +263,6 @@ AWS: name: default specification: applications: - - name: ignite-stateless - enabled: false - image_path: "epiphanyplatform/ignite:2.9.1" - use_local_image_registry: false - namespace: ignite - service: - rest_nodeport: 32300 - sql_nodeport: 32301 - thinclients_nodeport: 32302 - replicas: 1 - enabled_plugins: - - ignite-kubernetes - - ignite-rest-http - name: rabbitmq enabled: false image_path: rabbitmq:3.8.3 diff --git a/docs/home/howto/SECURITY_GROUPS.md b/docs/home/howto/SECURITY_GROUPS.md index 68f420d11b..f3e0b56855 100644 --- a/docs/home/howto/SECURITY_GROUPS.md +++ b/docs/home/howto/SECURITY_GROUPS.md @@ -285,8 +285,6 @@ specification: count: 0 rabbitmq: count: 0 - ignite: - count: 0 opendistro_for_elasticsearch: count: 0 single_machine: diff --git a/docs/home/howto/UPGRADE.md b/docs/home/howto/UPGRADE.md index 4a63212ef4..df25107313 100644 --- a/docs/home/howto/UPGRADE.md +++ b/docs/home/howto/UPGRADE.md @@ -194,8 +194,6 @@ specification: count: 1 rabbitmq: count: 0 - ignite: - count: 0 opendistro_for_elasticsearch: count: 0 name: clustername diff --git a/schema/any/defaults/epiphany-cluster.yml b/schema/any/defaults/epiphany-cluster.yml index 84d07d98b6..4bc8719823 100644 --- a/schema/any/defaults/epiphany-cluster.yml +++ b/schema/any/defaults/epiphany-cluster.yml @@ -41,10 +41,6 @@ specification: count: 0 machine: rabbitmq-machine configuration: default - ignite: - count: 0 - machine: ignite-machine - configuration: default opendistro_for_elasticsearch: count: 0 machine: logging-machine diff --git a/schema/aws/defaults/epiphany-cluster.yml b/schema/aws/defaults/epiphany-cluster.yml index 3c0aa03ce4..f1dbd958a2 100644 --- a/schema/aws/defaults/epiphany-cluster.yml +++ b/schema/aws/defaults/epiphany-cluster.yml @@ -80,13 +80,6 @@ specification: subnets: - availability_zone: eu-west-2a address_pool: 10.1.8.0/24 - ignite: - count: 0 - machine: ignite-machine - configuration: default - subnets: - - availability_zone: eu-west-2a - address_pool: 10.1.9.0/24 opendistro_for_elasticsearch: count: 0 machine: logging-machine diff --git a/schema/aws/defaults/infrastructure/virtual-machine.yml b/schema/aws/defaults/infrastructure/virtual-machine.yml index e260d9d5c3..49150661ae 100644 --- a/schema/aws/defaults/infrastructure/virtual-machine.yml +++ b/schema/aws/defaults/infrastructure/virtual-machine.yml @@ -377,23 +377,6 @@ specification: kind: infrastructure/virtual-machine title: "Virtual Machine Infra" provider: aws -name: ignite-machine -specification: - size: t3.medium - security: - rules: - - _merge: true - - name: ignite_traffic - description: Allow traffic to Apache Ignite cluster - direction: Inbound - protocol: ALL - destination_port_range: "0" - source_address_prefix: "10.1.0.0/20" - destination_address_prefix: "10.1.0.0/20" ---- -kind: infrastructure/virtual-machine -title: "Virtual Machine Infra" -provider: aws name: single-machine specification: size: t3.xlarge diff --git a/schema/azure/defaults/epiphany-cluster.yml b/schema/azure/defaults/epiphany-cluster.yml index b0a37480a1..331821bcea 100644 --- a/schema/azure/defaults/epiphany-cluster.yml +++ b/schema/azure/defaults/epiphany-cluster.yml @@ -68,12 +68,6 @@ specification: configuration: default subnets: - address_pool: 10.1.8.0/24 - ignite: - count: 0 - machine: ignite-machine - configuration: default - subnets: - - address_pool: 10.1.9.0/24 opendistro_for_elasticsearch: count: 0 machine: logging-machine diff --git a/schema/azure/defaults/infrastructure/virtual-machine.yml b/schema/azure/defaults/infrastructure/virtual-machine.yml index 3c265d5c88..d6efa0c600 100644 --- a/schema/azure/defaults/infrastructure/virtual-machine.yml +++ b/schema/azure/defaults/infrastructure/virtual-machine.yml @@ -439,26 +439,6 @@ specification: kind: infrastructure/virtual-machine title: "Virtual Machine Infra" provider: azure -name: ignite-machine -specification: - size: Standard_DS1_v2 - security: - rules: - - _merge: true - - name: vnet_traffic - description: Allow vnet traffic - priority: 300 - direction: Inbound - access: Allow - protocol: Tcp - source_port_range: "*" - destination_port_range: "*" - source_address_prefix: "10.1.0.0/20" - destination_address_prefix: "10.1.0.0/20" ---- -kind: infrastructure/virtual-machine -title: "Virtual Machine Infra" -provider: azure name: single-machine specification: size: Standard_DS3_v2 diff --git a/schema/common/defaults/configuration/applications.yml b/schema/common/defaults/configuration/applications.yml index dc9952263d..2c97d74029 100644 --- a/schema/common/defaults/configuration/applications.yml +++ b/schema/common/defaults/configuration/applications.yml @@ -4,22 +4,6 @@ name: default specification: applications: -## --- ignite --- - - - name: ignite-stateless - enabled: false - image_path: "epiphanyplatform/ignite:2.9.1" # it will be part of the image path: {{local_repository}}/{{image_path}} - use_local_image_registry: true - namespace: ignite - service: - rest_nodeport: 32300 - sql_nodeport: 32301 - thinclients_nodeport: 32302 - replicas: 1 - enabled_plugins: - - ignite-kubernetes # required to work on K8s - - ignite-rest-http - # Abstract these configs to separate default files and add # the ability to add custom application roles. diff --git a/schema/common/defaults/configuration/feature-mapping.yml b/schema/common/defaults/configuration/feature-mapping.yml index 6d7400eaa9..a08d6605be 100644 --- a/schema/common/defaults/configuration/feature-mapping.yml +++ b/schema/common/defaults/configuration/feature-mapping.yml @@ -51,8 +51,6 @@ specification: enabled: true - name: applications enabled: true - - name: ignite - enabled: true roles_mapping: kafka: @@ -125,11 +123,6 @@ specification: - node-exporter - filebeat - firewall - ignite: - - ignite - - node-exporter - - filebeat - - firewall opendistro_for_elasticsearch: - opendistro-for-elasticsearch - node-exporter diff --git a/schema/common/defaults/configuration/firewall.yml b/schema/common/defaults/configuration/firewall.yml index 6d3d0b6d17..8a9d66493c 100644 --- a/schema/common/defaults/configuration/firewall.yml +++ b/schema/common/defaults/configuration/firewall.yml @@ -14,7 +14,6 @@ specification: - 30104/tcp # auth-service - 30672/tcp # rabbitmq-amqp - 31672/tcp # rabbitmq-http (management) - - 32300-32302/tcp # ignite common: # for all inventory hosts enabled: true ports: @@ -28,14 +27,6 @@ specification: ports: - 443/tcp - 9000/tcp # stats and metrics - ignite: - enabled: true - ports: - - 8080/tcp # REST API - - 10800/tcp # thin client connection - - 11211/tcp # JDBC - - 47100/tcp # local communication - - 47500/tcp # local discovery image_registry: enabled: true ports: diff --git a/schema/common/defaults/configuration/ignite.yml b/schema/common/defaults/configuration/ignite.yml deleted file mode 100644 index 3f3eb96248..0000000000 --- a/schema/common/defaults/configuration/ignite.yml +++ /dev/null @@ -1,96 +0,0 @@ -kind: configuration/ignite -title: "Apache Ignite stateful installation" -name: default -specification: - enabled_plugins: - - ignite-rest-http - config: | - - - - - - - - - - - - - - - - - - - - - - - - - - - - IP_LIST_PLACEHOLDER - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/schema/common/defaults/configuration/image-registry.yml b/schema/common/defaults/configuration/image-registry.yml index eacdcaa4d4..e79ab198ac 100644 --- a/schema/common/defaults/configuration/image-registry.yml +++ b/schema/common/defaults/configuration/image-registry.yml @@ -13,8 +13,6 @@ specification: file_name: keycloak-14.0.0.tar - name: "rabbitmq:3.8.9" file_name: rabbitmq-3.8.9.tar - - name: "epiphanyplatform/ignite:2.9.1" - file_name: ignite-2.9.1.tar - name: "kubernetesui/dashboard:v2.3.1" file_name: dashboard-v2.3.1.tar - name: "kubernetesui/metrics-scraper:v1.0.7" @@ -134,8 +132,6 @@ specification: file_name: keycloak-14.0.0.tar - name: "rabbitmq:3.8.9" file_name: rabbitmq-3.8.9.tar - - name: "epiphanyplatform/ignite:2.9.1" - file_name: ignite-2.9.1.tar - name: "kubernetesui/dashboard:v2.3.1" file_name: dashboard-v2.3.1.tar - name: "kubernetesui/metrics-scraper:v1.0.7" diff --git a/schema/common/validation/configuration/feature-mapping.yml b/schema/common/validation/configuration/feature-mapping.yml index ae6cd504b0..85b954b095 100644 --- a/schema/common/validation/configuration/feature-mapping.yml +++ b/schema/common/validation/configuration/feature-mapping.yml @@ -55,10 +55,6 @@ properties: type: array items: type: string - ignite: - type: array - items: - type: string opendistro_for_elasticsearch: type: array items: diff --git a/schema/common/validation/configuration/firewall.yml b/schema/common/validation/configuration/firewall.yml index b1ea2e508c..82148a9453 100644 --- a/schema/common/validation/configuration/firewall.yml +++ b/schema/common/validation/configuration/firewall.yml @@ -53,15 +53,6 @@ properties: type: array items: type: string - ignite: - type: object - properties: - enabled: - type: boolean - ports: - type: array - items: - type: string image_registry: type: object properties: diff --git a/schema/common/validation/configuration/ignite.yml b/schema/common/validation/configuration/ignite.yml deleted file mode 100644 index 195e69d462..0000000000 --- a/schema/common/validation/configuration/ignite.yml +++ /dev/null @@ -1,11 +0,0 @@ -"$id": "#/specification" -title: "Ignite specification schema" -description: "Ignite specification schema" -type: object -properties: - enabled_plugins: - type: array - items: - type: string - config: - type: string diff --git a/tests/spec/spec/applications/applications_spec.rb b/tests/spec/spec/applications/applications_spec.rb index 661adbe937..9bbd613448 100644 --- a/tests/spec/spec/applications/applications_spec.rb +++ b/tests/spec/spec/applications/applications_spec.rb @@ -2,7 +2,6 @@ require 'securerandom' require 'applications/rabbitmq/rabbitmq' require 'applications/auth-service/auth-service' -require 'applications/ignite-stateless/ignite-stateless' require 'applications/pgpool/pgpool' if !readDataYaml("configuration/applications")["specification"]["applications"].select {|i| i["name"] == 'rabbitmq'}.empty? && @@ -19,16 +18,9 @@ end -if !readDataYaml("configuration/applications")["specification"]["applications"].select {|i| i["name"] == 'ignite-stateless'}.empty? && - readDataYaml("configuration/applications")["specification"]["applications"].detect {|i| i["name"] == 'ignite-stateless'}["enabled"] - - callIgniteDeploymentTests - -end - if !readDataYaml("configuration/applications")["specification"]["applications"].select {|i| i["name"] == 'pgpool'}.empty? && readDataYaml("configuration/applications")["specification"]["applications"].detect {|i| i["name"] == 'pgpool'}["enabled"] callPgpoolDeploymentTests - -end \ No newline at end of file + +end diff --git a/tests/spec/spec/applications/ignite-stateless/ignite-stateless.rb b/tests/spec/spec/applications/ignite-stateless/ignite-stateless.rb deleted file mode 100644 index dcf62eddf1..0000000000 --- a/tests/spec/spec/applications/ignite-stateless/ignite-stateless.rb +++ /dev/null @@ -1,122 +0,0 @@ -require 'spec_helper' - -def callIgniteDeploymentTests - - rest_api_host = '127.0.0.1' - cacheName = 'epi-test-k8s' - service_namespace = readDataYaml("configuration/applications")["specification"]["applications"].detect {|i| i["name"] == 'ignite-stateless'}["namespace"] - service_replicas = readDataYaml("configuration/applications")["specification"]["applications"].detect {|i| i["name"] == 'ignite-stateless'}["replicas"] - rest_nodeport = readDataYaml("configuration/applications")["specification"]["applications"].detect {|i| i["name"] == 'ignite-stateless'}["service"]["rest_nodeport"] - sql_nodeport = readDataYaml("configuration/applications")["specification"]["applications"].detect {|i| i["name"] == 'ignite-stateless'}["service"]["sql_nodeport"] - thinclients_nodeport = readDataYaml("configuration/applications")["specification"]["applications"].detect {|i| i["name"] == 'ignite-stateless'}["service"]["thinclients_nodeport"] - - describe 'Checking if th Ignite service exists' do - describe command("kubectl get services --namespace=#{service_namespace} --no-headers -o custom-columns=:metadata.name") do - its(:stdout) { should match /ignite/ } - end - end - - describe 'Checking if the ports are open' do - let(:disable_sudo) { false } - describe port(rest_nodeport) do - it { should be_listening } - end - describe port(sql_nodeport) do - it { should be_listening } - end - describe port(thinclients_nodeport) do - it { should be_listening } - end - end - - describe 'Checking the status of Ignite pods - all pods should be running' do - describe command("kubectl get pods --namespace=#{service_namespace} --field-selector=status.phase=Running --no-headers | wc -l") do - it "is expected to be equal" do - expect(subject.stdout.to_i).to eq service_replicas - end - its(:exit_status) { should eq 0 } - end - end - - describe 'Checking API connection' do - describe command("curl 'http://#{rest_api_host}:#{rest_nodeport}/ignite?cmd=version'") do - its(:stdout_as_json) { should include('successStatus' => 0) } - its(:exit_status) { should eq 0 } - end - end - - describe 'Checking if it is possible to create a cache' do - describe command("curl 'http://#{rest_api_host}:#{rest_nodeport}/ignite?cmd=getorcreate&cacheName=#{cacheName}'") do - its(:stdout_as_json) { should include('successStatus' => 0) } - its(:exit_status) { should eq 0 } - end - end - - describe 'Checking if it is possible to add key-value pairs to cache' do - describe command("curl 'http://#{rest_api_host}:#{rest_nodeport}/ignite?cmd=putall&k1=testKey1&k2=testKey2&k3=testKey3&v1=testValue1&v2=testValue2&v3=testValue3&cacheName=#{cacheName}'") do - its(:stdout_as_json) { should include('successStatus' => 0) } - its(:exit_status) { should eq 0 } - end - end - - describe 'Checking if it is possible to get values mapped to the specified keys from cache' do - describe command("curl 'http://#{rest_api_host}:#{rest_nodeport}/ignite?cmd=getall&k1=testKey1&k2=testKey2&k3=testKey3&cacheName=#{cacheName}'") do - its(:stdout_as_json) { should include('response' => include('testKey1' => 'testValue1')) } - its(:stdout_as_json) { should include('response' => include('testKey2' => 'testValue2')) } - its(:stdout_as_json) { should include('response' => include('testKey3' => 'testValue3')) } - its(:exit_status) { should eq 0 } - end - end - - describe 'Checking if it is possible to get the number of all entries cached across all nodes' do - describe command("curl 'http://#{rest_api_host}:#{rest_nodeport}/ignite?cmd=size&cacheName=#{cacheName}'") do - its(:stdout_as_json) { should include('response' => 3) } - its(:exit_status) { should eq 0 } - end - end - - describe 'Checking if it is possible to create a test table via API' do - describe command("curl 'http://#{rest_api_host}:#{rest_nodeport}/ignite?cmd=qryfldexe&pageSize=10&cacheName=#{cacheName}&qry=CREATE+TABLE+PUBLIC.EPI_TEST_TABLE(id+int,+name+varchar,+PRIMARY+KEY+(id))+WITH+\"CACHE_NAME=#{cacheName}-ddl\"'") do - its(:stdout_as_json) { should include('successStatus' => 0) } - its(:exit_status) { should eq 0 } - end - end - - describe 'Checking if it is possible to insert values into the test table via API' do - describe command("curl 'http://#{rest_api_host}:#{rest_nodeport}/ignite?cmd=qryfldexe&pageSize=10&cacheName=#{cacheName}&qry=INSERT+INTO+PUBLIC.EPI_TEST_TABLE(id,+name)+values+(1,+%27SUCCESS%27)'") do - its(:stdout_as_json) { should include('successStatus' => 0) } - its(:exit_status) { should eq 0 } - end - end - - describe 'Checking if it is possible to select values from the test table via API' do - describe command("curl 'http://#{rest_api_host}:#{rest_nodeport}/ignite?cmd=qryfldexe&pageSize=10&cacheName=#{cacheName}&qry=SELECT+NAME+FROM+PUBLIC.EPI_TEST_TABLE' | jq '.response.items'") do - its(:stdout) { should match /\bSUCCESS\b/ } - its(:exit_status) { should eq 0 } - end - end - - describe 'Checking if it is possible to select values from the test table using JDBC connection inside pods' do - describe command("for pod in $(kubectl get pods --namespace=#{service_namespace} --no-headers --field-selector=status.phase=Running -o custom-columns=':metadata.name'); do kubectl exec $pod -n=ignite -- bash -c 'echo \"SELECT name FROM EPI_TEST_TABLE;\" | $IGNITE_HOME/bin/sqlline.sh -u jdbc:ignite:thin://127.0.0.1/ 2>&1'; done") do - its(:stdout) { should match /(.*1 row selected.*){#{service_replicas}}/m } - its(:stdout) { should match /(.*SUCCESS.*){#{service_replicas}}/m } - its(:stdout) { should_not match /Error/ } - its(:exit_status) { should eq 0 } - end - end - - describe 'Checking if it is possible to drop the test table via API' do - describe command("curl 'http://#{rest_api_host}:#{rest_nodeport}/ignite?cmd=qryfldexe&pageSize=10&cacheName=#{cacheName}&qry=DROP+TABLE+IF+EXISTS+PUBLIC.EPI_TEST_TABLE'") do - its(:stdout_as_json) { should include('successStatus' => 0) } - its(:exit_status) { should eq 0 } - end - end - - describe 'Checking if it is possible to destroy a cache' do - describe command("curl 'http://#{rest_api_host}:#{rest_nodeport}/ignite?cmd=destcache&cacheName=#{cacheName}'") do - its(:stdout_as_json) { should include('successStatus' => 0) } - its(:exit_status) { should eq 0 } - end - end - -end diff --git a/tests/spec/spec/ignite/ignite_spec.rb b/tests/spec/spec/ignite/ignite_spec.rb deleted file mode 100644 index ad68d451c5..0000000000 --- a/tests/spec/spec/ignite/ignite_spec.rb +++ /dev/null @@ -1,165 +0,0 @@ -require 'spec_helper' - -$ignite_rest_api_host = '127.0.0.1' -$ignite_rest_api_port = 8080 -$ignite_jdbc_port = 10800 -$cacheName = 'epi-test' - -def checkAPIconnection - describe 'Checking API connection' do - describe command("curl 'http://#{$ignite_rest_api_host}:#{$ignite_rest_api_port}/ignite?cmd=version'") do - its(:stdout_as_json) { should include('successStatus' => 0) } - its(:exit_status) { should eq 0 } - end - end -end - -def createCache - describe 'Checking if it is possible to create a cache' do - describe command("curl 'http://#{$ignite_rest_api_host}:#{$ignite_rest_api_port}/ignite?cmd=getorcreate&cacheName=#{$cacheName}'") do - its(:stdout_as_json) { should include('successStatus' => 0) } - its(:exit_status) { should eq 0 } - end - end -end - -def addKeyValuePairs - describe 'Checking if it is possible to add key-value pairs to cache' do - describe command("curl 'http://#{$ignite_rest_api_host}:#{$ignite_rest_api_port}/ignite?cmd=putall&k1=testKey1&k2=testKey2&k3=testKey3&v1=testValue1&v2=testValue2&v3=testValue3&cacheName=#{$cacheName}'") do - its(:stdout_as_json) { should include('successStatus' => 0) } - its(:exit_status) { should eq 0 } - end - end -end - -def getKeyValuePairs - describe 'Checking if it is possible to get values mapped to the specified keys from cache' do - describe command("curl 'http://#{$ignite_rest_api_host}:#{$ignite_rest_api_port}/ignite?cmd=getall&k1=testKey1&k2=testKey2&k3=testKey3&cacheName=#{$cacheName}'") do - its(:stdout_as_json) { should include('response' => include('testKey1' => 'testValue1')) } - its(:stdout_as_json) { should include('response' => include('testKey2' => 'testValue2')) } - its(:stdout_as_json) { should include('response' => include('testKey3' => 'testValue3')) } - its(:exit_status) { should eq 0 } - end - end -end - -def getCacheSize - describe 'Checking if it is possible to get the number of all entries cached across all nodes' do - describe command("curl 'http://#{$ignite_rest_api_host}:#{$ignite_rest_api_port}/ignite?cmd=size&cacheName=#{$cacheName}'") do - its(:stdout_as_json) { should include('response' => 3) } - its(:exit_status) { should eq 0 } - end - end -end - -def destroyCache - describe 'Checking if it is possible to destroy a cache' do - describe command("curl 'http://#{$ignite_rest_api_host}:#{$ignite_rest_api_port}/ignite?cmd=destcache&cacheName=#{$cacheName}'") do - its(:stdout_as_json) { should include('successStatus' => 0) } - its(:exit_status) { should eq 0 } - end - end -end - -def checkJDBCconnection - describe 'Checking JDBC connection' do - describe command("echo \"SELECT 2 + 2 * 2 as RESULT ;\" | /opt/ignite/bin/sqlline.sh -u jdbc:ignite:thin://127.0.0.1/") do - let(:disable_sudo) { false } - its(:stdout) { should match /\b6\b/ } - its(:exit_status) { should eq 0 } - end - end -end - -def createTable - describe 'Checking if it is possible to create a test table' do - describe command("echo \"CREATE TABLE EPI_TEST_TABLE (id int, name varchar, PRIMARY KEY (id)) WITH \\\"CACHE_NAME=#{$cacheName}-ddl\\\";\" | /opt/ignite/bin/sqlline.sh -u jdbc:ignite:thin://127.0.0.1/ 2>&1") do - let(:disable_sudo) { false } - its(:stdout) { should match /No rows affected/ } - its(:stdout) { should_not match /Error/ } - its(:exit_status) { should eq 0 } - end - end -end - -def insertValuesIntoTable - describe 'Checking if it is possible to insert values into the test table' do - describe command("echo \"INSERT INTO EPI_TEST_TABLE(id, name) values (1, 'SUCCESS');\" | /opt/ignite/bin/sqlline.sh -u jdbc:ignite:thin://127.0.0.1/ 2>&1") do - let(:disable_sudo) { false } - its(:stdout) { should match /1 row affected/ } - its(:stdout) { should_not match /Error/ } - its(:exit_status) { should eq 0 } - end - end -end - -def selectValuesFromTable - describe 'Checking if it is possible to select values from the test table' do - describe command("echo \"SELECT id, name FROM EPI_TEST_TABLE;\" | /opt/ignite/bin/sqlline.sh -u jdbc:ignite:thin://127.0.0.1/ 2>&1") do - let(:disable_sudo) { false } - its(:stdout) { should match /1 row selected/ } - its(:stdout) { should match /SUCCESS/ } - its(:stdout) { should_not match /Error/ } - its(:exit_status) { should eq 0 } - end - end -end - -def getValueFromTableViaAPI - describe 'Checking if it is possible to get values from table created with DDL statement ' do - describe command("curl 'http://#{$ignite_rest_api_host}:#{$ignite_rest_api_port}/ignite?cmd=qryfldexe&pageSize=10&cacheName=#{$cacheName}-ddl&qry=select+name+from+EPI_TEST_TABLE' | jq '.response.items'") do - its(:stdout) { should match /\bSUCCESS\b/ } - its(:exit_status) { should eq 0 } - end - end -end - -def dropTable - describe 'Checking if it is possible to drop the test table' do - describe command("echo \"DROP TABLE IF EXISTS EPI_TEST_TABLE;\" | /opt/ignite/bin/sqlline.sh -u jdbc:ignite:thin://127.0.0.1/ 2>&1") do - let(:disable_sudo) { false } - its(:stdout) { should match /No rows affected/ } - its(:stdout) { should_not match /Error/ } - its(:exit_status) { should eq 0 } - end - end -end - -describe 'Checking if Ignite service is running' do - describe service('ignite') do - it { should be_enabled } - it { should be_running } - end -end - -describe 'Checking if the ports are open' do - let(:disable_sudo) { false } - describe port($ignite_jdbc_port) do - it { should be_listening } - end - describe port($ignite_rest_api_port) do - it { should be_listening } - end -end - -nodes = listInventoryHosts("ignite") - -checkAPIconnection -checkJDBCconnection - -if nodes[0].include? host_inventory['hostname'] - createCache - addKeyValuePairs - createTable - insertValuesIntoTable -end - -getKeyValuePairs -getCacheSize -selectValuesFromTable -getValueFromTableViaAPI - -if nodes[nodes.length-1].include? host_inventory['hostname'] - destroyCache - dropTable -end From 16cafa2d0df4453b05658f029dc65192ace5f050 Mon Sep 17 00:00:00 2001 From: to-bar <46519524+to-bar@users.noreply.github.com> Date: Fri, 14 Jan 2022 18:53:44 +0100 Subject: [PATCH 2/2] Fix validation for epicli backup (#2874) --- docs/changelogs/CHANGELOG-1.3.md | 1 + schema/common/validation/configuration/backup.yml | 1 + schema/common/validation/configuration/recovery.yml | 1 + 3 files changed, 3 insertions(+) diff --git a/docs/changelogs/CHANGELOG-1.3.md b/docs/changelogs/CHANGELOG-1.3.md index a07e81f997..f76c73985f 100644 --- a/docs/changelogs/CHANGELOG-1.3.md +++ b/docs/changelogs/CHANGELOG-1.3.md @@ -43,6 +43,7 @@ - [#2737](https://github.com/epiphany-platform/epiphany/issues/2737) - Fix asserting number of postgres nodes - [#1175](https://github.com/epiphany-platform/epiphany/issues/1175) - Task 'Join to Kubernetes cluster' may fail when Ansible vault already exists - [#2420](https://github.com/epiphany-platform/epiphany/issues/2420) - Changing Grafana admin password in the apply mode +- [#2873](https://github.com/epiphany-platform/epiphany/issues/2873) - Epicli backup fails on schema validation ### Updated diff --git a/schema/common/validation/configuration/backup.yml b/schema/common/validation/configuration/backup.yml index 1f196f6638..23ecbc6797 100644 --- a/schema/common/validation/configuration/backup.yml +++ b/schema/common/validation/configuration/backup.yml @@ -1,3 +1,4 @@ +$schema: 'http://json-schema.org/draft-07/schema#' type: object required: - name diff --git a/schema/common/validation/configuration/recovery.yml b/schema/common/validation/configuration/recovery.yml index ebdbd93092..6225f7c269 100644 --- a/schema/common/validation/configuration/recovery.yml +++ b/schema/common/validation/configuration/recovery.yml @@ -1,3 +1,4 @@ +$schema: 'http://json-schema.org/draft-07/schema#' type: object required: - name