From 24b48bfba78f5239ba6554e525c8c6f7a0517d4d Mon Sep 17 00:00:00 2001 From: cicharka Date: Fri, 14 Jan 2022 15:20:21 +0100 Subject: [PATCH] Removal of Logstash component --- ansible/playbooks/logstash.yml | 9 ---- .../roles/logstash/defaults/main.yml | 4 -- .../files/logstash-export.conf.template | 23 --------- .../playbooks/roles/logstash/tasks/main.yml | 33 ------------ .../roles/preflight/defaults/main.yml | 2 - .../centos-7/requirements.aarch64.txt | 1 - .../centos-7/requirements.x86_64.txt | 1 - .../redhat-7/requirements.x86_64.txt | 1 - .../ubuntu-20.04/requirements.x86_64.txt | 1 - docs/changelogs/CHANGELOG-2.0.md | 1 + docs/home/COMPONENTS.md | 1 - docs/home/DEPRECATION-NOTE.md | 4 +- docs/home/HOWTO.md | 2 - docs/home/howto/LOGGING.md | 50 ------------------- .../configuration/feature-mapping.yml | 2 - .../defaults/configuration/logstash.yml | 4 -- .../validation/configuration/logstash.yml | 4 -- 17 files changed, 2 insertions(+), 141 deletions(-) delete mode 100644 ansible/playbooks/logstash.yml delete mode 100644 ansible/playbooks/roles/logstash/defaults/main.yml delete mode 100644 ansible/playbooks/roles/logstash/files/logstash-export.conf.template delete mode 100644 ansible/playbooks/roles/logstash/tasks/main.yml delete mode 100644 schema/common/defaults/configuration/logstash.yml delete mode 100644 schema/common/validation/configuration/logstash.yml diff --git a/ansible/playbooks/logstash.yml b/ansible/playbooks/logstash.yml deleted file mode 100644 index f2f838c890..0000000000 --- a/ansible/playbooks/logstash.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -- hosts: all - tasks: [] - -- hosts: logstash - become: true - become_method: sudo - roles: - - logstash diff --git a/ansible/playbooks/roles/logstash/defaults/main.yml b/ansible/playbooks/roles/logstash/defaults/main.yml deleted file mode 100644 index 908f68e9b3..0000000000 --- a/ansible/playbooks/roles/logstash/defaults/main.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -versions: - RedHat: "7.12.0" - Debian: "1:7.12.0*" diff --git a/ansible/playbooks/roles/logstash/files/logstash-export.conf.template b/ansible/playbooks/roles/logstash/files/logstash-export.conf.template deleted file mode 100644 index 0fd99f00c7..0000000000 --- a/ansible/playbooks/roles/logstash/files/logstash-export.conf.template +++ /dev/null @@ -1,23 +0,0 @@ -input { - elasticsearch { - hosts => "HOST_IP:ES_HTTP_PORT" - ssl => "BOOLEAN" - ca_file => "CA_FILE.pem" - user => "USER" - password => "PASSWORD" - index => "INDEX_NAME" - query => ' - { - "query": { - "match_all": {} - } - } - ' - } -} -output { - csv { - fields => ["FIELDS_TO_EXPORT"] - path => "PATH" - } -} diff --git a/ansible/playbooks/roles/logstash/tasks/main.yml b/ansible/playbooks/roles/logstash/tasks/main.yml deleted file mode 100644 index 3791428099..0000000000 --- a/ansible/playbooks/roles/logstash/tasks/main.yml +++ /dev/null @@ -1,33 +0,0 @@ ---- -- name: Install OS packages - package: - name: "{{ _packages[ansible_os_family] }}" - state: present - vars: - _packages: - # Logstash is installed with bundled JDK - Debian: - - logstash-oss={{ versions[ansible_os_family] }} - RedHat: - - logstash-oss-{{ versions[ansible_os_family] }} - module_defaults: - yum: { lock_timeout: "{{ yum_lock_timeout }}" } - -- name: Populate service facts - service_facts: - -- name: Create logstash systemd service - command: /usr/share/logstash/bin/system-install /etc/logstash/startup.options systemd - when: - - "'logstash.service' not in ansible_facts.services" - -- name: Ensure Logstash is disabled - systemd: - name: logstash - enabled: false - state: stopped - -- name: Provide logstash-export.conf template file - copy: - src: logstash-export.conf.template - dest: /etc/logstash/logstash-export.conf.template diff --git a/ansible/playbooks/roles/preflight/defaults/main.yml b/ansible/playbooks/roles/preflight/defaults/main.yml index fba5d9f83c..860f7db731 100644 --- a/ansible/playbooks/roles/preflight/defaults/main.yml +++ b/ansible/playbooks/roles/preflight/defaults/main.yml @@ -42,7 +42,6 @@ unsupported_roles: - elasticsearch - kibana - filebeat - - logstash - prometheus - grafana - node_exporter @@ -80,7 +79,6 @@ unsupported_roles: - elasticsearch - kibana - filebeat - - logstash - prometheus - grafana - node_exporter diff --git a/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.aarch64.txt b/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.aarch64.txt index 2b9692dbcf..7ecc3fe7f3 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.aarch64.txt +++ b/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.aarch64.txt @@ -58,7 +58,6 @@ libxcb # for grafana libXcursor # for grafana libXt # for grafana logrotate -logstash-oss-7.12.0 net-tools nfs-utils nmap-ncat diff --git a/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.x86_64.txt b/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.x86_64.txt index 2f384fb0f0..fe4ac78f84 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.x86_64.txt +++ b/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.x86_64.txt @@ -59,7 +59,6 @@ libxcb # for grafana libXcursor # for grafana libXt # for grafana logrotate -logstash-oss-7.12.0 net-tools nfs-utils nmap-ncat diff --git a/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.x86_64.txt b/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.x86_64.txt index bd5f2dca66..86698c69fa 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.x86_64.txt +++ b/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.x86_64.txt @@ -57,7 +57,6 @@ libxcb # for grafana libXcursor # for grafana libXt # for grafana logrotate -logstash-oss-7.12.0 net-tools nfs-utils nmap-ncat diff --git a/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-20.04/requirements.x86_64.txt b/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-20.04/requirements.x86_64.txt index 44e8c0db06..a37ec22874 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-20.04/requirements.x86_64.txt +++ b/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-20.04/requirements.x86_64.txt @@ -54,7 +54,6 @@ libdpkg-perl libfontconfig1 logrotate -logstash-oss 1:7.12.0 netcat net-tools nfs-common diff --git a/docs/changelogs/CHANGELOG-2.0.md b/docs/changelogs/CHANGELOG-2.0.md index ea10bcc71b..5f5095ea28 100644 --- a/docs/changelogs/CHANGELOG-2.0.md +++ b/docs/changelogs/CHANGELOG-2.0.md @@ -11,6 +11,7 @@ ### Removed - [#2834](https://github.com/epiphany-platform/epiphany/issues/2834) - Removal of Hashicorp Vault component +- [#2833](https://github.com/epiphany-platform/epiphany/issues/2833) - Removal of Logstash component - [#2836](https://github.com/epiphany-platform/epiphany/issues/2836) - Removal of Istio component - [#2837](https://github.com/epiphany-platform/epiphany/issues/2837) - Removal of Apache Ignite component diff --git a/docs/home/COMPONENTS.md b/docs/home/COMPONENTS.md index 8e564009d3..eeba7ccd71 100644 --- a/docs/home/COMPONENTS.md +++ b/docs/home/COMPONENTS.md @@ -24,7 +24,6 @@ Note that versions are default versions and can be changed in certain cases thro | Opendistro for Elasticsearch Kibana | 1.13.1 | https://opendistro.github.io/for-elasticsearch-docs/docs/kibana/ | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | | Filebeat | 7.9.2 | https://github.com/elastic/beats | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | | Filebeat Helm Chart | 7.9.2 | https://github.com/elastic/helm-charts | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | -| Logstash OSS | 7.12.0 | https://github.com/elastic/logstash | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | | Prometheus | 2.31.1 | https://github.com/prometheus/prometheus | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | | Grafana | 8.3.2 | https://github.com/grafana/grafana | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | | Node Exporter | 1.3.1 | https://github.com/prometheus/node_exporter | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | diff --git a/docs/home/DEPRECATION-NOTE.md b/docs/home/DEPRECATION-NOTE.md index 710a1af92d..f27d2e17ed 100644 --- a/docs/home/DEPRECATION-NOTE.md +++ b/docs/home/DEPRECATION-NOTE.md @@ -26,11 +26,9 @@ Apache Ignite is prone to 1.x log4j issue. Removed: Epiphany 2.0 Issue: [2837](https://github.com/epiphany-platform/epiphany/issues/2837) -The list of components to be deprecated: - - Logstash The functionality that the Logstash is serving in Epiphany - exporting csv can right now be achieved using the Open Distro for Elasticsearch plugin. The Logstash in this version is prone to log4j issue and this is also the reason why should be removed. -Planed removal: Epiphany 2.0 +Removed: Epiphany 2.0 Issue: [2833](https://github.com/epiphany-platform/epiphany/issues/2833) diff --git a/docs/home/HOWTO.md b/docs/home/HOWTO.md index 81bb4659bd..17c8ed78da 100644 --- a/docs/home/HOWTO.md +++ b/docs/home/HOWTO.md @@ -91,7 +91,6 @@ - [How to set up PostgreSQL audit logging](./howto/DATABASES.md#how-to-set-up-postgresql-audit-logging) - [How to start working with OpenDistro for Elasticsearch](./howto/DATABASES.md#how-to-start-working-with-opendistro-for-elasticsearch) - [How to manage Opendistro for Elasticsearch data](./howto/LOGGING.md#how-to-manage-opendistro-for-elasticsearch-data) - - [How to export Elasticsearch data to csv format](./howto/LOGGING.md#how-to-export-elasticsearch-data-to-csv-format) - [Backup and Recovery](./howto/BACKUP.md) - [Epiphany backup and restore](./howto/BACKUP.md#epiphany-backup-and-restore) @@ -111,7 +110,6 @@ - [Centralized Logging](./howto/LOGGING.md) - [Centralized logging setup](./howto/LOGGING.md#centralized-logging-setup) - - [How to export Elasticsearch data to csv format](./howto/LOGGING.md#how-to-export-elasticsearch-data-to-csv-format) - [How to add multiline support for Filebeat logs](./howto/LOGGING.md#how-to-add-multiline-support-for-filebeat-logs) - [How to deploy Filebeat as Daemonset in K8s](./howto/LOGGING.md#how-to-deploy-filebeat-as-daemonset-in-k8s) diff --git a/docs/home/howto/LOGGING.md b/docs/home/howto/LOGGING.md index 806a183eeb..e419b2a543 100644 --- a/docs/home/howto/LOGGING.md +++ b/docs/home/howto/LOGGING.md @@ -33,9 +33,6 @@ logging: ... ``` -> Optional feature (role) available for logging: **logstash** -> more details here: [link](https://github.com/epiphany-platform/epiphany/blob/develop/docs/home/howto/LOGGING.md#how-to-export-elasticsearch-data-to-csv-format) - The `logging` role replaced `elasticsearch` role. This change was done to enable Elasticsearch usage also for data storage - not only for logs as it was till 0.5.0. @@ -223,53 +220,6 @@ Check more details about the plugin and how to export reports in the You can easily check enabled default plugins for Kibana using the following command on the logging machine: `./bin/kibana-plugin list` in Kibana directory. -## How to export Elasticsearch data to CSV format - -Since v0.8 Epiphany provides the possibility to export data from Elasticsearch to CSV using Logstash *(logstash-oss)* along -with *logstash-input-elasticsearch* and *logstash-output-csv* plugins. - -To install Logstash in your cluster add **logstash** to feature mapping for *logging, opendistro_for_elasticsearch* or * -elasticsearch* group. - ---- -**NOTE** - -To check plugin versions following command can be used - -```shell -/usr/share/logstash/bin/logstash-plugin list --verbose -``` - ---- - -Epiphany provides a basic configuration file `(logstash-export.conf.template)` as template for your data export. This -file has to be modified according to your Elasticsearch configuration and data you want to export. - ---- -**NOTE** - -Exporting data is not automated. It has to be invoked manually. Logstash daemon is disabled by default after -installation. - ---- - -Run Logstash to export data: -`/usr/share/logstash/bin/logstash -f /etc/logstash/logstash-export.conf` - -More details about configuration -of [input](https://www.elastic.co/guide/en/logstash/current/plugins-inputs-elasticsearch.html) -and [output](https://www.elastic.co/guide/en/logstash/current/plugins-outputs-csv.html) plugins. - ---- -**NOTE** - -At the moment input plugin doesn't officially support skipping certificate validation for secure connection to -Elasticsearch. For non-production environment you can easily disable it by adding new line: - -`ssl_options[:verify] = false` right after other ssl_options definitions in file: - -`/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-elasticsearch-*/lib/logstash/inputs/elasticsearch.rb` - --- ## How to add multiline support for Filebeat logs diff --git a/schema/common/defaults/configuration/feature-mapping.yml b/schema/common/defaults/configuration/feature-mapping.yml index a08d6605be..8050b2b9e3 100644 --- a/schema/common/defaults/configuration/feature-mapping.yml +++ b/schema/common/defaults/configuration/feature-mapping.yml @@ -25,8 +25,6 @@ specification: enabled: true - name: filebeat enabled: true - - name: logstash - enabled: true - name: prometheus enabled: true - name: grafana diff --git a/schema/common/defaults/configuration/logstash.yml b/schema/common/defaults/configuration/logstash.yml deleted file mode 100644 index 6d463a81d4..0000000000 --- a/schema/common/defaults/configuration/logstash.yml +++ /dev/null @@ -1,4 +0,0 @@ -kind: configuration/logstash -title: "Logstash" -name: default -specification: {} diff --git a/schema/common/validation/configuration/logstash.yml b/schema/common/validation/configuration/logstash.yml deleted file mode 100644 index b7e3f6cf9c..0000000000 --- a/schema/common/validation/configuration/logstash.yml +++ /dev/null @@ -1,4 +0,0 @@ -"$id": "#/specification" -title: "Logstash specification schema" -description: "Logstash specification schema" -type: object