diff --git a/.dockerignore b/.dockerignore index bacfb3817f..91f05020c8 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,15 +1,13 @@ -# Ignore all unwanted files from the docker build process - -.git -.git* -*.log -certs/ -**/data/* - -*.md - -tests/ -Dockerfile - -core/data/ -core/build/ +# Exclude unnecessary files from the build context +# This file is meant to be used as a whitelist + +# Exclude everything +* + +# Include the following +!README.md +!LICENSE +!core/src/epicli/.devcontainer/requirements.txt +!core/src/epicli/cli/ +!core/src/epicli/data/ +!core/src/epicli/setup.py diff --git a/CHANGELOG-0.8.md b/CHANGELOG-0.8.md index 042ccd993f..e1251fbf98 100644 --- a/CHANGELOG-0.8.md +++ b/CHANGELOG-0.8.md @@ -5,8 +5,10 @@ ### Added - [#1302](https://github.com/epiphany-platform/epiphany/issues/1302) - Ability to update control plane certificates expiration date +- [#1324](https://github.com/epiphany-platform/epiphany/issues/1324) - Added Logstash to export data from Elasticsearch to csv format - [#1300](https://github.com/epiphany-platform/epiphany/issues/1300) - Configure OpenSSH according to Mozilla Infosec guidance - [#1543](https://github.com/epiphany-platform/epiphany/issues/1543) - Add support for Azure availability sets +- [#1609](https://github.com/epiphany-platform/epiphany/issues/1609) - Build epicli image using Dockerfile only (without shell script) ### Updated diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000..1aaee7e1d1 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,47 @@ +# === Build epicli wheel file === + +FROM python:3.7-slim AS build-epicli-wheel + +COPY . /src +WORKDIR /src/core/src/epicli + +RUN python setup.py bdist_wheel + +# === Build final image === + +FROM python:3.7-slim + +ARG USERNAME=epiuser +ARG USER_UID=1000 +ARG USER_GID=$USER_UID + +ENV EPICLI_DOCKER_SHARED_DIR=/shared + +COPY --from=build-epicli-wheel /src/core/src/epicli/dist/ /epicli/ + +RUN apt-get update \ + && apt-get install --no-install-recommends -y \ + gcc libffi-dev make musl-dev openssh-client ruby-full sudo tar unzip vim \ + && gem install \ + rake rspec_junit_formatter serverspec \ + && pip install --disable-pip-version-check --no-cache-dir \ + /epicli/epicli-*-py3-none-any.whl \ +\ + && apt-get autoremove -y \ + && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* \ +\ + && groupadd --gid $USER_GID $USERNAME \ + && useradd -s /bin/bash --uid $USER_UID --gid $USER_GID -m $USERNAME \ + && echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \ + && chmod 0440 /etc/sudoers.d/$USERNAME \ +\ + && mkdir -p $EPICLI_DOCKER_SHARED_DIR \ + && chown $USERNAME $EPICLI_DOCKER_SHARED_DIR \ + && chmod g+w $EPICLI_DOCKER_SHARED_DIR + +WORKDIR $EPICLI_DOCKER_SHARED_DIR + +USER $USERNAME + +ENTRYPOINT ["/bin/bash"] diff --git a/core/src/epicli/.devcontainer/Dockerfile b/core/src/epicli/.devcontainer/Dockerfile index 2dc9152d16..dbe41ffc84 100644 --- a/core/src/epicli/.devcontainer/Dockerfile +++ b/core/src/epicli/.devcontainer/Dockerfile @@ -14,31 +14,32 @@ COPY requirements.txt / RUN chmod +x /config-pre.sh \ && /bin/bash -c /config-pre.sh \ - +\ && apt-get update \ - && apt-get -y install --no-install-recommends apt-utils dialog 2>&1 \ - - && apt-get -y install git git-lfs procps lsb-release gcc make musl-dev libffi-dev tar unzip vim \ - - && apt-get -y install ruby-full \ - - && gem install serverspec rake rspec_junit_formatter \ - - && pip --disable-pip-version-check --no-cache-dir install -r /requirements.txt --default-timeout=100 \ - - && pip --disable-pip-version-check --no-cache-dir install pipenv pylint pytest wheel setuptools twine --default-timeout=100 \ - + && apt-get install --no-install-recommends -y \ + apt-utils dialog 2>&1 \ + && apt-get install --no-install-recommends -y \ + gcc libffi-dev make musl-dev openssh-client ruby-full sudo tar unzip vim \ + \ + git git-lfs lsb-release procps \ +\ + && gem install \ + rake rspec_junit_formatter serverspec \ +\ + && pip install --disable-pip-version-check --no-cache-dir --default-timeout=100 \ + --requirement /requirements.txt \ + && pip install --disable-pip-version-check --no-cache-dir --default-timeout=100 \ + pipenv pylint pytest setuptools twine wheel \ +\ + && apt-get autoremove -y \ + && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* \ +\ && groupadd --gid $USER_GID $USERNAME \ && useradd -s /bin/bash --uid $USER_UID --gid $USER_GID -m $USERNAME \ - - && apt-get install -y sudo \ && echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \ && chmod 0440 /etc/sudoers.d/$USERNAME \ - - && apt-get autoremove -y \ - && apt-get clean -y \ - && rm -rf /var/lib/apt/lists/* \ - +\ && chmod +x /config-post.sh \ && /bin/bash -c /config-post.sh diff --git a/core/src/epicli/Dockerfile b/core/src/epicli/Dockerfile deleted file mode 100644 index 2fd68a481d..0000000000 --- a/core/src/epicli/Dockerfile +++ /dev/null @@ -1,37 +0,0 @@ -FROM python:3.7-slim - -ARG EPICLI_VERSION -ARG USERNAME=epiuser -ARG USER_UID=1000 -ARG USER_GID=$USER_UID - -ENV EPICLI_DOCKER_SHARED_DIR=/shared - -COPY /dist/ /epicli -WORKDIR /epicli - -RUN apt-get update \ - && apt-get -y install gcc make musl-dev libffi-dev tar unzip openssh-client vim \ - && apt-get -y install ruby-full \ - && gem install serverspec rake rspec_junit_formatter - -RUN pip install epicli-${EPICLI_VERSION}-py3-none-any.whl - -RUN groupadd --gid $USER_GID $USERNAME \ - && useradd -s /bin/bash --uid $USER_UID --gid $USER_GID -m $USERNAME \ - && apt-get install -y sudo \ - && echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \ - && chmod 0440 /etc/sudoers.d/$USERNAME \ - && apt-get autoremove -y \ - && apt-get clean -y \ - && rm -rf /var/lib/apt/lists/* - -RUN mkdir $EPICLI_DOCKER_SHARED_DIR \ - ;chown :$USERNAME $EPICLI_DOCKER_SHARED_DIR \ - && chmod g+w $EPICLI_DOCKER_SHARED_DIR - -WORKDIR $EPICLI_DOCKER_SHARED_DIR - -USER $USERNAME - -ENTRYPOINT ["/bin/bash"] \ No newline at end of file diff --git a/core/src/epicli/build-docker.bat b/core/src/epicli/build-docker.bat deleted file mode 100644 index b7433c3842..0000000000 --- a/core/src/epicli/build-docker.bat +++ /dev/null @@ -1,5 +0,0 @@ -:: Script to build the Epicli docker image. It wil build the Epicli distribtion wheel -:: and from use that to provision the docker image. -call build-wheel.bat -set /p EPICLI_VERSION= "HOST_IP:ES_HTTP_PORT" + ssl => "BOOLEAN" + ca_file => "CA_FILE.pem" + user => "USER" + password => "PASSWORD" + index => "INDEX_NAME" + query => ' + { + "query": { + "match_all": {} + } + } + ' + } +} +output { + csv { + fields => ["FIELDS_TO_EXPORT"] + path => "PATH" + } +} diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/logstash/tasks/Debian.yml b/core/src/epicli/data/common/ansible/playbooks/roles/logstash/tasks/Debian.yml new file mode 100644 index 0000000000..ca9bbefea8 --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/logstash/tasks/Debian.yml @@ -0,0 +1,7 @@ +--- +- name: Install Logstash package + apt: + name: + - logstash-oss={{ versions[ansible_os_family] }} + update_cache: yes + state: present diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/logstash/tasks/RedHat.yml b/core/src/epicli/data/common/ansible/playbooks/roles/logstash/tasks/RedHat.yml new file mode 100644 index 0000000000..d7129b36c0 --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/logstash/tasks/RedHat.yml @@ -0,0 +1,7 @@ +--- +- name: Install Logstash package + yum: + name: + - logstash-oss-{{ versions[ansible_os_family] }} + update_cache: yes + state: present diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/logstash/tasks/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/logstash/tasks/main.yml new file mode 100644 index 0000000000..a58e189e2d --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/logstash/tasks/main.yml @@ -0,0 +1,14 @@ +--- +- name: Install logstash-oss + include_tasks: "{{ ansible_os_family }}.yml" + +- name: Ensure Logstash is disabled + systemd: + name: logstash + enabled: no + state: stopped + +- name: Provide logstash-export.conf template file + copy: + src: logstash-export.conf.template + dest: /etc/logstash/logstash-export.conf.template diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.txt b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.txt index 8614f34066..9c6f04d875 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.txt +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.txt @@ -57,6 +57,7 @@ libxcb # for grafana libXcursor # for grafana libXt # for grafana logrotate +logstash-oss-7.8.1 net-tools nfs-utils nmap-ncat diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.txt b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.txt index fb4276d787..f770d802f8 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.txt +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.txt @@ -56,6 +56,7 @@ libxcb # for grafana libXcursor # for grafana libXt # for grafana logrotate +logstash-oss-7.8.1 net-tools nfs-utils nmap-ncat diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.txt b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.txt index 7a261a5bf2..f409f30c73 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.txt +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.txt @@ -29,6 +29,7 @@ init-system-helpers jq libfontconfig1 logrotate +logstash-oss 1:7.8.1 netcat net-tools nfs-common diff --git a/core/src/epicli/data/common/defaults/configuration/feature-mapping.yml b/core/src/epicli/data/common/defaults/configuration/feature-mapping.yml index c80090a713..689d9b6828 100644 --- a/core/src/epicli/data/common/defaults/configuration/feature-mapping.yml +++ b/core/src/epicli/data/common/defaults/configuration/feature-mapping.yml @@ -27,6 +27,8 @@ specification: enabled: yes - name: filebeat enabled: yes + - name: logstash + enabled: yes - name: prometheus enabled: yes - name: grafana diff --git a/core/src/epicli/data/common/defaults/configuration/logstash.yml b/core/src/epicli/data/common/defaults/configuration/logstash.yml new file mode 100644 index 0000000000..6d463a81d4 --- /dev/null +++ b/core/src/epicli/data/common/defaults/configuration/logstash.yml @@ -0,0 +1,4 @@ +kind: configuration/logstash +title: "Logstash" +name: default +specification: {} diff --git a/core/src/epicli/data/common/validation/configuration/logstash.yml b/core/src/epicli/data/common/validation/configuration/logstash.yml new file mode 100644 index 0000000000..e476595b41 --- /dev/null +++ b/core/src/epicli/data/common/validation/configuration/logstash.yml @@ -0,0 +1 @@ +$ref: '#/definitions/unvalidated_specification' diff --git a/docs/home/COMPONENTS.md b/docs/home/COMPONENTS.md index e6669ce973..b499efab18 100644 --- a/docs/home/COMPONENTS.md +++ b/docs/home/COMPONENTS.md @@ -21,7 +21,8 @@ Note that versions are default versions and can be changed in certain cases thro | Kibana | 6.5.4 | https://github.com/elastic/kibana | https://github.com/elastic/kibana/blob/master/LICENSE.txt | | Opendistro for Elasticsearch | 1.9.0 | https://opendistro.github.io/for-elasticsearch/ | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | | Opendistro for Elasticsearch Kibana | 1.9.0 | https://opendistro.github.io/for-elasticsearch-docs/docs/kibana/ | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | -| Filebeat | 7.8.1 | https://github.com/elastic/beats/tree/6.8 | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | +| Filebeat | 7.8.1 | https://github.com/elastic/beats | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | +| Logstash OSS | 7.8.1 | https://github.com/elastic/logstash | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | | Prometheus | 2.10.0 | https://github.com/prometheus/prometheus | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | | Grafana | 6.2.5 | https://github.com/grafana/grafana | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | | node_exporter | 0.16.0 | https://github.com/prometheus/node_exporter | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | diff --git a/docs/home/HOWTO.md b/docs/home/HOWTO.md index d616e8d87b..0b1877d5e0 100644 --- a/docs/home/HOWTO.md +++ b/docs/home/HOWTO.md @@ -60,11 +60,11 @@ - [How to use TLS/SSL certificate with RabbitMQ](./howto/SECURITY.md#how-to-use-tlsssl-certificate-with-rabbitmq) - [How to use TLS/SSL with Kafka](./howto/SECURITY.md#how-to-use-tlsssl-with-kafka) - [How to use TLS/SSL certificates for Kafka authentication](./howto/SECURITY.md#how-to-use-tlsssl-certificates-for-kafka-authentication) - - [How to use TLS/SSL certificates for Kafka authorization](./howto/SECURITY.md#how-to-use-tlsssl-certificates-for-kafka-authorization) + - [How to use TLS/SSL certificates for Kafka authorization](./howto/SECURITY.md#how-to-use-tlsssl-certificates-for-kafka-authorization) - [How to enable Azure disk encryption](./howto/SECURITY.md#how-to-enable-azure-disk-encryption) - [How to enable AWS disk encryption](./howto/SECURITY.md#how-to-enable-AWS-disk-encryption) - [How to use Kubernetes Secrets](./howto/SECURITY.md#how-to-use-kubernetes-secrets) - - [How to authenticate to Azure AD app](./howto/SECURITY.md#how-to-authenticate-to-azure-ad-app) + - [How to authenticate to Azure AD app](./howto/SECURITY.md#how-to-authenticate-to-azure-ad-app) - [How to run epicli with password](./howto/SECURITY.md#how-to-run-epicli-with-password) - [How to enable kubectl on Kubernetes master](./howto/SECURITY.md#how-to-enable-kubectl-on-Kubernetes-master) - [How to turn on Hashicorp Vault functionality](./howto/SECURITY.md#how-to-turn-on-hashicorp-vault-functionality) @@ -80,6 +80,7 @@ - [How to set up PGBouncer, PgPool and PostgreSQL parameters](./howto/DATABASES.md#how-to-set-up-pgbouncer-pgpool-and-postgresql-parameters) - [How to set up PostgreSQL audit logging](./howto/DATABASES.md#how-to-set-up-postgresql-audit-logging) - [How to start working with OpenDistro for Elasticsearch](./howto/DATABASES.md#how-to-start-working-with-opendistro-for-elasticsearch) + - [How to export Elasticsearch data to csv format](./howto/LOGGING.md##how-to-export-elasticsearch-data-to-csv-format) - [How to start working with Apache Ignite Stateful setup](./howto/DATABASES.md#how-to-start-working-with-apache-ignite-stateful-setup) - [How to start working with Apache Ignite Stateless setup](./howto/DATABASES.md#how-to-start-working-with-apache-ignite-stateless-setup) @@ -101,3 +102,4 @@ - [Centralized Logging](./howto/LOGGING.md) - [Centralized logging setup](./howto/LOGGING.md#centralized-logging-setup) + - [How to export Elasticsearch data to csv format](./howto/LOGGING.md#how-to-export-elasticsearch-data-to-csv-format) diff --git a/docs/home/howto/LOGGING.md b/docs/home/howto/LOGGING.md index a30434e189..c84715d560 100644 --- a/docs/home/howto/LOGGING.md +++ b/docs/home/howto/LOGGING.md @@ -30,6 +30,9 @@ specification: - firewall ... ``` +>Optional feature (role) available for logging: **logstash** +>more details here: [link](https://github.com/epiphany-platform/epiphany/blob/develop/docs/home/howto/LOGGING.md#how-to-export-elasticsearch-data-to-csv-format) + The `logging` role replaced `elasticsearch` role. This change was done to enable Elasticsearch usage also for data storage - not only for logs as it was till 0.5.0. Default configuration of `logging` and `opendistro_for_elasticsearch` roles is identical (./DATABASES.md#how-to-start-working-with-opendistro-for-elasticsearch). To modify configuration of centralized logging adjust and use the following defaults in your manifest: @@ -46,3 +49,28 @@ specification: repo: /var/lib/elasticsearch-snapshots logs: /var/log/elasticsearch ``` + +### How to export Elasticsearch data to csv format +Since v0.8 Epiphany provide posibility to export data from Elasticsearch to CSV using Logstash *(logstash-oss v7.8.1*) along with *logstash-input-elasticsearch (v4.6.2)* and *logstash-output-csv (v3.0.8)* plugin. + +To install Logstash in your cluster add **logstash** to feature mapping for *logging, opendistro_for_elasticsearch* or *elasticsearch* group. + +Epiphany provides a basic configuration file `(logstash-export.conf.template)` as template for your data export. +This file has to be modified according to your Elasticsearch configuration and data you want to export. + +`Note: Exporting data is not automated. It has to be invoked manually. Logstash daemon is disabled by default after installation.` + +Run Logstash to export data: +`/usr/share/logstash/bin/logstash -f /etc/logstash/logstash-export.conf` + +More details about configuration of input plugin: +https://www.elastic.co/guide/en/logstash/current/plugins-inputs-elasticsearch.html + +More details about configuration of output plugin: +https://www.elastic.co/guide/en/logstash/current/plugins-outputs-csv.html + +Note: Currently input plugin doesn't officialy support skipping certificate validation for secure connection to Elasticsearch. + +For non-production environment you can easly disable it by adding new line: +`ssl_options[:verify] = false` right after other ssl_options definitions in file: +`/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-elasticsearch-4.6.2/lib/logstash/inputs/elasticsearch.rb` diff --git a/docs/home/howto/PREREQUISITES.md b/docs/home/howto/PREREQUISITES.md index 65a5d5f4cf..7fcf2ec265 100644 --- a/docs/home/howto/PREREQUISITES.md +++ b/docs/home/howto/PREREQUISITES.md @@ -6,59 +6,38 @@ There are 2 ways to get the image, build it locally yourself or pull it from the 1. Install the following dependencies: - - Python 3.7 - - PIP - Docker -2. Install the following Python dependencies using PIP: +2. Open a terminal in the root directory of the Epiphany source code and run: - ```bash - pip install wheel setuptools twine - ``` - -3. Open a terminal in `/core/src/epicli` and run: - - On Linux/Mac: - - ```bash - ./build-docker.sh debian|alpine - ``` - - On windows: - - ```bash - ./build-docker.bat debian|alpine - ``` +```bash +TAG=$(cat core/src/epicli/cli/version.txt.py) +docker build --file Dockerfile --tag epicli:${TAG} . +``` -*Note: Use the debian or alpine flag to indicate which base image you want to use for the Epicli container.* - ### Pull Epicli image from the registry ```bash docker pull epiphanyplatform/epicli:TAG ``` +Where `TAG` should be replaced with an existing tag. + *Check [here](https://cloud.docker.com/u/epiphanyplatform/repository/docker/epiphanyplatform/epicli) for the available tags.* ### Running the Epicli image To run the image: -Locally build: - -```bash -docker run -it -v LOCAL_DIR:/shared --rm epicli -``` - -Pulled: - ```bash docker run -it -v LOCAL_DIR:/shared --rm epiphanyplatform/epicli:TAG ``` -*Check [here](https://cloud.docker.com/u/epiphanyplatform/repository/docker/epiphanyplatform/epicli) for the available tags.* +Where: +- `LOCAL_DIR` should be replaced with the local path to the directory for Epicli input (SSH keys, data yamls) and output (logs, build states), +- `TAG` should be replaced with an existing tag. -Where `LOCAL_DIR` should be replaced with the local path to the directory for Epicli input (SSH keys, data yamls) and output (logs, build states). +*Check [here](https://cloud.docker.com/u/epiphanyplatform/repository/docker/epiphanyplatform/epicli) for the available tags.* ## Epicli development