From efa5c90b8d489e7f7d5ca1e2b9d811eeccbbf1bd Mon Sep 17 00:00:00 2001 From: Luuk van Venrooij <11056665+seriva@users.noreply.github.com> Date: Fri, 11 Oct 2019 17:27:16 +0200 Subject: [PATCH] Merge offline into develop. (#603) * Added download role * Kubernetes install packages, dependencies list added * Loading images from download cache- Kubernetes role * Docker installation from offline * Added download role * Commit to store prototype (temporary changes) * Updated prototype * sync -> copy + improvements * The same package for multiple OS distros handled * Grafana offline mode install, docker offline fixes * ELK, HAProxy, Postgresql moved to offline installation * RabbitMQ package installation offline mode * HAProxy experimental binary install (offline mode) * RabbitMQ offline, HAProxy offline binary install (#473) * RabbitMQ package installation offline mode * HAProxy experimental binary install (offline mode) * Added links to kibana, haproxy, filebeat, elasticsearch, grafana, docker packages (#474) * Added links to haproxy_exporter and rabbitmq packages (#476) * Offline mode for (file install): elacticsearch-curator, haproxy_exporter, jmx_exporter, kafka, node_exporter * Fixes for downloading packages and installing Docker on Ubuntu (#480) * Updated variable names for dest dirs * Fixes for downloading packages * Prometheus, AlertManager and ZooKeeper move to offline mode * Tasks for downloading all files and improvements (#412) * Added links to postgresql, zookeeper, kafka, prometheus, exporter packages (#484) * Exact version match (#485) * Exact version match (#412) * Simplified setting properties (#412) * Fix: Prevent docs duplication in manifest * Remote_src fix in unarchive, haproxy and haproxy_exporter validated * [Offline-mode] ELK stack updated * Fixes for offline mode with packages and files * Kubernetes installation of deb package fix * Download all packages + fix for admin_user (#504) * Feature/offline-mode - added image download capability (#486) * image download capability added * multiple fixes and multi-arch support added * Added links to Kubernetes packages (#502) * Removing the test environment destruction function * Added links to kibana, haproxy, filebeat, elasticsearch, grafana, docker packages * Added links to haproxy_exporter and rabbitmq packages * Added links to haproxy_exporter and rabbitmq packages * Added links to postgresql packages * Added links to postgresql packages * Added links to postgresql packages * Added links to zookeeper, kafka, exporter packages * Added links to Kubernetes packages * replaced sha512 with file_name * More fixes to offline mode (#503) * Fix: Prevent docs duplication in manifest * Remote_src fix in unarchive, haproxy and haproxy_exporter validated * [Offline-mode] ELK stack updated * Fixes for offline mode with packages and files * Kubernetes installation of deb package fix * Removed test data (#412) (#505) * Feature/offline-mode - image download tweaks, added configuration example (#507) * change url to full docker registry path with version tag, added configuration examples * removal of unnecessary quotations, silenced ansible warning * Kafka, Kafka exporter, postgresql and install role * Add offline-mode flag to epicli * Add offline mode to AnsibleVarsGenerator * Add offline flag for epicli (#510) * Add offline-mode flag to epicli * Add offline mode to AnsibleVarsGenerator * Installing packages + files + images using artifacts repository * Installing packages + files + images using artifacts repository (#512) * Rewrote: postgress, haproxy and haproxy_exporter. * Prometheus, Grafana, Exporters, Kafka, Zookeeper to use private repo * Kafka exporter moved to private repo * Rewrote: postgress, haproxy and haproxy_exporter. (#514) * Missing flannel image definition in K8s configuration * Download role cleanup * Install role cleanup * Add stub of repositories role * Fixed feature mapping yaml * Updated elk, Kibana, Filebeat, elk-curator. * Added file_name for haproxy-exporter * Roles to use private repo (#515) * Installing packages + files + images using artifacts repository * Prometheus, Grafana, Exporters, Kafka, Zookeeper to use private repo * Kafka exporter moved to private repo * Missing flannel image definition in K8s configuration * Download role cleanup * Install role cleanup * Fixed feature mapping yaml * Feature/offline mode (#516) - Updated elk, Kibana, Filebeat, elk-curator - Added file_name for haproxy-exporter * Add setup and teardown to Epiphany role * Add integration to repository role with epicli * Change repository configuration to take files from kubernetes master server * Fixed remove of node_exporter endpoint accessibility on kubernetes master for default configuration * Add offline/online mode for Red Hat (#519) * Added prepare command to prepare offline repo scripts. * - Updated RabbitMQ roll to pull stuff from repo. * Add scopeo copying (#521) * Add offline/online mode for Red Hat * Add skopeo copy and stub of task for Debian * Copy scopeo to upload dir * Add new structure of files to repository role (#522) * Add offline/online mode for Red Hat * Add skopeo copy and stub of task for Debian * Copy scopeo to upload dir * Changed repository script structure * Move repository scripts * Add directories copy for new data structure * Fixes for Red Hat script * Refactor of repository role. * Refactored stuff. * download-requirements.sh for CentOS-7 (#513) (#526) download-requirements.sh for CentOS-7 * Started refactoring repository role. (#527) * Refactor of repository role. * Refactored stuff. * Run download-requirements.sh from any location (#528) Run download-requirements.sh from any location * Move Skopeo for copying to repo host and add message for long running repo task. * Feature/offline mode (#529) - Move Skopeo for copying to repo host and add message for long running repo task. * Offline progress - Changed offline_mode to offline_requirements accepting a path param. - Removed old offline mode code - Removed useless checks for AnsibleVarsGenerator.py * Minor message update. * Feature/offline mode (#530) - Changed offline_mode to offline_requirements accepting a path param. - Removed old offline mode code - Removed useless checks for AnsibleVarsGenerator.py * Offline progress - Fix for local epi-repo path. - Cleaned up old code. * Feature/offline mode (#531) - Fix for local epi-repo path. - Cleaned up old code. * Refactoring repository role (#532) - Refactoring repository role * Changes in repository role (#533) * Redirect STDERR to STDOUT for logging * Fix for packages containing 'error' in name * Added Perl as dependency for vim * Fix for 'ERROR 403: Forbidden' on AWS * Added python-firewall for firewalld * Added dependencies for vim * - Minor fixes after merge with develop * Fixed version numbers. * More dependencies for vim (#538) * Merged develop into feature/offline + some fixes (#539) * Updated documentation - Added changelog - Added versions to for components - Minor documentation updates - Removed unused documentation * Fixed links. * Fixed changelog. * Added node_exporter port known issue. * Feature/skopeo (#475) * Updated documentation - Added changelog - Added versions to for components - Minor documentation updates - Removed unused documentation * Fixed links. * Fixed changelog. * Added node_exporter port known issue. * - Added Skopeo minor update to devcontainer. * subnets, network interfaces, security rules, ansible inventory (#469) Fixes for subnets Fixed versions for Terraform providers for both Azure and AWS Added VMs Added network interfaces Added security rules Added ansible inventory generation * File generating hashes for directory (#477) * Item: #422 Desc: Add missing ports for prometheus and grafana * Item: #0000 Desc: File hash generator for directory * Item: #0000 Desc: Moved folder to new structure * Testruns (#506) - Added running of python unit tests via debug config in VSCode - Added running of serverspec tests via debug config in VSCode - Documentation * Release prep part 1 (#501) * Updated documentation - Added changelog - Added versions to for components - Minor documentation updates - Removed unused documentation * Fixed links. * Fixed changelog. * Added node_exporter port known issue. * Preparation for release. * Feature/azure-k8n-storage (#487) - Added shared storage for k8n * Azure rehat fixes (#511) Fix for container-selinux package on redhat. Fixes for HAProxy and Posgress on Azure. Fix for running tests from VSCode. * Fix typo rabbitmq (#518) * Fix/rabbitmq error (#523) - Fixed typo for rabbitmq machine * Fixed link to container-selinux package (legacy) (#525) * Fixed link to container-selinux package * Refactor of repository role. * Refactored stuff. * Move Skopeo for copying to repo host and add message for long running repo task. * Offline progress - Changed offline_mode to offline_requirements accepting a path param. - Removed old offline mode code - Removed useless checks for AnsibleVarsGenerator.py * Minor message update. * Offline progress - Fix for local epi-repo path. - Cleaned up old code. * - Minor fixes after merge with develop * Fixed version numbers. * Debian scripts for download * Reverted temp changes * Review changes * Changed scripts fore Debian packages (#540) * Installing packages + files + images using artifacts repository * Prometheus, Grafana, Exporters, Kafka, Zookeeper to use private repo * Kafka exporter moved to private repo * Missing flannel image definition in K8s configuration * Download role cleanup * Install role cleanup * Fix for dependencies to be downloaded (#544) * uniq -> 'sort --unique' * Updated package dependencies * Remove trailing spaces * java-1.8.0-openjdk -> java-1.8.0-openjdk-headless * Refactored repository role, changes for RHEL (#547) * Refactored repository role, changes for RHEL #536 * Changed directories structure for tasks #536 * Added repository role to feature mappings #536 * Fix for removing flag file if expired #536 * Added wget #536 * Minor fix for prepare when output dir is not supplied. * Minor fix for prepare paths (#550) - Minor fix for prepare when output dir is not supplied. * Offline mode - progress for RedHat (#552) * Added --log-to-journal option #536 * download-requirements script logs to journal #536 * Added python-slip-dbus for firewalld on AWS #536 * Disabled repository role in feature mappings #536 * Static version for docker-ce #536 * download-reqs: yum makecache fast -> yum makecache #536 * Do not install epel-release #536 * Download latest versions of dependencies #536 * Optimizations and better naming #536 * Colons -> hyphens in file names of images #536 * jmx_prometheus_javaagent v0.12.0 #536 * Better task names in zookeeper role * Offline mode progress (#555) * yum makecache -> yum makecache fast #536 * Added versions for docker-ce & docker-ce-cli #536 * Added property 'download_done_flag_expire_minutes' #536 * enabled-system-repos.txt in /var/tmp #536 * Clean up temporary files #536 * Added execution time #536 * Fixes for Azure in repository role #536 * Fixes for centos offline mode * Fixes for offline centos (#577) * Installing packages + files + images using artifacts repository * Prometheus, Grafana, Exporters, Kafka, Zookeeper to use private repo * Kafka exporter moved to private repo * Missing flannel image definition in K8s configuration * Download role cleanup * Install role cleanup * Fixed feature mapping yaml * Debian scripts for download * Reverted temp changes * Review changes * Fixes for centos offline mode * Fixes for offline mode for RedHat after testing (#579) * Improvements after testing #536 * Rabbitmq and Erlang in fixed version #536 * Added --cacheonly to yum repolist #536 * Removed old structure #536 * Local docker registry for controlplane images * Change comment for custom_image_registry Co-Authored-By: to-bar <46519524+to-bar@users.noreply.github.com> * Skip downloading when image exists #536 * Fix in repository teardown #536 * Missing coredns images, flannel installation fix * WIP: Feature/offline Ubuntu part (#580) * offline repo for ubuntu - merge scripts into repository role - initial support * requirements and repo creation fixes * ubuntu offline-online installation works * apt-cache policy not needed here * cleanups * added missing bash error handling * added removal of 3rd party repos, other cleanups, install wget and gpg for minimal OS * fix for running script from different location (#583) * Fixes for offline mode for RedHat and CentOS (#585) * Fix for hosts without yum-utils package #536 * Fixes for repo prereqs and image permissions #536 * Offline mode Ubuntu part - fixes (#587) * fix for running script from different location * fixed skopeo .tar permissions to 644 * added missing dependencies for jq * Fix/offline ubu (#588) * fix for running script from different location * fixed skopeo .tar permissions to 644 * added missing dependencies for jq * bootstrap apache and dpkg-dev installation in air-gap mode * Requirements and repo creation for centos * Added missing file * Requirements and repo creation for centos (#589) * Requirements and repo creation for centos * Added missing file * Added NSG rules for image repository and package repository * Added security rules to AWS and fixed on Azure * Added security rules to AWS and fixed on Azure (#591) * Offline mode Ubuntu part - fixes for air-gap mode, install libdpkg-perl (#592) * fixes for air-gap mode, install libdpkg-perl * unified stderr to stdout bash pipe with rhel version of this script * convoluted comments are convoluted * added missing dependencies * Change name for RedHat filebeat * File name typo fix (#593) * Changed name to fix again... * Fixed name for RedHat in rabbitmq tasks. * Epicli validation and RabbitMQ and Postgress cluster fixes. (#595) * Ensure httpd is running * Ensure httpd is running (#597) * Ensure httpd is running * Added killing of the previous instance for Ansible * Set up sysctl params for K8s * Fix/offline ubu (#599) * fixes for air-gap mode, install libdpkg-perl * unified stderr to stdout bash pipe with rhel version of this script * convoluted comments are convoluted * added missing dependencies * added dependencies for postgresql-10 and gpg/gnupg * Fixed default HAProxy config * Fix/offline last minute improvements (#601) * fixes for air-gap mode, install libdpkg-perl * unified stderr to stdout bash pipe with rhel version of this script * convoluted comments are convoluted * added missing dependencies * added dependencies for postgresql-10 and gpg/gnupg * restore system repos if missing, logging improvements * Fix in enable-system-repos.sh for RHEL (#604) * Merged develop into offline. (#605) * Fix/aws ports and naming (#576) * FIxes: - Init issues for different providers - Link in docs * Added ports for clustering Postgres and RabbitMQ * RabbitMQ test fix for AWS hostnames (#578) * Test fix - added regexp to eliminate false negatives in particular cases (#586) * Feature/doc updates (#584) * - Updated changelog * Added article for up/downscaling, clustering components. * Added article for offline installation. * removal of unneeded code from Ubuntu 16.04 era (#606) --- core/src/epicli/.gitignore | 2 +- core/src/epicli/cli/engine/BuildEngine.py | 4 +- core/src/epicli/cli/engine/PrepareEngine.py | 53 ++ .../cli/engine/ansible/AnsibleRunner.py | 24 +- .../engine/ansible/AnsibleVarsGenerator.py | 18 +- .../engine/schema/ConfigurationAppender.py | 23 +- core/src/epicli/cli/epicli.py | 25 + core/src/epicli/cli/helpers/Config.py | 22 +- .../infrastructure/virtual-machine.yml | 32 +- .../infrastructure/virtual-machine.yml | 20 + .../common/ansible/playbooks/download.yml | 8 + .../ansible/playbooks/elasticsearch.yml | 3 + .../playbooks/elasticsearch_curator.yml | 3 + .../common/ansible/playbooks/filebeat.yml | 4 + .../data/common/ansible/playbooks/grafana.yml | 3 + .../ansible/playbooks/group_vars/all.yml | 5 + .../data/common/ansible/playbooks/haproxy.yml | 4 +- .../ansible/playbooks/haproxy_exporter.yml | 4 +- .../ansible/playbooks/image_registry.yml | 12 + .../common/ansible/playbooks/jmx_exporter.yml | 4 + .../data/common/ansible/playbooks/kafka.yml | 4 + .../ansible/playbooks/kafka_exporter.yml | 4 + .../data/common/ansible/playbooks/kibana.yml | 4 + .../ansible/playbooks/kubernetes_master.yml | 4 + .../ansible/playbooks/node_exporter.yml | 3 + .../common/ansible/playbooks/postgresql.yml | 3 + .../common/ansible/playbooks/prometheus.yml | 4 + .../common/ansible/playbooks/repository.yml | 5 + .../ansible/playbooks/repository_setup.yml | 10 + .../ansible/playbooks/repository_teardown.yml | 10 + .../playbooks/roles/common/tasks/Debian.yml | 6 +- .../playbooks/roles/common/tasks/RedHat.yml | 12 +- .../playbooks/roles/common/tasks/main.yml | 37 - .../playbooks/roles/docker/defaults/main.yml | 2 + .../playbooks/roles/docker/tasks/Debian.yml | 26 - .../playbooks/roles/docker/tasks/RedHat.yml | 28 - .../playbooks/roles/docker/tasks/main.yml | 21 +- .../roles/docker/templates/daemon.json.j2 | 5 + .../roles/download/tasks/download_file.yml | 12 + .../roles/download/tasks/download_image.yml | 12 + .../playbooks/roles/download/tasks/main.yml | 3 + .../roles/elasticsearch/tasks/Debian.yml | 38 +- .../roles/elasticsearch/tasks/RedHat.yml | 37 +- .../roles/elasticsearch/tasks/main.yml | 21 +- .../elasticsearch_curator/tasks/Debian.yml | 13 +- .../elasticsearch_curator/tasks/RedHat.yml | 10 +- .../elasticsearch_curator/tasks/main.yml | 2 +- .../playbooks/roles/filebeat/tasks/Debian.yml | 11 +- .../playbooks/roles/filebeat/tasks/RedHat.yml | 11 +- .../playbooks/roles/filebeat/tasks/main.yml | 4 +- .../playbooks/roles/grafana/tasks/install.yml | 71 +- .../playbooks/roles/grafana/tasks/main.yml | 14 - .../grafana/templates/grafana_rh.repo.j2 | 10 - .../playbooks/roles/grafana/vars/debian.yml | 8 - .../playbooks/roles/grafana/vars/redhat.yml | 3 - .../playbooks/roles/haproxy/tasks/RedHat.yml | 25 - .../roles/haproxy_exporter/tasks/main.yml | 69 +- .../roles/image_registry/meta/main.yml | 3 + .../roles/image_registry/tasks/main.yml | 32 + .../roles/jmx_exporter/tasks/main.yml | 42 +- .../playbooks/roles/kafka/tasks/main.yml | 4 +- .../roles/kafka/tasks/setup-kafka.yml | 115 ++-- .../roles/kafka_exporter/tasks/main.yml | 70 +- .../playbooks/roles/kibana/tasks/Debian.yml | 44 +- .../playbooks/roles/kibana/tasks/RedHat.yml | 48 +- .../playbooks/roles/kibana/tasks/main.yml | 33 +- .../tasks/install-packages-Debian.yml | 15 +- .../tasks/install-packages-RedHat.yml | 19 +- .../kubernetes_common/tasks/load-image.yml | 35 + .../roles/kubernetes_common/tasks/main.yml | 32 + .../tasks/cni-plugins/calico.yml | 10 +- .../tasks/cni-plugins/flannel.yml | 8 +- .../kubernetes_master/tasks/single-master.yml | 20 +- .../calico.yml => templates/calico.yml.j2} | 10 +- .../templates/canal-deployment.yml.j2 | 8 +- .../coredns-config.yml.j2} | 2 +- .../kube-flannel.yml.j2} | 20 +- .../templates/kubeadm-config.yml.j2 | 9 +- .../kubernetes-dashboard.yml.j2} | 4 +- .../roles/node_exporter/tasks/main.yml | 67 +- .../roles/postgresql/tasks/Debian.yml | 2 +- .../roles/postgresql/tasks/RedHat.yml | 26 - .../playbooks/roles/postgresql/tasks/main.yml | 1 - .../prometheus/tasks/install-alertmanager.yml | 67 +- .../roles/prometheus/tasks/install.yml | 173 +++-- .../roles/prometheus/tasks/preflight.yml | 4 +- .../playbooks/roles/rabbitmq/tasks/Debian.yml | 27 +- .../playbooks/roles/rabbitmq/tasks/RedHat.yml | 19 +- .../files/client/Debian/add-epirepo-client.sh | 7 + .../create-enabled-system-repos-list.sh | 7 + .../client/Debian/disable-epirepo-client.sh | 5 + .../client/Debian/disable-system-repos.sh | 9 + .../client/Debian/enable-system-repos.sh | 5 + .../files/client/RedHat/add-epirepo-client.sh | 19 + .../create-enabled-system-repos-list.sh | 8 + .../client/RedHat/disable-epirepo-client.sh | 5 + .../client/RedHat/disable-system-repos.sh | 16 + .../client/RedHat/enable-system-repos.sh | 22 + .../centos-7/download-requirements.sh | 611 +++++++++++++++++ .../centos-7/requirements.txt | 138 ++++ .../redhat-7/download-requirements.sh | 634 ++++++++++++++++++ .../redhat-7/requirements.txt | 134 ++++ .../ubuntu-18.04/add-repositories.sh | 22 + .../ubuntu-18.04/common.sh | 81 +++ .../ubuntu-18.04/download-requirements.sh | 153 +++++ .../ubuntu-18.04/requirements.txt | 144 ++++ .../files/server/Debian/create-repository.sh | 27 + .../Debian/disable-repository-server.sh | 4 + .../files/server/Debian/dpkg-scanpackages | 295 ++++++++ .../files/server/RedHat/create-repository.sh | 20 + .../RedHat/disable-repository-server.sh | 4 + .../roles/repository/tasks/Debian/setup.yml | 23 + .../repository/tasks/Debian/teardown.yml | 12 + .../roles/repository/tasks/RedHat/setup.yml | 33 + .../repository/tasks/RedHat/teardown.yml | 12 + .../tasks/download-requirements.yml | 37 + .../roles/repository/tasks/setup.yml | 76 +++ .../roles/repository/tasks/teardown.yml | 11 + .../roles/zookeeper/defaults/main.yml | 5 - .../playbooks/roles/zookeeper/tasks/main.yml | 128 ++-- .../zookeeper/templates/zookeeper.service.j2 | 2 +- .../common/ansible/playbooks/zookeeper.yml | 4 + .../configuration/feature-mapping.yml | 13 +- .../configuration/haproxy-exporter.yml | 13 +- .../common/defaults/configuration/haproxy.yml | 13 +- .../defaults/configuration/image-registry.yml | 9 + .../defaults/configuration/jmx-exporter.yml | 5 +- .../defaults/configuration/kafka-exporter.yml | 16 +- .../common/defaults/configuration/kafka.yml | 7 +- .../configuration/kubernetes-master.yml | 40 +- .../configuration/kubernetes-node.yml | 7 +- .../defaults/configuration/node-exporter.yml | 10 +- .../defaults/configuration/postgresql.yml | 1 + .../defaults/configuration/prometheus.yml | 21 +- .../defaults/configuration/repository.yml | 7 + .../defaults/configuration/shared-config.yml | 14 + .../defaults/configuration/zookeeper.yml | 2 +- .../configuration/image-registry.yml | 1 + .../validation/configuration/repository.yml | 1 + .../configuration/shared-config.yml | 1 + 140 files changed, 3643 insertions(+), 1055 deletions(-) create mode 100644 core/src/epicli/cli/engine/PrepareEngine.py create mode 100644 core/src/epicli/data/common/ansible/playbooks/download.yml create mode 100644 core/src/epicli/data/common/ansible/playbooks/group_vars/all.yml create mode 100644 core/src/epicli/data/common/ansible/playbooks/image_registry.yml create mode 100644 core/src/epicli/data/common/ansible/playbooks/repository.yml create mode 100644 core/src/epicli/data/common/ansible/playbooks/repository_setup.yml create mode 100644 core/src/epicli/data/common/ansible/playbooks/repository_teardown.yml delete mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/docker/tasks/Debian.yml delete mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/docker/tasks/RedHat.yml create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/download_file.yml create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/download_image.yml create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/main.yml delete mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/grafana/templates/grafana_rh.repo.j2 delete mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/grafana/vars/debian.yml delete mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/grafana/vars/redhat.yml create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/image_registry/meta/main.yml create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/image_registry/tasks/main.yml create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_common/tasks/load-image.yml rename core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/{files/calico.yml => templates/calico.yml.j2} (98%) rename core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/{files/coredns-config.yml => templates/coredns-config.yml.j2} (98%) rename core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/{files/kube-flannel.yml => templates/kube-flannel.yml.j2} (93%) rename core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/{files/kubernetes-dashboard.yml => templates/kubernetes-dashboard.yml.j2} (97%) create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/Debian/add-epirepo-client.sh create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/Debian/create-enabled-system-repos-list.sh create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/Debian/disable-epirepo-client.sh create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/Debian/disable-system-repos.sh create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/Debian/enable-system-repos.sh create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/add-epirepo-client.sh create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/create-enabled-system-repos-list.sh create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/disable-epirepo-client.sh create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/disable-system-repos.sh create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/enable-system-repos.sh create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/download-requirements.sh create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.txt create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/download-requirements.sh create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.txt create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/add-repositories.sh create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/common.sh create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/download-requirements.sh create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.txt create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/repository/files/server/Debian/create-repository.sh create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/repository/files/server/Debian/disable-repository-server.sh create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/repository/files/server/Debian/dpkg-scanpackages create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/repository/files/server/RedHat/create-repository.sh create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/repository/files/server/RedHat/disable-repository-server.sh create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/Debian/setup.yml create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/Debian/teardown.yml create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/RedHat/setup.yml create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/RedHat/teardown.yml create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/download-requirements.yml create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/setup.yml create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/teardown.yml create mode 100644 core/src/epicli/data/common/defaults/configuration/image-registry.yml create mode 100644 core/src/epicli/data/common/defaults/configuration/repository.yml create mode 100644 core/src/epicli/data/common/defaults/configuration/shared-config.yml create mode 100644 core/src/epicli/data/common/validation/configuration/image-registry.yml create mode 100644 core/src/epicli/data/common/validation/configuration/repository.yml create mode 100644 core/src/epicli/data/common/validation/configuration/shared-config.yml diff --git a/core/src/epicli/.gitignore b/core/src/epicli/.gitignore index ec7a9fe64d..7a68a2f912 100644 --- a/core/src/epicli/.gitignore +++ b/core/src/epicli/.gitignore @@ -28,7 +28,7 @@ share/python-wheels/ *.egg MANIFEST -requirements.txt +/requirements.txt # PyInstaller # Usually these files are written by a python script from a template diff --git a/core/src/epicli/cli/engine/BuildEngine.py b/core/src/epicli/cli/engine/BuildEngine.py index 81f9533e03..33f5992217 100644 --- a/core/src/epicli/cli/engine/BuildEngine.py +++ b/core/src/epicli/cli/engine/BuildEngine.py @@ -74,7 +74,7 @@ def process_configuration_docs(self): def collect_infrastructure_config(self): with provider_class_loader(self.cluster_model.provider, 'InfrastructureConfigCollector')( - [*self.input_docs, *self.configuration_docs, *self.infrastructure_docs]) as config_collector: + [*self.configuration_docs, *self.infrastructure_docs]) as config_collector: config_collector.run() def validate(self): @@ -107,7 +107,7 @@ def apply(self): self.collect_infrastructure_config() # Merge all the docs - docs = [*self.input_docs, *self.configuration_docs, *self.infrastructure_docs] + docs = [*self.configuration_docs, *self.infrastructure_docs] # Save docs to manifest file save_manifest(docs, self.cluster_model.specification.name) diff --git a/core/src/epicli/cli/engine/PrepareEngine.py b/core/src/epicli/cli/engine/PrepareEngine.py new file mode 100644 index 0000000000..a9c55f173a --- /dev/null +++ b/core/src/epicli/cli/engine/PrepareEngine.py @@ -0,0 +1,53 @@ +import os +import stat +import inspect +import shutil +from os.path import dirname + +from cli.helpers.Step import Step +from cli.helpers.data_loader import DATA_FOLDER_PATH +from cli.helpers.Config import Config +from cli.helpers.build_saver import copy_files_recursively + + +class PrepareEngine(Step): + PREPARE_PATH = DATA_FOLDER_PATH + '/common/ansible/playbooks/roles/repository/files/download-requirements' + + def __init__(self, input_data): + super().__init__(__name__) + self.os = input_data.os + + def __enter__(self): + super().__enter__() + return self + + def __exit__(self, exc_type, exc_value, traceback): + super().__exit__(exc_type, exc_value, traceback) + + def prepare(self): + prepare_src = os.path.join(self.PREPARE_PATH, self.os) + skopeo_src = os.path.join(dirname(dirname(inspect.getfile(os))), 'skopeo_linux') + prepare_dst = os.path.join(Config().output_dir, 'prepare_scripts') + + if not os.path.exists(prepare_src): + supported_os = os.listdir(self.PREPARE_PATH) + raise Exception(f'Unsupported OS: {self.os}. Currently supported: {supported_os}') + + if not os.path.exists(skopeo_src): + raise Exception('Skopeo dependency not found') + + # copy files to output dir + copy_files_recursively(prepare_src, prepare_dst) + shutil.copy(skopeo_src, prepare_dst) + + # make sure the scripts and skopeo are executable + self.make_file_executable(os.path.join(prepare_dst, 'skopeo_linux')) + self.make_file_executable(os.path.join(prepare_dst, 'download-requirements.sh')) + + self.logger.info(f'Prepared files for downloading the offline requirements in: {prepare_dst}') + return 0 + + @staticmethod + def make_file_executable(file): + executable_stat = os.stat(file) + os.chmod(file, executable_stat.st_mode | stat.S_IEXEC) diff --git a/core/src/epicli/cli/engine/ansible/AnsibleRunner.py b/core/src/epicli/cli/engine/ansible/AnsibleRunner.py index 668dfca621..d282577538 100644 --- a/core/src/epicli/cli/engine/ansible/AnsibleRunner.py +++ b/core/src/epicli/cli/engine/ansible/AnsibleRunner.py @@ -1,5 +1,8 @@ +import inspect import os import time +import shutil +from os.path import dirname from cli.engine.ansible.AnsibleCommand import AnsibleCommand from cli.engine.ansible.AnsibleInventoryCreator import AnsibleInventoryCreator @@ -8,10 +11,11 @@ from cli.helpers.build_saver import get_inventory_path, get_ansible_path, copy_files_recursively from cli.helpers.naming_helpers import to_role_name from cli.helpers.data_loader import DATA_FOLDER_PATH +from cli.helpers.Config import Config class AnsibleRunner(Step): - ANSIBLE_PLAYBOOKS_PATH = DATA_FOLDER_PATH + "/common/ansible/playbooks/" + ANSIBLE_PLAYBOOKS_PATH = DATA_FOLDER_PATH + '/common/ansible/playbooks/' def __init__(self, cluster_model, config_docs): super().__init__(__name__) @@ -42,21 +46,25 @@ def run(self): copy_files_recursively(AnsibleRunner.ANSIBLE_PLAYBOOKS_PATH, get_ansible_path(self.cluster_model.specification.name)) - # todo: install packages to run ansible on Red Hat hosts - self.ansible_command.run_task_with_retries(hosts="all", inventory=inventory_path, module="raw", - args="cat /etc/lsb-release | grep -i DISTRIB_ID | grep -i ubuntu && " - "sudo apt-get update && sudo apt-get install -y python-simplejson " - "|| echo 'Cannot find information about Ubuntu distribution'", retries=5) + # copy skopeo so Ansible can move it to the repositry machine + if not Config().offline_requirements: + shutil.copy(os.path.join(dirname(dirname(inspect.getfile(os))), 'skopeo_linux'), '/tmp') self.ansible_vars_generator.run() - + self.logger.info('Setting up repository for cluster provisioning. This will take a while...') self.ansible_command.run_playbook_with_retries(inventory=inventory_path, - playbook_path=self.playbook_path('common'), + playbook_path=self.playbook_path('repository_setup'), retries=5) + self.ansible_command.run_playbook(inventory=inventory_path, + playbook_path=self.playbook_path('common')) + enabled_roles = self.inventory_creator.get_enabled_roles() for role in enabled_roles: self.ansible_command.run_playbook(inventory=inventory_path, playbook_path=self.playbook_path(to_role_name(role))) + + self.ansible_command.run_playbook(inventory=inventory_path, + playbook_path=self.playbook_path('repository_teardown')) diff --git a/core/src/epicli/cli/engine/ansible/AnsibleVarsGenerator.py b/core/src/epicli/cli/engine/ansible/AnsibleVarsGenerator.py index e1724d7894..273e20b2a0 100644 --- a/core/src/epicli/cli/engine/ansible/AnsibleVarsGenerator.py +++ b/core/src/epicli/cli/engine/ansible/AnsibleVarsGenerator.py @@ -51,6 +51,8 @@ def populate_group_vars(self, ansible_dir): main_vars = ObjDict() main_vars = self.add_admin_user_name(main_vars) main_vars = self.add_validate_certs(main_vars) + main_vars = self.add_shared_config(main_vars) + main_vars = self.add_offline_requirements(main_vars) vars_dir = os.path.join(ansible_dir, 'group_vars') if not os.path.exists(vars_dir): @@ -59,22 +61,24 @@ def populate_group_vars(self, ansible_dir): vars_file_name = 'all.yml' vars_file_path = os.path.join(vars_dir, vars_file_name) - with open(vars_file_path, 'w') as stream: + with open(vars_file_path, 'a') as stream: dump(main_vars, stream) def add_admin_user_name(self, document): - if document is None: - raise Exception('Config is empty for: ' + 'group_vars/all.yml') - document['admin_user'] = self.cluster_model.specification.admin_user return document def add_validate_certs(self, document): - if document is None: - raise Exception('Config is empty for: ' + 'group_vars/all.yml') - document['validate_certs'] = Config().validate_certs + return document + + def add_offline_requirements(self, document): + document['offline_requirements'] = Config().offline_requirements + return document + def add_shared_config(self, document): + shared_config_doc = select_first(self.config_docs, lambda x: x.kind == 'configuration/shared-config') + document.update(shared_config_doc.specification) return document def add_provider_info(self, document): diff --git a/core/src/epicli/cli/engine/schema/ConfigurationAppender.py b/core/src/epicli/cli/engine/schema/ConfigurationAppender.py index 4170b5c891..3e65371c8a 100644 --- a/core/src/epicli/cli/engine/schema/ConfigurationAppender.py +++ b/core/src/epicli/cli/engine/schema/ConfigurationAppender.py @@ -6,6 +6,8 @@ class ConfigurationAppender(Step): + REQUIRED_DOCS = ['configuration/feature-mapping', 'configuration/shared-config', 'epiphany-cluster'] + def __init__(self, input_docs): super().__init__(__name__) self.cluster_model = select_single(input_docs, lambda x: x.kind == 'epiphany-cluster') @@ -14,22 +16,25 @@ def __init__(self, input_docs): def run(self): configuration_docs = [] + for document_kind in ConfigurationAppender.REQUIRED_DOCS: + doc = select_first(self.input_docs, lambda x: x.kind == document_kind) + if doc is None: + doc = load_yaml_obj(types.DEFAULT, 'common', document_kind) + self.logger.info("Adding: " + doc.kind) + configuration_docs.append(doc) + else: + configuration_docs.append(doc) + for component_key, component_value in self.cluster_model.specification.components.items(): if component_value.count < 1: continue - features_map = select_first(self.input_docs, lambda x: x.kind == 'configuration/feature-mapping') - if features_map is None: - features_map = select_first(configuration_docs, lambda x: x.kind == 'configuration/feature-mapping') - - if features_map is None: - features_map = load_yaml_obj(types.DEFAULT, 'common', 'configuration/feature-mapping') - self.logger.info("Adding: " + features_map.kind) - configuration_docs.append(features_map) - + features_map = select_first(configuration_docs, lambda x: x.kind == 'configuration/feature-mapping') config_selector = component_value.configuration for feature_key in features_map.specification.roles_mapping[component_key]: config = select_first(self.input_docs, lambda x: x.kind == 'configuration/' + feature_key and x.name == config_selector) + if config is not None: + configuration_docs.append(config) if config is None: config = select_first(configuration_docs, lambda x: x.kind == 'configuration/' + feature_key and x.name == config_selector) diff --git a/core/src/epicli/cli/epicli.py b/core/src/epicli/cli/epicli.py index 2b6158c457..53108897cb 100644 --- a/core/src/epicli/cli/epicli.py +++ b/core/src/epicli/cli/epicli.py @@ -8,6 +8,7 @@ from cli.engine.PatchEngine import PatchEngine from cli.engine.DeleteEngine import DeleteEngine from cli.engine.InitEngine import InitEngine +from cli.engine.PrepareEngine import PrepareEngine from cli.helpers.Log import Log from cli.helpers.Config import Config from cli.version import VERSION @@ -60,6 +61,7 @@ def main(): backup_parser(subparsers) recovery_parser(subparsers) delete_parser(subparsers) + prepare_parser(subparsers) # check if there were any variables and display full help if len(sys.argv) < 2: @@ -78,6 +80,8 @@ def main(): config.log_type = args.log_type config.log_count = args.log_count config.validate_certs = True if args.validate_certs == 'true' else False + if 'offline_requirements' in args and not args.offline_requirements is None: + config.offline_requirements = args.offline_requirements config.debug = args.debug config.auto_approve = args.auto_approve @@ -114,6 +118,8 @@ def apply_parser(subparsers): help='File with infrastructure/configuration definitions to use.') sub_parser.add_argument('--no-infra', dest='no_infra', action="store_true", help='Skip infrastructure provisioning.') + sub_parser.add_argument('--offline-requirements', dest='offline_requirements', type=str, + help='Path to the folder with pre-prepared offline requirements.') def run_apply(args): adjust_paths_from_file(args) @@ -198,11 +204,30 @@ def run_recovery(args): sub_parser.set_defaults(func=run_recovery) +def prepare_parser(subparsers): + sub_parser = subparsers.add_parser('prepare', description='Creates a folder with all prerequisites to setup the offline requirements to install a cluster offline.') + sub_parser.add_argument('--os', type=str, required=True, dest='os', + help='The OS to prepare the offline requirements for.') + + def run_prepare(args): + adjust_paths_from_output_dir() + with PrepareEngine(args) as engine: + return engine.prepare() + + sub_parser.set_defaults(func=run_prepare) + + def experimental_query(): if not query_yes_no('This is an experimental feature and could change at any time. Do you want to continue?'): sys.exit(0) +def adjust_paths_from_output_dir(): + if not Config().output_dir: + Config().output_dir = os.getcwd() # Default to working dir so we can at least write logs. + dump_config(Config()) + + def adjust_paths_from_file(args): if not os.path.isabs(args.file): args.file = os.path.join(os.getcwd(), args.file) diff --git a/core/src/epicli/cli/helpers/Config.py b/core/src/epicli/cli/helpers/Config.py index 5bd6583e35..ddce0b0912 100644 --- a/core/src/epicli/cli/helpers/Config.py +++ b/core/src/epicli/cli/helpers/Config.py @@ -1,5 +1,6 @@ import os + class Config: class __ConfigBase: def __init__(self): @@ -18,6 +19,7 @@ def __init__(self): self._validate_certs = True self._debug = False self._auto_approve = False + self._offline_requirements = '' @property def docker_cli(self): @@ -93,8 +95,8 @@ def debug(self): @debug.setter def debug(self, debug): if not debug is None: - self._debug = debug - + self._debug = debug + @property def auto_approve(self): return self._auto_approve @@ -102,8 +104,20 @@ def auto_approve(self): @auto_approve.setter def auto_approve(self, auto_approve): if not auto_approve is None: - self._auto_approve = auto_approve - + self._auto_approve = auto_approve + + @property + def offline_requirements(self): + return self._offline_requirements + + @offline_requirements.setter + def offline_requirements(self, offline_requirements): + if not offline_requirements is None: + # To make sure Ansible copies the content of the folder the the repository host. + if not offline_requirements.endswith('/'): + offline_requirements = f'{offline_requirements}/' + self._offline_requirements = offline_requirements + instance = None def __new__(cls): diff --git a/core/src/epicli/data/aws/defaults/infrastructure/virtual-machine.yml b/core/src/epicli/data/aws/defaults/infrastructure/virtual-machine.yml index ea28c68cbb..66b490d0cf 100644 --- a/core/src/epicli/data/aws/defaults/infrastructure/virtual-machine.yml +++ b/core/src/epicli/data/aws/defaults/infrastructure/virtual-machine.yml @@ -51,6 +51,26 @@ specification: destination_port_range: "9100" source_address_prefix: "10.1.0.0/20" destination_address_prefix: "0.0.0.0/0" + - name: image_registry + description: Allow image registry traffic + priority: 303 + direction: Inbound + access: Allow + protocol: Tcp + source_port_range: "*" + destination_port_range: "80" + source_address_prefix: "10.1.0.0/20" + destination_address_prefix: "0.0.0.0/0" + - name: package_repository + description: Allow package repository traffic + priority: 304 + direction: Inbound + access: Allow + protocol: Tcp + source_port_range: "*" + destination_port_range: "5000" + source_address_prefix: "10.1.0.0/20" + destination_address_prefix: "0.0.0.0/0" - name: out description: Allow out priority: 101 @@ -167,7 +187,7 @@ specification: source_port_range: "*" destination_port_range: "0" source_address_prefix: "0.0.0.0/0" - destination_address_prefix: "0.0.0.0/0" + destination_address_prefix: "0.0.0.0/0" --- kind: infrastructure/virtual-machine version: 0.4.0 @@ -243,6 +263,16 @@ specification: destination_port_range: "22" source_address_prefix: "0.0.0.0/0" destination_address_prefix: "0.0.0.0/0" + - name: repository + description: Allow repository traffic + priority: 302 + direction: Inbound + access: Allow + protocol: Tcp + source_port_range: "*" + destination_port_range: "80" + source_address_prefix: "10.1.0.0/20" + destination_address_prefix: "0.0.0.0/0" - name: node_exporter description: Allow node_exporter traffic priority: 302 diff --git a/core/src/epicli/data/azure/defaults/infrastructure/virtual-machine.yml b/core/src/epicli/data/azure/defaults/infrastructure/virtual-machine.yml index 8aec07e5a7..dca2fce330 100644 --- a/core/src/epicli/data/azure/defaults/infrastructure/virtual-machine.yml +++ b/core/src/epicli/data/azure/defaults/infrastructure/virtual-machine.yml @@ -139,6 +139,26 @@ specification: destination_port_range: "0" source_address_prefix: "10.1.2.0/24" destination_address_prefix: "0.0.0.0/0" + - name: package_repository + description: Allow package repository traffic + priority: 205 + direction: Inbound + access: Allow + protocol: Tcp + source_port_range: "*" + destination_port_range: "80" + source_address_prefix: "10.1.0.0/20" + destination_address_prefix: "0.0.0.0/0" + - name: image_repository + description: Allow image repository traffic + priority: 206 + direction: Inbound + access: Allow + protocol: Tcp + source_port_range: "*" + destination_port_range: "5000" + source_address_prefix: "10.1.0.0/20" + destination_address_prefix: "0.0.0.0/0" # - name: node2-subnet-traffic # description: Allow node subnet traffic # priority: 102 diff --git a/core/src/epicli/data/common/ansible/playbooks/download.yml b/core/src/epicli/data/common/ansible/playbooks/download.yml new file mode 100644 index 0000000000..3169d7ee0b --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/download.yml @@ -0,0 +1,8 @@ +--- +# Ansible playbook that downloads files and images + +- hosts: 127.0.0.1 + gather_facts: no + connection: local + roles: + - download \ No newline at end of file diff --git a/core/src/epicli/data/common/ansible/playbooks/elasticsearch.yml b/core/src/epicli/data/common/ansible/playbooks/elasticsearch.yml index ec9e960685..457fd925c7 100644 --- a/core/src/epicli/data/common/ansible/playbooks/elasticsearch.yml +++ b/core/src/epicli/data/common/ansible/playbooks/elasticsearch.yml @@ -1,5 +1,8 @@ --- # Ansible playbook that makes sure the base items for all nodes are installed +- hosts: all + gather_facts: yes + tasks: [ ] - hosts: elasticsearch become: true diff --git a/core/src/epicli/data/common/ansible/playbooks/elasticsearch_curator.yml b/core/src/epicli/data/common/ansible/playbooks/elasticsearch_curator.yml index ff232a1f5e..b85a7ef1b7 100644 --- a/core/src/epicli/data/common/ansible/playbooks/elasticsearch_curator.yml +++ b/core/src/epicli/data/common/ansible/playbooks/elasticsearch_curator.yml @@ -1,5 +1,8 @@ --- # Ansible playbook that makes sure elasticsearch-curator will be installed +- hosts: all + gather_facts: yes + tasks: [ ] - hosts: elasticsearch_curator become: true diff --git a/core/src/epicli/data/common/ansible/playbooks/filebeat.yml b/core/src/epicli/data/common/ansible/playbooks/filebeat.yml index 7802d22f51..13fccec98e 100644 --- a/core/src/epicli/data/common/ansible/playbooks/filebeat.yml +++ b/core/src/epicli/data/common/ansible/playbooks/filebeat.yml @@ -1,6 +1,10 @@ --- # Ansible playbook that installs and configures Filebeat +- hosts: all + gather_facts: yes + tasks: [ ] + - hosts: elasticsearch:kibana # To gather facts tasks: [ ] diff --git a/core/src/epicli/data/common/ansible/playbooks/grafana.yml b/core/src/epicli/data/common/ansible/playbooks/grafana.yml index 907cb69f8b..77eca32da8 100644 --- a/core/src/epicli/data/common/ansible/playbooks/grafana.yml +++ b/core/src/epicli/data/common/ansible/playbooks/grafana.yml @@ -1,4 +1,7 @@ --- +- hosts: all + gather_facts: yes + tasks: [ ] - hosts: grafana become: true diff --git a/core/src/epicli/data/common/ansible/playbooks/group_vars/all.yml b/core/src/epicli/data/common/ansible/playbooks/group_vars/all.yml new file mode 100644 index 0000000000..a0f0814b97 --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/group_vars/all.yml @@ -0,0 +1,5 @@ +local_repository_url: "http://{{ hostvars[groups['repository'][0]]['ansible_default_ipv4']['address'] }}/epirepo" +repository_url: "{{ custom_repository_url | default(local_repository_url, true) }}" + +local_image_registry: "{{ groups['image_registry'] | first }}:5000" +image_registry_address: "{{ local_image_registry }}" # TODO support custom_image_registry_address with user defined repo diff --git a/core/src/epicli/data/common/ansible/playbooks/haproxy.yml b/core/src/epicli/data/common/ansible/playbooks/haproxy.yml index 83e00bb182..b1233c983b 100644 --- a/core/src/epicli/data/common/ansible/playbooks/haproxy.yml +++ b/core/src/epicli/data/common/ansible/playbooks/haproxy.yml @@ -1,8 +1,8 @@ --- # Ansible playbook that makes sure the base items for all nodes are installed - - hosts: all - tasks: [ ] + gather_facts: yes + tasks: [ ] - hosts: haproxy become: true diff --git a/core/src/epicli/data/common/ansible/playbooks/haproxy_exporter.yml b/core/src/epicli/data/common/ansible/playbooks/haproxy_exporter.yml index c2a6367627..1177e49694 100644 --- a/core/src/epicli/data/common/ansible/playbooks/haproxy_exporter.yml +++ b/core/src/epicli/data/common/ansible/playbooks/haproxy_exporter.yml @@ -1,8 +1,8 @@ --- # Ansible playbook that makes sure the base items for all nodes are installed - - hosts: all - tasks: [ ] + gather_facts: yes + tasks: [ ] - hosts: haproxy_exporter become: true diff --git a/core/src/epicli/data/common/ansible/playbooks/image_registry.yml b/core/src/epicli/data/common/ansible/playbooks/image_registry.yml new file mode 100644 index 0000000000..fee200322e --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/image_registry.yml @@ -0,0 +1,12 @@ +--- +# Ansible playbook that makes sure the base items for all nodes are installed + +- hosts: all + gather_facts: yes + tasks: [ ] + +- hosts: image_registry + become: true + become_method: sudo + roles: + - image_registry diff --git a/core/src/epicli/data/common/ansible/playbooks/jmx_exporter.yml b/core/src/epicli/data/common/ansible/playbooks/jmx_exporter.yml index 0ea6105973..cb29ee08b9 100644 --- a/core/src/epicli/data/common/ansible/playbooks/jmx_exporter.yml +++ b/core/src/epicli/data/common/ansible/playbooks/jmx_exporter.yml @@ -1,5 +1,9 @@ --- +- hosts: all + gather_facts: yes + tasks: [ ] + - hosts: jmx_exporter become: true become_method: sudo diff --git a/core/src/epicli/data/common/ansible/playbooks/kafka.yml b/core/src/epicli/data/common/ansible/playbooks/kafka.yml index 5e594c88a3..3b7aa6da2d 100644 --- a/core/src/epicli/data/common/ansible/playbooks/kafka.yml +++ b/core/src/epicli/data/common/ansible/playbooks/kafka.yml @@ -1,6 +1,10 @@ --- # Ansible playbook that makes sure the base items for all nodes are installed +- hosts: all + gather_facts: yes + tasks: [ ] + - hosts: kafka become: true become_method: sudo diff --git a/core/src/epicli/data/common/ansible/playbooks/kafka_exporter.yml b/core/src/epicli/data/common/ansible/playbooks/kafka_exporter.yml index ded5987967..b727f6fbc7 100644 --- a/core/src/epicli/data/common/ansible/playbooks/kafka_exporter.yml +++ b/core/src/epicli/data/common/ansible/playbooks/kafka_exporter.yml @@ -1,5 +1,9 @@ --- +- hosts: all + gather_facts: yes + tasks: [ ] + - hosts: kafka_exporter become: true become_method: sudo diff --git a/core/src/epicli/data/common/ansible/playbooks/kibana.yml b/core/src/epicli/data/common/ansible/playbooks/kibana.yml index 0cacde8764..882d4c66ff 100644 --- a/core/src/epicli/data/common/ansible/playbooks/kibana.yml +++ b/core/src/epicli/data/common/ansible/playbooks/kibana.yml @@ -1,6 +1,10 @@ --- # Ansible playbook that makes sure the base items for all nodes are installed +- hosts: all + gather_facts: yes + tasks: [ ] + - hosts: kibana become: true become_method: sudo diff --git a/core/src/epicli/data/common/ansible/playbooks/kubernetes_master.yml b/core/src/epicli/data/common/ansible/playbooks/kubernetes_master.yml index 4ad62c4747..271bff96f2 100644 --- a/core/src/epicli/data/common/ansible/playbooks/kubernetes_master.yml +++ b/core/src/epicli/data/common/ansible/playbooks/kubernetes_master.yml @@ -1,6 +1,10 @@ --- # Ansible playbook that makes sure the base items for all nodes are installed +- hosts: all + gather_facts: yes + tasks: [ ] + - hosts: kubernetes_master become: true become_method: sudo diff --git a/core/src/epicli/data/common/ansible/playbooks/node_exporter.yml b/core/src/epicli/data/common/ansible/playbooks/node_exporter.yml index 8a3c897dfb..2d8d9b97e0 100644 --- a/core/src/epicli/data/common/ansible/playbooks/node_exporter.yml +++ b/core/src/epicli/data/common/ansible/playbooks/node_exporter.yml @@ -1,4 +1,7 @@ --- +- hosts: all + gather_facts: yes + tasks: [ ] - hosts: node_exporter become: true diff --git a/core/src/epicli/data/common/ansible/playbooks/postgresql.yml b/core/src/epicli/data/common/ansible/playbooks/postgresql.yml index 0a9e26422b..2b4496aba5 100644 --- a/core/src/epicli/data/common/ansible/playbooks/postgresql.yml +++ b/core/src/epicli/data/common/ansible/playbooks/postgresql.yml @@ -1,5 +1,8 @@ --- # Ansible playbook that installs and performs basic configuration of postgresql +- hosts: all + gather_facts: yes + tasks: [ ] - hosts: postgresql become: true diff --git a/core/src/epicli/data/common/ansible/playbooks/prometheus.yml b/core/src/epicli/data/common/ansible/playbooks/prometheus.yml index 780bfa3566..d45bc35088 100644 --- a/core/src/epicli/data/common/ansible/playbooks/prometheus.yml +++ b/core/src/epicli/data/common/ansible/playbooks/prometheus.yml @@ -1,5 +1,9 @@ --- +- hosts: all + gather_facts: yes + tasks: [ ] + - hosts: prometheus become: true become_method: sudo diff --git a/core/src/epicli/data/common/ansible/playbooks/repository.yml b/core/src/epicli/data/common/ansible/playbooks/repository.yml new file mode 100644 index 0000000000..3fcc144ef1 --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/repository.yml @@ -0,0 +1,5 @@ +--- +# This playbook is empty by purpose, just to enable repository role in configuration/feature-mapping +# to populate defaults/configuration to Ansible vars +- hosts: [] + tasks: [] \ No newline at end of file diff --git a/core/src/epicli/data/common/ansible/playbooks/repository_setup.yml b/core/src/epicli/data/common/ansible/playbooks/repository_setup.yml new file mode 100644 index 0000000000..21b2875478 --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/repository_setup.yml @@ -0,0 +1,10 @@ +--- +# Ansible playbook for disabling/enabling repositories before/after Epiphany installation + +- hosts: all + become: true + become_method: sudo + tasks: + - import_role: + name: repository + tasks_from: setup diff --git a/core/src/epicli/data/common/ansible/playbooks/repository_teardown.yml b/core/src/epicli/data/common/ansible/playbooks/repository_teardown.yml new file mode 100644 index 0000000000..d2a01fd20e --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/repository_teardown.yml @@ -0,0 +1,10 @@ +--- +# Ansible playbook for disabling/enabling repositories before/after Epiphany installation + +- hosts: all + become: true + become_method: sudo + tasks: + - import_role: + name: repository + tasks_from: teardown diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/common/tasks/Debian.yml b/core/src/epicli/data/common/ansible/playbooks/roles/common/tasks/Debian.yml index 50110206ac..c5c051e838 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/common/tasks/Debian.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/common/tasks/Debian.yml @@ -1,8 +1,6 @@ --- # Common Debian family of specific tasks - - - name: Install selinux packages as prerequisite for SELinux module apt: name: @@ -11,9 +9,6 @@ update_cache: yes state: present - - - - name: Install debian family packages apt: name: @@ -39,6 +34,7 @@ - ethtool - telnet - ufw + - jq update_cache: yes state: present diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/common/tasks/RedHat.yml b/core/src/epicli/data/common/ansible/playbooks/roles/common/tasks/RedHat.yml index 8cd21dde80..c61939f989 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/common/tasks/RedHat.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/common/tasks/RedHat.yml @@ -1,18 +1,12 @@ --- # Common Redhat family of specific tasks -# Install RedHat extras if on redhat family. This needs to be done first -- name: Install Extras packages - yum: - name: https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm - state: latest - # Subscriptions with certs: subscription-manager repos --enable "rhel-*-optional-rpms" --enable "rhel-*-extras-rpms" # w/o subscription https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm - name: Install RedHat family packages yum: - name: + name: - libselinux-python - libsemanage-python - firewalld @@ -30,12 +24,14 @@ - sysstat - python-setuptools - openssl + - yum-utils - yum-versionlock - logrotate - ebtables - ethtool - telnet - update_cache: yes + - jq + update_cache: yes state: present register: result retries: 3 diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/common/tasks/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/common/tasks/main.yml index bc783c3365..a7cdcd7061 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/common/tasks/main.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/common/tasks/main.yml @@ -20,30 +20,6 @@ - name: Print environment variables debug: msg={{ env_output.stdout_lines }} -# TODO: Checking sha512 of jq -- name: Gather prereqs - get_url: - url: https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 - dest: /usr/local/bin/jq - mode: 0755 - validate_certs: "{{ validate_certs | bool}}" - -- name: Get sha512 sum of archive - stat: - path: "/usr/local/bin/jq" - checksum_algorithm: sha512 - get_checksum: yes - register: jq_download_stat - -- name: Display sha of archive - debug: - msg: "Jq SHA512: {{ jq_download_stat.stat.checksum }}" - -- name: Verify sha512 of archive before installation. - fail: - msg: "File checksum is not correct." - when: jq_download_stat.stat.checksum != "aaa016d57ab8351360d02186809ade9cdecd3eb20df7a8cf05cd5d1037c4d36efae9e1bb0102d175c91b530b0309f24b48d579544249da7cbd50f721332617b9" - - name: Adjust swappiness parameter sysctl: name: vm.swappiness @@ -109,8 +85,6 @@ changed_when: false when: ansible_selinux is defined and ansible_selinux != False and ansible_selinux.status == 'enabled' - - - name: Motd cross-bones copy: src: motd.tail @@ -134,17 +108,6 @@ owner: root group: root -- name: PIP via easy_install - easy_install: - name: pip - state: latest - when: not ansible_distribution_version == "18.04" - -- name: PIP install - Ubuntu 18.04 - apt: name=python-pip update_cache=yes state=present - when: ansible_os_family == "Debian" and ansible_distribution_version == "18.04" - - - include_tasks: epiuser.yml tags: - epiuser diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/docker/defaults/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/docker/defaults/main.yml index 0a4bc44e9f..3ba1cf5165 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/docker/defaults/main.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/docker/defaults/main.yml @@ -3,3 +3,5 @@ docker_logging: log_opts: max_file_size: 10m # The maximum size of the log before it is rolled. A positive integer plus a modifier representing the unit of measure (k, m, or g) max_files: 2 # The maximum number of log files that can be present + +docker_version: 18.09.6 \ No newline at end of file diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/docker/tasks/Debian.yml b/core/src/epicli/data/common/ansible/playbooks/roles/docker/tasks/Debian.yml deleted file mode 100644 index b6568fe19e..0000000000 --- a/core/src/epicli/data/common/ansible/playbooks/roles/docker/tasks/Debian.yml +++ /dev/null @@ -1,26 +0,0 @@ ---- -- name: Ensure dependencies are installed - apt: - name: - - apt-transport-https - - ca-certificates - state: present - -- name: Add Docker apt key - apt_key: - url: "https://download.docker.com/linux/{{ ansible_distribution|lower }}/gpg" - id: 9DC858229FC7DD38854AE2D88D81803C0EBFCD88 - state: present - register: add_repository_key - -- name: Add Docker-CE stable repo - apt_repository: - repo: "deb [arch=amd64] https://download.docker.com/linux/{{ ansible_distribution|lower }} {{ ansible_distribution_release }} stable" - state: present - update_cache: true - -- name: Install Docker - apt: - name: "docker-ce=5:18.09.6~3-0~{{ ansible_distribution|lower }}-{{ ansible_distribution_release }}" - state: present - update_cache: true \ No newline at end of file diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/docker/tasks/RedHat.yml b/core/src/epicli/data/common/ansible/playbooks/roles/docker/tasks/RedHat.yml deleted file mode 100644 index 4560ac582f..0000000000 --- a/core/src/epicli/data/common/ansible/playbooks/roles/docker/tasks/RedHat.yml +++ /dev/null @@ -1,28 +0,0 @@ ---- -- name: Add Docker-CE stable repo - yum_repository: - name: docker-ce-stable - description: Docker CE Stable Repo - baseurl: https://download.docker.com/linux/centos/7/$basearch/stable - gpgkey: https://download.docker.com/linux/centos/gpg - gpgcheck: yes - -- name: Install container-selinux for RHEL 7.6 and later - yum: - name: http://mirror.centos.org/centos/7/extras/x86_64/Packages/container-selinux-2.107-1.el7_6.noarch.rpm - state: present - update_cache: yes - when: ansible_distribution_version is version('7.6', '>=') - -- name: Install container-selinux for 7.5 and older - yum: - name: http://mirror.centos.org/centos/7/extras/x86_64/Packages/container-selinux-2.107-3.el7.noarch.rpm - state: present - update_cache: yes - when: ansible_distribution_version is version('7.5', '<=') - -- name: Install Docker - yum: - name: docker-ce-18.09.6-3.el7 - state: present - update_cache: yes diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/docker/tasks/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/docker/tasks/main.yml index 78108d6a5d..eaea6119e9 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/docker/tasks/main.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/docker/tasks/main.yml @@ -1,6 +1,25 @@ --- # Docker (used by master & worker as dependency) -- include_tasks: "{{ ansible_os_family }}.yml" +- name: Install Docker for Debian family + package: + name: "{{ item }}" + state: present + loop: + - apt-transport-https + - ca-certificates + - containerd.io + - docker-ce-cli=5:18.09.* + - docker-ce=5:18.09.* + when: ansible_os_family == "Debian" + +- name: Install Docker for RedHat family + package: + name: "{{ item }}" + state: present + loop: + - docker-ce-cli-18.09.* + - docker-ce-18.09.* + when: ansible_os_family == "RedHat" - include_tasks: configure-docker.yml diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/docker/templates/daemon.json.j2 b/core/src/epicli/data/common/ansible/playbooks/roles/docker/templates/daemon.json.j2 index 213544beab..4ec3d3d69c 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/docker/templates/daemon.json.j2 +++ b/core/src/epicli/data/common/ansible/playbooks/roles/docker/templates/daemon.json.j2 @@ -1,4 +1,9 @@ { + {% if custom_image_registry_address|length == 0 %} + "insecure-registries" : ["{{ image_registry_address }}"], + {% endif %} + + "log-driver": "json-file", "log-opts": { "max-size": "{{ docker_logging.log_opts.max_file_size }}", diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/download_file.yml b/core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/download_file.yml new file mode 100644 index 0000000000..f943e4753c --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/download_file.yml @@ -0,0 +1,12 @@ +--- + +- name: "Download file {{ file_name }}" + get_url: + url: "{{ repository_url }}/files/{{ file_name }}" + dest: "{{ download_directory }}" + validate_certs: "{{ validate_certs }}" + register: get_url_result + until: "'OK' in get_url_result.msg or 'file already exists' in get_url_result.msg" + retries: "3" + delay: "2" + become: false diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/download_image.yml b/core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/download_image.yml new file mode 100644 index 0000000000..d8e12f15ee --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/download_image.yml @@ -0,0 +1,12 @@ +--- + +- name: Download image {{ file_name }} + get_url: + url: "{{ repository_url }}/images/{{ file_name }}" + dest: "{{ download_directory }}" + validate_certs: "{{ validate_certs }}" + register: get_url_result + until: "'OK' in get_url_result.msg or 'file already exists' in get_url_result.msg" + retries: "3" + delay: "2" + become: false \ No newline at end of file diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/main.yml new file mode 100644 index 0000000000..8696dc18c9 --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/main.yml @@ -0,0 +1,3 @@ +--- + +# This role is only included in other roles to download from repository \ No newline at end of file diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/elasticsearch/tasks/Debian.yml b/core/src/epicli/data/common/ansible/playbooks/roles/elasticsearch/tasks/Debian.yml index 5f9d5e555b..f4676ada49 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/elasticsearch/tasks/Debian.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/elasticsearch/tasks/Debian.yml @@ -1,32 +1,14 @@ ---- -## Elasticsearch master +# splitted to separate tasks to make it work on Debian/Ubuntu: +# https://github.com/elastic/elasticsearch/issues/33607#issue-359124678 -- name: Install debian family packages - apt: +- name: Install prerequisites for Elasticsearch + apt: name: - - openjdk-8-jre - update_cache: yes + - openjdk-8-jre-headless state: present -- name: Install Elasticsearch Package - apt: - deb: "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-{{ specification.elasticsearch_version }}.deb" - when: - - groups['elasticsearch'][0] == inventory_hostname - -- name: Replace with interface - replace: - path: /etc/elasticsearch/elasticsearch.yml - regexp: '^#network.host: 192.168.0.1$' - replace: "network.host: {{ hostvars[groups['elasticsearch'][0]]['ansible_default_ipv4']['address'] }}" - backup: no - when: - - groups['elasticsearch'][0] == inventory_hostname - -- name: Start services - service: - name: elasticsearch - state: started - enabled: yes - when: - - groups['elasticsearch'][0] == inventory_hostname +- name: Install Elasticsearch package + apt: + name: + - elasticsearch-oss + state: present \ No newline at end of file diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/elasticsearch/tasks/RedHat.yml b/core/src/epicli/data/common/ansible/playbooks/roles/elasticsearch/tasks/RedHat.yml index ff92dfc27f..223175ae53 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/elasticsearch/tasks/RedHat.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/elasticsearch/tasks/RedHat.yml @@ -1,36 +1,7 @@ ---- -## Elasticsearch master - -- name: Install RedHat family packages +- name: Install Elasticsearch package yum: name: - - java-1.8.0-openjdk + - java-1.8.0-openjdk-headless + - elasticsearch-oss update_cache: yes - state: present - register: result - retries: 3 - delay: 1 - until: result is succeeded - -- name: Install Elasticsearch Package - yum: - name: "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-{{ specification.elasticsearch_version }}.rpm" - when: - - groups['elasticsearch'][0] == inventory_hostname - -- name: Replace with interface - replace: - path: /etc/elasticsearch/elasticsearch.yml - regexp: '^#network.host: 192.168.0.1$' - replace: "network.host: {{ hostvars[groups['elasticsearch'][0]]['ansible_default_ipv4']['address'] }}" - backup: no - when: - - groups['elasticsearch'][0] == inventory_hostname - -- name: Start services - service: - name: elasticsearch - state: started - enabled: yes - when: - - groups['elasticsearch'][0] == inventory_hostname + state: present \ No newline at end of file diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/elasticsearch/tasks/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/elasticsearch/tasks/main.yml index 4f0f1c848d..1119f13cab 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/elasticsearch/tasks/main.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/elasticsearch/tasks/main.yml @@ -1,5 +1,22 @@ --- -## Elasticsearch master - include_tasks: "set-pam-limits.yml" -- include_tasks: "{{ ansible_os_family }}.yml" + +- include_tasks: "{{ ansible_os_family }}.yml" + +- name: Replace with interface + replace: + path: /etc/elasticsearch/elasticsearch.yml + regexp: '^#network.host: 192.168.0.1$' + replace: "network.host: {{ hostvars[groups['elasticsearch'][0]]['ansible_default_ipv4']['address'] }}" + backup: no + when: + - groups['elasticsearch'][0] == inventory_hostname + +- name: Start services + service: + name: elasticsearch + state: started + enabled: yes + when: + - groups['elasticsearch'][0] == inventory_hostname diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/elasticsearch_curator/tasks/Debian.yml b/core/src/epicli/data/common/ansible/playbooks/roles/elasticsearch_curator/tasks/Debian.yml index 5b107565eb..be4d6609c5 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/elasticsearch_curator/tasks/Debian.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/elasticsearch_curator/tasks/Debian.yml @@ -1,7 +1,6 @@ ---- - -- name: Install Elasticsearch-Curator Package - apt: - deb: "https://packages.elastic.co/curator/5/debian9/pool/main/e/elasticsearch-curator/elasticsearch-curator_{{ specification.debian_curator_version }}.deb" - - +- name: Install Elasticsearch-Curator package + apt: + name: + - elasticsearch-curator + update_cache: yes + state: present \ No newline at end of file diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/elasticsearch_curator/tasks/RedHat.yml b/core/src/epicli/data/common/ansible/playbooks/roles/elasticsearch_curator/tasks/RedHat.yml index 047b9a2faf..f5ef0ddd31 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/elasticsearch_curator/tasks/RedHat.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/elasticsearch_curator/tasks/RedHat.yml @@ -1,6 +1,6 @@ ---- - -- name: Install Elasticsearch-Curator Package +- name: Install Elasticsearch-Curator package yum: - name: "https://packages.elastic.co/curator/5/centos/7/Packages/elasticsearch-curator-{{ specification.redhat_curator_version }}.rpm" - + name: + - elasticsearch-curator + update_cache: yes + state: present \ No newline at end of file diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/elasticsearch_curator/tasks/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/elasticsearch_curator/tasks/main.yml index f3c7dd6fba..23b7c5a042 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/elasticsearch_curator/tasks/main.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/elasticsearch_curator/tasks/main.yml @@ -1,6 +1,6 @@ --- -- include_tasks: "{{ ansible_os_family }}.yml" +- include_tasks: "{{ ansible_os_family }}.yml" - name: Configure cron jobs for Elasticsearch Curator cron: diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/filebeat/tasks/Debian.yml b/core/src/epicli/data/common/ansible/playbooks/roles/filebeat/tasks/Debian.yml index a763b459ea..bd4602789f 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/filebeat/tasks/Debian.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/filebeat/tasks/Debian.yml @@ -1,7 +1,6 @@ ---- -## Filebeat Debian - -- name: Install Filebeat package +- name: Install Filbeat package apt: - deb: "https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-oss-{{ specification.filebeat_version }}-amd64.deb" - state: present + name: + - filebeat + update_cache: yes + state: present \ No newline at end of file diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/filebeat/tasks/RedHat.yml b/core/src/epicli/data/common/ansible/playbooks/roles/filebeat/tasks/RedHat.yml index a1ea711375..2613b00908 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/filebeat/tasks/RedHat.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/filebeat/tasks/RedHat.yml @@ -1,7 +1,6 @@ ---- -## Filebeat Red Hat - -- name: Install Filebeat package - yum: - name: "https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-oss-{{ specification.filebeat_version }}-x86_64.rpm" +- name: Install Filbeat package + yum: + name: + - filebeat + update_cache: yes state: present diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/filebeat/tasks/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/filebeat/tasks/main.yml index d59f5c3687..fa7b8026de 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/filebeat/tasks/main.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/filebeat/tasks/main.yml @@ -1,8 +1,6 @@ --- # Filebeat - -- name: Include {{ ansible_os_family }} family tasks - include_tasks: "{{ ansible_os_family }}.yml" +- include_tasks: "{{ ansible_os_family }}.yml" - name: Include configuration tasks include_tasks: configure-filebeat.yml diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/grafana/tasks/install.yml b/core/src/epicli/data/common/ansible/playbooks/roles/grafana/tasks/install.yml index d54c71f8bf..a1f1b4ba25 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/grafana/tasks/install.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/grafana/tasks/install.yml @@ -1,12 +1,4 @@ --- -- name: Install dependencies - package: - name: "{{ grafana_dependencies }}" - state: present - register: _install_dep_packages - until: _install_dep_packages is succeeded - retries: 5 - delay: 2 - name: Remove conflicting grafana packages package: @@ -14,50 +6,27 @@ state: absent register: _old_grafana_pkgs -- name: Clean apt cache - command: apt clean - when: - - _old_grafana_pkgs is changed - - ansible_pkg_mgr == "apt" - -- name: Add Grafana repository file [RHEL/CentOS] - template: - src: "grafana_rh.repo.j2" - dest: "/etc/yum.repos.d/grafana.repo" - force: true - backup: true - when: ansible_pkg_mgr in ['yum', 'dnf'] - -- block: - - name: Import Grafana GPG signing key [Debian/Ubuntu] - apt_key: - url: "https://packages.grafana.com/gpg.key" - state: present - validate_certs: false - register: _add_apt_key - until: _add_apt_key is succeeded - retries: 5 - delay: 2 +- name: Install dependencies for Debian family + package: + name: "{{ item }}" + state: present + loop: + - apt-transport-https + - adduser + - ca-certificates + - libfontconfig + - gnupg2 + when: ansible_os_family == "Debian" - - name: Add Grafana repository [Debian/Ubuntu] - apt_repository: - repo: deb https://packages.grafana.com/oss/deb stable main - state: present - update_cache: true - register: _update_apt_cache - until: _update_apt_cache is succeeded - retries: 5 - delay: 2 - when: - - ansible_pkg_mgr == "apt" - environment: "{{ grafana_environment }}" +#TODO: this is a quick workaround, we should tackle versioning in a smarter way +- name: Install Grafana + package: + name: "grafana-{{ grafana_version }}" + state: present + when: ansible_os_family == "RedHat" - name: Install Grafana package: - name: "{{ grafana_package }}" - state: "{{ (grafana_version == 'latest') | ternary('latest', 'present') }}" - register: _install_packages - until: _install_packages is succeeded - retries: 5 - delay: 2 - notify: restart grafana \ No newline at end of file + name: "grafana={{ grafana_version }}" + state: present + when: ansible_os_family == "Debian" \ No newline at end of file diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/grafana/tasks/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/grafana/tasks/main.yml index c7886b1559..d70bad335d 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/grafana/tasks/main.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/grafana/tasks/main.yml @@ -1,18 +1,4 @@ --- -- name: Gather variables for each operating system - include_vars: "{{ item }}" - with_first_found: - - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml" - - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml" - - "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml" - - "{{ ansible_distribution | lower }}.yml" - - "{{ ansible_os_family | lower }}.yml" - tags: - - grafana_install - - grafana_configure - - grafana_datasources - - grafana_notifications - - grafana_dashboards - include: preflight.yml tags: diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/grafana/templates/grafana_rh.repo.j2 b/core/src/epicli/data/common/ansible/playbooks/roles/grafana/templates/grafana_rh.repo.j2 deleted file mode 100644 index fe06dcc964..0000000000 --- a/core/src/epicli/data/common/ansible/playbooks/roles/grafana/templates/grafana_rh.repo.j2 +++ /dev/null @@ -1,10 +0,0 @@ -{{ ansible_managed | comment }} -[grafana] -name=grafana -baseurl=https://packages.grafana.com/oss/rpm -#repo_gpgcheck=1 -enabled=1 -gpgcheck=1 -gpgkey=https://packages.grafana.com/gpg.key -sslverify=1 -sslcacert=/etc/pki/tls/certs/ca-bundle.crt \ No newline at end of file diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/grafana/vars/debian.yml b/core/src/epicli/data/common/ansible/playbooks/roles/grafana/vars/debian.yml deleted file mode 100644 index 734db9bce3..0000000000 --- a/core/src/epicli/data/common/ansible/playbooks/roles/grafana/vars/debian.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -grafana_package: "grafana{{ (grafana_version != 'latest') | ternary('=' ~ grafana_version, '') }}" -grafana_dependencies: - - apt-transport-https - - adduser - - ca-certificates - - libfontconfig - - gnupg2 diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/grafana/vars/redhat.yml b/core/src/epicli/data/common/ansible/playbooks/roles/grafana/vars/redhat.yml deleted file mode 100644 index 0a65d61913..0000000000 --- a/core/src/epicli/data/common/ansible/playbooks/roles/grafana/vars/redhat.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -grafana_package: "grafana{{ (grafana_version != 'latest') | ternary('-' ~ grafana_version, '') }}" -grafana_dependencies: [] \ No newline at end of file diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/haproxy/tasks/RedHat.yml b/core/src/epicli/data/common/ansible/playbooks/roles/haproxy/tasks/RedHat.yml index 8b181d86b8..d341c44c33 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/haproxy/tasks/RedHat.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/haproxy/tasks/RedHat.yml @@ -11,31 +11,6 @@ ignore_errors: true changed_when: false -- name: Check if repository file exists - stat: - path: /etc/yum.repos.d/redhat-rhui.repo - register: rhui_exist - when: - - specification.provider == "aws" - -- name: Check if Software Collections repository on EC2 exists - shell: grep -c -i rhui-REGION-rhel-server-rhscl /etc/yum.repos.d/redhat-rhui.repo - register: sc_repo_count - failed_when: "sc_repo_count.rc == 2" - when: - - specification.provider == "aws" - - rhui_exist - -- name: Enable Software Collections on EC2 - ini_file: - dest: /etc/yum.repos.d/redhat-rhui.repo - section: rhui-REGION-rhel-server-rhscl - option: enabled - value: 1 - when: - - specification.provider == "aws" - - sc_repo_count.stdout|int == 1 - - name: Install haproxy family packages yum: name: diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/haproxy_exporter/tasks/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/haproxy_exporter/tasks/main.yml index 01dc71b621..28d1191f35 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/haproxy_exporter/tasks/main.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/haproxy_exporter/tasks/main.yml @@ -14,52 +14,16 @@ group: haproxy_exporter createhome: false -- name: Ensure download directory exists - become: false - file: - path: "{{ specification.download_directory }}/haproxy_exporter" - state: directory - delegate_to: localhost - -- name: Download haproxy_exporter binary to local folder - become: false - get_url: - url: "{{ specification.download_urls[ansible_architecture] }}" - dest: "{{ specification.download_directory }}/haproxy_exporter.tar.gz" - validate_certs: "{{ validate_certs | bool}}" - register: _download_archive - until: _download_archive is succeeded - retries: 5 - delay: 2 - delegate_to: localhost - -- name: Get sha512 sum of archive - become: false - stat: - path: "{{ specification.download_directory }}/haproxy_exporter.tar.gz" - checksum_algorithm: sha512 - get_checksum: yes - register: haproxy_exporter_stat - delegate_to: localhost - -- name: Display sha of archive - debug: - msg: "Haproxy Exporter SHA512: {{ haproxy_exporter_stat.stat.checksum }}" - -- name: Verify sha512 of archive before installation. - fail: - msg: "File checksum is not correct." - when: haproxy_exporter_stat.stat.checksum not in specification.download_shas +- name: Set HAProxy Exporter file name to install + set_fact: + exporter_file_name: "{{ specification.file_name }}" -- name: Unpack haproxy_exporter binary - become: false - unarchive: - src: "{{ specification.download_directory }}/haproxy_exporter.tar.gz" - dest: "{{ specification.download_directory }}/haproxy_exporter" - creates: "{{ specification.download_directory }}/haproxy_exporter/haproxy_exporter" - extra_opts: [--strip-components=1] - delegate_to: localhost - check_mode: false +- name: Download HAProxy Exporter binaries + include_role: + name: download + tasks_from: download_file + vars: + file_name: "{{ exporter_file_name }}" - name: Create /opt/haproxy_exporter directories become: yes @@ -73,13 +37,18 @@ with_items: - /opt/haproxy_exporter -- name: Copy haproxy_exporter binary - copy: - src: "{{ specification.download_directory }}/haproxy_exporter/haproxy_exporter" - dest: "/opt/haproxy_exporter/haproxy_exporter" +- name: Unpack haproxy_exporter binary + become: yes + unarchive: + remote_src: yes + src: "{{ download_directory }}/{{ exporter_file_name }}" + dest: "/opt/haproxy_exporter" + creates: "/opt/haproxy_exporter/haproxy_exporter" + extra_opts: [--strip-components=1] owner: root group: haproxy_exporter - mode: 0755 + mode: 0750 + check_mode: false - name: Load stats credentials from HAProxy shell: >- diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/image_registry/meta/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/image_registry/meta/main.yml new file mode 100644 index 0000000000..cb7d8e0460 --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/image_registry/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - role: docker diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/image_registry/tasks/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/image_registry/tasks/main.yml new file mode 100644 index 0000000000..db09739cc5 --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/image_registry/tasks/main.yml @@ -0,0 +1,32 @@ +--- +- name: Check if image is already loaded + shell: "docker images {{ specification.repository_image.name }} --format {{ '{{' }}.ID{{ '}}' }}" + register: image_check + ignore_errors: true + changed_when: false + +- name: Load image if does not exists + block: + - name: Download file + include_role: + name: download + tasks_from: download_image + vars: + file_name: "{{ specification.repository_image.file_name }}" + + - name: Load image {{ specification.repository_image.name }} + become: yes + shell: "docker load --input {{ download_directory }}/{{ specification.repository_image.file_name }}" + when: image_check.stdout | length == 0 + +- name: Check if registry is running + become: yes + shell: docker ps | grep registry:2 | cat + register: regitry_up_check + check_mode: no + +# todo run registry with SSL - generate/copy certs, mount it to registry container +- name: Run registry + become: yes + shell: "docker run -d -e REGISTRY_HTTP_ADDR=0.0.0.0:5000 -p 5000:5000 --restart=always --name epiphany-registry {{ specification.repository_image.name }}" + when: regitry_up_check.stdout | length == 0 diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/jmx_exporter/tasks/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/jmx_exporter/tasks/main.yml index 08bae61d7d..c814fa5234 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/jmx_exporter/tasks/main.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/jmx_exporter/tasks/main.yml @@ -25,37 +25,29 @@ with_items: - "{{ specification.jmx_jars_directory }}" -# TODO: Checking sha512 -- name: Prometheus jmx | download jar - become: yes - get_url: - url: "{{ specification.download_url }}" - dest: "/opt/jmx-exporter/jmx_prometheus_javaagent-{{ specification.jmx_exporter_version }}.jar" - force: no +- name: Set JMX Exporter file name to install + set_fact: + exporter_file_name: "{{ specification.file_name }}" + +- name: Download JMX Exporter binaries + include_role: + name: download + tasks_from: download_file + vars: + file_name: "{{ exporter_file_name }}" + +- name: Copy JMX Exporter binaries + copy: + src: "{{ download_directory }}/{{ exporter_file_name }}" + dest: "/opt/jmx-exporter/{{ exporter_file_name }}" owner: "{{ specification.jmx_exporter_user }}" group: "{{ specification.jmx_exporter_group }}" - validate_certs: "{{ validate_certs | bool }}" - -- name: Get sha512 sum of archive - stat: - path: "/opt/jmx-exporter/jmx_prometheus_javaagent-{{ specification.jmx_exporter_version }}.jar" - checksum_algorithm: sha512 - get_checksum: yes - register: jmx_prometheus_download_stat - -- name: Display sha of archive - debug: - msg: "Alertmanager SHA512: {{ jmx_prometheus_download_stat.stat.checksum }}" - -- name: Verify sha512 of archive before installation. - fail: - msg: "File checksum is not correct." - when: jmx_prometheus_download_stat.stat.checksum != specification.download_sha + remote_src: yes - name: Prometheus jmx | symlink jar become: yes file: - src: "/opt/jmx-exporter/jmx_prometheus_javaagent-{{ specification.jmx_exporter_version }}.jar" + src: "/opt/jmx-exporter/{{ exporter_file_name }}" path: "{{ specification.jmx_path }}" force: yes state: link diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/kafka/tasks/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/kafka/tasks/main.yml index 88f7c15c2c..c34005f1e4 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/kafka/tasks/main.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/kafka/tasks/main.yml @@ -8,8 +8,8 @@ - include_tasks: setup-kafka.yml -- include_tasks: verify-kafka.yml - +# - include_tasks: verify-kafka.yml # todo change testing kafka to bash or remove it? (since we test it using serverspec) + - include_tasks: metrics.yml when: exporter.stat.exists diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/kafka/tasks/setup-kafka.yml b/core/src/epicli/data/common/ansible/playbooks/roles/kafka/tasks/setup-kafka.yml index a9afa5b422..854b252be5 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/kafka/tasks/setup-kafka.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/kafka/tasks/setup-kafka.yml @@ -1,53 +1,68 @@ --- -- name: Install debian family packages - apt: - name: - - openjdk-8-jre - update_cache: yes - state: present - when: ansible_os_family == "Debian" +- name: Setup group + group: + name: "{{ specification.kafka_var.group }}" + system: yes + +- name: Setup user + user: + name: "{{ specification.kafka_var.user }}" + system: yes + group: "{{ specification.kafka_var.group }}" + shell: "/usr/sbin/nologin" -- name: Install RedHat family packages - yum: - name: - - java-1.8.0-openjdk - update_cache: yes +- name: Install Java package + package: + name: "java-1.8.0-openjdk-headless" state: present - register: result - retries: 3 - delay: 1 - until: result is succeeded when: ansible_os_family == "RedHat" -- name: Check for Kafka package - stat: - path: "/tmp/kafka_{{ specification.kafka_var.scala.version }}-{{ specification.kafka_var.version }}.tgz" - register: kafka_check - -- name: Fetch Kafka binary package - get_url: - url: "https://archive.apache.org/dist/kafka/{{ specification.kafka_var.version }}/kafka_{{ specification.kafka_var.scala.version }}-{{ specification.kafka_var.version }}.tgz" - dest: "/tmp/kafka_{{ specification.kafka_var.scala.version }}-{{ specification.kafka_var.version }}.tgz" - validate_certs: "{{ validate_certs | bool }}" - when: not kafka_check.stat.exists - -- name: Get sha512 sum of archive - stat: - path: "/tmp/kafka_{{ specification.kafka_var.scala.version }}-{{ specification.kafka_var.version }}.tgz" - checksum_algorithm: sha512 - get_checksum: yes - register: kafka_download_stat - -- name: Display sha of archive - debug: - msg: "Kafka SHA512: {{ kafka_download_stat.stat.checksum }}" +- name: Install Java package + package: + name: "openjdk-8-jre-headless" + state: present + when: ansible_os_family == "Debian" -- name: Verify sha512 of archive before installation - fail: - msg: "File checksum is not correct." - when: kafka_download_stat.stat.checksum != specification.kafka_var.sha +- name: Set Kafka file name to install + set_fact: + kafka_file_name: "{{ specification.kafka_var.file_name }}" + +- name: Download Kafka binaries + include_role: + name: download + tasks_from: download_file + vars: + file_name: "{{ kafka_file_name }}" + +# - name: Check for Kafka package +# stat: +# path: "/tmp/kafka_{{ specification.kafka_var.scala.version }}-{{ specification.kafka_var.version }}.tgz" +# register: kafka_check + +# - name: Fetch Kafka binary package +# get_url: +# url: "https://archive.apache.org/dist/kafka/{{ specification.kafka_var.version }}/kafka_{{ specification.kafka_var.scala.version }}-{{ specification.kafka_var.version }}.tgz" +# dest: "/tmp/kafka_{{ specification.kafka_var.scala.version }}-{{ specification.kafka_var.version }}.tgz" +# validate_certs: "{{ validate_certs | bool }}" +# when: not kafka_check.stat.exists + +# - name: Get sha512 sum of archive +# stat: +# path: "/tmp/kafka_{{ specification.kafka_var.scala.version }}-{{ specification.kafka_var.version }}.tgz" +# checksum_algorithm: sha512 +# get_checksum: yes +# register: kafka_download_stat + +# - name: Display sha of archive +# debug: +# msg: "Kafka SHA512: {{ kafka_download_stat.stat.checksum }}" + +# - name: Verify sha512 of archive before installation +# fail: +# msg: "File checksum is not correct." +# when: kafka_download_stat.stat.checksum != specification.kafka_var.sha - name: Add Kafka's bin dir to the PATH copy: @@ -55,18 +70,6 @@ dest: "/etc/profile.d/kafka_path.sh" mode: 0755 -- name: Setup group - group: - name: "{{ specification.kafka_var.group }}" - system: yes - -- name: Setup user - user: - name: "{{ specification.kafka_var.user }}" - system: yes - group: "{{ specification.kafka_var.group }}" - shell: "/usr/sbin/nologin" - - name: Check for Kafka package stat: path: /opt/kafka_{{ specification.kafka_var.scala.version }}-{{ specification.kafka_var.version }}/bin/kafka-server-start.sh @@ -74,9 +77,9 @@ - name: Uncompress the Kafka tar unarchive: - copy: no + remote_src: yes creates: /opt/kafka_{{ specification.kafka_var.scala.version }}-{{ specification.kafka_var.version }} - src: /tmp/kafka_{{ specification.kafka_var.scala.version }}-{{ specification.kafka_var.version }}.tgz + src: "{{ download_directory }}/{{ kafka_file_name }}" dest: /opt when: not kafka_package.stat.exists diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/kafka_exporter/tasks/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/kafka_exporter/tasks/main.yml index 68aa78aa3c..737e77e520 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/kafka_exporter/tasks/main.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/kafka_exporter/tasks/main.yml @@ -13,69 +13,37 @@ group: kafka_exporter createhome: false -- name: Download kafka_exporter binary to local folder - become: false - get_url: - url: "{{ specification.download_urls[ansible_architecture] }}" - dest: "{{ specification.download_directory }}/kafka_exporter.tar.gz" - validate_certs: "{{ validate_certs | bool }}" - register: _download_archive - until: _download_archive is succeeded - retries: 5 - delay: 2 - delegate_to: localhost +- name: Set Kafka Exporter file name to install + set_fact: + exporter_file_name: "{{ specification.file_name }}" -- name: Get sha512 sum of archive - become: false - stat: - path: "{{ specification.download_directory }}/kafka_exporter.tar.gz" - checksum_algorithm: sha512 - get_checksum: yes - register: kafka_exporter_download_stat - delegate_to: localhost +- name: Download Kafka Exporter binaries + include_role: + name: download + tasks_from: download_file + vars: + file_name: "{{ exporter_file_name }}" -- name: Display sha of archive - debug: - msg: "Kafka Exporter SHA512: {{ kafka_exporter_download_stat.stat.checksum }}" - -- name: Verify sha512 of archive before installation. - fail: - msg: "File checksum is not correct." - when: kafka_exporter_download_stat.stat.checksum not in specification.download_shas - -- name: Ensure download directory exists - become: false - file: - path: "{{ specification.download_directory }}/kafka_exporter" - state: directory - delegate_to: localhost - -- name: Unpack kafka_exporter binary - become: false - unarchive: - src: "{{ specification.download_directory }}/kafka_exporter.tar.gz" - dest: "{{ specification.download_directory }}/kafka_exporter" - creates: "{{ specification.download_directory }}/kafka_exporter/kafka_exporter" - extra_opts: [--strip-components=1] - delegate_to: localhost - check_mode: false - -- name: Create /opt/kafka_exporter directories +- name: Create /opt/kafka_exporter directory become: yes file: path: "{{ item }}" recurse: yes owner: root - group: "kafka_exporter" + group: kafka_exporter mode: 0750 state: directory with_items: - /opt/kafka_exporter -- name: Propagate kafka_exporter binaries - copy: - src: "{{ specification.download_directory }}/kafka_exporter/kafka_exporter" - dest: "/opt/kafka_exporter/kafka_exporter" +- name: Unpack kafka_exporter binary + become: true + unarchive: + remote_src: yes + src: "{{ download_directory }}/{{ exporter_file_name }}" + dest: "/opt/kafka_exporter" + creates: "/opt/kafka_exporter/kafka_exporter" + extra_opts: [--strip-components=1] mode: 0755 owner: root group: kafka_exporter diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/kibana/tasks/Debian.yml b/core/src/epicli/data/common/ansible/playbooks/roles/kibana/tasks/Debian.yml index 080d942eb0..1be9d0e028 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/kibana/tasks/Debian.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/kibana/tasks/Debian.yml @@ -1,38 +1,6 @@ ---- -## Kibana - -- name: Package install - apt: - deb: "https://artifacts.elastic.co/downloads/kibana/kibana-oss-{{ specification.kibana_version }}-amd64.deb" - when: - - groups['kibana'][0] == inventory_hostname - -- name: Replace with interface - replace: - path: /etc/kibana/kibana.yml - regexp: '^#elasticsearch.url: "http://localhost:9200"$' - replace: "elasticsearch.url: \"http://{{ hostvars[groups['kibana'][0]]['ansible_default_ipv4']['address'] }}:9200\"" - backup: no - when: - - groups['kibana'][0] == inventory_hostname - -- name: Replace with interface - replace: - path: /etc/kibana/kibana.yml - regexp: '^#server.host: "localhost"$' - replace: "server.host: \"{{ hostvars[groups['kibana'][0]]['ansible_default_ipv4']['address'] }}\"" - backup: no - when: - - groups['kibana'][0] == inventory_hostname - -- include_tasks: setup-logging.yml - when: - - groups['kibana'][0] == inventory_hostname - -- name: Start kibana service - service: - name: kibana - state: started - enabled: yes - when: - - groups['kibana'][0] == inventory_hostname +- name: Install Kibana package + apt: + name: + - kibana-oss + update_cache: yes + state: present \ No newline at end of file diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/kibana/tasks/RedHat.yml b/core/src/epicli/data/common/ansible/playbooks/roles/kibana/tasks/RedHat.yml index 5df069aa66..c0664086d6 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/kibana/tasks/RedHat.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/kibana/tasks/RedHat.yml @@ -1,44 +1,6 @@ ---- -## Kibana master - -- name: Run echo - shell: "echo 'I am kibana on RedHat'" - register: output - when: - - groups['kibana'][0] == inventory_hostname - -- name: Install the latest version of Kibana +- name: Install Kibana package yum: - name: "https://artifacts.elastic.co/downloads/kibana/kibana-oss-{{ specification.kibana_version }}-x86_64.rpm" - when: - - groups['kibana'][0] == inventory_hostname - -- name: Replace with interface - replace: - path: /etc/kibana/kibana.yml - regexp: '^#elasticsearch.url: "http://localhost:9200"$' - replace: "elasticsearch.url: \"http://{{ hostvars[groups['kibana'][0]]['ansible_default_ipv4']['address'] }}:9200\"" - backup: no - when: - - groups['kibana'][0] == inventory_hostname - -- name: Replace with interface - replace: - path: /etc/kibana/kibana.yml - regexp: '^#server.host: "localhost"$' - replace: "server.host: \"{{ hostvars[groups['kibana'][0]]['ansible_default_ipv4']['address'] }}\"" - backup: no - when: - - groups['kibana'][0] == inventory_hostname - -- include_tasks: setup-logging.yml - when: - - groups['kibana'][0] == inventory_hostname - -- name: Start kibana service - service: - name: kibana - state: started - enabled: yes - when: - - groups['kibana'][0] == inventory_hostname + name: + - kibana-oss + update_cache: yes + state: present \ No newline at end of file diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/kibana/tasks/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/kibana/tasks/main.yml index b7784c818f..0dd21b6e8b 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/kibana/tasks/main.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/kibana/tasks/main.yml @@ -1,4 +1,33 @@ --- -## Elasticsearch master - - include_tasks: "{{ ansible_os_family }}.yml" + +- name: Replace with interface + replace: + path: /etc/kibana/kibana.yml + regexp: '^#elasticsearch.url: "http://localhost:9200"$' + replace: "elasticsearch.url: \"http://{{ hostvars[groups['kibana'][0]]['ansible_default_ipv4']['address'] }}:9200\"" + backup: no + when: + - groups['kibana'][0] == inventory_hostname + +- name: Replace with interface + replace: + path: /etc/kibana/kibana.yml + regexp: '^#server.host: "localhost"$' + replace: "server.host: \"{{ hostvars[groups['kibana'][0]]['ansible_default_ipv4']['address'] }}\"" + backup: no + when: + - groups['kibana'][0] == inventory_hostname + +- include_tasks: setup-logging.yml + when: + - groups['kibana'][0] == inventory_hostname + +- name: Start kibana service + service: + name: kibana + state: started + enabled: yes + when: + - groups['kibana'][0] == inventory_hostname + diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_common/tasks/install-packages-Debian.yml b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_common/tasks/install-packages-Debian.yml index c68ed3d08a..54c4cc2abb 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_common/tasks/install-packages-Debian.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_common/tasks/install-packages-Debian.yml @@ -1,21 +1,9 @@ --- -- name: APT Key - apt_key: - url: https://packages.cloud.google.com/apt/doc/apt-key.gpg - state: present - -- name: Add kubernetes repository - apt_repository: - repo: deb http://apt.kubernetes.io/ kubernetes-xenial main - state: present - filename: kubernetes - - name: Install NFS package for Debian family apt: name: - nfs-common - update_cache: yes state: present - name: Install Kubernetes packages for Debian family @@ -25,5 +13,4 @@ - kubelet={{specification.version}}-00 - kubectl={{specification.version}}-00 - kubeadm={{specification.version}}-00 - update_cache: yes - state: present + state: present \ No newline at end of file diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_common/tasks/install-packages-RedHat.yml b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_common/tasks/install-packages-RedHat.yml index ecb1493020..79d5b05bfb 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_common/tasks/install-packages-RedHat.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_common/tasks/install-packages-RedHat.yml @@ -1,14 +1,5 @@ --- - -- name: Add kubernetes repository - yum_repository: - name: kubernetes - description: Kubernetes - file: kubernetes - baseurl: https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64 - gpgcheck: yes - gpgkey: https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg - + - name: Install NFS package for RedHat family yum: name: @@ -20,8 +11,8 @@ yum: name: - kubernetes-cni-0.7.5-0 - - kubelet-{{specification.version}} - - kubectl-{{specification.version}} - - kubeadm-{{specification.version}} + - kubelet-{{specification.version}}-0 + - kubectl-{{specification.version}}-0 + - kubeadm-{{specification.version}}-0 update_cache: yes - state: present + state: present \ No newline at end of file diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_common/tasks/load-image.yml b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_common/tasks/load-image.yml new file mode 100644 index 0000000000..450fbcbbea --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_common/tasks/load-image.yml @@ -0,0 +1,35 @@ +--- +- name: Create tag name with local image registry + set_fact: + new_image_tag: "{{image_registry_address}}/{{ docker_image.name }}" + changed_when: false + +- name: Check if image is already loaded + shell: "docker images {{ new_image_tag }} --format {{ '{{' }}.ID{{ '}}' }}" + register: image_check + ignore_errors: true + changed_when: false + +- name: Load image if does not exists + block: + - name: Download file + include_role: + name: download + tasks_from: download_image + vars: + file_name: "{{ docker_image.file_name }}" + + - name: Load image {{ docker_image.name }} + become: yes + shell: "docker load --input {{ download_directory }}/{{ docker_image.file_name }}" + + - name: Tag image {{ docker_image.name }} with {{ new_image_tag }} + become: yes + shell: "docker tag {{ docker_image.name }} {{ new_image_tag }}" + + - name: Push image to registry {{ docker_image.name }} + become: yes + shell: "docker push {{ new_image_tag }}" + + when: + - image_check.stdout | length == 0 diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_common/tasks/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_common/tasks/main.yml index 8bf67323c0..fa57d01c90 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_common/tasks/main.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_common/tasks/main.yml @@ -1,2 +1,34 @@ --- + - include_tasks: install-packages.yml + +- name: Include load-image.yml + include_tasks: "load-image.yml" + vars: + docker_image: "{{ item }}" + loop: "{{ specification.images_to_load }}" + when: not custom_image_registry_address + +- name: Enable ip forwarding + sysctl: + name: net.ipv4.ip_forward + value: "1" + state: present + reload: yes + +- name: Check if bridge-nf-call-iptables key exists + command: "sysctl net.bridge.bridge-nf-call-iptables" + failed_when: false + changed_when: false + register: sysctl_bridge_nf_call_iptables + +- name: Enable bridge-nf-call tables + sysctl: + name: "{{ item }}" + state: present + value: "1" + reload: yes + when: sysctl_bridge_nf_call_iptables.rc == 0 + with_items: + - net.bridge.bridge-nf-call-iptables + - net.bridge.bridge-nf-call-ip6tables \ No newline at end of file diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/cni-plugins/calico.yml b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/cni-plugins/calico.yml index 70647e171e..cbac0895b0 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/cni-plugins/calico.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/cni-plugins/calico.yml @@ -1,11 +1,13 @@ --- -- name: Copy calico config - copy: - src: calico.yml - dest: /home/{{ admin_user.name }}/ + +- name: Create calico deployment + template: + dest: "/home/{{ admin_user.name }}/calico.yml" + src: calico.yml.j2 owner: "{{ admin_user.name }}" group: "{{ admin_user.name }}" + - name: Apply calico definition shell: kubectl apply --kubeconfig=/home/{{ admin_user.name }}/.kube/config -f /home/{{ admin_user.name }}/calico.yml become_user: "{{ admin_user.name }}" \ No newline at end of file diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/cni-plugins/flannel.yml b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/cni-plugins/flannel.yml index 53f02c5d55..5ba8167b50 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/cni-plugins/flannel.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/cni-plugins/flannel.yml @@ -1,8 +1,8 @@ --- -- name: Copy flannel config - copy: - src: kube-flannel.yml - dest: /home/{{ admin_user.name }}/ +- name: Create flannel deployment + template: + dest: "/home/{{ admin_user.name }}/kube-flannel.yml" + src: kube-flannel.yml.j2 owner: "{{ admin_user.name }}" group: "{{ admin_user.name }}" diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/single-master.yml b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/single-master.yml index 9d2a095aeb..f7e98b77e1 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/single-master.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/single-master.yml @@ -8,9 +8,7 @@ path: /etc/kubernetes/manifests/kube-apiserver.yaml register: kube -- name: kubeadm config images pull --kubernetes-version={{ specification.version }} - shell: "kubeadm config images pull --kubernetes-version={{ specification.version }}" - # when: not kube.stat.exists +# TODO ensure there are images in repo - if not, fail - name: Creates directory file: @@ -55,10 +53,10 @@ owner: "{{ admin_user.name }}" group: "{{ admin_user.name }}" -- name: Copy dashboard yaml - copy: - src: kubernetes-dashboard.yml - dest: /home/{{ admin_user.name }}/ +- name: Create dashboard deployment + template: + dest: "/home/{{ admin_user.name }}/kubernetes-dashboard.yml" + src: kubernetes-dashboard.yml.j2 owner: "{{ admin_user.name }}" group: "{{ admin_user.name }}" @@ -83,10 +81,10 @@ owner: "{{ admin_user.name }}" group: "{{ admin_user.name }}" -- name: Copy coredns-config.yml - copy: - src: coredns-config.yml - dest: /home/{{ admin_user.name }}/ +- name: Create CoreDNS deployment + template: + dest: "/home/{{ admin_user.name }}/coredns-config.yml" + src: coredns-config.yml.j2 owner: "{{ admin_user.name }}" group: "{{ admin_user.name }}" diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/files/calico.yml b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/templates/calico.yml.j2 similarity index 98% rename from core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/files/calico.yml rename to core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/templates/calico.yml.j2 index f1f29e13a7..54385980b3 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/files/calico.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/templates/calico.yml.j2 @@ -515,7 +515,7 @@ spec: # It can be deleted if this is a fresh installation, or if you have already # upgraded to use calico-ipam. - name: upgrade-ipam - image: calico/cni:v3.8.0 + image: {{ image_registry_address }}/calico/cni:v3.8.1 command: ["/opt/cni/bin/calico-ipam", "-upgrade"] env: - name: KUBERNETES_NODE_NAME @@ -535,7 +535,7 @@ spec: # This container installs the CNI binaries # and CNI network config file on each node. - name: install-cni - image: calico/cni:v3.8.0 + image: {{ image_registry_address }}/calico/cni:v3.8.1 command: ["/install-cni.sh"] env: # Name of the CNI config file to create. @@ -569,7 +569,7 @@ spec: # Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes # to communicate with Felix over the Policy Sync API. - name: flexvol-driver - image: calico/pod2daemon-flexvol:v3.8.0 + image: {{ image_registry_address }}/calico/pod2daemon-flexvol:v3.8.1 volumeMounts: - name: flexvol-driver-host mountPath: /host/driver @@ -578,7 +578,7 @@ spec: # container programs network policy and routes on each # host. - name: calico-node - image: calico/node:v3.8.0 + image: {{ image_registry_address }}/calico/node:v3.8.1 env: # Use Kubernetes API as the backing datastore. - name: DATASTORE_TYPE @@ -752,7 +752,7 @@ spec: priorityClassName: system-cluster-critical containers: - name: calico-kube-controllers - image: calico/kube-controllers:v3.8.0 + image: {{ image_registry_address }}/calico/kube-controllers:v3.8.1 env: # Choose which controllers to run. - name: ENABLED_CONTROLLERS diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/templates/canal-deployment.yml.j2 b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/templates/canal-deployment.yml.j2 index 8717258e53..a41db4eed1 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/templates/canal-deployment.yml.j2 +++ b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/templates/canal-deployment.yml.j2 @@ -399,7 +399,7 @@ spec: # This container installs the CNI binaries # and CNI network config file on each node. - name: install-cni - image: calico/cni:v3.8.1 + image: {{ image_registry_address }}/calico/cni:v3.8.1 command: ["/install-cni.sh"] env: # Name of the CNI config file to create. @@ -427,7 +427,7 @@ spec: # Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes # to communicate with Felix over the Policy Sync API. - name: flexvol-driver - image: calico/pod2daemon-flexvol:v3.8.1 + image: {{ image_registry_address }}/calico/pod2daemon-flexvol:v3.8.1 volumeMounts: - name: flexvol-driver-host mountPath: /host/driver @@ -436,7 +436,7 @@ spec: # container programs network policy and routes on each # host. - name: calico-node - image: calico/node:v3.8.1 + image: {{ image_registry_address }}/calico/node:v3.8.1 env: # Use Kubernetes API as the backing datastore. - name: DATASTORE_TYPE @@ -520,7 +520,7 @@ spec: # This container runs flannel using the kube-subnet-mgr backend # for allocating subnets. - name: kube-flannel - image: quay.io/coreos/flannel:v0.11.0 + image: {{ image_registry_address }}/quay.io/coreos/flannel:v0.11.0 command: [ "/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr" ] securityContext: privileged: true diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/files/coredns-config.yml b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/templates/coredns-config.yml.j2 similarity index 98% rename from core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/files/coredns-config.yml rename to core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/templates/coredns-config.yml.j2 index 1e7849e87d..6760d46561 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/files/coredns-config.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/templates/coredns-config.yml.j2 @@ -103,7 +103,7 @@ spec: beta.kubernetes.io/os: linux containers: - name: coredns - image: coredns/coredns:1.5.0 + image: {{ image_registry_address }}/coredns/coredns:1.5.0 imagePullPolicy: IfNotPresent resources: limits: diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/files/kube-flannel.yml b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/templates/kube-flannel.yml.j2 similarity index 93% rename from core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/files/kube-flannel.yml rename to core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/templates/kube-flannel.yml.j2 index 859d55bb61..ad51445749 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/files/kube-flannel.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/templates/kube-flannel.yml.j2 @@ -154,7 +154,7 @@ spec: serviceAccountName: flannel initContainers: - name: install-cni - image: quay.io/coreos/flannel:v0.11.0-amd64 + image: {{ image_registry_address }}/quay.io/coreos/flannel:v0.11.0-amd64 command: - cp args: @@ -168,7 +168,7 @@ spec: mountPath: /etc/kube-flannel/ containers: - name: kube-flannel - image: quay.io/coreos/flannel:v0.11.0-amd64 + image: {{ image_registry_address }}/quay.io/coreos/flannel:v0.11.0-amd64 command: - /opt/bin/flanneld args: @@ -234,7 +234,7 @@ spec: serviceAccountName: flannel initContainers: - name: install-cni - image: quay.io/coreos/flannel:v0.11.0-arm64 + image: {{ image_registry_address }}/quay.io/coreos/flannel:v0.11.0-arm64 command: - cp args: @@ -248,7 +248,7 @@ spec: mountPath: /etc/kube-flannel/ containers: - name: kube-flannel - image: quay.io/coreos/flannel:v0.11.0-arm64 + image: {{ image_registry_address }}/quay.io/coreos/flannel:v0.11.0-arm64 command: - /opt/bin/flanneld args: @@ -314,7 +314,7 @@ spec: serviceAccountName: flannel initContainers: - name: install-cni - image: quay.io/coreos/flannel:v0.11.0-arm + image: {{ image_registry_address }}/quay.io/coreos/flannel:v0.11.0-arm command: - cp args: @@ -328,7 +328,7 @@ spec: mountPath: /etc/kube-flannel/ containers: - name: kube-flannel - image: quay.io/coreos/flannel:v0.11.0-arm + image: {{ image_registry_address }}/quay.io/coreos/flannel:v0.11.0-arm command: - /opt/bin/flanneld args: @@ -394,7 +394,7 @@ spec: serviceAccountName: flannel initContainers: - name: install-cni - image: quay.io/coreos/flannel:v0.11.0-ppc64le + image: {{ image_registry_address }}/quay.io/coreos/flannel:v0.11.0-ppc64le command: - cp args: @@ -408,7 +408,7 @@ spec: mountPath: /etc/kube-flannel/ containers: - name: kube-flannel - image: quay.io/coreos/flannel:v0.11.0-ppc64le + image: {{ image_registry_address }}/quay.io/coreos/flannel:v0.11.0-ppc64le command: - /opt/bin/flanneld args: @@ -474,7 +474,7 @@ spec: serviceAccountName: flannel initContainers: - name: install-cni - image: quay.io/coreos/flannel:v0.11.0-s390x + image: {{ image_registry_address }}/quay.io/coreos/flannel:v0.11.0-s390x command: - cp args: @@ -488,7 +488,7 @@ spec: mountPath: /etc/kube-flannel/ containers: - name: kube-flannel - image: quay.io/coreos/flannel:v0.11.0-s390x + image: {{ image_registry_address }}/quay.io/coreos/flannel:v0.11.0-s390x command: - /opt/bin/flanneld args: diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/templates/kubeadm-config.yml.j2 b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/templates/kubeadm-config.yml.j2 index aa3a2f978d..9d4ee08580 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/templates/kubeadm-config.yml.j2 +++ b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/templates/kubeadm-config.yml.j2 @@ -20,5 +20,12 @@ networking: dnsDomain: {{ specification.advanced.networking.dnsDomain }} podSubnet: {{ specification.advanced.networking.podSubnet }} serviceSubnet: {{ specification.advanced.networking.serviceSubnet }} -imageRepository: {{ specification.advanced.imageRepository }} + +{% if custom_image_registry_address|length == 0 %} +imageRepository: {{ image_registry_address }}/{{ specification.advanced.imageRepository }} +{% else %} +imageRepository: {{ custom_image_registry_address }}/{{ specification.advanced.imageRepository }} +{% endif %} + + certificatesDir: {{ specification.advanced.certificatesDir }} \ No newline at end of file diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/files/kubernetes-dashboard.yml b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/templates/kubernetes-dashboard.yml.j2 similarity index 97% rename from core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/files/kubernetes-dashboard.yml rename to core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/templates/kubernetes-dashboard.yml.j2 index a33a5c2515..fea5d584a7 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/files/kubernetes-dashboard.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/templates/kubernetes-dashboard.yml.j2 @@ -188,7 +188,7 @@ spec: spec: containers: - name: kubernetes-dashboard - image: kubernetesui/dashboard:v2.0.0-beta1 + image: {{ image_registry_address }}/kubernetesui/dashboard:v2.0.0-beta1 imagePullPolicy: Always ports: - containerPort: 8443 @@ -263,7 +263,7 @@ spec: spec: containers: - name: kubernetes-metrics-scraper - image: kubernetesui/metrics-scraper:v1.0.0 + image: {{ image_registry_address }}/kubernetesui/metrics-scraper:v1.0.0 ports: - containerPort: 8000 protocol: TCP diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/node_exporter/tasks/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/node_exporter/tasks/main.yml index f4af0ccd78..03c90b08a2 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/node_exporter/tasks/main.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/node_exporter/tasks/main.yml @@ -13,53 +13,16 @@ group: node_exporter createhome: false -- name: Ensure download directory exists - become: false - file: - path: "{{ specification.download_directory }}/node_exporter" - state: directory - delegate_to: localhost - -# TODO: Checking download node_exporter -- name: Download node_exporter binary to local folder - become: false - get_url: - url: "{{ specification.download_urls[ansible_architecture] }}" - dest: "{{ specification.download_directory }}/node_exporter.tar.gz" - validate_certs: "{{ validate_certs | bool}}" - register: _download_archive - until: _download_archive is succeeded - retries: 5 - delay: 2 - delegate_to: localhost - -- name: Get sha512 sum of archive - become: false - stat: - path: "{{ specification.download_directory }}/node_exporter.tar.gz" - checksum_algorithm: sha512 - get_checksum: yes - register: node_exporter_stat - delegate_to: localhost +- name: Set Node Exporter file name to install + set_fact: + exporter_file_name: "{{ specification.file_name }}" -- name: Display sha of archive - debug: - msg: "Node Exporter SHA512: {{ node_exporter_stat.stat.checksum }}" - -- name: Verify sha512 of archive before installation. - fail: - msg: "File checksum is not correct." - when: node_exporter_stat.stat.checksum not in specification.download_shas - -- name: Unpack node_exporter binary - become: false - unarchive: - src: "{{ specification.download_directory }}/node_exporter.tar.gz" - dest: "{{ specification.download_directory }}/node_exporter" - creates: "{{ specification.download_directory }}/node_exporter/node_exporter" - extra_opts: [--strip-components=1] - delegate_to: localhost - check_mode: false +- name: Download Node Exporter binaries + include_role: + name: download + tasks_from: download_file + vars: + file_name: "{{ exporter_file_name }}" - name: Create /opt/node_exporter directories become: yes @@ -73,10 +36,14 @@ with_items: - /opt/node_exporter -- name: Propagate node_exporter binaries - copy: - src: "{{ specification.download_directory }}/node_exporter/node_exporter" - dest: "/opt/node_exporter/node_exporter" +- name: Unpack node_exporter binary + become: true + unarchive: + remote_src: yes + src: "{{ download_directory }}/{{ exporter_file_name }}" + dest: "/opt/node_exporter" + creates: "/opt/node_exporter/node_exporter" + extra_opts: [--strip-components=1] mode: 0755 owner: root group: node_exporter diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/postgresql/tasks/Debian.yml b/core/src/epicli/data/common/ansible/playbooks/roles/postgresql/tasks/Debian.yml index aa36549a56..53991a06bf 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/postgresql/tasks/Debian.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/postgresql/tasks/Debian.yml @@ -8,7 +8,7 @@ - postgresql-contrib-10 - python-psycopg2 # required for postresql ansible management update_cache: yes - state: present + state: present - name: Changing pg_hba.conf replace: diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/postgresql/tasks/RedHat.yml b/core/src/epicli/data/common/ansible/playbooks/roles/postgresql/tasks/RedHat.yml index 8e5d187a0e..e15fc5902b 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/postgresql/tasks/RedHat.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/postgresql/tasks/RedHat.yml @@ -1,31 +1,5 @@ --- # Postgresql Red Hat family of specific tasks - -- name: Check if repository file exists - stat: - path: /etc/yum.repos.d/redhat-rhui.repo - register: rhui_exist - when: - - specification.provider == "aws" - -- name: Check if Software Collections repository on EC2 exists - shell: grep -c -i rhui-REGION-rhel-server-rhscl /etc/yum.repos.d/redhat-rhui.repo - register: sc_repo_count - failed_when: "sc_repo_count.rc == 2" - when: - - specification.provider == "aws" - - rhui_exist - -- name: Enable Software Collections on EC2 - ini_file: - dest: /etc/yum.repos.d/redhat-rhui.repo - section: rhui-REGION-rhel-server-rhscl - option: enabled - value: 1 - when: - - specification.provider == "aws" - - sc_repo_count.stdout|int == 1 - - name: Install postgresql family packages yum: name: diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/postgresql/tasks/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/postgresql/tasks/main.yml index 51bc61e23d..e47a430997 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/postgresql/tasks/main.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/postgresql/tasks/main.yml @@ -1,5 +1,4 @@ --- # Common main as the entry point - - include_tasks: "{{ ansible_os_family }}.yml" diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/prometheus/tasks/install-alertmanager.yml b/core/src/epicli/data/common/ansible/playbooks/roles/prometheus/tasks/install-alertmanager.yml index fb44e8a4cc..fcf6f8ee0c 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/prometheus/tasks/install-alertmanager.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/prometheus/tasks/install-alertmanager.yml @@ -1,56 +1,23 @@ --- -- name: Ensure download directory exists - become: false - file: - path: "{{ specification.download_directory }}/alert_manager_libraries" - state: directory - delegate_to: localhost - -# TODO: Checking alertmanager -- name: Download Alertmanager binary to local folder - become: false - get_url: - url: "{{ specification.alerts.alertmanager.download_urls[ansible_architecture] }}" - dest: "{{ specification.download_directory }}/alertmanager.tar.gz" - validate_certs: "{{ validate_certs | bool}}" - register: _download_archive - until: _download_archive is succeeded - retries: 5 - delay: 2 - delegate_to: localhost - -- name: Get sha512 sum of archive - become: false - stat: - path: "{{ specification.download_directory }}/alertmanager.tar.gz" - checksum_algorithm: sha512 - get_checksum: yes - register: alertmanager_download_stat - delegate_to: localhost - -- name: Display sha of archive - debug: - msg: "Alertmanager SHA512: {{ alertmanager_download_stat.stat.checksum }}" - -- name: Verify sha512 of archive before installation. - fail: - msg: "File checksum is not correct." - when: alertmanager_download_stat.stat.checksum not in specification.alerts.alertmanager.download_shas - -- name: Unpack Alertmanager binary - become: false +- name: Set Prometheus file name to install + set_fact: + binary_file_name: "{{ specification.alerts.alertmanager.file_name }}" + +- name: Download Prometheus binaries + include_role: + name: download + tasks_from: download_file + vars: + file_name: "{{ binary_file_name }}" + +- name: Unpack AlertManager binary + become: true unarchive: - src: "{{ specification.download_directory }}/alertmanager.tar.gz" - dest: "{{ specification.download_directory }}/alert_manager_libraries" - creates: "{{ specification.download_directory }}/alert_manager_libraries/alertmanager" + remote_src: yes + src: "{{ download_directory }}/{{ binary_file_name }}" + dest: "/usr/local/bin" + creates: "/usr/local/bin/alertmanager" extra_opts: [--strip-components=1] - delegate_to: localhost - check_mode: false - -- name: Propagate Alertmanager - copy: - src: "{{ specification.download_directory }}/alert_manager_libraries/alertmanager" - dest: "/usr/local/bin/alertmanager" mode: 0755 owner: root group: prometheus diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/prometheus/tasks/install.yml b/core/src/epicli/data/common/ansible/playbooks/roles/prometheus/tasks/install.yml index 3d912cd0a1..b54acd3b81 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/prometheus/tasks/install.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/prometheus/tasks/install.yml @@ -34,89 +34,118 @@ - "{{ specification.config_directory }}/rules" - "{{ specification.config_directory }}/file_sd" -- name: Ensure download directory exists - become: false - file: - path: "{{ specification.download_directory }}/prometheus_binaries" - state: directory - delegate_to: localhost - -# TODO: Checking prometheus -- name: Download Prometheus binary to local folder - become: false - get_url: - url: "{{ specification.download_urls[ansible_architecture] }}" - dest: "{{ specification.download_directory }}/prometheus.tar.gz" - validate_certs: "{{ validate_certs | bool}}" - register: _download_archive - until: _download_archive is succeeded - retries: 5 - delay: 2 - delegate_to: localhost - -- name: Get sha512 sum of archive - become: false - stat: - path: "{{ specification.download_directory }}/prometheus.tar.gz" - checksum_algorithm: sha512 - get_checksum: yes - register: prometheus_download_stat - delegate_to: localhost - -- name: Display sha of archive - debug: - msg: "Prometheus SHA512: {{ prometheus_download_stat.stat.checksum }}" - -- name: Verify sha512 of archive before installation. - fail: - msg: "File checksum is not correct." - when: prometheus_download_stat.stat.checksum not in specification.download_shas +# - name: Ensure download directory exists +# become: false +# file: +# path: "{{ specification.download_directory }}/prometheus_binaries" +# state: directory +# delegate_to: localhost + +# # TODO: Checking prometheus +# - name: Download Prometheus binary to local folder +# become: false +# get_url: +# url: "{{ specification.download_urls[ansible_architecture] }}" +# dest: "{{ specification.download_directory }}/prometheus.tar.gz" +# validate_certs: "{{ validate_certs | bool}}" +# register: _download_archive +# until: _download_archive is succeeded +# retries: 5 +# delay: 2 +# delegate_to: localhost + +# - name: Get sha512 sum of archive +# become: false +# stat: +# path: "{{ specification.download_directory }}/prometheus.tar.gz" +# checksum_algorithm: sha512 +# get_checksum: yes +# register: prometheus_download_stat +# delegate_to: localhost + +# - name: Display sha of archive +# debug: +# msg: "Prometheus SHA512: {{ prometheus_download_stat.stat.checksum }}" + +# - name: Verify sha512 of archive before installation. +# fail: +# msg: "File checksum is not correct." +# when: prometheus_download_stat.stat.checksum not in specification.download_shas + +# - name: Unpack Prometheus binary +# become: false +# unarchive: +# src: "{{ specification.download_directory }}/prometheus.tar.gz" +# dest: "{{ specification.download_directory }}/prometheus_binaries" +# creates: "{{ specification.download_directory }}/prometheus_binaries/prometheus" +# extra_opts: [--strip-components=1] +# delegate_to: localhost +# check_mode: false + +# - name: Propagate Prometheus and promtool binaries +# copy: +# src: "{{ specification.download_directory }}/prometheus_binaries/{{ item }}" +# dest: "/usr/local/bin/{{ item }}" +# mode: 0755 +# owner: root +# group: prometheus +# with_items: +# - prometheus +# - promtool +# check_mode: false +# notify: +# - restart prometheus + +# - name: Propagate console libraries templates +# copy: +# src: "{{ specification.download_directory }}/prometheus_binaries/{{ item }}" +# dest: "{{ specification.config_directory }}/{{ item }}/" +# mode: 0755 +# with_items: +# - console_libraries +# - consoles +# check_mode: false +# notify: +# - restart prometheus + +# - name: Remove prometheus binaries from old location +# file: +# path: "{{ item }}" +# state: absent +# with_items: +# - /opt/prometheus/prometheus +# - /opt/prometheus/promtool +# - /opt/prometheus + +- name: Set Prometheus file name to install + set_fact: + binary_file_name: "{{ specification.file_name }}" + +- name: Package + debug: msg="{{ binary_file_name }}" + +- name: Download Prometheus binaries + include_role: + name: download + tasks_from: download_file + vars: + file_name: "{{ binary_file_name }}" - name: Unpack Prometheus binary - become: false + become: true unarchive: - src: "{{ specification.download_directory }}/prometheus.tar.gz" - dest: "{{ specification.download_directory }}/prometheus_binaries" - creates: "{{ specification.download_directory }}/prometheus_binaries/prometheus" + src: "{{ download_directory }}/{{ binary_file_name }}" + remote_src: yes + dest: "/usr/local/bin" + creates: "/usr/local/bin/prometheus" extra_opts: [--strip-components=1] - delegate_to: localhost - check_mode: false - -- name: Propagate Prometheus and promtool binaries - copy: - src: "{{ specification.download_directory }}/prometheus_binaries/{{ item }}" - dest: "/usr/local/bin/{{ item }}" mode: 0755 owner: root group: prometheus - with_items: - - prometheus - - promtool check_mode: false notify: - restart prometheus -- name: Propagate console libraries templates - copy: - src: "{{ specification.download_directory }}/prometheus_binaries/{{ item }}" - dest: "{{ specification.config_directory }}/{{ item }}/" - mode: 0755 - with_items: - - console_libraries - - consoles - check_mode: false - notify: - - restart prometheus - -- name: Remove prometheus binaries from old location - file: - path: "{{ item }}" - state: absent - with_items: - - /opt/prometheus/prometheus - - /opt/prometheus/promtool - - /opt/prometheus - - name: Create systemd service unit template: src: prometheus.service.j2 diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/prometheus/tasks/preflight.yml b/core/src/epicli/data/common/ansible/playbooks/roles/prometheus/tasks/preflight.yml index a91df64166..39d5984f3f 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/prometheus/tasks/preflight.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/prometheus/tasks/preflight.yml @@ -84,12 +84,12 @@ when: prometheus_version == "latest" #commented out due to issues when ran on macOS -#- name: "Get checksum for {{ go_arch_map[ansible_architecture] | default(ansible_architecture) }} architecture" +#- name: "Get checksum for {{ shared.architecture_map[ansible_architecture] | default(ansible_architecture) }} architecture" # set_fact: # prometheus_checksum: "{{ item.split(' ')[0] }}" # with_items: # - "{{ lookup('url', 'https://github.com/prometheus/prometheus/releases/download/v' + prometheus_version + '/sha256sums.txt', wantlist=True) | list }}" -# when: "('linux-' + (go_arch_map[ansible_architecture] | default(ansible_architecture)) + '.tar.gz') in item" +# when: "('linux-' + (shared.architecture_map[ansible_architecture] | default(ansible_architecture)) + '.tar.gz') in item" - name: Get systemd version shell: systemctl --version | awk '$1 == "systemd" {print $2}' diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/rabbitmq/tasks/Debian.yml b/core/src/epicli/data/common/ansible/playbooks/roles/rabbitmq/tasks/Debian.yml index 6b2d677b42..97c0596e19 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/rabbitmq/tasks/Debian.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/rabbitmq/tasks/Debian.yml @@ -1,28 +1,11 @@ ---- -- name: Get APT Key - apt_key: - url: https://github.com/rabbitmq/signing-keys/releases/download/2.0/rabbitmq-release-signing-key.asc - state: present - -- name: Add Erlang repository - apt_repository: - repo: deb http://dl.bintray.com/rabbitmq-erlang/debian bionic erlang-21.x - state: present - filename: bintray.erlang.list - update_cache: true - -- name: Install dependencies for RabbitMQ - apt: - update_cache: true +- name: Install Rabitmq packages + apt: name: - init-system-helpers - socat - erlang-nox - adduser - logrotate - state: present - -- name: Install RabbitMQ package - apt: - deb: "https://github.com/rabbitmq/rabbitmq-server/releases/download/v{{ specification.version }}/rabbitmq-server_{{ specification.version }}-1_all.deb" - state: present + - rabbitmq-server + update_cache: yes + state: present \ No newline at end of file diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/rabbitmq/tasks/RedHat.yml b/core/src/epicli/data/common/ansible/playbooks/roles/rabbitmq/tasks/RedHat.yml index a4dab851a2..9dda9c0283 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/rabbitmq/tasks/RedHat.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/rabbitmq/tasks/RedHat.yml @@ -1,19 +1,8 @@ ---- -## RabbitMQ Red Hat - -- name: Install Erlang package - yum: - name: "https://github.com/rabbitmq/erlang-rpm/releases/download/v21.2.2/erlang-21.2.2-1.el7.centos.x86_64.rpm" - state: present - -- name: Install dependency for RabbitMQ +- name: Install Rabbitmq packages yum: name: - socat - logrotate - state: present - -- name: Install RabbitMQ package - yum: - name: "https://github.com/rabbitmq/rabbitmq-server/releases/download/v{{ specification.version }}/rabbitmq-server-{{ specification.version }}-1.el7.noarch.rpm" - state: present + - erlang-21.3.* # order matters, check RabbitMQ/Erlang version compatibility matrix before modification + - rabbitmq-server-3.7.10 + state: present \ No newline at end of file diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/Debian/add-epirepo-client.sh b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/Debian/add-epirepo-client.sh new file mode 100644 index 0000000000..466f42efc8 --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/Debian/add-epirepo-client.sh @@ -0,0 +1,7 @@ +#!/bin/bash -eu + +REPOSITORY_URL=$1 + +echo "deb [trusted=yes] $REPOSITORY_URL/packages ./" > /etc/apt/sources.list.d/epirepo.list + +apt update \ No newline at end of file diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/Debian/create-enabled-system-repos-list.sh b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/Debian/create-enabled-system-repos-list.sh new file mode 100644 index 0000000000..dcdc4209c5 --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/Debian/create-enabled-system-repos-list.sh @@ -0,0 +1,7 @@ +#!/bin/bash -eu + +REPOS_BACKUP_FILE=/var/tmp/enabled-system-repos.tar + +if [ ! -f "$REPOS_BACKUP_FILE" ]; then + tar --ignore-failed-read --absolute-names -cvpf ${REPOS_BACKUP_FILE} /etc/apt/sources.list /etc/apt/sources.list.d/ 2>&1 +fi diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/Debian/disable-epirepo-client.sh b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/Debian/disable-epirepo-client.sh new file mode 100644 index 0000000000..e6b85d9b5e --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/Debian/disable-epirepo-client.sh @@ -0,0 +1,5 @@ +#!/bin/bash -eu + +rm -f /etc/apt/sources.list.d/epirepo.list +apt-get clean +apt update \ No newline at end of file diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/Debian/disable-system-repos.sh b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/Debian/disable-system-repos.sh new file mode 100644 index 0000000000..a87d67c23a --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/Debian/disable-system-repos.sh @@ -0,0 +1,9 @@ +#!/bin/bash -eu + +REPOS_BACKUP_FILE=/var/tmp/enabled-system-repos.tar + +if [ -f "$REPOS_BACKUP_FILE" ]; then + rm -f /etc/apt/sources.list /etc/apt/sources.list.d/* +else + echo "${REPOS_BACKUP_FILE} file not found. You don't seem to have a backup of the repositories. Cowardly refusing to delete system files." +fi diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/Debian/enable-system-repos.sh b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/Debian/enable-system-repos.sh new file mode 100644 index 0000000000..12fa5c1aba --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/Debian/enable-system-repos.sh @@ -0,0 +1,5 @@ +#!/bin/bash -eu + +REPOS_BACKUP_FILE=/var/tmp/enabled-system-repos.tar + +tar -C / --absolute-name -xvf ${REPOS_BACKUP_FILE} 2>&1 \ No newline at end of file diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/add-epirepo-client.sh b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/add-epirepo-client.sh new file mode 100644 index 0000000000..e6c21587e2 --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/add-epirepo-client.sh @@ -0,0 +1,19 @@ +#!/bin/bash -eu + +REPOSITORY_URL=$1 + +CURL_CMD="curl --head --location --connect-timeout 30 --silent --show-error $REPOSITORY_URL" +CURL_OUTPUT=$($CURL_CMD 2>&1) || { echo "Command failed: $CURL_CMD"; echo "Output was: $CURL_OUTPUT"; exit 2; } + +egrep 'HTTP/.{1,3} 200 OK' <<< "$CURL_OUTPUT" || { echo "HTTP 200 status code not found"; exit 3; } + +cat << EOF > /etc/yum.repos.d/epirepo.repo +[epirepo] +name=epirepo +baseurl=$REPOSITORY_URL/packages/ +enabled=1 +gpgcheck=0 +EOF + +yum makecache fast +yum repolist \ No newline at end of file diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/create-enabled-system-repos-list.sh b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/create-enabled-system-repos-list.sh new file mode 100644 index 0000000000..34d2791bf4 --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/create-enabled-system-repos-list.sh @@ -0,0 +1,8 @@ +#!/bin/bash -eu + +ENABLED_REPOS_LIST_FILE=/var/tmp/enabled-system-repos.txt + +if [ ! -f "$ENABLED_REPOS_LIST_FILE" ]; then + # 'yum repoinfo' or 'yum repolist -v' not used since they may require Internet access, even with --cacheonly + yum --cacheonly repolist enabled | awk '/^$/ {next}; /repo id/ {f=1; next}; /^repolist/ {f=0}; f {sub(/\/.*/,""); print $1}' > $ENABLED_REPOS_LIST_FILE +fi diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/disable-epirepo-client.sh b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/disable-epirepo-client.sh new file mode 100644 index 0000000000..24124a975a --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/disable-epirepo-client.sh @@ -0,0 +1,5 @@ +#!/bin/bash -eu + +yum-config-manager --disable epirepo +yum clean all --disablerepo='*' --enablerepo=epirepo +yum repolist \ No newline at end of file diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/disable-system-repos.sh b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/disable-system-repos.sh new file mode 100644 index 0000000000..d50a666400 --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/disable-system-repos.sh @@ -0,0 +1,16 @@ +#!/bin/bash -eu + +REPOS_LIST_FILE=/var/tmp/enabled-system-repos.txt +YUM_REPOS_BACKUP_FILE=/etc/yum.repos.d/yum.repos.d-epi-backup.tar + +if yum-config-manager --version > /dev/null 2>&1; then + cat $REPOS_LIST_FILE | while read repository + do + echo "Disabling repository: $repository" + yum-config-manager --disable $repository + done +elif [ ! -f $YUM_REPOS_BACKUP_FILE ]; then # for hosts where yum-utils is not available + echo "Disabling all yum repositories by backup & remove files in /etc/yum.repos.d" + tar -cv --verify --directory="/" --file=$YUM_REPOS_BACKUP_FILE /etc/yum.repos.d && + rm -f /etc/yum.repos.d/*.repo +fi \ No newline at end of file diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/enable-system-repos.sh b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/enable-system-repos.sh new file mode 100644 index 0000000000..74bc995f1f --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/enable-system-repos.sh @@ -0,0 +1,22 @@ +#!/bin/bash -eu + +REPOS_LIST_FILE=/var/tmp/enabled-system-repos.txt +YUM_REPOS_BACKUP_FILE=/etc/yum.repos.d/yum.repos.d-epi-backup.tar + +if [ -f $YUM_REPOS_BACKUP_FILE ]; then # hosts without yum-config-manager + echo "Restoring /etc/yum.repos.d/*.repo from: $YUM_REPOS_BACKUP_FILE" + if tar -xv --file $YUM_REPOS_BACKUP_FILE --directory /etc/yum.repos.d \ + --strip-components=2 etc/yum.repos.d/*.repo; then + echo "yum repositories restored" + rm -f $YUM_REPOS_BACKUP_FILE + else + echo "Extracting tar failed: $YUM_REPOS_BACKUP_FILE" + exit 2 + fi +else # hosts with yum-config-manager + cat $REPOS_LIST_FILE | while read repository + do + echo "Enabling repository: $repository" + yum-config-manager --enable $repository + done +fi \ No newline at end of file diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/download-requirements.sh b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/download-requirements.sh new file mode 100644 index 0000000000..55b620c3f9 --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/download-requirements.sh @@ -0,0 +1,611 @@ +#!/usr/bin/env bash + +# VERSION 1.0.3 + +# NOTE: You can run only one instance of this script, new instance kills the previous one +# This limitation is for Ansible + +set -euo pipefail + +# === Functions (in alphabetical order) === + +# params: +add_repo() { + local repo_id="$1" + local repo_url="$2" + + if ! is_repo_enabled "$repo_id"; then + echol "Adding repository: $repo_id" + yum-config-manager --add-repo "$repo_url" || + exit_with_error "Command failed: yum-config-manager --add-repo \"$repo_url\"" + echo "$repo_id.repo" >> "$ADDED_REPOSITORIES_FILE_PATH" + # to accept import of GPG keys + yum -y repolist > /dev/null || + exit_with_error "Command failed: yum -y repolist" + fi +} + +# params: +add_repo_as_file() { + local repo_id="$1" + local config_file_contents="$2" + local config_file_name="$repo_id.repo" + + if ! is_repo_enabled "$repo_id"; then + echol "Adding repository: $repo_id" + cat <<< "$config_file_contents" > "/etc/yum.repos.d/$config_file_name" || + exit_with_error "Function add_repo_as_file failed for repo: $repo_id" + echo "$config_file_name" >> "$ADDED_REPOSITORIES_FILE_PATH" + # to accept import of GPG keys + yum -y repolist > /dev/null || exit_with_error "Command failed: yum -y repolist" + fi +} + +# params: ... [path_N_to_backup] +backup_files() { + local backup_file_path="$1" + shift + local paths_to_backup="$@" + + # --directory='/' is for tar --verify + tar --create --verbose --verify --directory="/" --file="$backup_file_path" $paths_to_backup +} + +# params: +create_directory() { + local dir_path="$1" + + if [[ -d "$dir_path" ]]; then + echol "Directory $dir_path already exists" + else + echol "Creating directory: $dir_path" + mkdir -p "$dir_path" || exit_with_error "Command failed: mkdir -p \"$dir_path\"" + fi +} + +# params: +download_file() { + local file_url="$1" + local dest_dir="$2" + + local file_name=$(basename "$file_url") + local dest_path="$dest_dir/$file_name" + + # wget with --timestamping sometimes failes on AWS with ERROR 403: Forbidden + # so we remove existing file to overwrite it, to be optimized + [[ ! -f $dest_path ]] || remove_file "$dest_path" + + echol "Downloading file: $file" + + wget --no-verbose --directory-prefix="$dest_dir" "$file_url" || + exit_with_error "Command failed: wget --no-verbose --directory-prefix=\"$dest_dir\" \"$file_url\"" +} + +# params: +download_image() { + local image_name="$1" + local dest_dir="$2" + + local splited_image=(${image_name//:/ }) + local repository=${splited_image[0]} + local tag=${splited_image[1]} + local repo_basename=$(basename -- "$repository") + local dest_path="${dest_dir}/${repo_basename}-${tag}.tar" + + if [[ -f $dest_path ]]; then + echol "Image file: "$dest_path" already exists. Skipping..." + else + # use temporary file for downloading to be safe from sudden interruptions (network, ctrl+c) + local tmp_file_path=$(mktemp) + local skopeo_cmd="$SKOPEO_BIN --insecure-policy copy docker://$image_name docker-archive:$tmp_file_path:$repository:$tag" + echol "Downloading image: $image" + { $skopeo_cmd && chmod 644 $tmp_file_path && mv $tmp_file_path $dest_path; } || + exit_with_error "skopeo failed, command was: $skopeo_cmd && chmod 644 $tmp_file_path && mv $tmp_file_path $dest_path" + fi +} + +# params: ... [package_N] +download_packages() { + local dest_dir="$1" + shift + local packages="$@" + + if [[ -n $packages ]]; then + # when using --archlist=x86_64 yumdownloader (yum-utils-1.1.31-52) also downloads i686 packages + yumdownloader --quiet --archlist=x86_64 --exclude='*i686' --destdir="$dest_dir" $packages || + exit_with_error "yumdownloader failed for: $packages" + fi +} + +echol() { + echo -e "$@" + if [[ $CREATE_LOGFILE == 'yes' ]]; then + local timestamp=$(date +"%b %e %H:%M:%S") + echo -e "${timestamp}: $@" >> "$LOG_FILE_PATH" + fi +} + +# params: +enable_repo() { + local repo_id="$1" + + if ! yum repolist enabled | grep --quiet "$repo_id"; then + echol "Enabling repository: $repo_id" + yum-config-manager --enable "$repo_id" || + exit_with_error "Command failed: yum-config-manager --enable \"$repo_id\"" + fi +} + +exit_with_error() { + echol "ERROR: $1" + exit 1 +} + +# params: +get_package_dependencies_with_arch() { + # $1 reserved for result + local package="$2" + + local query_output=$(repoquery --requires --resolve --queryformat '%{name}.%{arch}' --archlist=x86_64,noarch "$package") || + exit_with_error "repoquery failed for dependencies of package: $package with exit code: $?, output was: $query_output" + + if [[ -z $query_output ]]; then + echol "No dependencies found for package: $package" + elif grep --ignore-case --perl-regexp '\b(? +get_package_with_version_arch() { + # $1 reserved for result + local package="$2" + + local query_output=$(repoquery --queryformat '%{ui_nevra}' --archlist=x86_64,noarch "$package") || + exit_with_error "repoquery failed for package: $package with exit code: $?, output was: $query_output" + + # yumdownloader doesn't set error code if repoquery returns empty output + [[ -n $query_output ]] || exit_with_error "repoquery failed: package $package not found" + if grep --ignore-case --perl-regexp '\b(? +get_packages_with_version_arch() { + local result_var_name="$1" + shift + local packages=("$@") + local packages_with_version_arch=() + + for package in "${packages[@]}"; do + get_package_with_version_arch 'QUERY_OUTPUT' "$package" + packages_with_version_arch+=("$QUERY_OUTPUT") + done + + eval $result_var_name='("${packages_with_version_arch[@]}")' +} + +# params: +get_requirements_from_group() { + # $1 reserved for result + local group_name="$2" + local requirements_file_path="$3" + + local all_requirements=$(grep --only-matching '^[^#]*' "$requirements_file_path" | sed -e 's/[[:space:]]*$//') + local requirements_from_group=$(awk "/^$/ {next}; /\[${group_name}\]/ {f=1; next}; /^\[/ {f=0}; f {print \$0}" <<< "$all_requirements") || + exit_with_error "Function get_requirements_from_group failed for group: $group_name" + + [[ -n $requirements_from_group ]] || echol "No requirements found for group: $group_name" + + eval $1='$requirements_from_group' +} + +# params: +get_unique_array() { + local result_var_name="$1" + shift + local array=("$@") + + # filter out duplicates + array=($(echo "${array[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' ')) + + eval $result_var_name='("${array[@]}")' +} + +# params: [package_name] +install_package() { + local package_name_or_url="$1" + local package_name="$1" + + [ $# -gt 1 ] && package_name="$2" + + echol "Installing package: $package_name" + if yum install -y "$package_name_or_url"; then + echo "$package_name" >> "$INSTALLED_PACKAGES_FILE_PATH" + else + exit_with_error "Command failed: yum install -y \"$package_name_or_url\"" + fi +} + +# params: +is_package_installed() { + local package="$1" + + if rpm --query --quiet "$package"; then + echol "Package $package already installed" + return 0 + else + return 1 + fi +} + +# params: +is_repo_enabled() { + local repo_id="$1" + + if yum repolist | grep --quiet "$repo_id"; then + echol "Repository $repo_id already enabled" + return 0 + else + return 1 + fi +} + +# params: +remove_package() { + local package="$1" + + if rpm --query --quiet "$package"; then + echol "Removing package: $package" + yum remove -y "$package" || exit_with_error "Command failed: yum remove -y \"$package\"" + fi +} + +# params: +remove_added_repos() { + local added_repos_list_file="$1" + + if [ -f "$added_repos_list_file" ]; then + for repo_config_file in $(cat $added_repos_list_file | sort --unique); do + remove_file "/etc/yum.repos.d/$repo_config_file" + done + remove_file "$added_repos_list_file" + fi +} + +# params: +remove_file() { + local file_path="$1" + + echol "Removing file: $file_path" + rm -f "$file_path" || exit_with_error "Command failed: rm -f \"$file_path\"" +} + +# params: +remove_installed_packages() { + local installed_packages_list_file="$1" + + if [ -f "$installed_packages_list_file" ]; then + for package in $(cat $installed_packages_list_file | sort --unique); do + remove_package "$package" + done + remove_file "$installed_packages_list_file" + fi +} + +usage() { + echo "usage: ./$(basename $0) " + echo " ./$(basename $0) /tmp/downloads" + [ -z "$1" ] || exit "$1" +} + +# === Start === + +[ $# -gt 0 ] || usage 1 >&2 +readonly START_TIME=$(date +%s) + +# --- Parse arguments --- + +POSITIONAL_ARGS=() +CREATE_LOGFILE='yes' +while [[ $# -gt 0 ]]; do +case $1 in + --no-logfile) + CREATE_LOGFILE='no' + shift # past argument + ;; + *) # unknown option + POSITIONAL_ARGS+=("$1") # save it in an array for later + shift + ;; +esac +done +set -- "${POSITIONAL_ARGS[@]}" # restore positional arguments + +# --- Global variables --- + +# dirs +readonly DOWNLOADS_DIR="$1" # root directory for downloads +readonly FILES_DIR="$DOWNLOADS_DIR/files" +readonly PACKAGES_DIR="$DOWNLOADS_DIR/packages" +readonly IMAGES_DIR="$DOWNLOADS_DIR/images" +readonly REPO_PREREQ_PACKAGES_DIR="$PACKAGES_DIR/repo-prereqs" +readonly SCRIPT_DIR="$(dirname $(readlink -f $0))" # want absolute path + +# files +readonly REQUIREMENTS_FILE_PATH="$SCRIPT_DIR/requirements.txt" +readonly SCRIPT_FILE_NAME=$(basename $0) +readonly LOG_FILE_NAME=${SCRIPT_FILE_NAME/sh/log} +readonly LOG_FILE_PATH="$SCRIPT_DIR/$LOG_FILE_NAME" +readonly YUM_CONFIG_BACKUP_FILE_PATH="$SCRIPT_DIR/${SCRIPT_FILE_NAME}-yum-repos-backup-tmp-do-not-remove.tar" +readonly SKOPEO_BIN="$SCRIPT_DIR/skopeo_linux" +readonly ADDED_REPOSITORIES_FILE_PATH="$SCRIPT_DIR/${SCRIPT_FILE_NAME}-added-repositories-list-do-not-remove.tmp" +readonly INSTALLED_PACKAGES_FILE_PATH="$SCRIPT_DIR/${SCRIPT_FILE_NAME}-installed-packages-list-do-not-remove.tmp" +readonly PID_FILE_PATH=/var/run/${SCRIPT_FILE_NAME/sh/pid} + +# --- Checks --- + +[ $EUID -eq 0 ] || { echo "You have to run as root" && exit 1; } + +[[ -f $REQUIREMENTS_FILE_PATH ]] || exit_with_error "File not found: $REQUIREMENTS_FILE_PATH" +[[ -f $SKOPEO_BIN ]] || exit_with_error "File not found: $SKOPEO_BIN" +[[ -x $SKOPEO_BIN ]] || exit_with_error "$SKOPEO_BIN have to be executable" + +# --- Want to have only one instance for Ansible --- + +if [ -f $PID_FILE_PATH ]; then + readonly PID_FROM_FILE=$(cat $PID_FILE_PATH 2> /dev/null) + if [[ -n $PID_FROM_FILE ]] && kill -0 $PID_FROM_FILE > /dev/null 2>&1; then + echol "Found running process with pid: $PID_FROM_FILE, cmd: $(ps -p $PID_FROM_FILE -o cmd=)" + if ps -p $PID_FROM_FILE -o cmd= | grep --quiet $SCRIPT_FILE_NAME; then + echol "Killing old instance using SIGTERM" + kill -s SIGTERM $PID_FROM_FILE # try gracefully + if sleep 3 && kill -0 $PID_FROM_FILE > /dev/null 2>&1; then + echol "Still running, killing old instance using SIGKILL" + kill -s SIGKILL $PID_FROM_FILE # forcefully + fi + else + remove_file $PID_FILE_PATH + exit_with_error "Process with pid: $PID_FILE_PATH seems to be not an instance of this script" + fi + else + echol "Process with pid: $PID_FROM_FILE not found" + fi + remove_file $PID_FILE_PATH +fi + +echol "PID is: $$, creating file: $PID_FILE_PATH" +echo $$ > $PID_FILE_PATH || exit_with_error "Command failed: echo $$ > $PID_FILE_PATH" + +# --- Parse requirements file --- + +# Requirements are grouped using sections: [packages-repo-prereqs], [packages], [files], [images] +get_requirements_from_group 'REPO_PREREQ_PACKAGES' 'packages-repo-prereqs' "$REQUIREMENTS_FILE_PATH" +get_requirements_from_group 'PACKAGES' 'packages' "$REQUIREMENTS_FILE_PATH" +get_requirements_from_group 'FILES' 'files' "$REQUIREMENTS_FILE_PATH" +get_requirements_from_group 'IMAGES' 'images' "$REQUIREMENTS_FILE_PATH" + +# === Packages === + +# --- Backup yum repositories --- + +if [ -f $YUM_CONFIG_BACKUP_FILE_PATH ]; then + echol "Backup aleady exists: $YUM_CONFIG_BACKUP_FILE_PATH" +else + echol "Backuping /etc/yum.repos.d/ to $YUM_CONFIG_BACKUP_FILE_PATH" + if backup_files $YUM_CONFIG_BACKUP_FILE_PATH '/etc/yum.repos.d/'; then + echol "Backup done" + else + if [ -f $YUM_CONFIG_BACKUP_FILE_PATH ]; then + remove_file $YUM_CONFIG_BACKUP_FILE_PATH + fi + exit_with_error "Backup of yum repositories failed" + fi +fi + +# --- Install required packages unless present --- + +# repos can be enabled or disabled using the yum-config-manager command, which is provided by yum-utils package +for package in 'yum-utils' 'wget'; do + if ! is_package_installed "$package"; then + install_package "$package" + fi +done + +# --- Enable OS repos --- + +# -> CentOS-7 - Extras # for container-selinux and centos-release-scl packages +enable_repo 'extras' + +# --- Add repos --- + +ELASTIC_REPO_CONF=$(cat <<'EOF' +[elastic-6] +name=Elastic repository for 6.x packages +baseurl=https://artifacts.elastic.co/packages/oss-6.x/yum +gpgcheck=1 +gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch +enabled=1 +autorefresh=1 +type=rpm-md +EOF +) + +ELASTICSEARCH_CURATOR_REPO_CONF=$(cat <<'EOF' +[curator-5] +name=CentOS/RHEL 7 repository for Elasticsearch Curator 5.x packages +baseurl=https://packages.elastic.co/curator/5/centos/7 +gpgcheck=1 +gpgkey=https://packages.elastic.co/GPG-KEY-elasticsearch +enabled=1 +EOF +) + +GRAFANA_REPO_CONF=$(cat <<'EOF' +[grafana] +name=grafana +baseurl=https://packages.grafana.com/oss/rpm +repo_gpgcheck=1 +enabled=1 +gpgcheck=1 +gpgkey=https://packages.grafana.com/gpg.key +sslverify=1 +sslcacert=/etc/pki/tls/certs/ca-bundle.crt +EOF +) + +KUBERNETES_REPO_CONF=$(cat <<'EOF' +[kubernetes] +name=Kubernetes +baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64 +enabled=1 +gpgcheck=1 +repo_gpgcheck=1 +gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg +EOF +) + +RABBITMQ_ERLANG_REPO_CONF=$(cat <<'EOF' +[rabbitmq_erlang] +name=rabbitmq_erlang +baseurl=https://packagecloud.io/rabbitmq/erlang/el/7/$basearch +repo_gpgcheck=1 +gpgcheck=1 +enabled=1 +gpgkey=https://packagecloud.io/rabbitmq/erlang/gpgkey +EOF +) + +RABBITMQ_SERVER_REPO_CONF=$(cat <<'EOF' +[rabbitmq_rabbitmq-server] +name=rabbitmq_rabbitmq-server +baseurl=https://packagecloud.io/rabbitmq/rabbitmq-server/el/7/$basearch +repo_gpgcheck=1 +gpgcheck=1 +enabled=1 +gpgkey=https://packagecloud.io/rabbitmq/rabbitmq-server/gpgkey +EOF +) + +add_repo 'docker-ce' 'https://download.docker.com/linux/centos/docker-ce.repo' +add_repo_as_file 'elastic-6' "$ELASTIC_REPO_CONF" +add_repo_as_file 'curator-5' "$ELASTICSEARCH_CURATOR_REPO_CONF" +add_repo_as_file 'grafana' "$GRAFANA_REPO_CONF" +add_repo_as_file 'kubernetes' "$KUBERNETES_REPO_CONF" +add_repo_as_file 'rabbitmq_erlang' "$RABBITMQ_ERLANG_REPO_CONF" +add_repo_as_file 'rabbitmq_rabbitmq-server' "$RABBITMQ_SERVER_REPO_CONF" + +# -> Software Collections (SCL) https://wiki.centos.org/AdditionalResources/Repositories/SCL +if ! is_package_installed 'centos-release-scl'; then + # from extras repo + install_package 'centos-release-scl-rh' + install_package 'centos-release-scl' +fi + +# some packages are from EPEL repo +if ! is_package_installed 'epel-release'; then + install_package 'https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm' 'epel-release' +fi + +echol "Executing: yum -y makecache fast" && yum -y makecache fast + +# --- Download packages --- + +# 1) packages required to create repository + +create_directory "$REPO_PREREQ_PACKAGES_DIR" + +# prepare lists +PREREQ_PACKAGES=() +for package in $REPO_PREREQ_PACKAGES; do + echol "Processing package: $package" + get_package_with_version_arch 'QUERY_OUTPUT' "$package" + PREREQ_PACKAGES+=("$QUERY_OUTPUT") +done + +# download requirements (fixed versions) +if [[ ${#PREREQ_PACKAGES[@]} -gt 0 ]]; then + echol "Downloading repository prerequisite packages (${#PREREQ_PACKAGES[@]})..." + download_packages "$REPO_PREREQ_PACKAGES_DIR" "${PREREQ_PACKAGES[@]}" +fi + +# 2) non-prerequisite packages + +create_directory "$PACKAGES_DIR" + +# prepare lists +NON_PREREQ_PACKAGES=() +DEPENDENCIES_OF_NON_PREREQ_PACKAGES=() +for package in $PACKAGES; do + echol "Processing package: $package" + get_package_with_version_arch 'QUERY_OUTPUT' "$package" + NON_PREREQ_PACKAGES+=("$QUERY_OUTPUT") + get_package_dependencies_with_arch 'DEPENDENCIES' "$package" + if [[ ${#DEPENDENCIES[@]} -gt 0 ]]; then + for dependency in "${DEPENDENCIES[@]}"; do + DEPENDENCIES_OF_NON_PREREQ_PACKAGES+=("$dependency") + done + fi +done + +if [[ ${#NON_PREREQ_PACKAGES[@]} -gt 0 ]]; then + # download requirements (fixed versions) + echol "Downloading packages (${#NON_PREREQ_PACKAGES[@]})..." + download_packages "$PACKAGES_DIR" "${NON_PREREQ_PACKAGES[@]}" + # download dependencies (latest versions) + get_unique_array 'DEPENDENCIES' "${DEPENDENCIES_OF_NON_PREREQ_PACKAGES[@]}" + get_packages_with_version_arch 'DEPENDENCIES' "${DEPENDENCIES[@]}" + echol "Downloading dependencies of packages (${#DEPENDENCIES[@]})..." + download_packages "$PACKAGES_DIR" "${DEPENDENCIES[@]}" +fi + +# --- Clean up yum repos --- + +remove_added_repos "$ADDED_REPOSITORIES_FILE_PATH" + +# --- Restore yum repos --- + +if [ -f $YUM_CONFIG_BACKUP_FILE_PATH ]; then + echol "Restoring /etc/yum.repos.d/*.repo from: $YUM_CONFIG_BACKUP_FILE_PATH" + echol "Executing: tar --extract --verbose --file $YUM_CONFIG_BACKUP_FILE_PATH" + if tar --extract --verbose --file $YUM_CONFIG_BACKUP_FILE_PATH --directory /etc/yum.repos.d \ + --strip-components=2 etc/yum.repos.d/*.repo; then + echol "Restored: yum repositories" + remove_file $YUM_CONFIG_BACKUP_FILE_PATH + else + exit_with_error "Extracting tar failed: $YUM_CONFIG_BACKUP_FILE_PATH" + fi +fi + +# === Files === + +create_directory "$FILES_DIR" + +for file in $FILES; do + download_file "$file" "$FILES_DIR" +done + +# === Images === + +create_directory "$IMAGES_DIR" + +for image in $IMAGES; do + download_image "$image" "$IMAGES_DIR" +done + +# --- Clean up packages --- +remove_installed_packages "$INSTALLED_PACKAGES_FILE_PATH" + +remove_file $PID_FILE_PATH + +readonly END_TIME=$(date +%s) + +echol "$(basename $0) finished, execution time: $(date -u -d @$((END_TIME-START_TIME)) +'%Hh:%Mm:%Ss')" \ No newline at end of file diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.txt b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.txt new file mode 100644 index 0000000000..531d1214ec --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.txt @@ -0,0 +1,138 @@ +# Put this file in the same directory as download script + +[packages-repo-prereqs] +apr # for httpd +apr-util # for httpd +createrepo +deltarpm # for createrepo +httpd +httpd-tools # for httpd +libxml2-python # for createrepo +mailcap # for httpd +mod_ssl # for httpd +python-chardet # for createrepo +python-deltarpm # for createrepo +python-kitchen # for createrepo +yum-utils + + +[packages] +audit # for docker-ce +bash-completion +ca-certificates +cifs-utils +conntrack-tools # for kubelet +containerd.io +container-selinux +cri-tools-1.13.0 +curl +dejavu-sans-fonts # for grafana +docker-ce-18.09.9 +docker-ce-cli-18.09.9 +ebtables +elasticsearch-curator-5.5.4 +elasticsearch-oss-6.4.0 +erlang-21.3.8.7 +ethtool +filebeat-6.5.4 # actually it's filebeat-oss +firewalld +fontconfig # for grafana +fping +grafana-6.2.5 +gssproxy # for nfs-utils +htop +iftop +ipset # for firewalld +java-1.8.0-openjdk-headless +javapackages-tools # for java-1.8.0-openjdk-headless +jq +kibana-oss-6.4.0 +kubeadm-1.14.6 +kubectl-1.14.6 +kubelet-1.14.6 +kubernetes-cni-0.7.5 +libini_config # for nfs-utils +libselinux-python +libsemanage-python +libX11 # for grafana +libxcb # for grafana +libXcursor # for grafana +libXt # for grafana +logrotate +net-tools +nfs-utils +nmap-ncat +openssl +perl # for vim +perl-Getopt-Long # for vim +perl-libs # for vim +perl-Pod-Perldoc # for vim +perl-Pod-Simple # for vim +perl-Pod-Usage # for vim +policycoreutils-python # for container-selinux +python-firewall # for firewalld +python-kitchen # for yum-utils +python-lxml # for java-1.8.0-openjdk-headless +python-psycopg2 +python-setuptools +python-slip-dbus # for firewalld +python-ipaddress +python-backports +quota # for nfs-utils +rabbitmq-server-3.7.10 +rh-haproxy18 +rh-haproxy18-haproxy-syspaths +rh-postgresql10-postgresql +rh-postgresql10-postgresql-contrib +rh-postgresql10-postgresql-contrib-syspaths +rh-postgresql10-postgresql-libs +rh-postgresql10-postgresql-server +rh-postgresql10-postgresql-server-syspaths +rh-postgresql10-postgresql-syspaths +samba-client +samba-client-libs # for samba-client +samba-common +socat +sysstat +tar +telnet +tmux +urw-base35-fonts # for grafana +vim-common # for vim +vim-enhanced +wget +xorg-x11-font-utils # for grafana +xorg-x11-server-utils # for grafana +yum-plugin-versionlock +yum-utils + +[files] +https://github.com/prometheus/haproxy_exporter/releases/download/v0.10.0/haproxy_exporter-0.10.0.linux-amd64.tar.gz +https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.12.0/jmx_prometheus_javaagent-0.12.0.jar +https://archive.apache.org/dist/kafka/2.0.0/kafka_2.12-2.0.0.tgz +https://github.com/danielqsj/kafka_exporter/releases/download/v1.2.0/kafka_exporter-1.2.0.linux-amd64.tar.gz +https://github.com/prometheus/node_exporter/releases/download/v0.16.0/node_exporter-0.16.0.linux-amd64.tar.gz +https://github.com/prometheus/prometheus/releases/download/v2.10.0/prometheus-2.10.0.linux-amd64.tar.gz +https://github.com/prometheus/alertmanager/releases/download/v0.17.0/alertmanager-0.17.0.linux-amd64.tar.gz +https://archive.apache.org/dist/zookeeper/zookeeper-3.4.12/zookeeper-3.4.12.tar.gz + +[images] +k8s.gcr.io/kube-apiserver:v1.14.6 +k8s.gcr.io/kube-controller-manager:v1.14.6 +k8s.gcr.io/kube-scheduler:v1.14.6 +k8s.gcr.io/kube-proxy:v1.14.6 +k8s.gcr.io/pause:3.1 +k8s.gcr.io/etcd:3.3.10 +k8s.gcr.io/coredns:1.3.1 +coredns/coredns:1.5.0 +quay.io/coreos/flannel:v0.11.0-amd64 +quay.io/coreos/flannel:v0.11.0 +calico/node:v3.8.1 +calico/pod2daemon-flexvol:v3.8.1 +kubernetesui/dashboard:v2.0.0-beta1 +kubernetesui/metrics-scraper:v1.0.0 +calico/cni:v3.8.1 +calico/kube-controllers:v3.8.1 +jboss/keycloak:4.8.3.Final +rabbitmq:3.7.10 +registry:2 \ No newline at end of file diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/download-requirements.sh b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/download-requirements.sh new file mode 100644 index 0000000000..dbcbff0094 --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/download-requirements.sh @@ -0,0 +1,634 @@ +#!/usr/bin/env bash + +# VERSION 1.0.3 + +# NOTE: You can run only one instance of this script, new instance kills the previous one +# This limitation is for Ansible + +set -euo pipefail + +# === Functions (in alphabetical order) === + +# params: +add_repo() { + local repo_id="$1" + local repo_url="$2" + + if ! is_repo_enabled "$repo_id"; then + echol "Adding repository: $repo_id" + yum-config-manager --add-repo "$repo_url" || + exit_with_error "Command failed: yum-config-manager --add-repo \"$repo_url\"" + echo "$repo_id.repo" >> "$ADDED_REPOSITORIES_FILE_PATH" + # to accept import of GPG keys + yum -y repolist > /dev/null || + exit_with_error "Command failed: yum -y repolist" + fi +} + +# params: +add_repo_as_file() { + local repo_id="$1" + local config_file_contents="$2" + local config_file_name="$repo_id.repo" + + if ! is_repo_enabled "$repo_id"; then + echol "Adding repository: $repo_id" + cat <<< "$config_file_contents" > "/etc/yum.repos.d/$config_file_name" || + exit_with_error "Function add_repo_as_file failed for repo: $repo_id" + echo "$config_file_name" >> "$ADDED_REPOSITORIES_FILE_PATH" + # to accept import of GPG keys + yum -y repolist > /dev/null || exit_with_error "Command failed: yum -y repolist" + fi +} + +# params: ... [path_N_to_backup] +backup_files() { + local backup_file_path="$1" + shift + local paths_to_backup="$@" + + # --directory='/' is for tar --verify + tar --create --verbose --verify --directory="/" --file="$backup_file_path" $paths_to_backup +} + +# params: +create_directory() { + local dir_path="$1" + + if [[ -d "$dir_path" ]]; then + echol "Directory $dir_path already exists" + else + echol "Creating directory: $dir_path" + mkdir -p "$dir_path" || exit_with_error "Command failed: mkdir -p \"$dir_path\"" + fi +} + +# params: +download_file() { + local file_url="$1" + local dest_dir="$2" + + local file_name=$(basename "$file_url") + local dest_path="$dest_dir/$file_name" + + # wget with --timestamping sometimes failes on AWS with ERROR 403: Forbidden + # so we remove existing file to overwrite it, to be optimized + [[ ! -f $dest_path ]] || remove_file "$dest_path" + + echol "Downloading file: $file" + + wget --no-verbose --directory-prefix="$dest_dir" "$file_url" || + exit_with_error "Command failed: wget --no-verbose --directory-prefix=\"$dest_dir\" \"$file_url\"" +} + +# params: +download_image() { + local image_name="$1" + local dest_dir="$2" + + local splited_image=(${image_name//:/ }) + local repository=${splited_image[0]} + local tag=${splited_image[1]} + local repo_basename=$(basename -- "$repository") + local dest_path="${dest_dir}/${repo_basename}-${tag}.tar" + + if [[ -f $dest_path ]]; then + echol "Image file: "$dest_path" already exists. Skipping..." + else + # use temporary file for downloading to be safe from sudden interruptions (network, ctrl+c) + local tmp_file_path=$(mktemp) + local skopeo_cmd="$SKOPEO_BIN --insecure-policy copy docker://$image_name docker-archive:$tmp_file_path:$repository:$tag" + echol "Downloading image: $image" + { $skopeo_cmd && chmod 644 $tmp_file_path && mv $tmp_file_path $dest_path; } || + exit_with_error "skopeo failed, command was: $skopeo_cmd && chmod 644 $tmp_file_path && mv $tmp_file_path $dest_path" + fi +} + +# params: ... [package_N] +download_packages() { + local dest_dir="$1" + shift + local packages="$@" + + if [[ -n $packages ]]; then + # when using --archlist=x86_64 yumdownloader (yum-utils-1.1.31-52) also downloads i686 packages + yumdownloader --quiet --archlist=x86_64 --exclude='*i686' --destdir="$dest_dir" $packages || + exit_with_error "yumdownloader failed for: $packages" + fi +} + +echol() { + echo -e "$@" + if [[ $CREATE_LOGFILE == 'yes' ]]; then + local timestamp=$(date +"%b %e %H:%M:%S") + echo -e "${timestamp}: $@" >> "$LOG_FILE_PATH" + fi +} + +# params: +enable_repo() { + local repo_id="$1" + + if ! yum repolist enabled | grep --quiet "$repo_id"; then + echol "Enabling repository: $repo_id" + yum-config-manager --enable "$repo_id" || + exit_with_error "Command failed: yum-config-manager --enable \"$repo_id\"" + fi +} + +exit_with_error() { + echol "ERROR: $1" + exit 1 +} + +# desc: find repo id (set $1) based on given pattern +# params: +find_rhel_repo_id() { + # $1 reserved for result + local rhel_on_prem_repo_id="$2" + local pattern="$3" + local repo_id + + if yum repolist all | egrep --quiet "$pattern"; then + repo_id=$(yum repolist all | egrep --only-matching "$pattern") + else + exit_with_error "RHEL yum repository not found, pattern was: $pattern" + fi + + eval $1='$repo_id' +} + +# params: +get_package_dependencies_with_arch() { + # $1 reserved for result + local package="$2" + + local query_output=$(repoquery --requires --resolve --queryformat '%{name}.%{arch}' --archlist=x86_64,noarch "$package") || + exit_with_error "repoquery failed for dependencies of package: $package with exit code: $?, output was: $query_output" + + if [[ -z $query_output ]]; then + echol "No dependencies found for package: $package" + elif grep --ignore-case --perl-regexp '\b(? +get_package_with_version_arch() { + # $1 reserved for result + local package="$2" + + local query_output=$(repoquery --queryformat '%{ui_nevra}' --archlist=x86_64,noarch "$package") || + exit_with_error "repoquery failed for package: $package with exit code: $?, output was: $query_output" + + # yumdownloader doesn't set error code if repoquery returns empty output + [[ -n $query_output ]] || exit_with_error "repoquery failed: package $package not found" + if grep --ignore-case --perl-regexp '\b(? +get_packages_with_version_arch() { + local result_var_name="$1" + shift + local packages=("$@") + local packages_with_version_arch=() + + for package in "${packages[@]}"; do + get_package_with_version_arch 'QUERY_OUTPUT' "$package" + packages_with_version_arch+=("$QUERY_OUTPUT") + done + + eval $result_var_name='("${packages_with_version_arch[@]}")' +} + +# params: +get_requirements_from_group() { + # $1 reserved for result + local group_name="$2" + local requirements_file_path="$3" + + local all_requirements=$(grep --only-matching '^[^#]*' "$requirements_file_path" | sed -e 's/[[:space:]]*$//') + local requirements_from_group=$(awk "/^$/ {next}; /\[${group_name}\]/ {f=1; next}; /^\[/ {f=0}; f {print \$0}" <<< "$all_requirements") || + exit_with_error "Function get_requirements_from_group failed for group: $group_name" + + [[ -n $requirements_from_group ]] || echol "No requirements found for group: $group_name" + + eval $1='$requirements_from_group' +} + +# params: +get_unique_array() { + local result_var_name="$1" + shift + local array=("$@") + + # filter out duplicates + array=($(echo "${array[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' ')) + + eval $result_var_name='("${array[@]}")' +} + +# params: [package_name] +install_package() { + local package_name_or_url="$1" + local package_name="$1" + + [ $# -gt 1 ] && package_name="$2" + + echol "Installing package: $package_name" + if yum install -y "$package_name_or_url"; then + echo "$package_name" >> "$INSTALLED_PACKAGES_FILE_PATH" + else + exit_with_error "Command failed: yum install -y \"$package_name_or_url\"" + fi +} + +# params: +is_package_installed() { + local package="$1" + + if rpm --query --quiet "$package"; then + echol "Package $package already installed" + return 0 + else + return 1 + fi +} + +# params: +is_repo_enabled() { + local repo_id="$1" + + if yum repolist | grep --quiet "$repo_id"; then + echol "Repository $repo_id already enabled" + return 0 + else + return 1 + fi +} + +# params: +remove_package() { + local package="$1" + + if rpm --query --quiet "$package"; then + echol "Removing package: $package" + yum remove -y "$package" || exit_with_error "Command failed: yum remove -y \"$package\"" + fi +} + +# params: +remove_added_repos() { + local added_repos_list_file="$1" + + if [ -f "$added_repos_list_file" ]; then + for repo_config_file in $(cat $added_repos_list_file | sort --unique); do + remove_file "/etc/yum.repos.d/$repo_config_file" + done + remove_file "$added_repos_list_file" + fi +} + +# params: +remove_file() { + local file_path="$1" + + echol "Removing file: $file_path" + rm -f "$file_path" || exit_with_error "Command failed: rm -f \"$file_path\"" +} + +# params: +remove_installed_packages() { + local installed_packages_list_file="$1" + + if [ -f "$installed_packages_list_file" ]; then + for package in $(cat $installed_packages_list_file | sort --unique); do + remove_package "$package" + done + remove_file "$installed_packages_list_file" + fi +} + +usage() { + echo "usage: ./$(basename $0) " + echo " ./$(basename $0) /tmp/downloads" + [ -z "$1" ] || exit "$1" +} + +# === Start === + +[ $# -gt 0 ] || usage 1 >&2 +readonly START_TIME=$(date +%s) + +# --- Parse arguments --- + +POSITIONAL_ARGS=() +CREATE_LOGFILE='yes' +while [[ $# -gt 0 ]]; do +case $1 in + --no-logfile) + CREATE_LOGFILE='no' + shift # past argument + ;; + *) # unknown option + POSITIONAL_ARGS+=("$1") # save it in an array for later + shift + ;; +esac +done +set -- "${POSITIONAL_ARGS[@]}" # restore positional arguments + +# --- Global variables --- + +# dirs +readonly DOWNLOADS_DIR="$1" # root directory for downloads +readonly FILES_DIR="$DOWNLOADS_DIR/files" +readonly PACKAGES_DIR="$DOWNLOADS_DIR/packages" +readonly IMAGES_DIR="$DOWNLOADS_DIR/images" +readonly REPO_PREREQ_PACKAGES_DIR="$PACKAGES_DIR/repo-prereqs" +readonly SCRIPT_DIR="$(dirname $(readlink -f $0))" # want absolute path + +# files +readonly REQUIREMENTS_FILE_PATH="$SCRIPT_DIR/requirements.txt" +readonly SCRIPT_FILE_NAME=$(basename $0) +readonly LOG_FILE_NAME=${SCRIPT_FILE_NAME/sh/log} +readonly LOG_FILE_PATH="$SCRIPT_DIR/$LOG_FILE_NAME" +readonly YUM_CONFIG_BACKUP_FILE_PATH="$SCRIPT_DIR/${SCRIPT_FILE_NAME}-yum-repos-backup-tmp-do-not-remove.tar" +readonly SKOPEO_BIN="$SCRIPT_DIR/skopeo_linux" +readonly ADDED_REPOSITORIES_FILE_PATH="$SCRIPT_DIR/${SCRIPT_FILE_NAME}-added-repositories-list-do-not-remove.tmp" +readonly INSTALLED_PACKAGES_FILE_PATH="$SCRIPT_DIR/${SCRIPT_FILE_NAME}-installed-packages-list-do-not-remove.tmp" +readonly PID_FILE_PATH=/var/run/${SCRIPT_FILE_NAME/sh/pid} + +# --- Checks --- + +[ $EUID -eq 0 ] || { echo "You have to run as root" && exit 1; } + +[[ -f $REQUIREMENTS_FILE_PATH ]] || exit_with_error "File not found: $REQUIREMENTS_FILE_PATH" +[[ -f $SKOPEO_BIN ]] || exit_with_error "File not found: $SKOPEO_BIN" +[[ -x $SKOPEO_BIN ]] || exit_with_error "$SKOPEO_BIN have to be executable" + +# --- Want to have only one instance for Ansible --- + +if [ -f $PID_FILE_PATH ]; then + readonly PID_FROM_FILE=$(cat $PID_FILE_PATH 2> /dev/null) + if [[ -n $PID_FROM_FILE ]] && kill -0 $PID_FROM_FILE > /dev/null 2>&1; then + echol "Found running process with pid: $PID_FROM_FILE, cmd: $(ps -p $PID_FROM_FILE -o cmd=)" + if ps -p $PID_FROM_FILE -o cmd= | grep --quiet $SCRIPT_FILE_NAME; then + echol "Killing old instance using SIGTERM" + kill -s SIGTERM $PID_FROM_FILE # try gracefully + if sleep 3 && kill -0 $PID_FROM_FILE > /dev/null 2>&1; then + echol "Still running, killing old instance using SIGKILL" + kill -s SIGKILL $PID_FROM_FILE # forcefully + fi + else + remove_file $PID_FILE_PATH + exit_with_error "Process with pid: $PID_FILE_PATH seems to be not an instance of this script" + fi + else + echol "Process with pid: $PID_FROM_FILE not found" + fi + remove_file $PID_FILE_PATH +fi + +echol "PID is: $$, creating file: $PID_FILE_PATH" +echo $$ > $PID_FILE_PATH || exit_with_error "Command failed: echo $$ > $PID_FILE_PATH" + +# --- Parse requirements file --- + +# Requirements are grouped using sections: [packages-repo-prereqs], [packages], [files], [images] +get_requirements_from_group 'REPO_PREREQ_PACKAGES' 'packages-repo-prereqs' "$REQUIREMENTS_FILE_PATH" +get_requirements_from_group 'PACKAGES' 'packages' "$REQUIREMENTS_FILE_PATH" +get_requirements_from_group 'FILES' 'files' "$REQUIREMENTS_FILE_PATH" +get_requirements_from_group 'IMAGES' 'images' "$REQUIREMENTS_FILE_PATH" + +# === Packages === + +# --- Backup yum repositories --- + +if [ -f $YUM_CONFIG_BACKUP_FILE_PATH ]; then + echol "Backup aleady exists: $YUM_CONFIG_BACKUP_FILE_PATH" +else + echol "Backuping /etc/yum.repos.d/ to $YUM_CONFIG_BACKUP_FILE_PATH" + if backup_files $YUM_CONFIG_BACKUP_FILE_PATH '/etc/yum.repos.d/'; then + echol "Backup done" + else + if [ -f $YUM_CONFIG_BACKUP_FILE_PATH ]; then + remove_file $YUM_CONFIG_BACKUP_FILE_PATH + fi + exit_with_error "Backup of yum repositories failed" + fi +fi + +# --- Install required packages unless present --- + +# repos can be enabled or disabled using the yum-config-manager command, which is provided by yum-utils package +for package in 'yum-utils' 'wget'; do + if ! is_package_installed "$package"; then + install_package "$package" + fi +done + +# --- Enable RHEL repos --- + +# -> rhel-7-server-extras-rpms # for container-selinux package, this repo has different id names on clouds +# About rhel-7-server-extras-rpms: https://access.redhat.com/solutions/3418891 + +ON_PREM_REPO_ID='rhel-7-server-extras-rpms' +REPO_ID_PATTERN="$ON_PREM_REPO_ID|rhui-REGION-rhel-server-extras|rhui-rhel-7-server-rhui-extras-rpms" # on-prem|AWS|Azure +find_rhel_repo_id 'REPO_ID' "$ON_PREM_REPO_ID" "$REPO_ID_PATTERN" +enable_repo "$REPO_ID" + +# -> rhel-server-rhscl-7-rpms # for Red Hat Software Collections (RHSCL), this repo has different id names on clouds +# About rhel-server-rhscl-7-rpms: https://access.redhat.com/solutions/472793 + +ON_PREM_REPO_ID='rhel-server-rhscl-7-rpms' +REPO_ID_PATTERN="$ON_PREM_REPO_ID|rhui-REGION-rhel-server-rhscl|rhui-rhel-server-rhui-rhscl-7-rpms" # on-prem|AWS|Azure +find_rhel_repo_id 'REPO_ID' "$ON_PREM_REPO_ID" "$REPO_ID_PATTERN" +enable_repo "$REPO_ID" + +# --- Add repos --- + +ELASTIC_REPO_CONF=$(cat <<'EOF' +[elastic-6] +name=Elastic repository for 6.x packages +baseurl=https://artifacts.elastic.co/packages/oss-6.x/yum +gpgcheck=1 +gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch +enabled=1 +autorefresh=1 +type=rpm-md +EOF +) + +ELASTICSEARCH_CURATOR_REPO_CONF=$(cat <<'EOF' +[curator-5] +name=CentOS/RHEL 7 repository for Elasticsearch Curator 5.x packages +baseurl=https://packages.elastic.co/curator/5/centos/7 +gpgcheck=1 +gpgkey=https://packages.elastic.co/GPG-KEY-elasticsearch +enabled=1 +EOF +) + +GRAFANA_REPO_CONF=$(cat <<'EOF' +[grafana] +name=grafana +baseurl=https://packages.grafana.com/oss/rpm +repo_gpgcheck=1 +enabled=1 +gpgcheck=1 +gpgkey=https://packages.grafana.com/gpg.key +sslverify=1 +sslcacert=/etc/pki/tls/certs/ca-bundle.crt +EOF +) + +KUBERNETES_REPO_CONF=$(cat <<'EOF' +[kubernetes] +name=Kubernetes +baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64 +enabled=1 +gpgcheck=1 +repo_gpgcheck=1 +gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg +EOF +) + +RABBITMQ_ERLANG_REPO_CONF=$(cat <<'EOF' +[rabbitmq_erlang] +name=rabbitmq_erlang +baseurl=https://packagecloud.io/rabbitmq/erlang/el/7/$basearch +repo_gpgcheck=1 +gpgcheck=1 +enabled=1 +gpgkey=https://packagecloud.io/rabbitmq/erlang/gpgkey +EOF +) + +RABBITMQ_SERVER_REPO_CONF=$(cat <<'EOF' +[rabbitmq_rabbitmq-server] +name=rabbitmq_rabbitmq-server +baseurl=https://packagecloud.io/rabbitmq/rabbitmq-server/el/7/$basearch +repo_gpgcheck=1 +gpgcheck=1 +enabled=1 +gpgkey=https://packagecloud.io/rabbitmq/rabbitmq-server/gpgkey +EOF +) + +add_repo 'docker-ce' 'https://download.docker.com/linux/centos/docker-ce.repo' +add_repo_as_file 'elastic-6' "$ELASTIC_REPO_CONF" +add_repo_as_file 'curator-5' "$ELASTICSEARCH_CURATOR_REPO_CONF" +add_repo_as_file 'grafana' "$GRAFANA_REPO_CONF" +add_repo_as_file 'kubernetes' "$KUBERNETES_REPO_CONF" +add_repo_as_file 'rabbitmq_erlang' "$RABBITMQ_ERLANG_REPO_CONF" +add_repo_as_file 'rabbitmq_rabbitmq-server' "$RABBITMQ_SERVER_REPO_CONF" + +# some packages are from EPEL repo +if ! is_package_installed 'epel-release'; then + install_package 'https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm' 'epel-release' +fi + +echol "Executing: yum -y makecache fast" && yum -y makecache fast + +# --- Download packages --- + +# 1) packages required to create repository + +create_directory "$REPO_PREREQ_PACKAGES_DIR" + +# prepare lists +PREREQ_PACKAGES=() +for package in $REPO_PREREQ_PACKAGES; do + echol "Processing package: $package" + get_package_with_version_arch 'QUERY_OUTPUT' "$package" + PREREQ_PACKAGES+=("$QUERY_OUTPUT") +done + +# download requirements (fixed versions) +if [[ ${#PREREQ_PACKAGES[@]} -gt 0 ]]; then + echol "Downloading repository prerequisite packages (${#PREREQ_PACKAGES[@]})..." + download_packages "$REPO_PREREQ_PACKAGES_DIR" "${PREREQ_PACKAGES[@]}" +fi + +# 2) non-prerequisite packages + +create_directory "$PACKAGES_DIR" + +# prepare lists +NON_PREREQ_PACKAGES=() +DEPENDENCIES_OF_NON_PREREQ_PACKAGES=() +for package in $PACKAGES; do + echol "Processing package: $package" + get_package_with_version_arch 'QUERY_OUTPUT' "$package" + NON_PREREQ_PACKAGES+=("$QUERY_OUTPUT") + get_package_dependencies_with_arch 'DEPENDENCIES' "$package" + if [[ ${#DEPENDENCIES[@]} -gt 0 ]]; then + for dependency in "${DEPENDENCIES[@]}"; do + DEPENDENCIES_OF_NON_PREREQ_PACKAGES+=("$dependency") + done + fi +done + +if [[ ${#NON_PREREQ_PACKAGES[@]} -gt 0 ]]; then + # download requirements (fixed versions) + echol "Downloading packages (${#NON_PREREQ_PACKAGES[@]})..." + download_packages "$PACKAGES_DIR" "${NON_PREREQ_PACKAGES[@]}" + # download dependencies (latest versions) + get_unique_array 'DEPENDENCIES' "${DEPENDENCIES_OF_NON_PREREQ_PACKAGES[@]}" + get_packages_with_version_arch 'DEPENDENCIES' "${DEPENDENCIES[@]}" + echol "Downloading dependencies of packages (${#DEPENDENCIES[@]})..." + download_packages "$PACKAGES_DIR" "${DEPENDENCIES[@]}" +fi + +# --- Clean up yum repos --- + +remove_added_repos "$ADDED_REPOSITORIES_FILE_PATH" + +# --- Restore yum repos --- + +if [ -f $YUM_CONFIG_BACKUP_FILE_PATH ]; then + echol "Restoring /etc/yum.repos.d/*.repo from: $YUM_CONFIG_BACKUP_FILE_PATH" + echol "Executing: tar --extract --verbose --file $YUM_CONFIG_BACKUP_FILE_PATH" + if tar --extract --verbose --file $YUM_CONFIG_BACKUP_FILE_PATH --directory /etc/yum.repos.d \ + --strip-components=2 etc/yum.repos.d/*.repo; then + echol "Restored: yum repositories" + remove_file $YUM_CONFIG_BACKUP_FILE_PATH + else + exit_with_error "Extracting tar failed: $YUM_CONFIG_BACKUP_FILE_PATH" + fi +fi + +# === Files === + +create_directory "$FILES_DIR" + +for file in $FILES; do + download_file "$file" "$FILES_DIR" +done + +# === Images === + +create_directory "$IMAGES_DIR" + +for image in $IMAGES; do + download_image "$image" "$IMAGES_DIR" +done + +# --- Clean up packages --- +remove_installed_packages "$INSTALLED_PACKAGES_FILE_PATH" + +remove_file $PID_FILE_PATH + +readonly END_TIME=$(date +%s) + +echol "$(basename $0) finished, execution time: $(date -u -d @$((END_TIME-START_TIME)) +'%Hh:%Mm:%Ss')" \ No newline at end of file diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.txt b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.txt new file mode 100644 index 0000000000..38dcd0e47d --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.txt @@ -0,0 +1,134 @@ +# Put this file in the same directory as download script + +[packages-repo-prereqs] +apr # for httpd +apr-util # for httpd +createrepo +deltarpm # for createrepo +httpd +httpd-tools # for httpd +mailcap # for httpd +mod_ssl # for httpd +python-chardet # for createrepo +python-deltarpm # for createrepo +python-kitchen # for createrepo +yum-utils + +[packages] +audit # for docker-ce +bash-completion +ca-certificates +cifs-utils +conntrack-tools # for kubelet +containerd.io +container-selinux +cri-tools-1.13.0 +curl +dejavu-sans-fonts # for grafana +docker-ce-18.09.9 +docker-ce-cli-18.09.9 +ebtables +elasticsearch-curator-5.5.4 +elasticsearch-oss-6.4.0 +erlang-21.3.8.7 +ethtool +filebeat-6.5.4 # actually it's filebeat-oss +firewalld +fontconfig # for grafana +fping +grafana-6.2.5 +gssproxy # for nfs-utils +htop +iftop +ipset # for firewalld +java-1.8.0-openjdk-headless +javapackages-tools # for java-1.8.0-openjdk-headless +jq +kibana-oss-6.4.0 +kubeadm-1.14.6 +kubectl-1.14.6 +kubelet-1.14.6 +kubernetes-cni-0.7.5 +libini_config # for nfs-utils +libselinux-python +libsemanage-python +libX11 # for grafana +libxcb # for grafana +libXcursor # for grafana +libXt # for grafana +logrotate +net-tools +nfs-utils +nmap-ncat +openssl +perl # for vim +perl-Getopt-Long # for vim +perl-libs # for vim +perl-Pod-Perldoc # for vim +perl-Pod-Simple # for vim +perl-Pod-Usage # for vim +policycoreutils-python # for container-selinux +python-firewall # for firewalld +python-kitchen # for yum-utils +python-lxml # for java-1.8.0-openjdk-headless +python-psycopg2 +python-setuptools +python-slip-dbus # for firewalld +quota # for nfs-utils +rabbitmq-server-3.7.10 +rh-haproxy18 +rh-haproxy18-haproxy-syspaths +rh-postgresql10-postgresql +rh-postgresql10-postgresql-contrib +rh-postgresql10-postgresql-contrib-syspaths +rh-postgresql10-postgresql-libs +rh-postgresql10-postgresql-server +rh-postgresql10-postgresql-server-syspaths +rh-postgresql10-postgresql-syspaths +samba-client +samba-client-libs # for samba-client +samba-common +socat +sysstat +tar +telnet +tmux +urw-base35-fonts # for grafana +vim-common # for vim +vim-enhanced +wget +xorg-x11-font-utils # for grafana +xorg-x11-server-utils # for grafana +yum-plugin-versionlock +yum-utils + +[files] +https://github.com/prometheus/haproxy_exporter/releases/download/v0.10.0/haproxy_exporter-0.10.0.linux-amd64.tar.gz +https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.12.0/jmx_prometheus_javaagent-0.12.0.jar +https://archive.apache.org/dist/kafka/2.0.0/kafka_2.12-2.0.0.tgz +https://github.com/danielqsj/kafka_exporter/releases/download/v1.2.0/kafka_exporter-1.2.0.linux-amd64.tar.gz +https://github.com/prometheus/node_exporter/releases/download/v0.16.0/node_exporter-0.16.0.linux-amd64.tar.gz +https://github.com/prometheus/prometheus/releases/download/v2.10.0/prometheus-2.10.0.linux-amd64.tar.gz +https://github.com/prometheus/alertmanager/releases/download/v0.17.0/alertmanager-0.17.0.linux-amd64.tar.gz +https://archive.apache.org/dist/zookeeper/zookeeper-3.4.12/zookeeper-3.4.12.tar.gz + +[images] +k8s.gcr.io/kube-apiserver:v1.14.6 +k8s.gcr.io/kube-controller-manager:v1.14.6 +k8s.gcr.io/kube-scheduler:v1.14.6 +k8s.gcr.io/kube-proxy:v1.14.6 +k8s.gcr.io/pause:3.1 +k8s.gcr.io/etcd:3.3.10 +k8s.gcr.io/coredns:1.3.1 +coredns/coredns:1.5.0 +quay.io/coreos/flannel:v0.11.0-amd64 +quay.io/coreos/flannel:v0.11.0 +calico/node:v3.8.1 +calico/pod2daemon-flexvol:v3.8.1 +kubernetesui/dashboard:v2.0.0-beta1 +kubernetesui/metrics-scraper:v1.0.0 +calico/cni:v3.8.1 +calico/kube-controllers:v3.8.1 +jboss/keycloak:4.8.3.Final +rabbitmq:3.7.10 +registry:2 \ No newline at end of file diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/add-repositories.sh b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/add-repositories.sh new file mode 100644 index 0000000000..913baae079 --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/add-repositories.sh @@ -0,0 +1,22 @@ +#!/bin/bash -eu + +wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | apt-key add - +echo "deb https://artifacts.elastic.co/packages/oss-6.x/apt stable main" | tee /etc/apt/sources.list.d/elastic-6.x.list + +wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | apt-key add - +echo "deb [arch=amd64] https://packages.elastic.co/curator/5/debian stable main" | tee /etc/apt/sources.list.d/elastic-curator-6.x.list + +wget -qO - https://packages.grafana.com/gpg.key | apt-key add - +echo "deb https://packages.grafana.com/oss/deb stable main" | tee /etc/apt/sources.list.d/grafana.list + +wget -qO - https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - +echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" | tee /etc/apt/sources.list.d/kubernetes.list + +wget -qO - https://github.com/rabbitmq/signing-keys/releases/download/2.0/rabbitmq-release-signing-key.asc | apt-key add - +echo "deb http://dl.bintray.com/rabbitmq-erlang/debian bionic erlang-21.x" | tee /etc/apt/sources.list.d/erlang-21.x.list +echo "deb https://dl.bintray.com/rabbitmq/debian bionic main" | tee /etc/apt/sources.list.d/rabbitmq.list + +wget -qO - https://download.docker.com/linux/ubuntu/gpg | apt-key add - +echo "deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable" | tee /etc/apt/sources.list.d/docker-ce.list + +apt update diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/common.sh b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/common.sh new file mode 100644 index 0000000000..f115e98eae --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/common.sh @@ -0,0 +1,81 @@ +# this file is just a bunch of functions meant to be called from other scripts + + +usage() { + echo "usage: ./$(basename $0) " + echo " ./$(basename $0) /tmp/downloads" + [ -z "$1" ] || exit "$1" +} + +echol() { + echo -e "$1" | tee --append $logfile +} + +# params: +remove_file() { + local file_path="$1" + + echol "Removing file: $file_path" + rm -f "$file_path" || exit_with_error "Command failed: rm -f \"$file_path\"" +} + +# params: +create_directory() { + local dir_path="$1" + + if [[ ! -d "$dir_path" ]]; then + mkdir -p $dir_path + fi +} + +# params: +# todo: skip on existing (maybe when checksum is correct?) +download_image() { + local image_name="$1" + local dest_dir="$2" + + local splited_image=(${image_name//:/ }) + local repository=${splited_image[0]} + local tag=${splited_image[1]} + local repo_basename=$(basename -- "$repository") + local dst_image="${dest_dir}/${repo_basename}-${tag}.tar" + + #[[ ! -f $dst_image ]] || remove_file "$dst_image" + if [[ -f ${dst_image} ]]; then + echo "Image: "${dst_image}" already exists. Skipping..." + else + local tmp_file=$(mktemp) + echo "Downloading image: $1" + echo "Skopeo command is: ./skopeo_linux --insecure-policy copy docker://{$image_name} docker-archive:${dst_image}:${repository}:${tag}" + # use temporary file for downloading to be safe from sudden interruptions (network, ctrl+c) + ./skopeo_linux --insecure-policy copy docker://${image_name} docker-archive:${tmp_file}:${repository}:${tag} && chmod 644 ${tmp_file} && mv ${tmp_file} ${dst_image} + fi +} + +# params: +download_file() { + local file_url="$1" + local dest_dir="$2" + + local file_name=$(basename "$file_url") + local dest_path="$dest_dir/$file_name" + + # wget with --timestamping sometimes failes on AWS with ERROR 403: Forbidden + # so we remove existing file to overwrite it + + # remove old files to force redownload after a while + # just a precaution so --continue won't append and corrupt files localy if file is updated on server without name change + if [[ $(find ${dest_path} -mmin +60 -print) ]]; then + echol "File ${dest_path} older than 1h, redownloading..." + remove_file "$dest_path" + fi + + echol "Downloading file: $file" + + # --no-use-server-timestamps - we don't use --timestamping and we need to expire files somehow + # --continue - don't download the same file multiple times, gracefully skip if file is fully downloaded + wget --no-use-server-timestamps --continue --show-progress --directory-prefix="${dest_dir}" "${file_url}" + + #wget --no-verbose --directory-prefix="$dest_dir" "$file_url" || + #exit_with_error "Command failed: wget --no-verbose --directory-prefix=\"$dest_dir\" \"$file_url\"" +} diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/download-requirements.sh b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/download-requirements.sh new file mode 100644 index 0000000000..9753ced55a --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/download-requirements.sh @@ -0,0 +1,153 @@ +#!/bin/bash + +set -euo pipefail + +if [[ $# -lt 1 ]]; then + usage + exit +fi + +if [[ "$EUID" -ne 0 ]]; then + echo "err: this script must be run as root" + exit +fi + +script_path="$( cd "$(dirname "$0")" ; pwd -P )" +input_file="${script_path}/requirements.txt" +dst_dir=$(readlink -m $1) # beautify input path - remove double slashes if occurs +dst_dir_packages="${dst_dir}/packages" +dst_dir_files="${dst_dir}/files" +dst_dir_images="${dst_dir}/images" +deplist="${script_path}/.dependencies" +logfile="${script_path}/log" +download_cmd="apt-get download" +add_repos="${script_path}/add-repositories.sh" + +# to download everything add "--recurse" here: +deplist_cmd() { + apt-cache depends --no-recommends --no-suggests --no-conflicts --no-breaks --no-replaces --no-enhances --no-pre-depends $1 +} + +# source common functions +. "${script_path}/common.sh" + +repos_backup_file="/tmp/epi-repository-setup-scripts/enable-system-repos.sh" +# restore system repositories in case they're missing if ansible role gets interrupted +if [[ ! -f /etc/apt/sources.list ]]; then + if [[ -f /var/tmp/enabled-system-repos.tar ]] && [[ -f ${repos_backup_file} ]]; then + echol "OS repositories seems missing, restoring..." + ${repos_backup_file} + else + echol "/etc/apt/sources.list seems missing, you either know what you're doing or you need to fix your repositories" + fi +fi + +# install prerequisites which might be missing +apt install -y wget gpg + +# some quick sanity check +echol "Dependency list: ${deplist}" +echol "Command used to download packages: ${download_cmd}" +echol "Destination directory for packages: ${dst_dir_packages}" + +# make sure destination dir exists +mkdir -p "${dst_dir_packages}" +mkdir -p "${dst_dir_files}" +mkdir -p "${dst_dir_images}" + +# add 3rd party repositories +. ${add_repos} + +# parse the input file, separete by tags: [packages], [files], [images] +packages=$(awk '/^$/ || /^#/ {next}; /\[packages\]/ {f=1; next}; /^\[/ {f=0}; f {print $0}' "${input_file}") +files=$(awk '/^$/ || /^#/ {next}; /\[files\]/ {f=1; next}; /^\[/ {f=0}; f {print $0}' "${input_file}") +images=$(awk '/^$/ || /^#/ {next}; /\[images\]/ {f=1; next}; /^\[/ {f=0}; f {print $0}' "${input_file}") + +printf "\n" + +# clear list of cached dependencies if .dependencies is older than 15 minutes +find "$script_path" -type f -wholename "${deplist}" -mmin +15 -exec rm "${deplist}" \; +# clear list of cached dependencies if requirements.txt was recently edited +find "$script_path" -type f -wholename "$input_file" -mmin -1 -exec rm "${deplist}" \; + +# PACKAGES +# if dependency list doesn't exist or is zero size then resolve dependency and store them in a deplist file +if [[ ! -f ${deplist} ]] || [[ ! -s ${deplist} ]] ; then + # clean dependency list if process gets interrupted + trap "rm -f ${deplist}; echol 'Dependency resolution interrupted, cleaning cache file'" SIGINT SIGTERM + echo Resolving dependencies to download. This might take a while and will be cached in ${deplist} + while IFS= read -r package; do + echol "Package read from requirements file: $package" + # if package has a specified version e.g. "name 1.0" store it as "name=1.0*" for compatibility with "apt-get download" + package=$(echo ${package} | awk '{if($2 != "") {print $1 "=" $2 "*"} else {print $1}}') + echol "Package to download: $package" + # store package itself in the list of dependencies... + echol "${package}" >> "${deplist}" + # .. and create depency list for the package + # (names only for dependencies, no version check here, not necessary as most dependencies are backward-compatible) + dependencies=$(deplist_cmd "${package}" | awk '/Depends/ && !/ Packages.gz && cd - + echo "deb [trusted=yes] file:${EPI_REPO_SERVER_PATH}/packages ./" > /etc/apt/sources.list.d/epilocal.list + apt update --assume-no # workaround for botched docker repository https://github.com/docker/for-linux/issues/812 + apt -y install apache2 dpkg-dev + rm -f /etc/apt/sources.list.d/epilocal.list + rm -f ${EPI_REPO_SERVER_PATH}/packages/Packages.gz + apt update --assume-no +else + apt -y install apache2 dpkg-dev +fi + +systemctl start apache2 + +# -m is important because it allow same packages with different versions +# 'cd' is needed here becuase 'dpkg-scanpackages' prepends path to "Filename" field in Packages.gz, otherwise it would break package URL for apt +cd /var/www/html/epirepo/packages && dpkg-scanpackages -m . | gzip -9c > Packages.gz diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/server/Debian/disable-repository-server.sh b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/server/Debian/disable-repository-server.sh new file mode 100644 index 0000000000..ff682636f1 --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/server/Debian/disable-repository-server.sh @@ -0,0 +1,4 @@ +#!/bin/bash -eu + +systemctl stop apache2 +systemctl disable apache2 \ No newline at end of file diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/server/Debian/dpkg-scanpackages b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/server/Debian/dpkg-scanpackages new file mode 100644 index 0000000000..05710ae69c --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/server/Debian/dpkg-scanpackages @@ -0,0 +1,295 @@ +#!/usr/bin/perl +# +# dpkg-scanpackages +# +# Copyright © 2006-2015 Guillem Jover +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +use warnings; +use strict; + +use Getopt::Long qw(:config posix_default bundling no_ignorecase); +use List::Util qw(none); +use File::Find; + +use Dpkg (); +use Dpkg::Gettext; +use Dpkg::ErrorHandling; +use Dpkg::Control; +use Dpkg::Version; +use Dpkg::Checksums; +use Dpkg::Compression::FileHandle; + +textdomain('dpkg-dev'); + +# Do not pollute STDOUT with info messages +report_options(info_fh => \*STDERR); + +my (@samemaint, @changedmaint); +my @spuriousover; +my %packages; +my %overridden; +my %hash; + +my %options = (help => sub { usage(); exit 0; }, + version => sub { version(); exit 0; }, + type => undef, + arch => undef, + hash => undef, + multiversion => 0, + 'extra-override'=> undef, + medium => undef, + ); + +my @options_spec = ( + 'help|?', + 'version', + 'type|t=s', + 'arch|a=s', + 'hash|h=s', + 'multiversion|m!', + 'extra-override|e=s', + 'medium|M=s', +); + +sub version { + printf g_("Debian %s version %s.\n"), $Dpkg::PROGNAME, $Dpkg::PROGVERSION; +} + +sub usage { + printf g_( +"Usage: %s [