diff --git a/group_vars/maintenance.yml b/group_vars/maintenance.yml new file mode 100644 index 000000000..0e13caed3 --- /dev/null +++ b/group_vars/maintenance.yml @@ -0,0 +1,517 @@ +--- +# packages +software_groups_to_install: + - admin + - debug + - editors + - pdf_export_deps + - services + - terminals + - utils + +# Autofs +autofs_service.install: true +autofs_service.enable: true +nfs_kernel_tuning: true + +autofs_mount_points: + - data + - gxtest + - gxkey + - jwd + - usrlocal + +# Miniconda role variables (galaxyproject.miniconda) +conda_prefix: /opt/miniconda + +# usegalaxy-eu.bashrc role variables +bashrc_users: + - uname: "{{ galaxy_user.name }}" + uhome: "{{ galaxy_user.home }}" + gname: "{{ galaxy_group.name }}" + + - uname: stats + uhome: /home/stats + gname: stats + +# hxr.postgres-connection role variables +pgc_users: + - uname: "{{ galaxy_user.name }}" + uhome: "{{ galaxy_user.home }}" + gname: "{{ galaxy_group.name }}" + pguser: "{{ postgres_user }}" + pgpass: "{{ postgres_pass }}" + pgdatabase: galaxy + + - uname: stats + uhome: /home/stats + gname: stats + pguser: "{{ galaxy_db_ro_user }}" + pgpass: "{{ galaxy_db_ro_pass }}" + pgdatabase: galaxy + +# fs-maintenance +# Only on the maintenance node +fsm_maintenance_dir: "/data/dnb01/maintenance" + +fsm_scripts: + temporary_dirs: + enable: true + src: "temporary_dirs.sh.j2" + dst: "{{ fsm_maintenance_dir }}/temporary_dirs.sh" + user: "{{ fsm_galaxy_user.username }}" + group: "{{ fsm_galaxy_user.groupname }}" + paths: + - /data/2/galaxy_db/tmp + - /data/dnb01/galaxy_db/tmp + - /data/dnb02/galaxy_db/tmp + - /data/dnb03/galaxy_db/tmp + - /data/jwd/tmp + - /data/1/galaxy_db/tmp + - /data/dnb05/galaxy_db/tmp + - /data/dnb06/galaxy_db/tmp + - /data/jwd02f/tmp + - /data/jwd04/tmp + time: "{{ fsm_intervals.long }}" + upload_dirs: + enable: true + src: "uploads.sh.j2" + dst: "{{ fsm_maintenance_dir }}/uploads.sh" + user: "{{ fsm_galaxy_user.username }}" + group: "{{ fsm_galaxy_user.groupname }}" + paths: + - "{{ galaxy_config['galaxy']['nginx_upload_store'] }}" + - "{{ galaxy_config['galaxy']['nginx_upload_job_files_store'] }}" + time: "{{ fsm_intervals.medium }}" + job_working_dirs: + enable: true + src: "job_working_dir.sh.j2" + dst: "{{ fsm_maintenance_dir }}/job_working_dir.sh" + user: "{{ fsm_galaxy_user.username }}" + group: "{{ fsm_galaxy_user.groupname }}" + paths: + - "{{ galaxy_config['galaxy']['job_working_directory'] }}" + - /data/dnb03/galaxy_db/job_working_directory + - /data/jwd/main + - /data/jwd01/main + - /data/jwd02f/main + - /data/jwd03f/main + - /data/jwd04/main + - /data/jwd05e/main + time: "{{ fsm_intervals.long }}" + +# Should run only on the headnode, because the gxadmin cron task uses galaxy's +# log dir /var/log/galaxy during cleanup +# fsm_cron_tasks: +# docker: +# enable: true +# name: "Docker clean up" +# minute: 30 +# hour: 2 +# dom: "*" +# month: "*" +# dow: "*" +# job: ". {{ galaxy_root }}/.bashrc && docker system prune -f > /dev/null" +# user: "{{ fsm_galaxy_user.username }}" +# gxadmin: +# enable: true +# name: "Gxadmin Galaxy clean up" +# minute: 0 +# hour: "*/6" +# dom: "*" +# month: "*" +# dow: "*" +# job: "{{ custom_telegraf_env }} /usr/bin/gxadmin galaxy cleanup 60" +# user: "{{ fsm_galaxy_user.username }}" + +fsm_htcondor_enable: true + +# Telegraf +telegraf_agent_hostname: "{{ hostname }}" +telegraf_agent_version: 1.17.2 +custom_telegraf_env: "/usr/bin/env GDPR_MODE=1 PGUSER={{ galaxy_user.name }} PGHOST={{ postgres_host }} GALAXY_ROOT={{ galaxy_server_dir }} GALAXY_CONFIG_FILE={{ galaxy_config_file }} GALAXY_LOG_DIR={{ galaxy_log_dir }} GXADMIN_PYTHON={{ galaxy_venv_dir }}/bin/python" +# Uncomment the outer block when the respective tasks are not run in the headnodes playbook anymore +# telegraf_plugins_extra: +# # Run only on the maintenance node +# postgres: +# plugin: "postgresql" +# config: +# - address = "{{ galaxy_db_connection }}" +# - databases = ["galaxy", "galaxy-test", "apollo", "chado"] + +# # Run only on the headnodes +# # Also when pushing it would be nice if they can have some sort of prefix +# # to differentiate the headnodes data +# # listen_galaxy_routes: +# # plugin: "statsd" +# # config: +# # - service_address = ":8125" +# # - percentiles = [90] +# # - metric_separator = "." +# # - allowed_pending_messages = 10000 +# # - percentile_limit = 100 + +# # Run only on the maintenance node +# monitor_nfsstat: +# plugin: "exec" +# config: +# - commands = ["/usr/bin/nfsstat-influx"] +# - timeout = "10s" +# - data_format = "influx" +# - interval = "15s" + +# # Some custom galaxy monitoring stuff +# galaxy_uploaded: +# plugin: "exec" +# config: +# - commands = ["{{ custom_telegraf_env }} /usr/bin/gxadmin iquery upload-gb-in-past-hour"] +# - timeout = "360s" +# - data_format = "influx" +# - interval = "1h" + +# # Run only on the headnodes +# galaxy_lastlog: +# plugin: "exec" +# config: +# - commands = ["{{ custom_telegraf_env }} /usr/bin/gxadmin gunicorn lastlog"] +# - timeout = "15s" +# - data_format = "influx" +# - interval = "15s" + +# # Run on the maintenance node +# galaxy_jobs_queued: +# plugin: "exec" +# config: +# - commands = ["{{ custom_telegraf_env }} /usr/bin/gxadmin iquery jobs-queued"] +# - timeout = "15s" +# - data_format = "influx" +# - interval = "1m" +# galaxy_jobs_queued_internal: +# plugin: "exec" +# config: +# - commands = ["{{ custom_telegraf_env }} /usr/bin/gxadmin iquery jobs-queued-internal-by-handler"] +# - timeout = "15s" +# - data_format = "influx" +# - interval = "1m" +# galaxy_jobs_queue_overview: +# plugin: "exec" +# config: +# - commands = ["{{ custom_telegraf_env }} /usr/bin/gxadmin iquery queue-overview --short-tool-id"] +# - timeout = "30s" +# - data_format = "influx" +# - interval = "1m" +# galaxy_oidc: +# plugin: "exec" +# config: +# - commands = ["{{ custom_telegraf_env }} /usr/bin/gxadmin iquery users-with-oidc"] +# - timeout = "15s" +# - data_format = "influx" +# - interval = "1m" +# galaxy_workflow: +# plugin: "exec" +# config: +# - commands = ["{{ custom_telegraf_env }} /usr/bin/gxadmin iquery workflow-invocation-status"] +# - timeout = "15s" +# - data_format = "influx" +# - interval = "1m" +# galaxy_workflow_totals: +# plugin: "exec" +# config: +# - commands = ["{{ custom_telegraf_env }} /usr/bin/gxadmin iquery workflow-invocation-totals"] +# - timeout = "15s" +# - data_format = "influx" +# - interval = "1m" +# # Uses logs from /var/log/nginx/ so it should be run only on the headnode +# # galaxy_active_users: +# # plugin: "exec" +# # config: +# # - commands = ["/usr/bin/gxadmin local cu"] +# # - timeout = "15s" +# # - data_format = "influx" +# # - interval = "1m" +# postgres_extra: +# plugin: "exec" +# config: +# - commands = [ +# "{{ custom_telegraf_env }} /usr/bin/gxadmin iquery pg-cache-hit", +# "{{ custom_telegraf_env }} /usr/bin/gxadmin iquery pg-index-size", +# "{{ custom_telegraf_env }} /usr/bin/gxadmin iquery pg-index-usage", +# "{{ custom_telegraf_env }} /usr/bin/gxadmin iquery pg-table-bloat", +# "{{ custom_telegraf_env }} /usr/bin/gxadmin iquery pg-table-size", +# "{{ custom_telegraf_env }} /usr/bin/gxadmin iquery pg-unused-indexes", +# "{{ custom_telegraf_env }} /usr/bin/gxadmin iquery pg-vacuum-stats", +# "{{ custom_telegraf_env }} /usr/bin/gxadmin iquery pg-stat-bgwriter", +# "{{ custom_telegraf_env }} /usr/bin/gxadmin iquery pg-stat-user-tables", +# ] +# - timeout = "60s" +# - data_format = "influx" +# - interval = "2m" + +# Custom pip installer +pip_venv_path: "{{ galaxy_venv_dir }}" +pip_install_dependencies: + # phdcomics + - beautifulsoup4 + # Used by code_files for apollo + - biopython + - bcbio-gff + # Needed for tool watching + - watchdog + # Needed gxadmin graphs + - data_hacks + # used by reports + - WeasyPrint + - nbconvert + - gitlab-arc-fs + # Needed for usegalaxy-eu.vgcn-monitoring Telegraf role + - pyyaml + - GitPython + - python-openstackclient + +yum_exclude_repos: + - condor* + - npm* + +all_yum_repositories: + - name: galaxy_gpel + description: Galaxy Packages for Enterprise Linux $releasever - $basearch + baseurl: https://depot.galaxyproject.org/yum/el/$releasever/$basearch/ + enabled: true + gpgcheck: false + retries: 1 + timeout: 10 + +# Galaxy monitoring +monitor_condor: true +monitor_condor_split_util: true + +# Rsync to NFS and to the headnodes +# Role: usegalaxy-eu.rsync-galaxy-sync +galaxy_nfs_location: /data/galaxy-sync +# Whether to execute the sync script or not during the playbook run +execute_galaxy_sync: false + +# Configure it only on the maintenance node +# TPV Script +tpv_config_dir_name: total_perspective_vortex + +# SystemD +galaxy_systemd_workflow_schedulers: 0 +galaxy_systemd_handlers: 0 + +# HTCondor +# condor_role: central-manager +# condor_minimal_version: 10.2.0 +# condor_enforce_role: true +# condor_copy_template: true +# condor_host: "condor-cm.galaxyproject.eu" +# condor_allow_write: "10.5.68.0/24, 132.230.223.0/24, 132.230.153.0/28" +# condor_daemons: +# - COLLECTOR +# - MASTER +# - NEGOTIATOR +# - SCHEDD +# condor_fs_domain: bi.uni-freiburg.de +# condor_uid_domain: bi.uni-freiburg.de +# condor_allow_negotiator: "$(CONDOR_HOST)" +# condor_allow_administrator: "$(ALLOW_NEGOTIATOR)" +# condor_system_periodic_hold: "{{ 30 * 24 * 60 * 60 }}" +# condor_network_interface: eth0 +# condor_extra: | +# MASTER_UPDATE_INTERVAL = 150 +# CLASSAD_LIFETIME = 300 +# SCHEDD_INTERVAL = 60 +# JOB_START_COUNT = 250 +# JOB_START_DELAY = 0 +# CLAIM_WORKLIFE = 120 +# NEGOTIATOR_UPDATE_INTERVAL = 100 +# SSH_TO_JOB_SSH_CMD = "ssh -oUser=galaxy -oIdentityFile=%i -oStrictHostKeyChecking=yes -oUserKnownHostsFile=%k -oGlobalKnownHostsFile=%k -oProxyCommand=%x %h" + + +# Configure it only on the maintenance node +# gie_proxy +# 13/02/2023: This is now GxIT (not IE anymore) +gie_proxy_install: true +gie_proxy_dir: "{{ galaxy_root }}/gie-proxy/proxy" +gie_proxy_git_version: main +gie_proxy_setup_nodejs: nodeenv +gie_proxy_virtualenv_command: "{{ galaxy_virtualenv_command }}" #"{{ pip_virtualenv_command }}" +gie_proxy_virtualenv_python: "{{ galaxy_virtualenv_python }}" +gie_proxy_nodejs_version: "18.14.0" +gie_proxy_virtualenv: "{{ galaxy_root }}/gie-proxy/venv" +gie_proxy_npm_executable: "{{ gie_proxy_virtualenv }}/bin/npm" +gie_proxy_setup_service: none +gie_proxy_sessions_path: "{{ galaxy_mutable_data_dir }}/interactivetools_map.sqlite" +gie_proxy_port: 8800 +gie_proxy_verbose: true + +# gxadmin +postgres_user: galaxy +postgres_host: sn05.galaxyproject.eu +postgres_port: 5432 + +# Configure it only on the maintenance node +# Slurp script +galaxy_slurper: galaxy +galaxy_slurp_influx_pass: "{{ influxdb.node.password }}" +galaxy_slurp_influx_user: "{{ influxdb.node.username }}" +galaxy_slurp_influx_url: "{{ influxdb.url }}" + +# postgres_pass defined in secret_group_vars/all.yml +gxadmin_commit: main +gxadmin_dir: /opt/gxadmin +gxadmin_bin_dir: /usr/bin +gxadmin_force: true + +# GALAXY +galaxy_group: + name: galaxy + gid: 999 +galaxy_system_group: galaxy +galaxy_log_dir: "/var/log/galaxy" +galaxy_layout: root-dir +galaxy_separate_privileges: true +galaxy_restart_handler_name: Restart Galaxy +galaxy_user: + name: galaxy + create_home: true + home: /opt/galaxy + uid: 999 + shell: /bin/bash + +# Galaxy configuration files will be written with these permissions (mode argument to Ansible copy/template module) +galaxy_config_perms: 0644 + +galaxy_root: /opt/galaxy +galaxy_workflow_scheduler_count: "{{ galaxy_systemd_workflow_schedulers }}" +galaxy_home_dir: /opt/galaxy +galaxy_mutable_config_dir: "{{ galaxy_root }}/mutable-config" +galaxy_mutable_data_dir: "{{ galaxy_root }}/mutable-data" +galaxy_shed_tools_dir: "{{ galaxy_root }}/shed_tools" +galaxy_tool_data_path: "{{ galaxy_root }}/tool-data" +__galaxy_shed_tools_dir: "{{ galaxy_shed_tools_dir }}" +galaxy_backup_configfiles: false + +galaxy_server_dir: "{{ galaxy_root }}/server" +galaxy_config_dir: "{{ galaxy_root }}/config" +galaxy_venv_dir: "{{ galaxy_root }}/venv" +galaxy_job_working_directory: "{{ galaxy_config['galaxy']['job_working_directory'] }}" +ucsc_build_sites: + +galaxy_virtualenv_command: "{{ conda_prefix }}/envs/_galaxy_/bin/virtualenv" +galaxy_virtualenv_python: "{{ conda_prefix }}/envs/_galaxy_/bin/python" +galaxy_virtualenv: "{{ galaxy_venv_dir }}" +galaxy_nonrepro_tools: "{{ galaxy_root }}/custom-tools" +galaxy_nonrepro_commit: master + + +# Custom override +# Our galaxy_tool_dependency_dir is on NFS, and permissions are set in such a +# way that they cannot be changed by the ansible connected user. +galaxy_dirs: + - "{{ galaxy_mutable_data_dir }}" + - "{{ galaxy_mutable_config_dir }}" + - "{{ galaxy_cache_dir }}" + - "{{ galaxy_shed_tools_dir }}" + #- "{{ galaxy_tool_dependency_dir }}" + - "{{ galaxy_file_path }}" + - "{{ galaxy_job_working_directory }}" + - "{{ galaxy_tool_data_path }}" + - "{{ galaxy_nonrepro_tools }}" + - "{{ galaxy_webhook_dir }}" + - "{{ galaxy_tour_dir }}" + - "{{ galaxy_log_dir }}" + - "{{ galaxy_config_dir }}/total_perspective_vortex" + - "{{ tpv_mutable_dir }}" + +galaxy_instance_hostname: usegalaxy.eu +galaxy_config_style: yaml + +galaxy_repo: "https://github.com/usegalaxy-eu/galaxy.git" +galaxy_commit_id: "release_23.0_europe" +galaxy_force_checkout: true # discard any modified files +#galaxy_admin_email_from: 'noreply@usegalaxy.eu' + +galaxy_singularity_images_cvmfs_path: "/cvmfs/singularity.galaxyproject.org/all/" + +galaxy_config: "{{ galaxy_config_hash }}" + +galaxy_config_file_src_dir: files/galaxy +galaxy_config_files: + # TODO(bgruening): sync this with cvmfs? can we have ours be just a small additional builds.txt? + - src: "{{ galaxy_config_file_src_dir }}/config/builds.txt" + dest: "{{ galaxy_config['galaxy']['builds_file_path'] }}" + - src: "{{ galaxy_config_file_src_dir }}/config/data_manager_conf.xml" + dest: "{{ galaxy_config['galaxy']['data_manager_config_file'] }}" + - src: "{{ galaxy_config_file_src_dir }}/config/datatypes_conf.xml" + dest: "{{ galaxy_config['galaxy']['datatypes_config_file'] }}" + - src: "{{ galaxy_config_file_src_dir }}/config/dependency_resolvers_conf.xml" + dest: "{{ galaxy_config['galaxy']['dependency_resolvers_config_file'] }}" + - src: "{{ galaxy_config_file_src_dir }}/config/email_domain_blocklist.conf" + dest: "{{ galaxy_config['galaxy']['email_domain_blocklist_file'] }}" + - src: "{{ galaxy_config_file_src_dir }}/config/user_preferences_extra_conf.yml" + dest: "{{ galaxy_config['galaxy']['user_preferences_extra_conf_path'] }}" + - src: "{{ galaxy_config_file_src_dir }}/config/job_resource_params_conf.xml" + dest: "{{ galaxy_config['galaxy']['job_resource_params_file'] }}" + - src: "{{ galaxy_config_file_src_dir }}/config/trs_servers_conf.yml" + dest: "{{ galaxy_config['galaxy']['trs_servers_config_file'] }}" + - src: "{{ galaxy_config_file_src_dir }}/config/themes_conf.yml" + dest: "{{ galaxy_config['galaxy']['themes_config_file'] }}" + # test? Unknown if this works. + - src: "{{ galaxy_config_file_src_dir }}/config/error_report.yml" + dest: "{{ galaxy_config_dir }}/error_report.yml" + - src: "{{ galaxy_config_file_src_dir }}/config/job_metrics_conf.yml" + dest: "{{ galaxy_config['galaxy']['job_metrics_config_file'] }}" + - src: "{{ galaxy_config_file_src_dir }}/config/migrated_tools_conf.xml" + dest: "{{ galaxy_config['galaxy']['migrated_tools_config'] }}" + - src: "{{ galaxy_config_file_src_dir }}/config/tool_data_table_conf.xml" + dest: "{{ galaxy_config_dir }}/tool_data_table_conf.xml" + - src: "{{ galaxy_config_file_src_dir }}/config/tool_sheds_conf.xml" + dest: "{{ galaxy_config['galaxy']['tool_sheds_config_file'] }}" + - src: "{{ galaxy_config_file_src_dir }}/config/ucsc_build_sites.txt" + dest: "{{ galaxy_config['galaxy']['ucsc_build_sites'] }}" + - src: "{{ galaxy_config_file_src_dir }}/config/echo_main_env.xml" + dest: "{{ galaxy_config_dir }}/echo_main_env.xml" + - src: "{{ galaxy_config_file_src_dir }}/tpv/tool_defaults.yml" + dest: "{{ tpv_mutable_dir }}/tool_defaults.yml" + - src: "{{ galaxy_config_file_src_dir }}/tpv/destinations.yml" + dest: "{{ tpv_mutable_dir }}/destinations.yml" + - src: "{{ galaxy_config_file_src_dir }}/tpv/tools.yml" + dest: "{{ tpv_mutable_dir }}/tools.yml" + - src: "{{ galaxy_config_file_src_dir }}/tpv/roles.yml" + dest: "{{ tpv_mutable_dir }}/roles.yml" + - src: "{{ galaxy_config_file_src_dir }}/tpv/interactive_tools.yml" + dest: "{{ tpv_mutable_dir }}/interactive_tools.yml" + - src: "{{ galaxy_config_file_src_dir }}/tpv/users.yml" + dest: "{{ tpv_mutable_dir }}/users.yml" + +galaxy_config_template_src_dir: templates/galaxy +galaxy_config_templates: + - src: "{{ galaxy_config_template_src_dir }}/config/build_sites.yml.j2" + dest: "{{ galaxy_config['galaxy']['build_sites_config_file'] }}" + - src: "{{ galaxy_config_template_src_dir }}/config/global_host_filters.py.j2" + dest: "{{ galaxy_server_dir }}/lib/galaxy/tool_util/toolbox/filters/global_host_filters.py" + - src: "{{ galaxy_config_template_src_dir }}/config/job_conf.yml.j2" + dest: "{{ galaxy_config_dir }}/job_conf.yml" + - src: "{{ galaxy_config_template_src_dir }}/config/container_resolvers_conf.xml.j2" + dest: "{{ galaxy_config_dir }}/container_resolvers_conf.xml" + - src: "{{ galaxy_config_template_src_dir }}/config/user_filters.py.j2" + dest: "{{ galaxy_server_dir }}/lib/galaxy/tool_util/toolbox/filters/user_filters.py" + - src: "{{ galaxy_config_template_src_dir }}/config/nagios_tool_conf.xml" + dest: "{{ galaxy_config_dir }}/nagios_tool_conf.xml" + - src: "{{ galaxy_config_template_src_dir }}/config/oidc_backends_config.xml" + dest: "{{ galaxy_config['galaxy']['oidc_backends_config_file'] }}" + - src: "{{ galaxy_config_template_src_dir }}/config/oidc_config.xml" + dest: "{{ galaxy_config['galaxy']['oidc_config_file'] }}" + - src: "{{ galaxy_config_template_src_dir }}/config/grt.yml.j2" + dest: "{{ galaxy_config_dir }}/grt.yml" + - src: "{{ galaxy_config_template_src_dir }}/config/pulsar_app.yml" + dest: "{{ galaxy_config_dir }}/pulsar_app.yml" + - src: "{{ galaxy_config_template_src_dir }}/config/tool_conf.xml.j2" + dest: "{{ galaxy_config_dir }}/tool_conf.xml" + - src: "{{ galaxy_config_template_src_dir }}/config/galaxy_workflow_scheduler.j2" + dest: "{{ galaxy_config_dir }}/workflow_schedulers_conf.xml" + - src: "{{ galaxy_config_template_src_dir }}/config/file_sources_conf.yml.j2" + dest: "{{ galaxy_config_dir }}/file_sources_conf.yml" + - src: "{{ galaxy_config_template_src_dir }}/config/object_store_conf.xml.j2" + dest: "{{ galaxy_config['galaxy']['object_store_config_file'] }}" diff --git a/hosts b/hosts index dda02f8c8..fe9bb433c 100644 --- a/hosts +++ b/hosts @@ -56,6 +56,9 @@ plausible.galaxyproject.eu [celerycluster] celery-0.galaxyproject.eu +[maintenance] +maintenance.galaxyproject.eu ansible_ssh_user=rocky + # Baremetal [galaxyservers] sn06.galaxyproject.eu diff --git a/maintenance.yml b/maintenance.yml new file mode 100644 index 000000000..4b963e993 --- /dev/null +++ b/maintenance.yml @@ -0,0 +1,178 @@ +--- +- name: UseGalaxy EU maintenance server + hosts: maintenance + become: true + become_user: root + vars: + # The full internal name. + hostname: maintenance.galaxyproject.eu + vars_files: + - group_vars/gxconfig.yml # The base galaxy configuration + - group_vars/toolbox.yml # User controlled toolbox + - secret_group_vars/sentry.yml # Sentry SDK init url + - secret_group_vars/aws.yml # AWS creds + - secret_group_vars/pulsar.yml # Pulsar + MQ Connections + - secret_group_vars/oidc.yml # OIDC credentials (ELIXIR, keycloak) + - secret_group_vars/object_store.yml # Object Store credentils (S3 etc ...) + - secret_group_vars/db-main.yml # DB URL + some postgres stuff + - secret_group_vars/file_sources.yml # file_sources_conf.yml creds + - secret_group_vars/all.yml # All of the other assorted secrets... + - secret_group_vars/keys.yml # SSH keys + - templates/galaxy/config/job_conf.yml + handlers: + # Though this handler doesn't do anything (for now), galaxyproject.galaxy role + # will fail if it's not defined + - name: Restart Galaxy + shell: | + echo 'Manual web handler restart required' && cd /opt/galaxy/ && source /opt/galaxy/.bashrc + collections: + - devsec.hardening + pre_tasks: + - name: Install Dependencies + package: + name: + [ + "git", + "postgresql", + "python3-psycopg2", + "python3-virtualenv", + "bc", + "python3", + "python3-devel", + ] + become: true + post_tasks: + - name: Append some users to the systemd-journal group + user: + name: "{{ item }}" + groups: systemd-journal + append: true + loop: + - "{{ galaxy_user.name }}" + - "telegraf" + - "stats" # special account to retrieve statistics from the server in read-only mode + - name: Set authorized SSH key (galaxy user) + ansible.posix.authorized_key: + user: "{{ galaxy_user.name }}" + state: present + key: "{{ item }}" + loop: + - "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOBINXdjILF6x3WuppXyq6J2a2oSLR6waZ6txgjYJogHdIKPbI0TdReCv4EVxxYRY/NqGpHbjkqfRTsf2VgoU3U= mk@galaxy-mira" + - "ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBACB5Q5blymkTIRSzVzXITOGvBuI7W0L9Ykwfz8LJGPraaGVPiezzFGvjhqwX+EyCqQPt7JprR5mimJRw/JN3nBXWAHjekvmB5FuILkk6m5fOiQJ5QhRMyQ5GfxODAvGbHpTuWHbYJLWD5fhcboKPxlXOWy4xY9kDZVuQvEKisNKYBsFLA== sanjay" + - name: Set authorized SSH key (stats user) + ansible.posix.authorized_key: + user: "stats" + state: present + key: "{{ item }}" + loop: + - "ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAA6oD5Ps9h6pKokzaAcCI6R08CKm2aSVv86h/O2HTEQkzeJq/Uvu4gbrONAM0FK5R693mHggIwaROKf1Z1+q4YNtACtYjV1c+6a9lcrGMM31y5RzO6mAW+rHNEXgZ1n3wqDcBQhSXhSekSen0R2QKwvmB7xeP4XX9qE10azZuafIFU9hQ== sanjay" + - "ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBADKblzzPBc3+dEfFvhJQHsHGkFFN6ORjfXo71P1OutwcKEMCIcNkZKJHhYkLLrfTDN5JJ5tK2L5AaSxdwETofwm4AG1xv3LuoYsXC6e3sjKi09BVmzef520pIMW+rvL+hESwSazZaUAC0wDcH4aNDTonZYcAY87rpMX7pNMkNPJvWilUA== mira" + roles: + ## Starting configuration of the operating system + - role: usegalaxy_eu.handy.os_setup + vars: + enable_hostname: true + enable_powertools: true # geerlingguy.repo-epel role doesn't enable PowerTools repository + enable_remap_user: true + enable_exclude_packages: true + enable_pam_limits: true # Prevent out of control processes + enable_install_software: true # Some extra admin tools (*top, vim, etc) + - geerlingguy.repo-epel # Install EPEL repository + + ## Istall miniconda, create a _galaxy_ environment and install Packages + ## Galaxy will use the virtualenv command from this conda environment (see + ## galaxy_virtualenv_command) in the group_vars/maintenance.yml + - role: galaxyproject.miniconda + vars: + miniconda_prefix: "{{ conda_prefix }}" + galaxy_conda_create_env: true + galaxy_conda_env_packages: + - python=3.8.8 + - pip + - virtualenv + + - usegalaxy-eu.autoupdates # keep all of our packages up to date + - influxdata.chrony # Keep our time in sync. + + ## Filesystems + - usegalaxy-eu.autofs # Setup the mount points which will be needed later + + ## Monitoring (Uncomment when in production) + # - hxr.monitor-cluster + # - hxr.monitor-email + # - usegalaxy-eu.monitoring + + ## Setup Galaxy user + - role: galaxyproject.galaxy + vars: + galaxy_create_user: true + galaxy_manage_clone: false + galaxy_manage_download: false + galaxy_manage_existing: true + galaxy_manage_paths: true + galaxy_manage_static_setup: false + galaxy_manage_mutable_setup: false + galaxy_manage_database: false + galaxy_fetch_dependencies: false + galaxy_build_client: false + + ## The bashrc needs to be created for several later features. + - role: usegalaxy-eu.bashrc + + # HTCondor (Uncomment when in production and latest version is installed) + # - usegalaxy_eu.htcondor + + # Misc. + - role: hxr.galaxy-nonreproducible-tools + become: true + become_user: galaxy + - usegalaxy-eu.dynmotd # nicer MOTD/welcome message + - usegalaxy-eu.rsync-galaxy-sync # syncs Galaxy codebase to NFS and to the headnodes + - role: usegalaxy-eu.webhooks # Clone webhook repository + become: true + become_user: galaxy + - role: usegalaxy-eu.tours # Clone tour repository + become: true + become_user: galaxy + + ## SSL / Security + # https://en.wikipedia.org/wiki/Trust_on_first_use + - ssh-host-sign # Sign the server host key to prevent TOFU for SSH + + ## GALAXY + - hxr.postgres-connection + - galaxyproject.gxadmin + # TODO move under monitoring + telegraf. + # Role that creates a cron to gather galaxy stats, needs to run only on one node (uncomment when in production) + # - usegalaxy-eu.galaxy-slurp + + # The REAL galaxy role + - role: galaxyproject.galaxy + vars: + galaxy_create_user: true + galaxy_manage_clone: true + galaxy_manage_download: false + galaxy_manage_existing: false + galaxy_manage_static_setup: true + galaxy_manage_mutable_setup: true + galaxy_manage_database: true + galaxy_fetch_dependencies: true + galaxy_build_client: true + + ## Extras! + - usegalaxy-eu.fix-galaxy-server-dir # Fix details into the galaxy server dirs + - hxr.install-to-venv # Some extra packages our site needs. + - usegalaxy_eu.gie_proxy # Setup the NodeJS proxy for GxIT (not IE anymore) + # - usegalaxy_eu.fs_maintenance # (Uncomment when in production) + # - usegalaxy-eu.htcondor_release # (Uncomment when in production and condor is installed) + # Various ugly fixes + # - usegalaxy-eu.fix-unscheduled-workflows (Uncomment when in production) + # - usegalaxy-eu.fix-ancient-ftp-data # Remove FTP data older than 3 months, create FTP user directories (Uncomment when in production) + # - usegalaxy-eu.fix-missing-api-keys # Workaround for IE users not have a key set. (Uncomment when in production) + # - usegalaxy-eu.fix-user-quotas # Automatically recalculate user quotas and attribute ELIXIR quota to ELIXIR AAI user on a regular basis (Uncomment when in production) + - usegalaxy_eu.tpv_auto_lint + - ssh_hardening #dev-sec.hardening collection + - dj-wasabi.telegraf # Uncomment when in production + - usegalaxy-eu.logrotate # Rotate logs + - usegalaxy-eu.fix-stop-ITs + - usegalaxy-eu.vgcn-monitoring diff --git a/requirements.yaml b/requirements.yaml index 74dc7dc46..ea637b63b 100644 --- a/requirements.yaml +++ b/requirements.yaml @@ -21,7 +21,7 @@ collections: source: https://galaxy.ansible.com type: galaxy - name: usegalaxy_eu.handy - version: 2.2.0 + version: 2.2.1 source: https://galaxy.ansible.com roles: @@ -89,7 +89,7 @@ roles: src: https://github.com/usegalaxy-eu/ansible-update-hosts version: 0.2.0 - name: usegalaxy_eu.gie_proxy - version: 0.0.2 + version: 0.0.4 - name: usegalaxy-eu.autofs src: https://github.com/usegalaxy-eu/ansible-autofs version: 1.0.0 diff --git a/roles/dj-wasabi.telegraf/tasks/RedHat.yml b/roles/dj-wasabi.telegraf/tasks/RedHat.yml index f43df7386..ac965a42e 100644 --- a/roles/dj-wasabi.telegraf/tasks/RedHat.yml +++ b/roles/dj-wasabi.telegraf/tasks/RedHat.yml @@ -7,7 +7,7 @@ description: InfluxDB Repository - RHEL $releasever baseurl: "https://repos.influxdata.com/rhel/{{ telegraf_redhat_releasever }}/$basearch/stable" gpgcheck: yes - gpgkey: https://repos.influxdata.com/influxdb.key + gpgkey: https://repos.influxdata.com/influxdata-archive_compat.key - name: "Install telegraf package | RedHat" yum: diff --git a/roles/usegalaxy-eu.rsync-galaxy-sync/README.md b/roles/usegalaxy-eu.rsync-galaxy-sync/README.md new file mode 100644 index 000000000..2efe85dd9 --- /dev/null +++ b/roles/usegalaxy-eu.rsync-galaxy-sync/README.md @@ -0,0 +1,30 @@ +Role Name +========= + +Adds a rsync script that performs a full sync of a Galaxy codebase to an NFS share and the head nodes + +Role Variables +-------------- + +`execute_galaxy_sync`: Whether to execute the sync script or not. Defaults to `false` +`galaxy_rsync_user_private_key_file`: The private key of the user that will be used to rsync the codebase. If this key does not exist then it will be added from the vault file. +`headnodes`: A list of headnodes to rsync the codebase to. Defaults to `sn07.galaxyproject.eu` (this is currently (24/03/2023) in testing phase so the default is `sn07`) +`headnodes_sync_location`: The location on the headnodes to rsync the codebase to. Defaults to the variable `galaxy_root` (which is defined in the group_vars files) + +Dependencies +------------ + +- `prsync` command (if not installed, it will be installed. The command is available in the `pssh` package) + +Example Playbook +---------------- + + - hosts: maintenance + roles: + - role: usegalaxy-eu.rsync-galaxy-sync + vars: + execute_galaxy_sync: false + galaxy_rsync_user_private_key: "/opt/galaxy/.ssh/galaxy_rsync_key" + headnodes: "sn07.galaxyproject.eu" + headnodes_sync_location: "/opt/galaxy" + galaxy_nfs_location: "/data/galaxy-sync" diff --git a/roles/usegalaxy-eu.rsync-galaxy-sync/defaults/main.yml b/roles/usegalaxy-eu.rsync-galaxy-sync/defaults/main.yml new file mode 100644 index 000000000..dc7837e54 --- /dev/null +++ b/roles/usegalaxy-eu.rsync-galaxy-sync/defaults/main.yml @@ -0,0 +1,9 @@ +--- +execute_galaxy_sync: false +galaxy_rsync_user_private_key_file: "{{ galaxy_user.home }}/.ssh/galaxy_rsync_key" + +# Uses the hostnames of the inventory group 'galaxyservers' +# headnodes: "{{ groups['galaxyservers'] | join(', ') }}" +headnodes: "sn07.galaxyproject.eu" +headnodes_sync_location: "{{ galaxy_root }}" +galaxy_nfs_location: "" diff --git a/roles/usegalaxy-eu.rsync-galaxy-sync/meta/main.yml b/roles/usegalaxy-eu.rsync-galaxy-sync/meta/main.yml new file mode 100644 index 000000000..9edf65de6 --- /dev/null +++ b/roles/usegalaxy-eu.rsync-galaxy-sync/meta/main.yml @@ -0,0 +1,14 @@ +--- +galaxy_info: + author: The Galaxy Project + description: Installs a Galaxy rsync script + company: The Galaxy Project + license: AFL v3.0 + min_ansible_version: 2.5 + platforms: + - name: EL + versions: + - 8 + - 9 + galaxy_tags: [] +dependencies: [] diff --git a/roles/usegalaxy-eu.rsync-galaxy-sync/tasks/main.yml b/roles/usegalaxy-eu.rsync-galaxy-sync/tasks/main.yml new file mode 100644 index 000000000..f510a608e --- /dev/null +++ b/roles/usegalaxy-eu.rsync-galaxy-sync/tasks/main.yml @@ -0,0 +1,80 @@ +--- +- name: Check if prsync is installed + command: prsync --version + register: prsync_installed + ignore_errors: true + +- name: Install prsync (part of pssh) + become: true + dnf: + name: pssh + state: present + when: not prsync_installed.stdout + +- name: Check if SSH key exists + stat: + path: "{{ galaxy_rsync_user_private_key_file }}" + register: ssh_key + +- name: Create SSH directory + file: + path: "{{ galaxy_user.home }}/.ssh" + state: directory + owner: "{{ galaxy_user.name }}" + group: "{{ galaxy_user.name }}" + mode: 0700 + when: not ssh_key.stat.exists + +- name: Add SSH key + copy: + content: "{{ galaxy_user_private_key }}" + dest: "{{ galaxy_rsync_user_private_key_file }}" + owner: "{{ galaxy_user.name }}" + group: "{{ galaxy_user.name }}" + mode: 0600 + when: not ssh_key.stat.exists + +- name: "Deploy galaxy-rsync script" + copy: + content: | + #!/bin/bash + headnodes="{{ headnodes }}" + cd {{ galaxy_root }}; + for dir in {config,custom-tools,dynamic_rules,gie-proxy,mutable-config,mutable-data,server,venv,tool-data}; do + if [ -d $dir ]; then + echo "Syncing $dir" + # Sync to NFS server in background + rsync -avr --delete --exclude node_modules/ --exclude .git --exclude __pycache__ $dir/ {{ galaxy_nfs_location }}/$dir/ & + + # Sync to headnodes only if the variable is set + if [ ! -z $headnodes ]; then + # Sync to head nodes in foreground (so we can see progress and wait until the sync is done before continuing with the next directory) + prsync -avr -H $headnodes --extra-arg='--delete' --extra-arg='--exclude=node_modules/' --extra-arg='--exclude=.git' --extra-arg='--exclude=__pycache__' --user "{{ galaxy_user.name }}" --ssh-args='-i "{{ galaxy_rsync_user_private_key_file }}"' $dir/ {{ headnodes_sync_location }}/$dir/ + fi + else + echo "Skipping $dir" + fi + done; + if [ -d shed_tools-local ]; then + echo "Syncing shed_tools-local" + # Sync to NFS server in background + rsync -avr --delete --exclude .hg shed_tools-local/ {{ galaxy_nfs_location }}/shed_tools/ & + + # Sync to headnodes only if the variable is set + if [ ! -z $headnodes ]; then + # Sync to head nodes in foreground (so we can see progress and wait until the sync is done before continuing with the next directory) + prsync -avr -H $headnodes --extra-arg='--delete' --extra-arg='--exclude=.hg' --user "{{ galaxy_user.name }}" --ssh-args='-i "{{ galaxy_rsync_user_private_key_file }}"' shed_tools-local/ {{ headnodes_sync_location }}/shed_tools/ + fi + else + echo "Skipping shed_tools-local" + fi + dest: /usr/bin/galaxy-rsync + owner: root + group: root + mode: 0755 + +- name: "Execute the script. Syncing in progress." + command: /usr/bin/galaxy-rsync + become: true + become_user: galaxy + when: execute_galaxy_sync