Upkeep for Playground deployment

ansible/apply

@@ -76,6 +76,10 @@ def run_playbook(args):
             ansible_args += ["-u", args.user]
         if args.start_at_task is not None:
             ansible_args += ["--start-at-task", args.start_at_task]
+        if args.check:
+            ansible_args += ["--check"]
+        if args.diff:
+            ansible_args += ["--diff"]
 
         env = os.environ.copy()
         # Set environment variable if running on macOS to avoid python crash
@@ -100,6 +104,14 @@ if __name__ == "__main__":
         "--start-at-task", help="start at a task with the provided name",
         default=None,
     )
+    parser.add_argument(
+        "--check", help="perform an Ansible check run",
+        action="store_true",
+    )
+    parser.add_argument(
+        "--diff", help="perform an Ansible diff run",
+        action="store_true",
+    )
     args = parser.parse_args()
 
     install_ansible()

ansible/playbooks/playground.yml

@@ -20,9 +20,10 @@
         - "{{ vars_playground_domain }}"
 
     - role: nginx
+      worker_connections: "{{ vars_playground_number_connections }}"
       proxied:
         - domain: "{{ vars_playground_domain }}"
-          to: "http://localhost:{{ vars_playground_env_ui_port }}"
+          to: "http://127.0.0.1:{{ vars_playground_env_ui_port }}"
           websockets:
             - '/websocket'
 

ansible/roles/nginx/README.md

@@ -9,6 +9,10 @@ this role as well.
 ```yaml
 - role: nginx
 
+  # The number of worker connections. [optional]
+  # https://nginx.org/en/docs/ngx_core_module.html#worker_connections
+  worker_connections: 123
+
   # Configures reverse proxies with HTTPS termination. [optional]
   proxied:
       # The domain to proxy from

ansible/roles/nginx/defaults/main.yml

@@ -1,4 +1,5 @@
 ---
 
 # See this role's README for documentation about these defaults.
+worker_connections: 768
 proxied: {}

ansible/roles/nginx/tasks/main.yml

@@ -25,3 +25,13 @@
     src: after-ssl-renew.sh
     dest: /etc/ssl/letsencrypt/after-renew.d
     mode: 0750
+
+- name: create systemd override file
+  file:
+    path: /etc/systemd/system/nginx.service.d
+    state: directory
+
+- name: create systemd override file
+  template:
+    src: override.conf
+    dest: /etc/systemd/system/nginx.service.d/override.conf

ansible/roles/nginx/templates/nginx.conf

@@ -8,7 +8,7 @@ pid /run/nginx.pid;
 include /etc/nginx/modules-enabled/*.conf;
 
 events {
-    worker_connections 768;
+    worker_connections {{ worker_connections }};
 }
 
 http {

ansible/roles/nginx/templates/override.conf

@@ -0,0 +1,6 @@
+[Service]
+# This assumes that the NGINX instance will usually be used as an
+# upstream proxy. Each incoming connection takes one FD for the client
+# and one FD for the proxy. We add a few extra FDs to account for
+# things like config and log files.
+LimitNOFILE={{ (worker_connections * 2) + 32 }}

ansible/roles/playground/defaults/main.yml

@@ -12,10 +12,16 @@ vars_playground_repository_url: https://github.com/rust-lang/rust-playground.git
 
 vars_playground_checkout_path: "{{ vars_playground_home_path }}/rust-playground"
 vars_playground_update_path: "{{ vars_playground_home_path }}/update.sh"
+vars_playground_gc_path: "{{ vars_playground_home_path }}/gc.sh"
 vars_playground_artifacts_path: "{{ vars_playground_home_path }}/playground-artifacts"
 vars_playground_executable_path: "{{ vars_playground_artifacts_path }}/ui"
 
 vars_playground_env_cors_enabled: 1
 vars_playground_env_ui_address: 127.0.0.1
 vars_playground_env_ui_port: 8080
 vars_playground_env_ui_root_path: "{{ vars_playground_artifacts_path }}/build"
+
+# The playground peaks at a bit less than 2000 open file descriptors
+# over a normal week but `2560` resulted in sporadic errors, so there
+# must be many non-WebSocket connections.
+vars_playground_number_connections: 5120

ansible/roles/playground/handlers/main.yml

@@ -6,6 +6,12 @@
     state: restarted
     daemon_reload: true
 
+- name: restart-containerd
+  systemd:
+    name: containerd
+    state: restarted
+    daemon_reload: true
+
 - name: start-playground-update
   systemd:
     name: playground-update
@@ -18,6 +24,18 @@
     state: restarted
     daemon_reload: true
 
+- name: start-playground-gc
+  systemd:
+    name: playground-gc
+    state: started
+    daemon_reload: true
+
+- name: restart-playground-gc-timer
+  systemd:
+    name: playground-gc.timer
+    state: restarted
+    daemon_reload: true
+
 - name: restart-playground
   systemd:
     name: playground

ansible/roles/playground/tasks/main.yml

@@ -1,5 +1,16 @@
 ---
 
+# Create a Systemd slice to manage resources (Memory, CPU) across the
+# processes spawned by the playground.
+
+- name: Configure playground slice
+  template:
+    src: playground.slice
+    dest: /etc/systemd/system/playground.slice
+    mode: 0644
+
+# --------------------
+
 - name: Add Docker APT repository GPG key
   apt_key:
     state: present
@@ -30,6 +41,30 @@
     mode: 0600
   notify: restart-docker
 
+- name: Add Docker to the Playground slice
+  file:
+    path: /etc/systemd/system/docker.service.d
+    state: directory
+
+- name: Add Docker to the Playground slice
+  template:
+    src: docker-override.conf
+    dest: /etc/systemd/system/docker.service.d/override.conf
+    mode: 0600
+  notify: restart-docker
+
+- name: Add ContainerD to the Playground slice
+  file:
+    path: /etc/systemd/system/containerd.service.d
+    state: directory
+
+- name: Add ContainerD to the Playground slice
+  template:
+    src: containerd-override.conf
+    dest: /etc/systemd/system/containerd.service.d/override.conf
+    mode: 0600
+  notify: restart-containerd
+
 # --------------------
 
 # Set up a partition with limited space to avoid temporary
@@ -119,6 +154,33 @@
     state: started
     enabled: true
 
+- name: Configure garbage collection script
+  template:
+    src: gc.sh
+    dest: "{{ vars_playground_gc_path }}"
+    mode: 0755
+  notify: start-playground-gc
+
+- name: Configure garbage collection script service
+  template:
+    src: playground-gc.service
+    dest: /etc/systemd/system/playground-gc.service
+    mode: 0644
+  notify: start-playground-gc
+
+- name: Configure garbage collection script service timer
+  template:
+    src: playground-gc.timer
+    dest: /etc/systemd/system/playground-gc.timer
+    mode: 0644
+  notify: restart-playground-gc-timer
+
+- name: Start and enable garbage collection script service timer
+  systemd:
+    name: playground-gc.timer
+    state: started
+    enabled: true
+
 - name: Configure playground service
   template:
     src: playground.service

ansible/roles/playground/templates/containerd-override.conf

@@ -0,0 +1,2 @@
+[Service]
+Slice=playground.slice

ansible/roles/playground/templates/daemon.json

@@ -1,3 +1,5 @@
 {
+  "cgroup-parent": "playground.slice",
+  "log-driver": "local",
   "storage-driver": "overlay2"
 }

ansible/roles/playground/templates/docker-override.conf

@@ -0,0 +1,2 @@
+[Service]
+Slice=playground.slice

ansible/roles/playground/templates/gc.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+#
+# {{ ansible_managed }}
+#
+
+# {% raw %}
+
+set -euv -o pipefail
+
+# How long a container must be running to be killed.
+# Number of seconds.
+MAX_TIME=3600
+
+now=$(date "+%s")
+to_kill=()
+
+readarray -t container_ids < <(docker ps --format '{{ .ID }}' --no-trunc)
+
+while read -r id started_at; do
+    started_at=$(date --date "${started_at}" "+%s")
+    running_time=$((now - started_at))
+
+    if [[ "${running_time}" -gt "${MAX_TIME}" ]]; then
+        to_kill+=("${id}")
+    fi
+done < <(docker inspect "${container_ids[@]}" --format '{{ .ID }} {{ .State.StartedAt }}')
+
+if [[ ${#to_kill[@]} -gt 0 ]]; then
+    docker kill "${to_kill[@]}"
+fi
+
+# {% endraw %}

ansible/roles/playground/templates/playground-gc.service

@@ -0,0 +1,10 @@
+#
+# {{ ansible_managed }}
+#
+
+[Unit]
+Description=Garbage collect dead playground containers
+
+[Service]
+Type=oneshot
+ExecStart={{ vars_playground_gc_path }}

ansible/roles/playground/templates/playground-gc.timer

@@ -0,0 +1,13 @@
+#
+# {{ ansible_managed }}
+#
+
+[Unit]
+Description = Garbage collect playground containers every 15 minutes
+
+[Timer]
+OnBootSec = 15min
+OnUnitActiveSec = 15min
+
+[Install]
+WantedBy = timers.target

ansible/roles/playground/templates/playground.service

@@ -6,6 +6,10 @@
 Description=The Rust Playground
 
 [Service]
+Slice=playground.slice
+
+Restart=on-failure
+
 Environment=TMPDIR={{ vars_playground_mountpoint_path }}
 Environment=RUST_LOG=info
 Environment=PLAYGROUND_CORS_ENABLED={{ vars_playground_env_cors_enabled }}
@@ -19,5 +23,7 @@ WorkingDirectory={{ vars_playground_artifacts_path }}
 
 ExecStart={{ vars_playground_executable_path }}
 
+LimitNOFILE={{ vars_playground_number_connections }}
+
 [Install]
 WantedBy=multi-user.target

ansible/roles/playground/templates/playground.slice

@@ -0,0 +1,3 @@
+[Unit]
+Description=Resource management group for playground processes
+Before=slices.target