Merge branch 'main' into oonimeasurements

.github/workflows/add_issues_to_project.yml

@@ -0,0 +1,14 @@
+on:
+  issues:
+    types:
+      - opened
+
+jobs:
+  add-to-project:
+    name: Add issue to project
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/add-to-project@RELEASE_VERSION
+        with:
+          project-url: https://github.com/orgs/ooni/projects/31
+          github-token: ${{ secrets.ADD_TO_PROJECT_GH_TOKEN }}

README.md

@@ -1,40 +1,26 @@
 # OONI Devops
 
-## Infrastructure Tiers
-
-We divide our infrastructure components into 3 tiers:
-
-- **Tier 0: Critical**: These are mission critical infrastructure components. If these become unavailable or have significant disruption, it will have a major impact.
-
-- **Tier 1: Essential**: These components are important, but not as critical as
-  tier 0. They are part of our core operations, but if they become unavailable
-  the impact is important, but not major.
-
-- **Tier 2: Non-Essential**: These are auxiliary components. Their
-  unavailability does not have a major impact.
-
-### Tier 0 (Critical) components
-
-- [ ] Probe Services (collector specifically)
-- [ ] Fastpath (part responsible for storing post-cans)
-- [x] DNS configuration
-- [ ] Monitoring
-- [ ] OONI bridges
-- [ ] OONI.org website
-- [x] Web Connectivity test helpers
-- [ ] Code signing
-
-### Tier 1 (Essential) components
-
-- [ ] OONI API measurement listing
-- [x] OONI Explorer
-- [x] OONI Run
-- [ ] OONI Data analysis pipeline
-- [ ] OONI Findings API
-- [x] Website analytics
-
-### Tier 2 (Non-Essential) components
-
-- [ ] Test list editor
-- [ ] Jupyter notebooks
-- [ ] Countly
+At a glance below is the overall architecture of OONI Infrastructure across our various locations:
+
+```mermaid
+flowchart TB
+    apiorg([api.ooni.org])-->alb
+    apiio([api.ooni.io])-->backend
+    ecs[Backend API ECS]<-->ch[(Clickhouse Cluster)]
+    subgraph Hetzner
+        backend[OONI Backend Monolith]<-->ch
+        monitoring[Monitoring host]
+        pipeline[Pipeline v5]
+    end
+    subgraph AWS
+    alb[API Load Balancer]<-->ecs
+    alb-->backend
+    ecs<-->s3[(OONI S3 Buckets)]
+    s3<-->backend
+    end
+    subgraph Digital Ocean
+        th[Web Connectivity Test helper]<-->alb
+    end
+```
+
+For more details [Infrastructure docs](https://docs.ooni.org/devops/infrastructure/)

ansible/README.md

@@ -1,46 +1,188 @@
-### Quickstart
+# Ansible
+
+**NOTE** We are currently in the process of migrating ansible configurations from [ooni/sysadmin](https://github.com/ooni/sysadmin) to [ooni/devops](https://github.com/ooni/devops).
+
+Ansible is used to configure the OSes on long term provisioned backend hosts and manage the configuration for these components.
+
+For example ansible will be used to configure the setup of VPSs and dedicated hosts that are provisioned manually or using terraform.
+
+In the case of hosts that are continously delivered, we instead use cloud-native configuration management tools.
+
+## Installation and setup
 
 It's recommended to make use of a virtualenv, for example managed using `pyenv virtualenv`:
 ```
 pyenv virtualenv ooni-devops
 pyenv activate ooni-devops
 ```
 
-Install deps:
+### Ansible setup
+
+You should then install the required python and ansible-galaxy depedencies with:
 ```
-pip install ansible dnspython boto3 passlib
+pip install -r requirements/python.yml
+ansible-galaxy install -r requirements/ansible-galaxy.yml
 ```
 
-Install ansible galaxy modules:
+In order to gain access to machines you will have to add your public key to the
+`ssh_users` variable inside of `ansible/group_vars/all/vars.yml`.
+
+It's recommended you generate an `ed25519` key using the following command:
 ```
-ansible-galaxy install -r requirements.yml
+ssh-keygen -t ed25519 -f ~/.ssh/id_ed25519_ooni
 ```
 
-Setup AWS credentials, you should add 2 profiles called `oonidevops_user_dev` and `oonidevops_user_prod` which have access to the development and production environment respectively
+### AWS configuration
+
+Refer to the [terraform docs](devops/terraform/) for setting up your AWS configuration.
+
+### SSH configuration
+
+You should configure your `~/.ssh/config` with the following:
 
 ```
-[oonidevops_user_dev]
-aws_access_key_id = XXX
-aws_secret_access_key = YYY
-source_profile = default
-region = eu-central-1
-# ARN of the dev role
-role_arn = arn:aws:iam::905418398257:role/oonidevops
+    IdentitiesOnly yes
+    ServerAliveInterval 120
+    UserKnownHostsFile ~/.ssh/known_hosts ~/REPLACE_ME/sysadmin/ext/known_hosts
+
+    host *.ooni.io
+      user YOUR_USERNAME
+
+    host *.ooni.nu
+      user YOUR_USERNAME
 
-[oonidevops_user_prod]
-aws_access_key_id = XXX
-aws_secret_access_key = YYY
-source_profile = default
-region = eu-central-1
-# ARN of the prod role
-role_arn = arn:aws:iam::471112720364:role/oonidevops
+    host *.ooni.org
+      user YOUR_USERNAME
 ```
 
-Run playbook:
+**TODO** restore ext/known_hosts setup
+
+Replace `~/REPLACE_ME/sysadmin/ext/known_hosts` to where you have cloned
+the `ooni/sysadmin` repo. This will ensure you use the host key
+fingeprints from this repo instead of just relying on TOFU.
+
+You should replace `YOUR_USERNAME` with your username from `adm_login`.
+
+On MacOS you may want to also add:
+
+    host *
+        UseKeychain yes
+
+To use the Keychain to store passwords.
+
+## Running ansible playbooks
+
+Playbooks are run via an wrapper script called `./play` which notifies the slack #ooni-bots channel that a deployment has been triggered.
+
 ```
-ansible-playbook playbook.yml -i inventory
+./play -i inventory deploy-<component>.yml -l <hostname> --diff -C
+./play -i inventory deploy-<component>.yml -l <hostname> --diff
 ```
 
+:::caution
+any minor error in configuration files or ansible's playbooks can be
+destructive for the backend infrastructure. Always test-run playbooks
+with `--diff` and `-C` at first and carefully verify configuration
+changes. After verification run the playbook without `-C` and verify
+again the applied changes.
+:::
+
+:::note
+[Etckeeper](#etckeeper)&thinsp;🔧 can be useful to verify configuration
+changes from a different point of view.
+:::
+
+In general there are two classes of playbooks:
+* Those starting with `deploy-*.yml`, which are used to deploy specific components or pieces of components related to OONI infrastructure. All of these playbooks are included inside of `playbook.yml` to faciliate testing and ensuring that every component in our infrastucture is fully deployable.
+* Those starting with `playbook-*` which are playbooks for specific tasks that may not be part of the main infrastructure deployment (eg. bootstrapping nodes once upon creation, creating snapshots of remote configurations, etc.)
+
+Some notable playbooks or roles are:
+
+The bootstrap playbook is in `playbook-bootstrap.yml` and is a playbook that should be run once when a new host is created.
+
+The nftables firewall is configured to read every `.nft` file under
+`/etc/ooni/nftables/` and `/etc/ooni/nftables/`. This allows roles to
+create small files to open a port each and keep the configuration as
+close as possible to the ansible step that deploys a service. See this in use inside of the `nftables` role.
+
+#### The root account
+
+Runbooks use ssh to log on the hosts using your own account and leveraging `sudo` to act as root.
+
+The only exception is when a new host is being deployed - in that case ansible will log in as root to create
+individual accounts and lock out the root user.
+
+When running the entire runbook ansible might try to run it as root.
+This can be avoided by selecting only the required tags using `-t <tagname>`.
+
+Ideally the root user should be disabled after succesfully creating user accounts.
+
+#### Roles layout
+
+Ansible playbooks use multiple roles (see
+[example](https://github.com/ooni/sysadmin/blob/master/ansible/deploy-backend.yml#L46))
+to deploy various components.
+
+Few roles use the `meta/main.yml` file to depend on other roles. See
+[example](https://github.com/ooni/sysadmin/blob/master/ansible/roles/ooni-backend/meta/main.yml)
+
+:::note
+The latter method should be used sparingly because ansible does not
+indicate where each task in a playbook is coming from. Moreover if a dependencies is specified twice inside of two roles, it will run twice.
+:::
+
+A diagram of the role dependencies for the deploy-backend.yml playbook:
+
+```mermaid
+
+flowchart LR
+        A(deploy-backend.yml) --> B(base-bullseye)
+        B -- meta --> G(adm)
+        A --> F(nftables)
+        A --> C(nginx-buster)
+        A --> D(dehydrated)
+        D -- meta --> C
+        E -- meta --> F
+        A --> E(ooni-backend)
+        style B fill:#eeffee
+        style C fill:#eeffee
+        style D fill:#eeffee
+        style E fill:#eeffee
+        style F fill:#eeffee
+        style G fill:#eeffee
+```
+
+A similar diagram for deploy-monitoring.yml:
+
+```mermaid
+
+flowchart LR
+        B -- meta --> G(adm)
+        M(deploy-monitoring.yml) --> B(base-bookworm)
+        M --> O(ooca-cert)
+        M --> F(nftables)
+        M --> D(dehydrated) -- meta --> N(nginx-buster)
+        M --> P(prometheus)
+        M --> X(blackbox-exporter)
+        M --> T(alertmanager)
+        style B fill:#eeffee
+        style D fill:#eeffee
+        style F fill:#eeffee
+        style G fill:#eeffee
+        style N fill:#eeffee
+        style O fill:#eeffee
+        style P fill:#eeffee
+        style T fill:#eeffee
+        style X fill:#eeffee
+```
+
+:::note
+When deploying files or updating files already existing on the hosts it can be useful to add a note e.g. "Deployed by ansible, see <role_name>".
+This helps track down how files on the host were modified and why.
+:::
+
+### Platform specific known bugs
+
 On macOS you might run into this issue: https://github.com/ansible/ansible/issues/76322
 
 The current workaround is to export the following environment variable before running ansible:

ansible/deploy-airflow.yml

@@ -0,0 +1,9 @@
+---
+- name: Deploy airflow frontend host
+  hosts:
+    - data1.htz-fsn.prod.ooni.nu
+  become: true
+  roles:
+    - oonidata_airflow
+  vars:
+    airflow_public_fqdn: "airflow.prod.ooni.io"

ansible/deploy-bootstrap.yml

@@ -0,0 +1,7 @@
+- name: Ensure all hosts are bootstrapped correctly
+  hosts: all
+  become: yes
+  roles:
+    - bootstrap
+  tags:
+    - bootstrap

ansible/deploy-clickhouse.yml

@@ -0,0 +1,13 @@
+---
+- name: Deploy oonidata clickhouse hosts
+  hosts:
+    - notebook.ooni.org
+    - data1.htz-fsn.prod.ooni.nu
+    #- data2.htz-fsn.prod.ooni.nu
+    - data3.htz-fsn.prod.ooni.nu
+  become: true
+  tags:
+    - clickhouse
+  roles:
+    - prometheus_node_exporter
+    - oonidata_clickhouse

ansible/deploy-monitoring-config.yml

@@ -0,0 +1,10 @@
+---
+- name: Update monitoring config
+  hosts: monitoring.ooni.org
+  become: true
+  tags:
+    - monitoring
+  roles:
+    - prometheus
+    - prometheus_blackbox_exporter
+    - prometheus_alertmanager

ansible/deploy-monitoring.yml

@@ -0,0 +1,12 @@
+---
+- name: Deploy monitoring host
+  hosts: monitoring.ooni.org
+  become: true
+  tags:
+    - monitoring
+  roles:
+    - monitoring
+  vars:
+    monitoring_htpasswd: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/monitoring_htpasswd', profile='oonidevops_user_prod') }}"
+
+- ansible.builtin.import_playbook: deploy-monitoring-config.yml

ansible/deploy-ooni-backend.yml

@@ -0,0 +1,21 @@
+---
+- hosts: backend-hel.ooni.org
+  roles:
+    - role: bootstrap
+    - role: base-backend
+    - role: nftables
+    - role: nginx
+      tags: nginx
+      vars:
+        nginx_user: "www-data"
+    - role: dehydrated
+      tags: dehydrated
+      expand: yes
+      vars: 
+        ssl_domains:
+          # with dehydrated the first entry is the cert FQDN
+          # and the other ones are alternative names
+          - "backend-hel.ooni.org"
+    - role: ooni-backend
+      vars: 
+        ssl_domain: backend-hel.ooni.org

ansible/deploy-tier0.yml

@@ -0,0 +1,12 @@
+---
+- name: Include monitoring playbook
+  ansible.builtin.import_playbook: deploy-monitoring.yml
+
+- name: Include ooni-backend playbook
+  ansible.builtin.import_playbook: deploy-ooni-backend.yml
+
+- name: Include clickhouse playbook
+  ansible.builtin.import_playbook: deploy-clickhouse.yml
+
+- name: Include airflow playbook
+  ansible.builtin.import_playbook: deploy-airflow.yml