Support for multiple PgBouncer processes using so_reuseport (#487)

molecule/default/converge.yml

@@ -16,6 +16,7 @@
         consul_node_role: server # if dcs_type: "consul"
         consul_bootstrap_expect: true # if dcs_type: "consul"
         postgresql_version: "15" # to test custom WAL dir
+        pgbouncer_processes: 2 # Test multiple pgbouncer processes (so_reuseport)
         cacheable: true
 
     - name: Set variables for custom PostgreSQL data and WAL directory test

molecule/pg_upgrade/converge.yml

@@ -16,6 +16,7 @@
         consul_node_role: server # if dcs_type: "consul"
         consul_bootstrap_expect: true # if dcs_type: "consul"
         postgresql_version: "14" # redefine the version to install for the upgrade test
+        pgbouncer_processes: 4 # Test multiple pgbouncer processes (so_reuseport)
         cacheable: true
 
     - name: Set variables for custom PostgreSQL data and WAL directory test

roles/pgbouncer/config/tasks/main.yml

@@ -11,10 +11,14 @@
 - name: Update pgbouncer.ini
   ansible.builtin.template:
     src: ../templates/pgbouncer.ini.j2
-    dest: "{{ pgbouncer_conf_dir }}/pgbouncer.ini"
+    dest: "{{ pgbouncer_conf_dir }}/pgbouncer{{ '-%d' % (idx + 1) if idx > 0 else '' }}.ini"
     owner: postgres
     group: postgres
     mode: "0640"
+  loop: "{{ range(0, (pgbouncer_processes | default(1) | int)) | list }}"
+  loop_control:
+    index_var: idx
+    label: "{{ 'pgbouncer' if idx == 0 else 'pgbouncer-%d' % (idx + 1) }}"
   notify: "restart pgbouncer"
   when: existing_pgcluster is not defined or not existing_pgcluster|bool
   tags: pgbouncer, pgbouncer_conf

roles/pgbouncer/handlers/main.yml

@@ -2,9 +2,13 @@
 
 - name: Restart pgbouncer service
   ansible.builtin.systemd:
-    name: pgbouncer
+    name: pgbouncer{{ '-%d' % (idx + 1) if idx > 0 else '' }}
     enabled: true
     state: restarted
+  loop: "{{ range(0, (pgbouncer_processes | default(1) | int)) | list }}"
+  loop_control:
+    index_var: idx
+    label: "{{ 'pgbouncer' if idx == 0 else 'pgbouncer-%d' % (idx + 1) }}"
   listen: "restart pgbouncer"
 
 - name: Wait for port "{{ pgbouncer_listen_port }}" to become open on the host
@@ -19,8 +23,12 @@
 
 - name: Reload pgbouncer service
   ansible.builtin.systemd:
-    name: pgbouncer
+    name: pgbouncer{{ '-%d' % (idx + 1) if idx > 0 else '' }}
     state: reloaded
+  loop: "{{ range(0, (pgbouncer_processes | default(1) | int)) | list }}"
+  loop_control:
+    index_var: idx
+    label: "{{ 'pgbouncer' if idx == 0 else 'pgbouncer-%d' % (idx + 1) }}"
   listen: "reload pgbouncer"
   ignore_errors: true  # Added to prevent test failures in CI.
 

roles/pgbouncer/tasks/main.yml

@@ -53,18 +53,26 @@
 - name: Configure pgbouncer systemd service file
   ansible.builtin.template:
     src: templates/pgbouncer.service.j2
-    dest: /etc/systemd/system/pgbouncer.service
+    dest: "/etc/systemd/system/pgbouncer{{ '-%d' % (idx + 1) if idx > 0 else '' }}.service"
     owner: postgres
     group: postgres
     mode: "0644"
+  loop: "{{ range(0, (pgbouncer_processes | default(1) | int)) | list }}"
+  loop_control:
+    index_var: idx
+    label: "{{ 'pgbouncer' if idx == 0 else 'pgbouncer-%d' % (idx + 1) }}"
   notify: "restart pgbouncer"
   tags: pgbouncer_service, pgbouncer
 
 - name: Ensure pgbouncer service is enabled
   ansible.builtin.systemd:
     daemon_reload: true
-    name: pgbouncer
+    name: "pgbouncer{{ '-%d' % (idx + 1) if idx > 0 else '' }}"
     enabled: true
+  loop: "{{ range(0, (pgbouncer_processes | default(1) | int)) | list }}"
+  loop_control:
+    index_var: idx
+    label: "{{ 'pgbouncer' if idx == 0 else 'pgbouncer-%d' % (idx + 1) }}"
   tags: pgbouncer_service, pgbouncer
 
 - block:  # workaround for pgbouncer from postgrespro repo
@@ -88,7 +96,7 @@
 - name: Enable log rotation with logrotate
   ansible.builtin.copy:
     content: |
-      /var/log/pgbouncer/pgbouncer.log {
+      {{ pgbouncer_log_dir }}/pgbouncer{{ '-%d' % (idx + 1) if idx > 0 else '' }}.log {
              daily
              rotate 7
              copytruncate
@@ -98,16 +106,24 @@
              missingok
              su root root
       }
-    dest: /etc/logrotate.d/pgbouncer
+    dest: "/etc/logrotate.d/pgbouncer{{ '-%d' % (idx + 1) if idx > 0 else '' }}"
+  loop: "{{ range(0, (pgbouncer_processes | default(1) | int)) | list }}"
+  loop_control:
+    index_var: idx
+    label: "{{ 'pgbouncer' if idx == 0 else 'pgbouncer-%d' % (idx + 1) }}"
   tags: pgbouncer_logrotate, pgbouncer
 
 - name: Configure pgbouncer.ini
   ansible.builtin.template:
     src: templates/pgbouncer.ini.j2
-    dest: "{{ pgbouncer_conf_dir }}/pgbouncer.ini"
+    dest: "{{ pgbouncer_conf_dir }}/pgbouncer{{ '-%d' % (idx + 1) if idx > 0 else '' }}.ini"
     owner: postgres
     group: postgres
     mode: "0640"
+  loop: "{{ range(0, (pgbouncer_processes | default(1) | int)) | list }}"
+  loop_control:
+    index_var: idx
+    label: "{{ 'pgbouncer' if idx == 0 else 'pgbouncer-%d' % (idx + 1) }}"
   notify: "restart pgbouncer"
   when: existing_pgcluster is not defined or not existing_pgcluster|bool
   tags: pgbouncer_conf, pgbouncer
@@ -128,10 +144,14 @@
     - name: Fetch pgbouncer.ini file from master
       run_once: true
       ansible.builtin.fetch:
-        src: "{{ pgbouncer_conf_dir }}/pgbouncer.ini"
+        src: "{{ pgbouncer_conf_dir }}/pgbouncer{{ '-%d' % (idx + 1) if idx > 0 else '' }}.ini"
         dest: files/
         validate_checksum: true
         flat: true
+      loop: "{{ range(0, (pgbouncer_processes | default(1) | int)) | list }}"
+      loop_control:
+        index_var: idx
+        label: "{{ 'pgbouncer' if idx == 0 else 'pgbouncer-%d' % (idx + 1) }}"
       delegate_to: "{{ groups.master[0] }}"
 
     - name: Fetch userlist.txt conf file from master
@@ -146,11 +166,15 @@
 
     - name: Copy pgbouncer.ini file to replica
       ansible.builtin.copy:
-        src: files/pgbouncer.ini
+        src: "files/pgbouncer{{ '-%d' % (idx + 1) if idx > 0 else '' }}.ini"
         dest: "{{ pgbouncer_conf_dir }}"
         owner: postgres
         group: postgres
         mode: "0640"
+      loop: "{{ range(0, (pgbouncer_processes | default(1) | int)) | list }}"
+      loop_control:
+        index_var: idx
+        label: "{{ 'pgbouncer' if idx == 0 else 'pgbouncer-%d' % (idx + 1) }}"
 
     - name: Copy userlist.txt conf file to replica
       ansible.builtin.copy:
@@ -165,8 +189,12 @@
       become: false
       run_once: true
       ansible.builtin.file:
-        path: files/pgbouncer.ini
+        path: "files/pgbouncer{{ '-%d' % (idx + 1) if idx > 0 else '' }}.ini"
         state: absent
+      loop: "{{ range(0, (pgbouncer_processes | default(1) | int)) | list }}"
+      loop_control:
+        index_var: idx
+        label: "{{ 'pgbouncer' if idx == 0 else 'pgbouncer-%d' % (idx + 1) }}"
       delegate_to: localhost
 
     - name: Remove userlist.txt conf file from localhost
@@ -180,30 +208,16 @@
 
     - name: Prepare pgbouncer.ini conf file (replace "listen_addr")
       ansible.builtin.lineinfile:
-        path: "{{ pgbouncer_conf_dir }}/pgbouncer.ini"
-        regexp: "{{ item.regexp }}"
-        line: "{{ item.line }}"
-        backrefs: true
-      loop:
-        - { regexp: '^listen_addr =', line: 'listen_addr = {{ hostvars[inventory_hostname].inventory_hostname }}' }
-      loop_control:
-        label: "{{ item.line }}"
-      notify: "restart pgbouncer"
-      when: with_haproxy_load_balancing|bool or
-            (cluster_vip is not defined or cluster_vip | length < 1)
-
-    - name: Prepare pgbouncer.ini conf file (replace "listen_addr")
-      ansible.builtin.lineinfile:
-        path: "{{ pgbouncer_conf_dir }}/pgbouncer.ini"
-        regexp: "{{ item.regexp }}"
-        line: "{{ item.line }}"
+        path: "{{ pgbouncer_conf_dir }}/pgbouncer{{ '-%d' % (idx + 1) if idx > 0 else '' }}.ini"
+        regexp: '^listen_addr ='
+        line: 'listen_addr = {{ pgbouncer_listen_addr }}'
         backrefs: true
-      loop:
-        - { regexp: '^listen_addr =', line: 'listen_addr = {{ hostvars[inventory_hostname].inventory_hostname }},{{ cluster_vip }}' }
+      loop: "{{ range(0, (pgbouncer_processes | default(1) | int)) | list }}"
       loop_control:
-        label: "{{ item.line }}"
+        index_var: idx
+        label: "{{ 'pgbouncer' if idx == 0 else 'pgbouncer-%d' % (idx + 1) }}"
       notify: "restart pgbouncer"
-      when: not with_haproxy_load_balancing|bool and (cluster_vip is defined and cluster_vip | length > 0 )
+      when: pgbouncer_listen_addr != "0.0.0.0"
   when: existing_pgcluster is defined and existing_pgcluster|bool
   tags: pgbouncer_conf, pgbouncer
 

roles/pgbouncer/templates/pgbouncer.ini.j2

@@ -6,11 +6,11 @@
 * = host=127.0.0.1 port={{ postgresql_port }}
 
 [pgbouncer]
-logfile = {{ pgbouncer_log_dir }}/pgbouncer.log
-pidfile = /run/pgbouncer/pgbouncer.pid
+logfile = {{ pgbouncer_log_dir }}/pgbouncer{{ '-%d' % (idx + 1) if idx > 0 else '' }}.log
+pidfile = /run/pgbouncer{{ '-%d' % (idx + 1) if idx > 0 else '' }}/pgbouncer.pid
 listen_addr = {{ pgbouncer_listen_addr | default('0.0.0.0') }}
 listen_port = {{ pgbouncer_listen_port | default(6432) }}
-unix_socket_dir = /var/run/postgresql
+unix_socket_dir = /var/run/pgbouncer{{ '-%d' % (idx + 1) if idx > 0 else '' }}
 auth_type = {{ pgbouncer_auth_type }}
 {% if pgbouncer_auth_user | bool %}
 auth_user = {{ pgbouncer_auth_username }}
@@ -34,6 +34,8 @@ max_db_connections = {{ pgbouncer_max_db_connections }}
 pkt_buf = 8192
 listen_backlog = 4096
 
+so_reuseport = 1
+
 log_connections = 0
 log_disconnections = 0
 

roles/pgbouncer/templates/pgbouncer.service.j2

@@ -9,16 +9,16 @@ User=postgres
 Group=postgres
 
 PermissionsStartOnly=true
-ExecStartPre=-/bin/mkdir -p /run/pgbouncer {{ pgbouncer_log_dir }}
-ExecStartPre=/bin/chown -R postgres:postgres /run/pgbouncer {{ pgbouncer_log_dir }}
+ExecStartPre=-/bin/mkdir -p /run/pgbouncer /var/run/pgbouncer{{ '-%d' % (idx + 1) if idx > 0 else '' }} {{ pgbouncer_log_dir }}
+ExecStartPre=/bin/chown -R postgres:postgres /run/pgbouncer /var/run/pgbouncer{{ '-%d' % (idx + 1) if idx > 0 else '' }} {{ pgbouncer_log_dir }}
 {% if ansible_os_family == "Debian" %}
-ExecStart=/usr/sbin/pgbouncer -d {{ pgbouncer_conf_dir }}/pgbouncer.ini
+ExecStart=/usr/sbin/pgbouncer -d {{ pgbouncer_conf_dir }}/pgbouncer{{ '-%d' % (idx + 1) if idx > 0 else '' }}.ini
 {% endif %}
 {% if ansible_os_family == "RedHat" %}
-ExecStart=/usr/bin/pgbouncer -d {{ pgbouncer_conf_dir }}/pgbouncer.ini
+ExecStart=/usr/bin/pgbouncer -d {{ pgbouncer_conf_dir }}/pgbouncer{{ '-%d' % (idx + 1) if idx > 0 else '' }}.ini
 {% endif %}
 ExecReload=/bin/kill -SIGHUP $MAINPID
-PIDFile=/run/pgbouncer/pgbouncer.pid
+PIDFile=/run/pgbouncer{{ '-%d' % (idx + 1) if idx > 0 else '' }}/pgbouncer.pid
 Restart=on-failure
 
 LimitNOFILE=100000

roles/pre-checks/tasks/pgbouncer.yml

@@ -40,13 +40,14 @@
   ansible.builtin.set_fact:
     pgbouncer_total_pool_size: >-
       {{
-        (pgbouncer_pool_size | int)
+        ((pgbouncer_pool_size | int)
         +
         (postgresql_databases
           | default([])
           | rejectattr('db', 'in', pgbouncer_pools | map(attribute='dbname') | list)
           | length
-        ) * (pgbouncer_default_pool_size | default(0) | int)
+        ) * (pgbouncer_default_pool_size | default(0) | int))
+        * (pgbouncer_processes | default(1) | int)
       }}
   when: pgbouncer_pool_size is defined
 

roles/upgrade/README.md

@@ -129,8 +129,8 @@ Please see the variable file vars/[upgrade.yml](../../vars/upgrade.yml)
 - **Check if PostgreSQL tablespaces exist**
   - Print tablespace location (if exists)
   - Note: If tablespaces are present they will be upgraded (step 5) on replicas using rsync
-- **Test PgBouncer access via localhost**
-  - test access via 'localhost' to be able to perform 'PAUSE' command
+- **Test PgBouncer access via unix socket**
+  - test access via unix socket to be able to perform 'PAUSE' command
 - **Make sure that the cluster ip address (VIP) is running**
   - Notes: if 'cluster_vip' is defined
 

roles/upgrade/tasks/pgbouncer_pause.yml

@@ -36,17 +36,24 @@
       and state <> 'idle'
       and query_start < clock_timestamp() - interval '{{ pg_slow_active_query_treshold_to_terminate }} ms'
       {{ "and backend_type = 'client backend'" if pg_old_version is version('10', '>=') else '' }}
+    pgb_unix_socket_dirs: >-
+      {% set unix_socket_dir = ['/var/run/pgbouncer'] %}
+      {%- for idx in range(1, pgbouncer_processes | default(1) | int) -%}
+        {% set _ = unix_socket_dir.append('/var/run/pgbouncer-' ~ (idx + 1) | string) %}
+      {%- endfor -%}
+      {{ unix_socket_dir | join(' ') }}
   ansible.builtin.shell: |
     set -o pipefail;
 
     pg_servers="{{ (groups['primary'] + groups['secondary']) | join('\n') }}"
-    pg_count=$(echo -e "$pg_servers" | wc -l)
+    pg_servers_count="{{ groups['primary'] | default([]) | length + groups['secondary'] | default([]) | length }}"
     pg_slow_active_count_query="{{ pg_slow_active_count_query }}"
     pg_slow_active_terminate_query="{{ pg_slow_active_terminate_query }}"
     # it is assumed that pgbouncer is installed on database servers
     pgb_servers="$pg_servers"
-    pgb_count="$pg_count"
-    pgb_pause_command="psql -h localhost -p {{ pgbouncer_listen_port }} -U {{ patroni_superuser_username }} -d pgbouncer -tAXc \"PAUSE\""
+    pgb_servers_count="$pg_servers_count"
+    pgb_count="{{ (groups['primary'] | default([]) | length + groups['secondary'] | default([]) | length) * (pgbouncer_processes | default(1) | int) }}"
+    pgb_pause_command="printf '%s\n' {{ pgb_unix_socket_dirs }} | xargs -I {} -P {{ pgbouncer_processes | default(1) | int }} -n 1 psql -h {} -p {{ pgbouncer_listen_port }} -U {{ patroni_superuser_username }} -d pgbouncer -tAXc 'PAUSE'"
     pgb_resume_command='kill -SIGUSR2 $(pidof pgbouncer)'
 
     start_time=$(date +%s)
@@ -56,7 +63,7 @@
       pgb_paused_count=0
 
       # wait for the active queries to complete on pg_servers
-      IFS=$'\n' pg_slow_active_counts=($(echo -e "$pg_servers" | xargs -I {} -P "$pg_count" -n 1 ssh -o StrictHostKeyChecking=no {} "psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc \"$pg_slow_active_count_query\""))
+      IFS=$'\n' pg_slow_active_counts=($(echo -e "$pg_servers" | xargs -I {} -P "$pg_servers_count" -n 1 ssh -o StrictHostKeyChecking=no {} "psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc \"$pg_slow_active_count_query\""))
 
       # sum up all the values in the array
       total_pg_slow_active_count=0
@@ -68,7 +75,7 @@
 
       if [[ "$total_pg_slow_active_count" == 0 ]]; then
         # pause pgbouncer on all pgb_servers. We send via ssh to all pgbouncers in parallel and collect results from all (maximum wait time 2 seconds)
-        IFS=$'\n' pause_results=($(echo -e "$pgb_servers" | xargs -I {} -P "$pgb_count" -n 1 ssh -o StrictHostKeyChecking=no {} "timeout 2 $pgb_pause_command 2>&1 || true"))
+        IFS=$'\n' pause_results=($(echo -e "$pgb_servers" | xargs -I {} -P "$pgb_servers_count" -n 1 ssh -o StrictHostKeyChecking=no {} "timeout 2 $pgb_pause_command 2>&1 || true"))
         echo "${pause_results[*]}"
         # analyze the pause_results array to count the number of paused pgbouncers
         pgb_paused_count=$(echo "${pause_results[*]}" | grep -o -e "PAUSE" -e "already suspended/paused" | wc -l)
@@ -80,14 +87,14 @@
         break # pause is performed on all pgb_servers, exit from the loop
       elif [[ "$pgb_paused_count" -gt 0 && "$pgb_paused_count" -ne "$pgb_count" ]]; then
         # pause is not performed on all pgb_servers, perform resume (we do not use timeout because we mast to resume all pgbouncers)
-        IFS=$'\n' resume_results=($(echo -e "$pgb_servers" | xargs -I {} -P "$pgb_count" -n 1 ssh -o StrictHostKeyChecking=no {} "$pgb_resume_command 2>&1 || true"))
+        IFS=$'\n' resume_results=($(echo -e "$pgb_servers" | xargs -I {} -P "$pgb_servers_count" -n 1 ssh -o StrictHostKeyChecking=no {} "$pgb_resume_command 2>&1 || true"))
         echo "${resume_results[*]}"
       fi
 
       # after 30 seconds of waiting, terminate active sessions on pg_servers and try pausing again
       if (( current_time - start_time >= {{ pgbouncer_pool_pause_terminate_after }} )); then
         echo "$(date): terminate active queries"
-        echo -e "$pg_servers" | xargs -I {} -P "$pg_count" -n 1 ssh -o StrictHostKeyChecking=no {} "psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc \"$pg_slow_active_terminate_query\""
+        echo -e "$pg_servers" | xargs -I {} -P "$pg_servers_count" -n 1 ssh -o StrictHostKeyChecking=no {} "psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc \"$pg_slow_active_terminate_query\""
       fi
 
       # if it was not possible to pause for 60 seconds, exit with an error

roles/upgrade/tasks/pre_checks.yml

@@ -334,10 +334,18 @@
     - tablespace_location.stdout_lines | length > 0
 
 # PgBouncer (if 'pgbouncer_pool_pause' is 'true')
-# test access via localhost to be able to perform 'PAUSE' command
-- name: '[Pre-Check] Test PgBouncer access via localhost'
-  ansible.builtin.command: >
-    psql -h localhost -p {{ pgbouncer_listen_port }} -U {{ patroni_superuser_username }} -d pgbouncer -tAXc "SHOW POOLS"
+# test access via unix socket to be able to perform 'PAUSE' command
+- name: '[Pre-Check] Test PgBouncer access via unix socket'
+  ansible.builtin.command: >-
+    psql -h /var/run/pgbouncer{{ '-%d' % (idx + 1) if idx > 0 else '' }}
+    -p {{ pgbouncer_listen_port }}
+    -U {{ patroni_superuser_username }}
+    -d pgbouncer
+    -tAXc "SHOW POOLS"
+  loop: "{{ range(0, (pgbouncer_processes | default(1) | int)) | list }}"
+  loop_control:
+    index_var: idx
+    label: "{{ 'pgbouncer' if idx == 0 else 'pgbouncer-%d' % (idx + 1) }}"
   changed_when: false
   when:
     - pgbouncer_install | bool

vars/main.yml

@@ -289,13 +289,14 @@ postgresql_pg_ident: []
 # the password file (~/.pgpass)
 postgresql_pgpass:
   - "localhost:{{ postgresql_port }}:*:{{ patroni_superuser_username }}:{{ patroni_superuser_password }}"
-  - "localhost:{{ pgbouncer_listen_port }}:*:{{ patroni_superuser_username }}:{{ patroni_superuser_password }}"
   - "{{ inventory_hostname }}:{{ postgresql_port }}:*:{{ patroni_superuser_username }}:{{ patroni_superuser_password }}"
+  - "*:{{ pgbouncer_listen_port }}:*:{{ patroni_superuser_username }}:{{ patroni_superuser_password }}"
 #  - hostname:port:database:username:password
 
 
 # PgBouncer parameters
 pgbouncer_install: true  # or 'false' if you do not want to install and configure the pgbouncer service
+pgbouncer_processes: 1  # Number of pgbouncer processes to be used. Multiple processes use the so_reuseport option for better performance.
 pgbouncer_conf_dir: "/etc/pgbouncer"
 pgbouncer_log_dir: "/var/log/pgbouncer"
 pgbouncer_listen_addr: "0.0.0.0"