From d9a3362ae0cd68d2f4b39ef0488284cbdd392d4a Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Thu, 3 Oct 2024 13:42:47 -0500 Subject: [PATCH 01/10] feat: apply sliding window rate limiting --- salt/haproxy/config/haproxy.cfg.jinja | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/salt/haproxy/config/haproxy.cfg.jinja b/salt/haproxy/config/haproxy.cfg.jinja index 2162c25c..b8cd79c0 100644 --- a/salt/haproxy/config/haproxy.cfg.jinja +++ b/salt/haproxy/config/haproxy.cfg.jinja @@ -117,6 +117,15 @@ frontend main bind :::80 bind 127.0.0.1:19001 # This is our TLS socket. + # Client rate limiting + # See: https://www.haproxy.com/blog/four-examples-of-haproxy-rate-limiting + stick-table type ip size 100k expire 15s store http_req_rate(10s) + http-request track-sc0 src + http-request deny deny_status 429 if { sc_http_req_rate(0) gt 20 } + + # Do we need to whitelist the LBs? + {# acl whitelist src 10.132.111.89 10.132.109.52 #} + # Custom logging format, this is the same as the normal "httplog" in # HAProxy except information about the TLS connection is included. log-format %ci:%cp\ [%t]\ %ft\ %b/%s\ %Tq/%Tw/%Tc/%Tr/%Tt\ %sslv/%sslc\ %ST\ %B\ %CC\ %CS\ %tsc\ %ac/%fc/%bc/%sc/%rc\ %sq/%bq\ %hr\ %hs\ %{+Q}r From 86785be6deedb11aeab9d6d46dfdb044beb31c4c Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Mon, 7 Oct 2024 16:09:41 -0500 Subject: [PATCH 02/10] chore: remove unneeded whitelist Closes https://github.com/python/psf-salt/pull/509#discussion_r1786742335 --- salt/haproxy/config/haproxy.cfg.jinja | 3 --- 1 file changed, 3 deletions(-) diff --git a/salt/haproxy/config/haproxy.cfg.jinja b/salt/haproxy/config/haproxy.cfg.jinja index b8cd79c0..6a110823 100644 --- a/salt/haproxy/config/haproxy.cfg.jinja +++ b/salt/haproxy/config/haproxy.cfg.jinja @@ -123,9 +123,6 @@ frontend main http-request track-sc0 src http-request deny deny_status 429 if { sc_http_req_rate(0) gt 20 } - # Do we need to whitelist the LBs? - {# acl whitelist src 10.132.111.89 10.132.109.52 #} - # Custom logging format, this is the same as the normal "httplog" in # HAProxy except information about the TLS connection is included. log-format %ci:%cp\ [%t]\ %ft\ %b/%s\ %Tq/%Tw/%Tc/%Tr/%Tt\ %sslv/%sslc\ %ST\ %B\ %CC\ %CS\ %tsc\ %ac/%fc/%bc/%sc/%rc\ %sq/%bq\ %hr\ %hs\ %{+Q}r From 7bcb25e6dbff4909383477389954dde7050af775 Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Thu, 17 Oct 2024 15:01:03 -0500 Subject: [PATCH 03/10] feat: granularize the rate limit configs --- pillar/base/haproxy.sls | 2 ++ salt/haproxy/config/haproxy.cfg.jinja | 19 ++++++++++++++----- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/pillar/base/haproxy.sls b/pillar/base/haproxy.sls index 6b2259ff..b6ee17e9 100644 --- a/pillar/base/haproxy.sls +++ b/pillar/base/haproxy.sls @@ -17,6 +17,7 @@ haproxy: - docs.python.org - doc.python.org check: "HEAD /_check HTTP/1.1\\r\\nHost:\\ docs.python.org" + rate_limit: 100 downloads: domains: @@ -75,6 +76,7 @@ haproxy: - {{ config.server_name }} verify_host: bugs.psf.io check: "HEAD / HTTP/1.1\\r\\nHost:\\ {{ config.server_name }}" + rate_limit: {{ config.get('rate_limit', 10) }} {% endfor %} moin: diff --git a/salt/haproxy/config/haproxy.cfg.jinja b/salt/haproxy/config/haproxy.cfg.jinja index 6a110823..2a1936a4 100644 --- a/salt/haproxy/config/haproxy.cfg.jinja +++ b/salt/haproxy/config/haproxy.cfg.jinja @@ -52,6 +52,12 @@ global # Lower the amount of space we reserve for header rewriting tune.maxrewrite 1024 + # rate limits only if there is a rate_limit in haproxy.sls + {% for service, config in haproxy.services.items() %} + {% if config.get('rate_limit') %} + stick-table type ip size 100k expire 30s store http_req_rate(1s) name {{ service }}_ratelimit + {% endif %} + {% endfor %} defaults log global @@ -117,11 +123,14 @@ frontend main bind :::80 bind 127.0.0.1:19001 # This is our TLS socket. - # Client rate limiting - # See: https://www.haproxy.com/blog/four-examples-of-haproxy-rate-limiting - stick-table type ip size 100k expire 15s store http_req_rate(10s) - http-request track-sc0 src - http-request deny deny_status 429 if { sc_http_req_rate(0) gt 20 } + # Apply rate limits per srvice + {% for service, config in haproxy.services.items() %} + {% if config.get('rate_limit') %} + acl is_{{ service }} hdr(host) -i {% for domain in config.domains %}{{ domain }} {% endfor %} + http-request track-sc{{ loop.index }} src table {{ service }}_ratelimit if is_{{ service }} + http-request deny deny_status 429 if is_{{ service }} { sc{{ loop.index }}_http_req_rate() gt {{ config.rate_limit }} } + {% endif %} + {% endfor %} # Custom logging format, this is the same as the normal "httplog" in # HAProxy except information about the TLS connection is included. From c29e87c0df64bc6608d67448a315dc2bc982dc7c Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Thu, 17 Oct 2024 15:03:10 -0500 Subject: [PATCH 04/10] fix: correct confi --- salt/haproxy/config/haproxy.cfg.jinja | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/salt/haproxy/config/haproxy.cfg.jinja b/salt/haproxy/config/haproxy.cfg.jinja index 2a1936a4..68ad4aaf 100644 --- a/salt/haproxy/config/haproxy.cfg.jinja +++ b/salt/haproxy/config/haproxy.cfg.jinja @@ -52,13 +52,6 @@ global # Lower the amount of space we reserve for header rewriting tune.maxrewrite 1024 - # rate limits only if there is a rate_limit in haproxy.sls - {% for service, config in haproxy.services.items() %} - {% if config.get('rate_limit') %} - stick-table type ip size 100k expire 30s store http_req_rate(1s) name {{ service }}_ratelimit - {% endif %} - {% endfor %} - defaults log global @@ -125,9 +118,16 @@ frontend main # Apply rate limits per srvice {% for service, config in haproxy.services.items() %} - {% if config.get('rate_limit') %} + {% if config.get('rate_limit') and loop.index <= 2 %} + stick-table type ip size 100k expire 30s store http_req_rate(1s) + {% endif %} + {% endfor %} + + # Apply rate limits + {% for service, config in haproxy.services.items() %} + {% if config.get('rate_limit') and loop.index <= 2 %} acl is_{{ service }} hdr(host) -i {% for domain in config.domains %}{{ domain }} {% endfor %} - http-request track-sc{{ loop.index }} src table {{ service }}_ratelimit if is_{{ service }} + http-request track-sc{{ loop.index }} src if is_{{ service }} http-request deny deny_status 429 if is_{{ service }} { sc{{ loop.index }}_http_req_rate() gt {{ config.rate_limit }} } {% endif %} {% endfor %} From e94a197dcb9c6aa27282e11568e34faec8c8e131 Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Wed, 23 Oct 2024 14:24:28 -0500 Subject: [PATCH 05/10] fix: combine rules, use one tracking counter --- salt/haproxy/config/haproxy.cfg.jinja | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/salt/haproxy/config/haproxy.cfg.jinja b/salt/haproxy/config/haproxy.cfg.jinja index 68ad4aaf..58b3f364 100644 --- a/salt/haproxy/config/haproxy.cfg.jinja +++ b/salt/haproxy/config/haproxy.cfg.jinja @@ -116,19 +116,20 @@ frontend main bind :::80 bind 127.0.0.1:19001 # This is our TLS socket. - # Apply rate limits per srvice + # Define a stick table for all services + stick-table type ip size 100k expire 30s store http_req_rate(10s) + # Track all requests using a single counter + # We could use the 3 available (sc0,1,2) to maybe tier requests + # into say <=100, 101-500, >= 501 if we needed to? + http-request track-sc0 src + # then create the ACL for services in haproxy.sls that have a 'rate_limit' key, + # constrained to the host header using the domain key in haproxy.sls + # then adds a rule to deny via HTTP 429 if the respective ACL is matched and the stick table http request rate + # is higher than the 'rate_limit' from haproxy.sls pillar date {% for service, config in haproxy.services.items() %} - {% if config.get('rate_limit') and loop.index <= 2 %} - stick-table type ip size 100k expire 30s store http_req_rate(1s) - {% endif %} - {% endfor %} - - # Apply rate limits - {% for service, config in haproxy.services.items() %} - {% if config.get('rate_limit') and loop.index <= 2 %} - acl is_{{ service }} hdr(host) -i {% for domain in config.domains %}{{ domain }} {% endfor %} - http-request track-sc{{ loop.index }} src if is_{{ service }} - http-request deny deny_status 429 if is_{{ service }} { sc{{ loop.index }}_http_req_rate() gt {{ config.rate_limit }} } + {% if config.get('rate_limit') %} + acl is_{{ service }} hdr(host) -i {% for domain in config.domains %}{{ domain }}{% endfor %} + http-request deny deny_status 429 if is_{{ service }} { sc0_http_req_rate() gt {{ config.rate_limit }} } {% endif %} {% endfor %} From 873f07c58711fe2376d040595008ea7ccbdd513c Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Wed, 23 Oct 2024 14:27:41 -0500 Subject: [PATCH 06/10] doc: add helpful comments in config, add trimming modifiers to fix whitespace --- salt/haproxy/config/haproxy.cfg.jinja | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/salt/haproxy/config/haproxy.cfg.jinja b/salt/haproxy/config/haproxy.cfg.jinja index 58b3f364..1956c989 100644 --- a/salt/haproxy/config/haproxy.cfg.jinja +++ b/salt/haproxy/config/haproxy.cfg.jinja @@ -126,12 +126,13 @@ frontend main # constrained to the host header using the domain key in haproxy.sls # then adds a rule to deny via HTTP 429 if the respective ACL is matched and the stick table http request rate # is higher than the 'rate_limit' from haproxy.sls pillar date - {% for service, config in haproxy.services.items() %} - {% if config.get('rate_limit') %} + {%- for service, config in haproxy.services.items() %} + {%- if config.get('rate_limit') %} + # Rate limit config for {{ service }} acl is_{{ service }} hdr(host) -i {% for domain in config.domains %}{{ domain }}{% endfor %} http-request deny deny_status 429 if is_{{ service }} { sc0_http_req_rate() gt {{ config.rate_limit }} } - {% endif %} - {% endfor %} + {%- endif %} + {%- endfor %} # Custom logging format, this is the same as the normal "httplog" in # HAProxy except information about the TLS connection is included. From 7ada80964fa4c171cb739fe0e14db185dd8b766a Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Wed, 23 Oct 2024 15:34:12 -0500 Subject: [PATCH 07/10] fix: add spacing for correct urls --- salt/haproxy/config/haproxy.cfg.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/haproxy/config/haproxy.cfg.jinja b/salt/haproxy/config/haproxy.cfg.jinja index 1956c989..d8df7a60 100644 --- a/salt/haproxy/config/haproxy.cfg.jinja +++ b/salt/haproxy/config/haproxy.cfg.jinja @@ -129,7 +129,7 @@ frontend main {%- for service, config in haproxy.services.items() %} {%- if config.get('rate_limit') %} # Rate limit config for {{ service }} - acl is_{{ service }} hdr(host) -i {% for domain in config.domains %}{{ domain }}{% endfor %} + acl is_{{ service }} hdr(host) -i {% for domain in config.domains %}{{ domain }} {% endfor %} http-request deny deny_status 429 if is_{{ service }} { sc0_http_req_rate() gt {{ config.rate_limit }} } {%- endif %} {%- endfor %} From 0b85e0b7eaecefaafa56c0439ca56e48ab18a3ae Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Wed, 6 Nov 2024 12:54:00 -0600 Subject: [PATCH 08/10] fix: use new syntax --- salt/haproxy/config/haproxy.cfg.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/haproxy/config/haproxy.cfg.jinja b/salt/haproxy/config/haproxy.cfg.jinja index d8df7a60..f1691c31 100644 --- a/salt/haproxy/config/haproxy.cfg.jinja +++ b/salt/haproxy/config/haproxy.cfg.jinja @@ -130,7 +130,7 @@ frontend main {%- if config.get('rate_limit') %} # Rate limit config for {{ service }} acl is_{{ service }} hdr(host) -i {% for domain in config.domains %}{{ domain }} {% endfor %} - http-request deny deny_status 429 if is_{{ service }} { sc0_http_req_rate() gt {{ config.rate_limit }} } + http-request deny deny_status 429 if is_{{ service }} { sc_http_req_rate(0) gt {{ config.rate_limit }} } {%- endif %} {%- endfor %} From dd17288519c99aa692dbe52caa2141ef9248e522 Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Wed, 6 Nov 2024 12:54:15 -0600 Subject: [PATCH 09/10] chore: remove unneeded limit --- pillar/base/haproxy.sls | 1 - 1 file changed, 1 deletion(-) diff --git a/pillar/base/haproxy.sls b/pillar/base/haproxy.sls index b6ee17e9..28b1ae96 100644 --- a/pillar/base/haproxy.sls +++ b/pillar/base/haproxy.sls @@ -17,7 +17,6 @@ haproxy: - docs.python.org - doc.python.org check: "HEAD /_check HTTP/1.1\\r\\nHost:\\ docs.python.org" - rate_limit: 100 downloads: domains: From 587e709e3a7cb2873555bd7ba49152fe392a85c8 Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Fri, 8 Nov 2024 09:12:11 -0600 Subject: [PATCH 10/10] fix: support ipv6 and ipv4 https://docs.haproxy.org/3.0/configuration.html#7.3.3-bc_src --- salt/haproxy/config/haproxy.cfg.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/haproxy/config/haproxy.cfg.jinja b/salt/haproxy/config/haproxy.cfg.jinja index f1691c31..d0048797 100644 --- a/salt/haproxy/config/haproxy.cfg.jinja +++ b/salt/haproxy/config/haproxy.cfg.jinja @@ -117,7 +117,7 @@ frontend main bind 127.0.0.1:19001 # This is our TLS socket. # Define a stick table for all services - stick-table type ip size 100k expire 30s store http_req_rate(10s) + stick-table type ipv6 size 100k expire 30s store http_req_rate(10s) # Track all requests using a single counter # We could use the 3 available (sc0,1,2) to maybe tier requests # into say <=100, 101-500, >= 501 if we needed to?