From 8b5821fdedf11694b3bcea81e302c22231d62b89 Mon Sep 17 00:00:00 2001 From: Eguzki Astiz Lezaun Date: Tue, 26 Sep 2023 15:07:17 +0200 Subject: [PATCH 1/3] [cli] detect number of available CPU shares from cgroups v2 --- gateway/src/apicast/cli/environment.lua | 38 ++++++++++++++++++++----- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/gateway/src/apicast/cli/environment.lua b/gateway/src/apicast/cli/environment.lua index b208ba152..fe6292e42 100644 --- a/gateway/src/apicast/cli/environment.lua +++ b/gateway/src/apicast/cli/environment.lua @@ -51,16 +51,35 @@ local function detect_kubernetes() return secrets or resty_env.value('KUBERNETES_PORT') end +-- CPU shares in Cgroups v1 or converted from weight in Cgroups v2 in millicores local function cpu_shares() - if not detect_kubernetes() then return end - local shares - local file = open('/sys/fs/cgroup/cpu/cpu.shares') - if file then - shares = file:read('*n') + -- This check is from https://github.com/kubernetes/kubernetes/blob/release-1.27/test/e2e/node/pod_resize.go#L305-L314 + -- alternatively, this method can be used: https://kubernetes.io/docs/concepts/architecture/cgroups/#check-cgroup-version + -- (`stat -fc %T /sys/fs/cgroup/` returns `cgroup2fs` or `tmpfs`) + if pl_path.exists("/sys/fs/cgroup/cgroup.controllers") then + -- Cgroups v2 + ngx.log(ngx.DEBUG, "detecting cpus in Cgroups v2") + -- Using the formula from https://github.com/kubernetes/kubernetes/blob/release-1.27/pkg/kubelet/cm/cgroup_manager_linux.go#L570-L574 + local file = open('/sys/fs/cgroup/cpu.weight') + + if file then + local weight = file:read('*n') + shares = (((weight - 1) * 262142) / 9999) + 2 + + file:close() + end + else + -- Cgroups v1 + ngx.log(ngx.DEBUG, "detecting cpus in Cgroups v1") + local file = open('/sys/fs/cgroup/cpu/cpu.shares') - file:close() + if file then + shares = file:read('*n') + + file:close() + end end return shares @@ -68,11 +87,16 @@ end local function cpus() local shares = cpu_shares() - if shares then return ceil(shares / 1024) end + if shares then + local res = ceil(shares / 1024) + ngx.log(ngx.DEBUG, "cpu_shares = "..res) + return res + end -- TODO: support /sys/fs/cgroup/cpuset/cpuset.cpus -- see https://github.com/sclorg/rhscl-dockerfiles/blob/ff912d8764af9a41096e63064bbc325395afa608/rhel7.sti-base/bin/cgroup-limits#L55-L75 local nproc = util.system('nproc') + ngx.log(ngx.DEBUG, "cpus from nproc = "..nproc) return tonumber(nproc) end From 5c99dfdec4ef75b465ef917ec9c24e558e48e6f7 Mon Sep 17 00:00:00 2001 From: Eguzki Astiz Lezaun Date: Tue, 26 Sep 2023 15:52:27 +0200 Subject: [PATCH 2/3] CHANGELOG.md: add entry --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b7d0157c..76bc30dd5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## [Unreleased] +### Added + +- Detect number of CPU shares when running on Cgroups V2 [PR #1410](https://github.com/3scale/apicast/pull/1410) [THREESCALE-10167](https://issues.redhat.com/browse/THREESCALE-10167) + +## [3.14.0] 2023-07-25 + ### Fixed - In boot mode on `init_worker` check configuration expiration [PR #1399](https://github.com/3scale/APIcast/pull/1399) [THREESCALE-9003](https://issues.redhat.com/browse/THREESCALE-9003) From 39e9f2c3c8c10222f649dfbe4609f5b9ed3cc7d9 Mon Sep 17 00:00:00 2001 From: Eguzki Astiz Lezaun Date: Tue, 26 Sep 2023 17:18:10 +0200 Subject: [PATCH 3/3] cpu detection: early closing of file pointer --- gateway/src/apicast/cli/environment.lua | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/gateway/src/apicast/cli/environment.lua b/gateway/src/apicast/cli/environment.lua index fe6292e42..4b201f28a 100644 --- a/gateway/src/apicast/cli/environment.lua +++ b/gateway/src/apicast/cli/environment.lua @@ -43,14 +43,6 @@ local function parse_nameservers() end end -local function detect_kubernetes() - local secrets = open('/run/secrets/kubernetes.io') - - if secrets then secrets:close() end - - return secrets or resty_env.value('KUBERNETES_PORT') -end - -- CPU shares in Cgroups v1 or converted from weight in Cgroups v2 in millicores local function cpu_shares() local shares @@ -66,9 +58,9 @@ local function cpu_shares() if file then local weight = file:read('*n') - shares = (((weight - 1) * 262142) / 9999) + 2 - file:close() + + shares = (((weight - 1) * 262142) / 9999) + 2 end else -- Cgroups v1