Merge #56461 #56786

56461: util, server: Add GetCgroupCPU, use it to reflect cpu usage r=knz a=itsbilal

Enhances the util/cgroups package to also support getting
CPU limits from the process' current cgroup. The cpu limit / share
for the current cgroup is denoted in two separate variables:
the cpu period, and the cpu quota. When (quota / period) = numCPUs,
this cgroup has access to all the CPUs on the system.

This PR also updates SampleEnvironment to call this method
and adjust the cpu usage % accordingly.

Release note (bug fix): Improve accuracy of reported CPU usage
when running in containers.

56786: build: upgrade to Go 1.15.5 r=jlinder a=otan

Pick up a security vulnerability.

* [x] Adjust the Pebble tests to run in new version.
* [x] Adjust version in Docker image ([source](./builder/Dockerfile#L199-L200)).
* [x] Rebuild and push the Docker image (following [Basic Process](#basic-process))
* [x] Bump the version in `builder.sh` accordingly ([source](./builder.sh#L6)).
* [x] Bump the version in `go-version-check.sh` ([source](./go-version-check.sh)), unless bumping to a new patch release.
* [x] Bump the go version in `go.mod`. You may also need to rerun `make vendor_rebuild` if vendoring has changed.
* [x] Bump the default installed version of Go in `bootstrap-debian.sh` ([source](./bootstrap/bootstrap-debian.sh#L40-42)).
* [x] Replace other mentions of the older version of go (grep for `golang:<old_version>` and `go<old_version>`).
* [ ] Update the `builder.dockerImage` parameter in the TeamCity [`Cockroach`](https://teamcity.cockroachdb.com/admin/editProject.html?projectId=Cockroach&tab=projectParams) and [`Internal`](https://teamcity.cockroachdb.com/admin/editProject.html?projectId=Internal&tab=projectParams) projects.

Release note: None

Co-authored-by: Bilal Akhtar <[email protected]>
Co-authored-by: Oliver Tan <[email protected]>

build/bootstrap/bootstrap-debian.sh

@@ -46,9 +46,9 @@ sudo tar -C /usr -zxf /tmp/cmake.tgz && rm /tmp/cmake.tgz
 
 # Install Go.
 trap 'rm -f /tmp/go.tgz' EXIT
-curl -fsSL https://dl.google.com/go/go1.15.4.linux-amd64.tar.gz > /tmp/go.tgz
+curl -fsSL https://dl.google.com/go/go1.15.5.linux-amd64.tar.gz > /tmp/go.tgz
 sha256sum -c - <<EOF
-010a88df924a81ec21b293b5da8f9b11c176d27c0ee3962dc1738d2352d3c02d /tmp/go.tgz
+c1076b90cf94b73ebed62a81d802cd84d43d02dea8c07abdc922c57a071c84f1 /tmp/go.tgz
 EOF
 sudo tar -C /usr/local -zxf /tmp/go.tgz && rm /tmp/go.tgz
 

build/builder.sh

@@ -3,7 +3,7 @@
 set -euo pipefail
 
 image=cockroachdb/builder
-version=20201106-211914
+version=20201117-153332
 
 function init() {
   docker build --tag="${image}" "$(dirname "${0}")/builder"

build/builder/Dockerfile

@@ -213,8 +213,8 @@ RUN curl -fsSL https://github.com/Kitware/CMake/releases/download/v3.17.0/cmake-
 # releases of Go will no longer be run in CI once it is changed. Consider
 # bumping the minimum allowed version of Go in /build/go-version-chech.sh.
 RUN apt-get install -y --no-install-recommends golang \
- && curl -fsSL https://storage.googleapis.com/golang/go1.15.4.src.tar.gz -o golang.tar.gz \
- && echo '063da6a9a4186b8118a0e584532c8c94e65582e2cd951ed078bfd595d27d2367 golang.tar.gz' | sha256sum -c - \
+ && curl -fsSL https://storage.googleapis.com/golang/go1.15.5.src.tar.gz -o golang.tar.gz \
+ && echo 'c1076b90cf94b73ebed62a81d802cd84d43d02dea8c07abdc922c57a071c84f1 golang.tar.gz' | sha256sum -c - \
  && tar -C /usr/local -xzf golang.tar.gz \
  && rm golang.tar.gz \
  && cd /usr/local/go/src \

build/go-version-check.sh

@@ -7,7 +7,7 @@
 
 required_version_major=1
 minimum_version_minor=15
-minimum_version_15_patch=3
+minimum_version_15_patch=5
 
 go=${1-go}
 

pkg/server/status/runtime.go

@@ -19,6 +19,7 @@ import (
 	"time"
 
 	"github.com/cockroachdb/cockroach/pkg/build"
+	"github.com/cockroachdb/cockroach/pkg/util/cgroups"
 	"github.com/cockroachdb/cockroach/pkg/util/hlc"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
 	"github.com/cockroachdb/cockroach/pkg/util/metric"
@@ -440,6 +441,8 @@ func (rsr *RuntimeStatSampler) SampleEnvironment(
 	if err := cpuTime.Get(pid); err != nil {
 		log.Errorf(ctx, "unable to get cpu usage: %v", err)
 	}
+	cgroupCPU, _ := cgroups.GetCgroupCPU()
+	cpuShare := cgroupCPU.CPUShares()
 
 	fds := gosigar.ProcFDUsage{}
 	if err := fds.Get(pid); err != nil {
@@ -500,7 +503,7 @@ func (rsr *RuntimeStatSampler) SampleEnvironment(
 	stime := int64(cpuTime.Sys) * 1e6
 	uPerc := float64(utime-rsr.last.utime) / dur
 	sPerc := float64(stime-rsr.last.stime) / dur
-	combinedNormalizedPerc := (sPerc + uPerc) / float64(runtime.NumCPU())
+	combinedNormalizedPerc := (sPerc + uPerc) / cpuShare
 	gcPausePercent := float64(uint64(gc.PauseTotal)-rsr.last.gcPauseTime) / dur
 	rsr.last.now = now
 	rsr.last.utime = utime