Merge pull request #15841 from knz/check-gosigar-mem

server: catch incomplete available memory detection by gosigar
cockroachdb · May 12, 2017 · be5890c · be5890c
2 parents 089a8b9 + f161689
commit be5890c
Showing 1 changed file with 52 additions and 37 deletions.
diff --git a/pkg/server/config.go b/pkg/server/config.go
@@ -228,52 +228,67 @@ func (cfg Config) HistogramWindowInterval() time.Duration {
 // GetTotalMemory returns either the total system memory or if possible the
 // cgroups available memory.
 func GetTotalMemory() (int64, error) {
-	mem := gosigar.Mem{}
-	if err := mem.Get(); err != nil {
+	totalMem, err := func() (int64, error) {
+		mem := gosigar.Mem{}
+		if err := mem.Get(); err != nil {
+			return 0, err
+		}
+		if mem.Total > math.MaxInt64 {
+			return 0, fmt.Errorf("inferred memory size %s exceeds maximum supported memory size %s",
+				humanize.IBytes(mem.Total), humanize.Bytes(math.MaxInt64))
+		}
+		return int64(mem.Total), nil
+	}()
+	if err != nil {
 		return 0, err
 	}
-	if mem.Total > math.MaxInt64 {
-		return 0, fmt.Errorf("inferred memory size %s exceeds maximum supported memory size %s",
-			humanize.IBytes(mem.Total), humanize.Bytes(math.MaxInt64))
-	}
-	totalMem := int64(mem.Total)
-	if runtime.GOOS == "linux" {
-		var err error
-		var buf []byte
-		if buf, err = ioutil.ReadFile(defaultCGroupMemPath); err != nil {
-			if log.V(1) {
-				log.Infof(context.TODO(), "can't read available memory from cgroups (%s), using system memory %s instead", err,
-					humanizeutil.IBytes(totalMem))
-			}
-			return totalMem, nil
+	checkTotal := func(x int64) (int64, error) {
+		if x <= 0 {
+			// https://github.com/elastic/gosigar/issues/72
+			return 0, fmt.Errorf("inferred memory size %d is suspicious, considering invalid", x)
 		}
-		var cgAvlMem uint64
-		if cgAvlMem, err = strconv.ParseUint(strings.TrimSpace(string(buf)), 10, 64); err != nil {
-			if log.V(1) {
-				log.Infof(context.TODO(), "can't parse available memory from cgroups (%s), using system memory %s instead", err,
-					humanizeutil.IBytes(totalMem))
-			}
-			return totalMem, nil
+		return x, nil
+	}
+	if runtime.GOOS != "linux" {
+		return checkTotal(totalMem)
+	}
+
+	var buf []byte
+	if buf, err = ioutil.ReadFile(defaultCGroupMemPath); err != nil {
+		if log.V(1) {
+			log.Infof(context.TODO(), "can't read available memory from cgroups (%s), using system memory %s instead", err,
+				humanizeutil.IBytes(totalMem))
 		}
-		if cgAvlMem > math.MaxInt64 {
-			if log.V(1) {
-				log.Infof(context.TODO(), "available memory from cgroups is too large and unsupported %s using system memory %s instead",
-					humanize.IBytes(cgAvlMem), humanizeutil.IBytes(totalMem))
+		return checkTotal(totalMem)
+	}
 
-			}
-			return totalMem, nil
+	cgAvlMem, err := strconv.ParseUint(strings.TrimSpace(string(buf)), 10, 64)
+	if err != nil {
+		if log.V(1) {
+			log.Infof(context.TODO(), "can't parse available memory from cgroups (%s), using system memory %s instead", err,
+				humanizeutil.IBytes(totalMem))
 		}
-		if cgAvlMem > mem.Total {
-			if log.V(1) {
-				log.Infof(context.TODO(), "available memory from cgroups %s exceeds system memory %s, using system memory",
-					humanize.IBytes(cgAvlMem), humanizeutil.IBytes(totalMem))
-			}
-			return totalMem, nil
+		return checkTotal(totalMem)
+	}
+
+	if cgAvlMem == 0 || cgAvlMem > math.MaxInt64 {
+		if log.V(1) {
+			log.Infof(context.TODO(), "available memory from cgroups (%s) is unsupported, using system memory %s instead",
+				humanize.IBytes(cgAvlMem), humanizeutil.IBytes(totalMem))
+
 		}
+		return checkTotal(totalMem)
+	}
 
-		return int64(cgAvlMem), nil
+	if totalMem > 0 && int64(cgAvlMem) > totalMem {
+		if log.V(1) {
+			log.Infof(context.TODO(), "available memory from cgroups (%s) exceeds system memory %s, using system memory",
+				humanize.IBytes(cgAvlMem), humanizeutil.IBytes(totalMem))
+		}
+		return checkTotal(totalMem)
 	}
-	return totalMem, nil
+
+	return checkTotal(int64(cgAvlMem))
 }
 
 // setOpenFileLimit sets the soft limit for open file descriptors to the hard