Scheduler Enhancements

api/docgen/docgen.go

@@ -231,16 +231,17 @@ func init() {
 				Hostname: "host",
 				Resources: storiface.WorkerResources{
 					MemPhysical: 256 << 30,
+					MemUsed:     2 << 30,
 					MemSwap:     120 << 30,
-					MemReserved: 2 << 30,
+					MemSwapUsed: 2 << 30,
 					CPUs:        64,
 					GPUs:        []string{"aGPU 1337"},
 				},
 			},
 			Enabled:    true,
 			MemUsedMin: 0,
 			MemUsedMax: 0,
-			GpuUsed:    false,
+			GpuUsed:    0,
 			CpuUse:     0,
 		},
 	})

api/version.go

@@ -58,7 +58,7 @@ var (
 	FullAPIVersion1 = newVer(2, 1, 0)
 
 	MinerAPIVersion0  = newVer(1, 2, 0)
-	WorkerAPIVersion0 = newVer(1, 1, 0)
+	WorkerAPIVersion0 = newVer(1, 4, 0)
 )
 
 //nolint:varcheck,deadcode

cmd/lotus-miner/sealing.go

@@ -4,6 +4,7 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
+	"math"
 	"os"
 	"sort"
 	"strings"
@@ -32,6 +33,17 @@ var sealingCmd = &cli.Command{
 	},
 }
 
+var barCols = float64(64)
+
+func barString(total, y, g float64) string {
+	yBars := int(math.Round(y / total * barCols))
+	gBars := int(math.Round(g / total * barCols))
+	eBars := int(barCols) - yBars - gBars
+	return color.YellowString(strings.Repeat("|", yBars)) +
+		color.GreenString(strings.Repeat("|", gBars)) +
+		strings.Repeat(" ", eBars)
+}
+
 var sealingWorkersCmd = &cli.Command{
 	Name:  "workers",
 	Usage: "list workers",
@@ -77,7 +89,7 @@ var sealingWorkersCmd = &cli.Command{
 		for _, stat := range st {
 			gpuUse := "not "
 			gpuCol := color.FgBlue
-			if stat.GpuUsed {
+			if stat.GpuUsed > 0 {
 				gpuCol = color.FgGreen
 				gpuUse = ""
 			}
@@ -89,56 +101,43 @@ var sealingWorkersCmd = &cli.Command{
 
 			fmt.Printf("Worker %s, host %s%s\n", stat.id, color.MagentaString(stat.Info.Hostname), disabled)
 
-			var barCols = uint64(64)
-			cpuBars := int(stat.CpuUse * barCols / stat.Info.Resources.CPUs)
-			cpuBar := strings.Repeat("|", cpuBars)
-			if int(barCols)-cpuBars >= 0 {
-				cpuBar += strings.Repeat(" ", int(barCols)-cpuBars)
-			}
-
 			fmt.Printf("\tCPU:  [%s] %d/%d core(s) in use\n",
-				color.GreenString(cpuBar), stat.CpuUse, stat.Info.Resources.CPUs)
-
-			ramBarsRes := int(stat.Info.Resources.MemReserved * barCols / stat.Info.Resources.MemPhysical)
-			ramBarsUsed := int(stat.MemUsedMin * barCols / stat.Info.Resources.MemPhysical)
-			ramRepeatSpace := int(barCols) - (ramBarsUsed + ramBarsRes)
-
-			colorFunc := color.YellowString
-			if ramRepeatSpace < 0 {
-				ramRepeatSpace = 0
-				colorFunc = color.RedString
+				barString(float64(stat.Info.Resources.CPUs), 0, float64(stat.CpuUse)), stat.CpuUse, stat.Info.Resources.CPUs)
+
+			ramTotal := stat.Info.Resources.MemPhysical
+			ramTasks := stat.MemUsedMin
+			ramUsed := stat.Info.Resources.MemUsed
+			var ramReserved uint64 = 0
+			if ramUsed > ramTasks {
+				ramReserved = ramUsed - ramTasks
 			}
-
-			ramBar := colorFunc(strings.Repeat("|", ramBarsRes)) +
-				color.GreenString(strings.Repeat("|", ramBarsUsed)) +
-				strings.Repeat(" ", ramRepeatSpace)
-
-			vmem := stat.Info.Resources.MemPhysical + stat.Info.Resources.MemSwap
-
-			vmemBarsRes := int(stat.Info.Resources.MemReserved * barCols / vmem)
-			vmemBarsUsed := int(stat.MemUsedMax * barCols / vmem)
-			vmemRepeatSpace := int(barCols) - (vmemBarsUsed + vmemBarsRes)
-
-			colorFunc = color.YellowString
-			if vmemRepeatSpace < 0 {
-				vmemRepeatSpace = 0
-				colorFunc = color.RedString
-			}
-
-			vmemBar := colorFunc(strings.Repeat("|", vmemBarsRes)) +
-				color.GreenString(strings.Repeat("|", vmemBarsUsed)) +
-				strings.Repeat(" ", vmemRepeatSpace)
+			ramBar := barString(float64(ramTotal), float64(ramReserved), float64(ramTasks))
 
 			fmt.Printf("\tRAM:  [%s] %d%% %s/%s\n", ramBar,
-				(stat.Info.Resources.MemReserved+stat.MemUsedMin)*100/stat.Info.Resources.MemPhysical,
-				types.SizeStr(types.NewInt(stat.Info.Resources.MemReserved+stat.MemUsedMin)),
+				(ramTasks+ramReserved)*100/stat.Info.Resources.MemPhysical,
+				types.SizeStr(types.NewInt(ramTasks+ramUsed)),
 				types.SizeStr(types.NewInt(stat.Info.Resources.MemPhysical)))
 
-			fmt.Printf("\tVMEM: [%s] %d%% %s/%s\n", vmemBar,
-				(stat.Info.Resources.MemReserved+stat.MemUsedMax)*100/vmem,
-				types.SizeStr(types.NewInt(stat.Info.Resources.MemReserved+stat.MemUsedMax)),
-				types.SizeStr(types.NewInt(vmem)))
+			vmemTotal := stat.Info.Resources.MemPhysical + stat.Info.Resources.MemSwap
+			vmemTasks := stat.MemUsedMax
+			vmemUsed := stat.Info.Resources.MemUsed + stat.Info.Resources.MemSwapUsed
+			var vmemReserved uint64 = 0
+			if vmemUsed > vmemTasks {
+				vmemReserved = vmemUsed - vmemTasks
+			}
+			vmemBar := barString(float64(vmemTotal), float64(vmemReserved), float64(vmemTasks))
 
+			fmt.Printf("\tVMEM: [%s] %d%% %s/%s\n", vmemBar,
+				(vmemTasks+vmemReserved)*100/vmemTotal,
+				types.SizeStr(types.NewInt(vmemTasks+vmemReserved)),
+				types.SizeStr(types.NewInt(vmemTotal)))
+
+			if len(stat.Info.Resources.GPUs) > 0 {
+				gpuBar := barString(float64(len(stat.Info.Resources.GPUs)), 0, stat.GpuUsed)
+				fmt.Printf("\tGPU:  [%s] %.f%% %.2f/%d gpu(s) in use\n", color.GreenString(gpuBar),
+					stat.GpuUsed*100/float64(len(stat.Info.Resources.GPUs)),
+					stat.GpuUsed, len(stat.Info.Resources.GPUs))
+			}
 			for _, gpu := range stat.Info.Resources.GPUs {
 				fmt.Printf("\tGPU: %s\n", color.New(gpuCol).Sprintf("%s, %sused", gpu, gpuUse))
 			}

cmd/lotus-seal-worker/info.go

@@ -58,8 +58,11 @@ var infoCmd = &cli.Command{
 
 		fmt.Printf("Hostname: %s\n", info.Hostname)
 		fmt.Printf("CPUs: %d; GPUs: %v\n", info.Resources.CPUs, info.Resources.GPUs)
-		fmt.Printf("RAM: %s; Swap: %s\n", types.SizeStr(types.NewInt(info.Resources.MemPhysical)), types.SizeStr(types.NewInt(info.Resources.MemSwap)))
-		fmt.Printf("Reserved memory: %s\n", types.SizeStr(types.NewInt(info.Resources.MemReserved)))
+		fmt.Printf("RAM: %s/%s; Swap: %s/%s\n",
+			types.SizeStr(types.NewInt(info.Resources.MemUsed)),
+			types.SizeStr(types.NewInt(info.Resources.MemPhysical)),
+			types.SizeStr(types.NewInt(info.Resources.MemSwapUsed)),
+			types.SizeStr(types.NewInt(info.Resources.MemSwap)))
 
 		fmt.Printf("Task types: ")
 		for _, t := range ttList(tt) {

extern/sector-storage/cgroups.go

@@ -0,0 +1,12 @@
+//go:build !linux
+// +build !linux
+
+package sectorstorage
+
+func cgroupV1Mem() (memoryMax, memoryUsed, swapMax, swapUsed uint64, err error) {
+	return 0, 0, 0, 0, nil
+}
+
+func cgroupV2Mem() (memoryMax, memoryUsed, swapMax, swapUsed uint64, err error) {
+	return 0, 0, 0, 0, nil
+}

extern/sector-storage/cgroups_linux.go

@@ -0,0 +1,117 @@
+//go:build linux
+// +build linux
+
+package sectorstorage
+
+import (
+	"bufio"
+	"bytes"
+	"math"
+	"os"
+	"path/filepath"
+
+	"github.com/containerd/cgroups"
+	cgroupv2 "github.com/containerd/cgroups/v2"
+)
+
+func cgroupV2MountPoint() (string, error) {
+	f, err := os.Open("/proc/self/mountinfo")
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		fields := bytes.Fields(scanner.Bytes())
+		if len(fields) >= 9 && bytes.Equal(fields[8], []byte("cgroup2")) {
+			return string(fields[4]), nil
+		}
+	}
+	return "", cgroups.ErrMountPointNotExist
+}
+
+func cgroupV1Mem() (memoryMax, memoryUsed, swapMax, swapUsed uint64, err error) {
+	path := cgroups.NestedPath("")
+	if pid := os.Getpid(); pid == 1 {
+		path = cgroups.RootPath
+	}
+	c, err := cgroups.Load(cgroups.SingleSubsystem(cgroups.V1, cgroups.Memory), path)
+	if err != nil {
+		return 0, 0, 0, 0, err
+	}
+	stats, err := c.Stat()
+	if err != nil {
+		return 0, 0, 0, 0, err
+	}
+	if stats.Memory == nil {
+		return 0, 0, 0, 0, nil
+	}
+	if stats.Memory.Usage != nil {
+		memoryMax = stats.Memory.Usage.Limit
+		// Exclude cached files
+		memoryUsed = stats.Memory.Usage.Usage - stats.Memory.InactiveFile - stats.Memory.ActiveFile
+	}
+	if stats.Memory.Swap != nil {
+		swapMax = stats.Memory.Swap.Limit
+		swapUsed = stats.Memory.Swap.Usage
+	}
+	return memoryMax, memoryUsed, swapMax, swapUsed, nil
+}
+
+func cgroupV2MemFromPath(mp, path string) (memoryMax, memoryUsed, swapMax, swapUsed uint64, err error) {
+	c, err := cgroupv2.LoadManager(mp, path)
+	if err != nil {
+		return 0, 0, 0, 0, err
+	}
+
+	stats, err := c.Stat()
+	if err != nil {
+		return 0, 0, 0, 0, err
+	}
+
+	if stats.Memory != nil {
+		memoryMax = stats.Memory.UsageLimit
+		// Exclude memory used caching files
+		memoryUsed = stats.Memory.Usage - stats.Memory.File
+		swapMax = stats.Memory.SwapLimit
+		swapUsed = stats.Memory.SwapUsage
+	}
+
+	return memoryMax, memoryUsed, swapMax, swapUsed, nil
+}
+
+func cgroupV2Mem() (memoryMax, memoryUsed, swapMax, swapUsed uint64, err error) {
+	memoryMax = math.MaxUint64
+	swapMax = math.MaxUint64
+
+	path, err := cgroupv2.PidGroupPath(os.Getpid())
+	if err != nil {
+		return 0, 0, 0, 0, err
+	}
+
+	mp, err := cgroupV2MountPoint()
+	if err != nil {
+		return 0, 0, 0, 0, err
+	}
+
+	for path != "/" {
+		cgMemoryMax, cgMemoryUsed, cgSwapMax, cgSwapUsed, err := cgroupV2MemFromPath(mp, path)
+		if err != nil {
+			return 0, 0, 0, 0, err
+		}
+		if cgMemoryMax != 0 && cgMemoryMax < memoryMax {
+			log.Debugf("memory limited by cgroup %s: %v", path, cgMemoryMax)
+			memoryMax = cgMemoryMax
+			memoryUsed = cgMemoryUsed
+		}
+		if cgSwapMax != 0 && cgSwapMax < swapMax {
+			log.Debugf("swap limited by cgroup %s: %v", path, cgSwapMax)
+			swapMax = cgSwapMax
+			swapUsed = cgSwapUsed
+		}
+		path = filepath.Dir(path)
+	}
+
+	return memoryMax, memoryUsed, swapMax, swapUsed, nil
+}