Merged PR 5999316: Update ADO to dbb347e

Sync ADO with upstream to enable including test GCS binaries as
part of dev-pipeline

Related work items: #1311, #1322, #1341, #1343, #1345, #1347, #1348, #1350, #1353, #1354, #1355, #1358, #1361, #1365, #1368, #1369, #1370

.github/workflows/ci.yml

@@ -189,6 +189,9 @@ jobs:
         with:
           go-version: '^1.17.0'
 
+      - name: Test
+        run: make test
+
       - name: Pull busybox image
         run: docker pull busybox
 
@@ -199,7 +202,5 @@ jobs:
         run: |
           docker export base_image_container | gzip > base.tar.gz
 
-      - name: Build And Test
-        run: |
-          BASE=./base.tar.gz
-          make all test
+      - name: Build
+        run: make BASE=./base.tar.gz all

Makefile

@@ -1,4 +1,5 @@
 BASE:=base.tar.gz
+DEV_BUILD:=0
 
 GO:=go
 GO_FLAGS:=-ldflags "-s -w" # strip Go binaries
@@ -15,6 +16,14 @@ endif
 GO_BUILD:=CGO_ENABLED=$(CGO_ENABLED) $(GO) build $(GO_FLAGS) $(GO_FLAGS_EXTRA)
 
 SRCROOT=$(dir $(abspath $(firstword $(MAKEFILE_LIST))))
+# additional directories to search for rule prerequisites and targets
+VPATH=$(SRCROOT)
+
+DELTA_TARGET=out/delta.tar.gz
+
+ifeq "$(DEV_BUILD)" "1"
+DELTA_TARGET=out/delta-dev.tar.gz
+endif
 
 # The link aliases for gcstools
 GCS_TOOLS=\
@@ -40,37 +49,52 @@ out/rootfs.vhd: out/rootfs.tar.gz bin/cmd/tar2ext4
 	gzip -f -d ./out/rootfs.tar.gz
 	bin/cmd/tar2ext4 -vhd -i ./out/rootfs.tar -o $@
 
+out/rootfs.tar.gz: out/initrd.img
+	rm -rf rootfs-conv
+	mkdir rootfs-conv
+	gunzip -c out/initrd.img | (cd rootfs-conv && cpio -imd)
+	tar -zcf $@ -C rootfs-conv .
+	rm -rf rootfs-conv
+
+out/initrd.img: $(BASE) $(DELTA_TARGET) $(SRCROOT)/hack/catcpio.sh
+	$(SRCROOT)/hack/catcpio.sh "$(BASE)" $(DELTA_TARGET) > out/initrd.img.uncompressed
+	gzip -c out/initrd.img.uncompressed > $@
+	rm out/initrd.img.uncompressed
+
+# This target includes utilities which may be useful for testing purposes.
+out/delta-dev.tar.gz: out/delta.tar.gz bin/internal/tools/snp-report
+	rm -rf rootfs-dev
+	mkdir rootfs-dev
+	tar -xzf out/delta.tar.gz -C rootfs-dev
+	cp bin/internal/tools/snp-report rootfs-dev/bin/
+	tar -zcf $@ -C rootfs-dev .
+	rm -rf rootfs-dev
+
 out/delta.tar.gz: bin/init bin/vsockexec bin/cmd/gcs bin/cmd/gcstools bin/cmd/hooks/wait-paths Makefile
 	@mkdir -p out
 	rm -rf rootfs
 	mkdir -p rootfs/bin/
+	mkdir -p rootfs/info/
 	cp bin/init rootfs/
 	cp bin/vsockexec rootfs/bin/
 	cp bin/cmd/gcs rootfs/bin/
 	cp bin/cmd/gcstools rootfs/bin/
 	cp bin/cmd/hooks/wait-paths rootfs/bin/
 	for tool in $(GCS_TOOLS); do ln -s gcstools rootfs/bin/$$tool; done
-	git -C $(SRCROOT) rev-parse HEAD > rootfs/gcs.commit && \
-	git -C $(SRCROOT) rev-parse --abbrev-ref HEAD > rootfs/gcs.branch
+	git -C $(SRCROOT) rev-parse HEAD > rootfs/info/gcs.commit && \
+	git -C $(SRCROOT) rev-parse --abbrev-ref HEAD > rootfs/info/gcs.branch && \
+	date --iso-8601=minute --utc > rootfs/info/tar.date
+	$(if $(and $(realpath $(subst .tar,.testdata.json,$(BASE))), $(shell which jq)), \
+		jq -r '.IMAGE_NAME' $(subst .tar,.testdata.json,$(BASE)) 2>/dev/null > rootfs/info/image.name && \
+		jq -r '.DATETIME' $(subst .tar,.testdata.json,$(BASE)) 2>/dev/null > rootfs/info/build.date)
 	tar -zcf $@ -C rootfs .
 	rm -rf rootfs
 
-out/rootfs.tar.gz: out/initrd.img
-	rm -rf rootfs-conv
-	mkdir rootfs-conv
-	gunzip -c out/initrd.img | (cd rootfs-conv && cpio -imd)
-	tar -zcf $@ -C rootfs-conv .
-	rm -rf rootfs-conv
-
-out/initrd.img: $(BASE) out/delta.tar.gz $(SRCROOT)/hack/catcpio.sh
-	$(SRCROOT)/hack/catcpio.sh "$(BASE)" out/delta.tar.gz > out/initrd.img.uncompressed
-	gzip -c out/initrd.img.uncompressed > $@
-	rm out/initrd.img.uncompressed
-
 -include deps/cmd/gcs.gomake
 -include deps/cmd/gcstools.gomake
 -include deps/cmd/hooks/wait-paths.gomake
 -include deps/cmd/tar2ext4.gomake
+-include deps/internal/tools/snp-report.gomake
 
 # Implicit rule for includes that define Go targets.
 %.gomake: $(SRCROOT)/Makefile
@@ -84,8 +108,6 @@ out/initrd.img: $(BASE) out/delta.tar.gz $(SRCROOT)/hack/catcpio.sh
 	@/bin/echo -e '-include $(@:%.gomake=%.godeps)' >> $@.new
 	mv $@.new $@
 
-VPATH=$(SRCROOT)
-
 bin/vsockexec: vsockexec/vsockexec.o vsockexec/vsock.o
 	@mkdir -p bin
 	$(CC) $(LDFLAGS) -o $@ $^

cmd/containerd-shim-runhcs-v1/delete.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/Microsoft/hcsshim/internal/hcs"
+	"github.com/Microsoft/hcsshim/internal/memory"
 	"github.com/Microsoft/hcsshim/internal/oc"
 	"github.com/Microsoft/hcsshim/internal/winapi"
 	"github.com/containerd/containerd/runtime/v2/task"
@@ -69,7 +70,7 @@ The delete command will be executed in the container's bundle as its cwd.
 		// This should be done as the first thing so that we don't miss any panic logs even if
 		// something goes wrong during delete op.
 		// The file can be very large so read only first 1MB of data.
-		readLimit := int64(1024 * 1024) // 1MB
+		readLimit := int64(memory.MiB) // 1MB
 		logBytes, err := limitedRead(filepath.Join(bundleFlag, "panic.log"), readLimit)
 		if err == nil && len(logBytes) > 0 {
 			if int64(len(logBytes)) == readLimit {

cmd/containerd-shim-runhcs-v1/events.go

@@ -17,17 +17,19 @@ type publisher interface {
 }
 
 type eventPublisher struct {
+	namespace       string
 	remotePublisher *shim.RemoteEventsPublisher
 }
 
-var _ = (publisher)(&eventPublisher{})
+var _ publisher = &eventPublisher{}
 
-func newEventPublisher(address string) (*eventPublisher, error) {
+func newEventPublisher(address, namespace string) (*eventPublisher, error) {
 	p, err := shim.NewPublisher(address)
 	if err != nil {
 		return nil, err
 	}
 	return &eventPublisher{
+		namespace:       namespace,
 		remotePublisher: p,
 	}, nil
 }
@@ -48,5 +50,5 @@ func (e *eventPublisher) publishEvent(ctx context.Context, topic string, event i
 		return nil
 	}
 
-	return e.remotePublisher.Publish(namespaces.WithNamespace(ctx, namespaceFlag), topic, event)
+	return e.remotePublisher.Publish(namespaces.WithNamespace(ctx, e.namespace), topic, event)
 }

cmd/containerd-shim-runhcs-v1/events_test.go

@@ -8,7 +8,7 @@ type fakePublisher struct {
 	events []interface{}
 }
 
-var _ = (publisher)(&fakePublisher{})
+var _ publisher = &fakePublisher{}
 
 func newFakePublisher() *fakePublisher {
 	return &fakePublisher{}

cmd/containerd-shim-runhcs-v1/serve.go

@@ -177,7 +177,7 @@ var serveCommand = cli.Command{
 		}
 
 		ttrpcAddress := os.Getenv(ttrpcAddressEnv)
-		ttrpcEventPublisher, err := newEventPublisher(ttrpcAddress)
+		ttrpcEventPublisher, err := newEventPublisher(ttrpcAddress, namespaceFlag)
 		if err != nil {
 			return err
 		}
@@ -281,11 +281,11 @@ func readOptions(r io.Reader) (*runhcsopts.Options, error) {
 	if len(d) > 0 {
 		var a types.Any
 		if err := proto.Unmarshal(d, &a); err != nil {
-			return nil, errors.Wrap(err, "failed unmarshaling into Any")
+			return nil, errors.Wrap(err, "failed unmarshalling into Any")
 		}
 		v, err := typeurl.UnmarshalAny(&a)
 		if err != nil {
-			return nil, errors.Wrap(err, "failed unmarshaling by typeurl")
+			return nil, errors.Wrap(err, "failed unmarshalling by typeurl")
 		}
 		return v.(*runhcsopts.Options), nil
 	}

cmd/containerd-shim-runhcs-v1/task_hcs.go

@@ -31,6 +31,7 @@ import (
 	"github.com/Microsoft/hcsshim/internal/hcsoci"
 	"github.com/Microsoft/hcsshim/internal/jobcontainers"
 	"github.com/Microsoft/hcsshim/internal/log"
+	"github.com/Microsoft/hcsshim/internal/memory"
 	"github.com/Microsoft/hcsshim/internal/oci"
 	"github.com/Microsoft/hcsshim/internal/processorinfo"
 	"github.com/Microsoft/hcsshim/internal/protocol/guestrequest"
@@ -42,8 +43,6 @@ import (
 	"github.com/Microsoft/hcsshim/pkg/annotations"
 )
 
-const bytesPerMB = 1024 * 1024
-
 func newHcsStandaloneTask(ctx context.Context, events publisher, req *task.CreateTaskRequest, s *specs.Spec) (shimTask, error) {
 	log.G(ctx).WithField("tid", req.ID).Debug("newHcsStandaloneTask")
 
@@ -1008,7 +1007,7 @@ func (ht *hcsTask) updateWCOWResources(ctx context.Context, data interface{}, an
 		return errors.New("must have resources be type *WindowsResources when updating a wcow container")
 	}
 	if resources.Memory != nil && resources.Memory.Limit != nil {
-		newMemorySizeInMB := *resources.Memory.Limit / bytesPerMB
+		newMemorySizeInMB := *resources.Memory.Limit / memory.MiB
 		memoryLimit := hcsoci.NormalizeMemorySize(ctx, ht.id, newMemorySizeInMB)
 		if err := ht.requestUpdateContainer(ctx, resourcepaths.SiloMemoryResourcePath, memoryLimit); err != nil {
 			return err

computestorage/helpers.go

@@ -8,10 +8,12 @@ import (
 	"path/filepath"
 	"syscall"
 
-	"github.com/Microsoft/go-winio/pkg/security"
 	"github.com/Microsoft/go-winio/vhd"
+	"github.com/Microsoft/hcsshim/internal/memory"
 	"github.com/pkg/errors"
 	"golang.org/x/sys/windows"
+
+	"github.com/Microsoft/hcsshim/internal/security"
 )
 
 const defaultVHDXBlockSizeInMB = 1
@@ -61,8 +63,8 @@ func SetupContainerBaseLayer(ctx context.Context, layerPath, baseVhdPath, diffVh
 	createParams := &vhd.CreateVirtualDiskParameters{
 		Version: 2,
 		Version2: vhd.CreateVersion2{
-			MaximumSize:      sizeInGB * 1024 * 1024 * 1024,
-			BlockSizeInBytes: defaultVHDXBlockSizeInMB * 1024 * 1024,
+			MaximumSize:      sizeInGB * memory.GiB,
+			BlockSizeInBytes: defaultVHDXBlockSizeInMB * memory.MiB,
 		},
 	}
 	handle, err := vhd.CreateVirtualDisk(baseVhdPath, vhd.VirtualDiskAccessNone, vhd.CreateVirtualDiskFlagNone, createParams)
@@ -137,8 +139,8 @@ func SetupUtilityVMBaseLayer(ctx context.Context, uvmPath, baseVhdPath, diffVhdP
 	createParams := &vhd.CreateVirtualDiskParameters{
 		Version: 2,
 		Version2: vhd.CreateVersion2{
-			MaximumSize:      sizeInGB * 1024 * 1024 * 1024,
-			BlockSizeInBytes: defaultVHDXBlockSizeInMB * 1024 * 1024,
+			MaximumSize:      sizeInGB * memory.GiB,
+			BlockSizeInBytes: defaultVHDXBlockSizeInMB * memory.MiB,
 		},
 	}
 	handle, err := vhd.CreateVirtualDisk(baseVhdPath, vhd.VirtualDiskAccessNone, vhd.CreateVirtualDiskFlagNone, createParams)

container.go

@@ -62,7 +62,7 @@ type container struct {
 	waitCh   chan struct{}
 }
 
-// createComputeSystemAdditionalJSON is read from the environment at initialisation
+// createContainerAdditionalJSON is read from the environment at initialization
 // time. It allows an environment variable to define additional JSON which
 // is merged in the CreateComputeSystem call to HCS.
 var createContainerAdditionalJSON []byte

ext4/dmverity/dmverity.go

@@ -14,14 +14,15 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/Microsoft/hcsshim/ext4/internal/compactext4"
+	"github.com/Microsoft/hcsshim/internal/memory"
 )
 
 const (
 	blockSize = compactext4.BlockSize
 	// MerkleTreeBufioSize is a default buffer size to use with bufio.Reader
-	MerkleTreeBufioSize = 1024 * 1024 // 1MB
+	MerkleTreeBufioSize = memory.MiB // 1MB
 	// RecommendedVHDSizeGB is the recommended size in GB for VHDs, which is not a hard limit.
-	RecommendedVHDSizeGB = 128 * 1024 * 1024 * 1024
+	RecommendedVHDSizeGB = 128 * memory.GiB
 	// VeritySignature is a value written to dm-verity super-block.
 	VeritySignature = "verity"
 )

ext4/internal/compactext4/compact.go

@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"github.com/Microsoft/hcsshim/ext4/internal/format"
+	"github.com/Microsoft/hcsshim/internal/memory"
 )
 
 // Writer writes a compact ext4 file system.
@@ -101,15 +102,15 @@ const (
 	maxInodesPerGroup       = BlockSize * 8 // Limited by the inode bitmap
 	inodesPerGroupIncrement = BlockSize / inodeSize
 
-	defaultMaxDiskSize = 16 * 1024 * 1024 * 1024        // 16GB
+	defaultMaxDiskSize = 16 * memory.GiB                // 16GB
 	maxMaxDiskSize     = 16 * 1024 * 1024 * 1024 * 1024 // 16TB
 
 	groupDescriptorSize      = 32 // Use the small group descriptor
 	groupsPerDescriptorBlock = BlockSize / groupDescriptorSize
 
-	maxFileSize             = 128 * 1024 * 1024 * 1024 // 128GB file size maximum for now
-	smallSymlinkSize        = 59                       // max symlink size that goes directly in the inode
-	maxBlocksPerExtent      = 0x8000                   // maximum number of blocks in an extent
+	maxFileSize             = 128 * memory.GiB // 128GB file size maximum for now
+	smallSymlinkSize        = 59               // max symlink size that goes directly in the inode
+	maxBlocksPerExtent      = 0x8000           // maximum number of blocks in an extent
 	inodeDataSize           = 60
 	inodeUsedSize           = 152 // fields through CrtimeExtra
 	inodeExtraSize          = inodeSize - inodeUsedSize

ext4/internal/compactext4/compact_test.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/Microsoft/hcsshim/ext4/internal/format"
+	"github.com/Microsoft/hcsshim/internal/memory"
 )
 
 type testFile struct {
@@ -318,9 +319,9 @@ func TestTime(t *testing.T) {
 
 func TestLargeFile(t *testing.T) {
 	testFiles := []testFile{
-		{Path: "small", File: &File{}, DataSize: 1024 * 1024},        // can't change type
-		{Path: "medium", File: &File{}, DataSize: 200 * 1024 * 1024}, // can't change type
-		{Path: "large", File: &File{}, DataSize: 600 * 1024 * 1024},  // can't change type
+		{Path: "small", File: &File{}, DataSize: memory.MiB},        // can't change type
+		{Path: "medium", File: &File{}, DataSize: 200 * memory.MiB}, // can't change type
+		{Path: "large", File: &File{}, DataSize: 600 * memory.MiB},  // can't change type
 	}
 	runTestsOnFiles(t, testFiles)
 }

hcn/hcnnamespace.go

@@ -296,11 +296,11 @@ func GetNamespaceContainerIds(namespaceId string) ([]string, error) {
 	var containerIds []string
 	for _, resource := range namespace.Resources {
 		if resource.Type == "Container" {
-			var contaienrResource NamespaceResourceContainer
-			if err := json.Unmarshal([]byte(resource.Data), &contaienrResource); err != nil {
+			var containerResource NamespaceResourceContainer
+			if err := json.Unmarshal([]byte(resource.Data), &containerResource); err != nil {
 				return nil, err
 			}
-			containerIds = append(containerIds, contaienrResource.Id)
+			containerIds = append(containerIds, containerResource.Id)
 		}
 	}
 	return containerIds, nil
@@ -377,7 +377,7 @@ func (namespace *HostComputeNamespace) Sync() error {
 	}
 	shimPath := runhcs.VMPipePath(cfg.HostUniqueID)
 	if err := runhcs.IssueVMRequest(shimPath, &req); err != nil {
-		// The shim is likey gone. Simply ignore the sync as if it didn't exist.
+		// The shim is likely gone. Simply ignore the sync as if it didn't exist.
 		if perr, ok := err.(*os.PathError); ok && perr.Err == syscall.ERROR_FILE_NOT_FOUND {
 			// Remove the reg key there is no point to try again
 			_ = cfg.Remove()