Merge pull request #883 from WeiZhang555/rfc-persist-data-standard-wip

persist: baseline persist data format

virtcontainers/api.go

@@ -224,6 +224,10 @@ func StartSandbox(ctx context.Context, sandboxID string) (VCSandbox, error) {
 		return nil, err
 	}
 
+	if err = s.storeSandbox(); err != nil {
+		return nil, err
+	}
+
 	return s, nil
 }
 
@@ -256,6 +260,10 @@ func StopSandbox(ctx context.Context, sandboxID string) (VCSandbox, error) {
 		return nil, err
 	}
 
+	if err = s.storeSandbox(); err != nil {
+		return nil, err
+	}
+
 	return s, nil
 }
 
@@ -394,6 +402,10 @@ func CreateContainer(ctx context.Context, sandboxID string, containerConfig Cont
 		return nil, nil, err
 	}
 
+	if err = s.storeSandbox(); err != nil {
+		return nil, nil, err
+	}
+
 	return s, c, nil
 }
 

virtcontainers/container.go

@@ -258,8 +258,18 @@ type SystemMountsInfo struct {
 type ContainerDevice struct {
 	// ID is device id referencing the device from sandbox's device manager
 	ID string
+
 	// ContainerPath is device path displayed in container
 	ContainerPath string
+
+	// FileMode permission bits for the device.
+	FileMode os.FileMode
+
+	// UID is user ID in the container namespace
+	UID uint32
+
+	// GID is group ID in the container namespace
+	GID uint32
 }
 
 // RootFs describes the container's rootfs.
@@ -364,7 +374,14 @@ func (c *Container) SetPid(pid int) error {
 func (c *Container) setStateFstype(fstype string) error {
 	c.state.Fstype = fstype
 
-	return c.storeState()
+	if !c.sandbox.supportNewStore() {
+		// experimental runtime use "persist.json" which doesn't need "state.json" anymore
+		if err := c.storeState(); err != nil {
+			return err
+		}
+	}
+
+	return nil
 }
 
 // GetAnnotations returns container's annotations
@@ -374,6 +391,11 @@ func (c *Container) GetAnnotations() map[string]string {
 
 // storeContainer stores a container config.
 func (c *Container) storeContainer() error {
+	if c.sandbox.supportNewStore() {
+		if err := c.sandbox.newStore.ToDisk(); err != nil {
+			return err
+		}
+	}
 	return c.store.Store(store.Configuration, *(c.config))
 }
 
@@ -423,10 +445,17 @@ func (c *Container) setContainerState(state types.StateString) error {
 	// update in-memory state
 	c.state.State = state
 
-	// update on-disk state
-	err := c.store.Store(store.State, c.state)
-	if err != nil {
-		return err
+	if c.sandbox.supportNewStore() {
+		// flush data to storage
+		if err := c.sandbox.newStore.ToDisk(); err != nil {
+			return err
+		}
+	} else {
+		// experimental runtime use "persist.json" which doesn't need "state.json" anymore
+		// update on-disk state
+		if err := c.store.Store(store.State, c.state); err != nil {
+			return err
+		}
 	}
 
 	return nil
@@ -588,9 +617,9 @@ func (c *Container) unmountHostMounts() error {
 	return nil
 }
 
-func filterDevices(sandbox *Sandbox, c *Container, devices []ContainerDevice) (ret []ContainerDevice) {
+func filterDevices(c *Container, devices []ContainerDevice) (ret []ContainerDevice) {
 	for _, dev := range devices {
-		major, _ := sandbox.devManager.GetDeviceByID(dev.ID).GetMajorMinor()
+		major, _ := c.sandbox.devManager.GetDeviceByID(dev.ID).GetMajorMinor()
 		if _, ok := cdromMajors[major]; ok {
 			c.Logger().WithFields(logrus.Fields{
 				"device": dev.ContainerPath,
@@ -670,27 +699,56 @@ func newContainer(sandbox *Sandbox, contConfig ContainerConfig) (*Container, err
 
 	c.store = ctrStore
 
-	state, err := c.store.LoadContainerState()
-	if err == nil {
-		c.state = state
+	// experimental runtime use "persist.json" instead of legacy "state.json" as storage
+	if c.sandbox.supportNewStore() {
+		if err := c.Restore(); err != nil &&
+			!os.IsNotExist(err) && err != errContainerPersistNotExist {
+			return nil, err
+		}
+	} else {
+		state, err := c.store.LoadContainerState()
+		if err == nil {
+			c.state = state
+		}
+	}
+
+	if err := c.Restore(); err != nil &&
+		!os.IsNotExist(err) && err != errContainerPersistNotExist {
+		return nil, err
 	}
 
 	var process Process
 	if err := c.store.Load(store.Process, &process); err == nil {
 		c.process = process
 	}
 
+	if err = c.createMounts(); err != nil {
+		return nil, err
+	}
+
+	if err = c.createDevices(contConfig); err != nil {
+		return nil, err
+	}
+
+	return c, nil
+}
+
+func (c *Container) createMounts() error {
 	mounts, err := c.loadMounts()
 	if err == nil {
 		// restore mounts from disk
 		c.mounts = mounts
 	} else {
 		// Create block devices for newly created container
 		if err := c.createBlockDevices(); err != nil {
-			return nil, err
+			return err
 		}
 	}
 
+	return nil
+}
+
+func (c *Container) createDevices(contConfig ContainerConfig) error {
 	// Devices will be found in storage after create stage has completed.
 	// We load devices from storage at all other stages.
 	storedDevices, err := c.loadDevices()
@@ -700,20 +758,22 @@ func newContainer(sandbox *Sandbox, contConfig ContainerConfig) (*Container, err
 		// If devices were not found in storage, create Device implementations
 		// from the configuration. This should happen at create.
 		for _, info := range contConfig.DeviceInfos {
-			dev, err := sandbox.devManager.NewDevice(info)
+			dev, err := c.sandbox.devManager.NewDevice(info)
 			if err != nil {
-				return &Container{}, err
+				return err
 			}
 
 			storedDevices = append(storedDevices, ContainerDevice{
 				ID:            dev.DeviceID(),
 				ContainerPath: info.ContainerPath,
+				FileMode:      info.FileMode,
+				UID:           info.UID,
+				GID:           info.GID,
 			})
 		}
-		c.devices = filterDevices(sandbox, c, storedDevices)
+		c.devices = filterDevices(c, storedDevices)
 	}
-
-	return c, nil
+	return nil
 }
 
 // rollbackFailingContainerCreation rolls back important steps that might have

virtcontainers/container_test.go

@@ -20,6 +20,7 @@ import (
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
 	"github.com/kata-containers/runtime/virtcontainers/device/drivers"
 	"github.com/kata-containers/runtime/virtcontainers/device/manager"
+	"github.com/kata-containers/runtime/virtcontainers/persist"
 	"github.com/kata-containers/runtime/virtcontainers/store"
 	"github.com/kata-containers/runtime/virtcontainers/types"
 	"github.com/stretchr/testify/assert"
@@ -236,6 +237,10 @@ func TestContainerAddDriveDir(t *testing.T) {
 	assert.Nil(t, err)
 	sandbox.store = sandboxStore
 
+	if sandbox.newStore, err = persist.GetDriver("fs"); err != nil || sandbox.newStore == nil {
+		t.Fatalf("failed to get fs persist driver")
+	}
+
 	contID := "100"
 	container := Container{
 		sandbox: sandbox,

virtcontainers/device/api/interface.go

@@ -10,6 +10,7 @@ import (
 	"github.com/sirupsen/logrus"
 
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 )
 
 var devLogger = logrus.WithField("subsystem", "device")
@@ -63,6 +64,9 @@ type Device interface {
 	Reference() uint
 	// Dereference removes one reference to device then returns final ref count
 	Dereference() uint
+
+	// Persist convert and return data in persist format
+	Dump() persistapi.DeviceState
 }
 
 // DeviceManager can be used to create a new device, this can be used as single

virtcontainers/device/drivers/block.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/kata-containers/runtime/virtcontainers/device/api"
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 	"github.com/kata-containers/runtime/virtcontainers/utils"
 )
 
@@ -146,5 +147,27 @@ func (device *BlockDevice) GetDeviceInfo() interface{} {
 	return device.BlockDrive
 }
 
+// Dump convert and return data in persist format
+func (device *BlockDevice) Dump() persistapi.DeviceState {
+	ds := device.GenericDevice.Dump()
+	ds.Type = string(device.DeviceType())
+
+	drive := device.BlockDrive
+	if drive != nil {
+		ds.BlockDrive = &persistapi.BlockDrive{
+			File:     drive.File,
+			Format:   drive.Format,
+			ID:       drive.ID,
+			Index:    drive.Index,
+			MmioAddr: drive.MmioAddr,
+			PCIAddr:  drive.PCIAddr,
+			SCSIAddr: drive.SCSIAddr,
+			NvdimmID: drive.NvdimmID,
+			VirtPath: drive.VirtPath,
+		}
+	}
+	return ds
+}
+
 // It should implement GetAttachCount() and DeviceID() as api.Device implementation
 // here it shares function from *GenericDevice so we don't need duplicate codes

virtcontainers/device/drivers/generic.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/kata-containers/runtime/virtcontainers/device/api"
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 )
 
 // GenericDevice refers to a device that is neither a VFIO device or block device.
@@ -115,3 +116,22 @@ func (device *GenericDevice) bumpAttachCount(attach bool) (skip bool, err error)
 		}
 	}
 }
+
+// Dump convert and return data in persist format
+func (device *GenericDevice) Dump() persistapi.DeviceState {
+	dss := persistapi.DeviceState{
+		ID:          device.ID,
+		Type:        string(device.DeviceType()),
+		RefCount:    device.RefCount,
+		AttachCount: device.AttachCount,
+	}
+
+	info := device.DeviceInfo
+	if info != nil {
+		dss.DevType = info.DevType
+		dss.Major = info.Major
+		dss.Minor = info.Minor
+		dss.DriverOptions = info.DriverOptions
+	}
+	return dss
+}

virtcontainers/device/drivers/vfio.go

@@ -17,6 +17,7 @@ import (
 
 	"github.com/kata-containers/runtime/virtcontainers/device/api"
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 	"github.com/kata-containers/runtime/virtcontainers/utils"
 )
 
@@ -139,6 +140,25 @@ func (device *VFIODevice) GetDeviceInfo() interface{} {
 	return device.VfioDevs
 }
 
+// Dump convert and return data in persist format
+func (device *VFIODevice) Dump() persistapi.DeviceState {
+	ds := device.GenericDevice.Dump()
+	ds.Type = string(device.DeviceType())
+
+	devs := device.VfioDevs
+	for _, dev := range devs {
+		if dev != nil {
+			ds.VFIODevs = append(ds.VFIODevs, &persistapi.VFIODev{
+				ID:       dev.ID,
+				Type:     string(dev.Type),
+				BDF:      dev.BDF,
+				SysfsDev: dev.SysfsDev,
+			})
+		}
+	}
+	return ds
+}
+
 // It should implement GetAttachCount() and DeviceID() as api.Device implementation
 // here it shares function from *GenericDevice so we don't need duplicate codes
 

virtcontainers/device/drivers/vhost_user_blk.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/kata-containers/runtime/virtcontainers/device/api"
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 	"github.com/kata-containers/runtime/virtcontainers/utils"
 )
 
@@ -71,5 +72,18 @@ func (device *VhostUserBlkDevice) GetDeviceInfo() interface{} {
 	return &device.VhostUserDeviceAttrs
 }
 
+// Dump convert and return data in persist format
+func (device *VhostUserBlkDevice) Dump() persistapi.DeviceState {
+	ds := device.GenericDevice.Dump()
+	ds.Type = string(device.DeviceType())
+	ds.VhostUserDev = &persistapi.VhostUserDeviceAttrs{
+		DevID:      device.DevID,
+		SocketPath: device.SocketPath,
+		Type:       string(device.Type),
+		MacAddress: device.MacAddress,
+	}
+	return ds
+}
+
 // It should implement GetAttachCount() and DeviceID() as api.Device implementation
 // here it shares function from *GenericDevice so we don't need duplicate codes

virtcontainers/device/drivers/vhost_user_net.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/kata-containers/runtime/virtcontainers/device/api"
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 	"github.com/kata-containers/runtime/virtcontainers/utils"
 )
 
@@ -72,5 +73,18 @@ func (device *VhostUserNetDevice) GetDeviceInfo() interface{} {
 	return &device.VhostUserDeviceAttrs
 }
 
+// Dump convert and return data in persist format
+func (device *VhostUserNetDevice) Dump() persistapi.DeviceState {
+	ds := device.GenericDevice.Dump()
+	ds.Type = string(device.DeviceType())
+	ds.VhostUserDev = &persistapi.VhostUserDeviceAttrs{
+		DevID:      device.DevID,
+		SocketPath: device.SocketPath,
+		Type:       string(device.Type),
+		MacAddress: device.MacAddress,
+	}
+	return ds
+}
+
 // It should implement GetAttachCount() and DeviceID() as api.Device implementation
 // here it shares function from *GenericDevice so we don't need duplicate codes