Skip to content
This repository has been archived by the owner on May 12, 2021. It is now read-only.

Commit

Permalink
device-manager: refactor device manger
Browse files Browse the repository at this point in the history 
Fixes #50

This commit imports a big logic change:
* host device to be attached or appended now is sandbox level resources,
one device should bind to sandbox/hypervisor first, then container could
reference it via device's unique ID.
* attach or detach device should go through the device manager interface
instead of the device interface.
* allocate device ID in global device mapper to guarantee every device
has a uniq device ID and there won't be any ID collision.

With this change, there will some changes on data format on disk for sandbox
and container, these changes also make a breakage of backward compatibility.

New persist data format:
* every sandbox will get a new "devices.json" file under "/run/vc/sbs/<sid>/"
which saves detailed device information, this also conforms to the concept that
device should be sandbox level resource.
* every container uses a "devices.json" file but with new data format:
```
[
  {
    "ID": "b80d4736e70a471f",
    "ContainerPath": "/dev/zero"
  },
  {
    "ID": "6765a06e0aa0897d",
    "ContainerPath": "/dev/null"
  }
]
```
`ID` should reference to a device in a sandbox, `ContainerPath` indicates device
path inside a container.

Signed-off-by: Zhang Wei <[email protected]>
  • Loading branch information
@WeiZhang555
WeiZhang555 committed Jul 31, 2018
1 parent eec7fa3 commit f905c16
Show file tree
Hide file tree
Showing 7 changed files with 210 additions and 67 deletions.
101 changes: 75 additions & 26 deletions virtcontainers/container.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,8 @@ import (
"github.com/sirupsen/logrus"
"golang.org/x/sys/unix"

"github.com/kata-containers/runtime/virtcontainers/device/api"
"github.com/kata-containers/runtime/virtcontainers/device/config"
"github.com/kata-containers/runtime/virtcontainers/device/drivers"
"github.com/kata-containers/runtime/virtcontainers/device/manager"
"github.com/kata-containers/runtime/virtcontainers/utils"
)

Expand Down Expand Up @@ -229,6 +228,14 @@ type SystemMountsInfo struct {
DevShmSize uint
}

// ContainerDevice describes a device associated with container
type ContainerDevice struct {
// ID is device id referencing the device from sandbox's device manager
ID string
// ContainerPath is device path displayed in container
ContainerPath string
}

// Container is composed of a set of containers and a runtime environment.
// A Container can be created, deleted, started, stopped, listed, entered, paused and restored.
type Container struct {
Expand All @@ -251,7 +258,7 @@ type Container struct {

mounts []Mount

devices []api.Device
devices []ContainerDevice

systemMountsInfo SystemMountsInfo
}
Expand Down Expand Up @@ -362,7 +369,7 @@ func (c *Container) storeDevices() error {
return c.sandbox.storage.storeContainerDevices(c.sandboxID, c.id, c.devices)
}

func (c *Container) fetchDevices() ([]api.Device, error) {
func (c *Container) fetchDevices() ([]ContainerDevice, error) {
return c.sandbox.storage.fetchContainerDevices(c.sandboxID, c.id)
}

Expand Down Expand Up @@ -430,29 +437,19 @@ func (c *Container) mountSharedDirMounts(hostSharedDir, guestSharedDir string) (
continue
}

var stat unix.Stat_t
if err := unix.Stat(m.Source, &stat); err != nil {
return nil, err
}

// Check if mount is a block device file. If it is, the block device will be attached to the host
// instead of passing this as a shared mount.
if c.checkBlockDeviceSupport() && stat.Mode&unix.S_IFBLK == unix.S_IFBLK {
// TODO: remove dependency of package drivers
b := &drivers.BlockDevice{
DeviceInfo: &config.DeviceInfo{
HostPath: m.Source,
ContainerPath: m.Destination,
DevType: "b",
},
}

if len(m.BlockDeviceID) > 0 {
// Attach this block device, all other devices passed in the config have been attached at this point
if err := b.Attach(c.sandbox); err != nil {
if err := c.sandbox.devManager.AttachDevice(m.BlockDeviceID, c.sandbox); err != nil &&
err != manager.ErrDeviceAttached {
return nil, err
}

c.mounts[idx].BlockDevice = b
if err := c.sandbox.storeSandboxDevices(); err != nil {
//TODO: roll back?
return nil, err
}
continue
}

Expand Down Expand Up @@ -562,7 +559,41 @@ func newContainer(sandbox *Sandbox, contConfig ContainerConfig) (*Container, err

mounts, err := c.fetchMounts()
if err == nil {
// restore mounts from disk
c.mounts = mounts
} else {
// for newly created container:
// iterate all mounts and create block device if it's block based.
for i, m := range c.mounts {
if len(m.BlockDeviceID) > 0 || m.Type != "bind" {
// Non-empty m.BlockDeviceID indicates there's already one device
// associated with the mount,so no need to create a new device for it
// and we only create block device for bind mount
continue
}

var stat unix.Stat_t
if err := unix.Stat(m.Source, &stat); err != nil {
return nil, fmt.Errorf("stat %q failed: %v", m.Source, err)
}

// Check if mount is a block device file. If it is, the block device will be attached to the host
// instead of passing this as a shared mount.
if c.checkBlockDeviceSupport() && stat.Mode&unix.S_IFBLK == unix.S_IFBLK {
b, err := c.sandbox.devManager.NewDevice(config.DeviceInfo{
HostPath: m.Source,
ContainerPath: m.Destination,
DevType: "b",
Major: int64(unix.Major(stat.Rdev)),
Minor: int64(unix.Minor(stat.Rdev)),
})
if err != nil {
return nil, fmt.Errorf("device manager failed to create new device for %q: %v", m.Source, err)
}

c.mounts[i].BlockDeviceID = b.DeviceID()
}
}
}

// Devices will be found in storage after create stage has completed.
Expand All @@ -578,9 +609,14 @@ func newContainer(sandbox *Sandbox, contConfig ContainerConfig) (*Container, err
if err != nil {
return &Container{}, err
}
c.devices = append(c.devices, dev)

c.devices = append(c.devices, ContainerDevice{
ID: dev.DeviceID(),
ContainerPath: info.ContainerPath,
})
}
}

return c, nil
}

Expand Down Expand Up @@ -1020,6 +1056,7 @@ func (c *Container) hotplugDrive() error {
return err
}

// TODO: use general device manager instead of BlockDrive directly
// Add drive with id as container id
devID := utils.MakeNameID("drive", c.id, maxDevIDSize)
drive := config.BlockDrive{
Expand Down Expand Up @@ -1076,18 +1113,30 @@ func (c *Container) removeDrive() (err error) {
}

func (c *Container) attachDevices() error {
for _, device := range c.devices {
if err := device.Attach(c.sandbox); err != nil {
for _, dev := range c.devices {
if err := c.sandbox.devManager.AttachDevice(dev.ID, c.sandbox); err != nil {
if err == manager.ErrDeviceAttached {
// skip if device is already attached before
continue
}
return err
}
}

if err := c.sandbox.storeSandboxDevices(); err != nil {
//TODO: roll back?
return err
}
return nil
}

func (c *Container) detachDevices() error {
for _, device := range c.devices {
if err := device.Detach(c.sandbox); err != nil {
for _, dev := range c.devices {
if err := c.sandbox.devManager.DetachDevice(dev.ID, c.sandbox); err != nil {
if err == manager.ErrDeviceNotAttached {
// skip if device is already attached before
continue
}
return err
}
}
Expand Down
85 changes: 75 additions & 10 deletions virtcontainers/filesystem.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,9 @@ const (

// devicesFileType represents a device file type
devicesFileType

// devicesIDFileType saves reference IDs to file, e.g. device IDs
devicesIDFileType
)

// configFile is the file name used for every JSON sandbox configuration.
Expand Down Expand Up @@ -126,6 +129,8 @@ type resourceStorage interface {
fetchSandboxState(sandboxID string) (State, error)
fetchSandboxNetwork(sandboxID string) (NetworkNamespace, error)
storeSandboxNetwork(sandboxID string, networkNS NetworkNamespace) error
fetchSandboxDevices(sandboxID string) ([]api.Device, error)
storeSandboxDevices(sandboxID string, devices []api.Device) error

// Hypervisor resources
fetchHypervisorState(sandboxID string, state interface{}) error
Expand All @@ -144,8 +149,8 @@ type resourceStorage interface {
storeContainerProcess(sandboxID, containerID string, process Process) error
fetchContainerMounts(sandboxID, containerID string) ([]Mount, error)
storeContainerMounts(sandboxID, containerID string, mounts []Mount) error
fetchContainerDevices(sandboxID, containerID string) ([]api.Device, error)
storeContainerDevices(sandboxID, containerID string, devices []api.Device) error
fetchContainerDevices(sandboxID, containerID string) ([]ContainerDevice, error)
storeContainerDevices(sandboxID, containerID string, devices []ContainerDevice) error
}

// filesystem is a resourceStorage interface implementation for a local filesystem.
Expand Down Expand Up @@ -227,6 +232,35 @@ type TypedDevice struct {
Data json.RawMessage
}

// storeDeviceIDFile is used to marshal and store device IDs to disk.
func (fs *filesystem) storeDeviceIDFile(file string, data interface{}) error {
if file == "" {
return errNeedFile
}

f, err := os.Create(file)
if err != nil {
return err
}
defer f.Close()

devices, ok := data.([]ContainerDevice)
if !ok {
return fmt.Errorf("Incorrect data type received, Expected []string")
}

jsonOut, err := json.Marshal(devices)
if err != nil {
return fmt.Errorf("Could not marshal devices: %s", err)
}

if _, err := f.Write(jsonOut); err != nil {
return err
}

return nil
}

// storeDeviceFile is used to provide custom marshalling for Device objects.
// Device is first marshalled into TypedDevice to include the type
// of the Device object.
Expand Down Expand Up @@ -347,7 +381,7 @@ func (fs *filesystem) fetchDeviceFile(fileData []byte, devices *[]api.Device) er
func resourceNeedsContainerID(sandboxSpecific bool, resource sandboxResource) bool {

switch resource {
case lockFileType, networkFileType, hypervisorFileType, agentFileType:
case lockFileType, networkFileType, hypervisorFileType, agentFileType, devicesFileType:
// sandbox-specific resources
return false
default:
Expand All @@ -370,7 +404,7 @@ func resourceDir(sandboxSpecific bool, sandboxID, containerID string, resource s
case configFileType:
path = configStoragePath
break
case stateFileType, networkFileType, processFileType, lockFileType, mountsFileType, devicesFileType, hypervisorFileType, agentFileType:
case stateFileType, networkFileType, processFileType, lockFileType, mountsFileType, devicesFileType, devicesIDFileType, hypervisorFileType, agentFileType:
path = runStoragePath
break
default:
Expand Down Expand Up @@ -421,6 +455,9 @@ func (fs *filesystem) resourceURI(sandboxSpecific bool, sandboxID, containerID s
case devicesFileType:
filename = devicesFile
break
case devicesIDFileType:
filename = devicesFile
break
default:
return "", "", errInvalidResource
}
Expand Down Expand Up @@ -466,6 +503,7 @@ func (fs *filesystem) commonResourceChecks(sandboxSpecific bool, sandboxID, cont
case processFileType:
case mountsFileType:
case devicesFileType:
case devicesIDFileType:
default:
return errInvalidResource
}
Expand Down Expand Up @@ -552,6 +590,19 @@ func (fs *filesystem) storeDeviceResource(sandboxSpecific bool, sandboxID, conta
return fs.storeDeviceFile(devicesFile, file)
}

func (fs *filesystem) storeDevicesIDResource(sandboxSpecific bool, sandboxID, containerID string, resource sandboxResource, file interface{}) error {
if resource != devicesIDFileType {
return errInvalidResource
}

devicesFile, _, err := fs.resourceURI(sandboxSpecific, sandboxID, containerID, resource)
if err != nil {
return err
}

return fs.storeDeviceIDFile(devicesFile, file)
}

func (fs *filesystem) storeResource(sandboxSpecific bool, sandboxID, containerID string, resource sandboxResource, data interface{}) error {
if err := fs.commonResourceChecks(sandboxSpecific, sandboxID, containerID, resource); err != nil {
return err
Expand All @@ -575,6 +626,8 @@ func (fs *filesystem) storeResource(sandboxSpecific bool, sandboxID, containerID

case []api.Device:
return fs.storeDeviceResource(sandboxSpecific, sandboxID, containerID, resource, file)
case []ContainerDevice:
return fs.storeDevicesIDResource(sandboxSpecific, sandboxID, containerID, resource, file)

default:
return fmt.Errorf("Invalid resource data type")
Expand Down Expand Up @@ -628,6 +681,18 @@ func (fs *filesystem) fetchSandboxNetwork(sandboxID string) (NetworkNamespace, e
return networkNS, nil
}

func (fs *filesystem) fetchSandboxDevices(sandboxID string) ([]api.Device, error) {
var devices []api.Device
if err := fs.fetchResource(true, sandboxID, "", devicesFileType, &devices); err != nil {
return []api.Device{}, err
}
return devices, nil
}

func (fs *filesystem) storeSandboxDevices(sandboxID string, devices []api.Device) error {
return fs.storeSandboxResource(sandboxID, devicesFileType, devices)
}

func (fs *filesystem) fetchHypervisorState(sandboxID string, state interface{}) error {
return fs.fetchResource(true, sandboxID, "", hypervisorFileType, state)
}
Expand Down Expand Up @@ -734,11 +799,11 @@ func (fs *filesystem) fetchContainerMounts(sandboxID, containerID string) ([]Mou
return mounts, nil
}

func (fs *filesystem) fetchContainerDevices(sandboxID, containerID string) ([]api.Device, error) {
var devices []api.Device
func (fs *filesystem) fetchContainerDevices(sandboxID, containerID string) ([]ContainerDevice, error) {
var devices []ContainerDevice

if err := fs.fetchResource(false, sandboxID, containerID, devicesFileType, &devices); err != nil {
return []api.Device{}, err
if err := fs.fetchResource(false, sandboxID, containerID, devicesIDFileType, &devices); err != nil {
return nil, err
}

return devices, nil
Expand All @@ -748,8 +813,8 @@ func (fs *filesystem) storeContainerMounts(sandboxID, containerID string, mounts
return fs.storeContainerResource(sandboxID, containerID, mountsFileType, mounts)
}

func (fs *filesystem) storeContainerDevices(sandboxID, containerID string, devices []api.Device) error {
return fs.storeContainerResource(sandboxID, containerID, devicesFileType, devices)
func (fs *filesystem) storeContainerDevices(sandboxID, containerID string, devices []ContainerDevice) error {
return fs.storeContainerResource(sandboxID, containerID, devicesIDFileType, devices)
}

func (fs *filesystem) deleteContainerResources(sandboxID, containerID string, resources []sandboxResource) error {
Expand Down
Loading

0 comments on commit f905c16

Please sign in to comment.