Skip to content

Commit

Permalink
devices: use device manager to manage all devices
Browse files Browse the repository at this point in the history 
Fixes kata-containers#50

Previously the devices are created with device manager and laterly
attached to hypervisor with "device.Attach()", this could work, but
there's no way to remember the reference count for every device, which
means if we plug one device to hypervisor twice, it's truly inserted
twice, but actually we only need to insert once but use it in many
places.

Use device manager as a consolidated entrypoint of device management can
give us a way to handle many "references" to single device, because it
can save all devices and remember it's use count.

Signed-off-by: Wei Zhang <[email protected]>
  • Loading branch information
@WeiZhang555
WeiZhang555 committed Jul 31, 2018
1 parent f4a7712 commit 1194154
Show file tree
Hide file tree
Showing 13 changed files with 360 additions and 165 deletions.
18 changes: 9 additions & 9 deletions virtcontainers/container.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -440,8 +440,7 @@ func (c *Container) mountSharedDirMounts(hostSharedDir, guestSharedDir string) (
if c.checkBlockDeviceSupport() && stat.Mode&unix.S_IFBLK == unix.S_IFBLK {
// TODO: remove dependency of package drivers
b := &drivers.BlockDevice{
DevType: config.DeviceBlock,
DeviceInfo: config.DeviceInfo{
DeviceInfo: &config.DeviceInfo{
HostPath: m.Source,
ContainerPath: m.Destination,
DevType: "b",
Expand Down Expand Up @@ -574,12 +573,13 @@ func newContainer(sandbox *Sandbox, contConfig ContainerConfig) (*Container, err
} else {
// If devices were not found in storage, create Device implementations
// from the configuration. This should happen at create.

devices, err := sandbox.devManager.NewDevices(contConfig.DeviceInfos)
if err != nil {
return &Container{}, err
for _, info := range contConfig.DeviceInfos {
dev, err := sandbox.devManager.NewDevice(info)
if err != nil {
return &Container{}, err
}
c.devices = append(c.devices, dev)
}
c.devices = devices
}
return c, nil
}
Expand Down Expand Up @@ -1022,7 +1022,7 @@ func (c *Container) hotplugDrive() error {

// Add drive with id as container id
devID := utils.MakeNameID("drive", c.id, maxDevIDSize)
drive := drivers.Drive{
drive := config.BlockDrive{
File: devicePath,
Format: "raw",
ID: devID,
Expand Down Expand Up @@ -1059,7 +1059,7 @@ func (c *Container) removeDrive() (err error) {
c.Logger().Info("unplugging block device")

devID := utils.MakeNameID("drive", c.id, maxDevIDSize)
drive := &drivers.Drive{
drive := &config.BlockDrive{
ID: devID,
}

Expand Down
21 changes: 20 additions & 1 deletion virtcontainers/device/api/interface.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ func DeviceLogger() *logrus.Entry {
// DeviceReceiver is an interface used for accepting devices
// a device should be attached/added/plugged to a DeviceReceiver
type DeviceReceiver interface {
// these are for hotplug/hot-unplug devices to/from hypervisor
HotplugAddDevice(Device, config.DeviceType) error
HotplugRemoveDevice(Device, config.DeviceType) error

Expand All @@ -51,11 +52,29 @@ type VhostUserDevice interface {
type Device interface {
Attach(DeviceReceiver) error
Detach(DeviceReceiver) error
// ID returns device identifier
DeviceID() string
// DeviceType indicates which kind of device it is
// e.g. block, vfio or vhost user
DeviceType() config.DeviceType
// GetDeviceInfo returns device information that the device is created based on
GetDeviceInfo() *config.DeviceInfo
// GetDeviceDrive returns device specific data used for hotplugging by hypervisor
// Caller could cast the return value to device specific struct
// e.g. Block device returns *config.BlockDrive and
// vfio device returns *config.VFIODrive
GetDeviceDrive() interface{}
// IsAttached checks if the device is attached
IsAttached() bool
}

// DeviceManager can be used to create a new device, this can be used as single
// device management object.
type DeviceManager interface {
NewDevices(devInfos []config.DeviceInfo) ([]Device, error)
NewDevice(config.DeviceInfo) (Device, error)
AttachDevice(string, DeviceReceiver) error
DetachDevice(string, DeviceReceiver) error
IsDeviceAttached(string) bool
GetDeviceByID(string) Device
GetAllDevices() []Device
}
38 changes: 36 additions & 2 deletions virtcontainers/device/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,10 +49,10 @@ var SysIOMMUPath = "/sys/kernel/iommu_groups"

// DeviceInfo is an embedded type that contains device data common to all types of devices.
type DeviceInfo struct {
// Device path on host
// Hostpath is device path on host
HostPath string

// Device path inside the container
// ContainerPath is device path inside container
ContainerPath string

// Type of device: c, b, u or p
Expand Down Expand Up @@ -87,6 +87,40 @@ type DeviceInfo struct {
DriverOptions map[string]string
}

// BlockDrive represents a block storage drive which may be used in case the storage
// driver has an underlying block storage device.
type BlockDrive struct {
// File is the path to the disk-image/device which will be used with this drive
File string

// Format of the drive
Format string

// ID is used to identify this drive in the hypervisor options.
ID string

// Index assigned to the drive. In case of virtio-scsi, this is used as SCSI LUN index
Index int

// PCIAddr is the PCI address used to identify the slot at which the drive is attached.
PCIAddr string

// SCSI Address of the block device, in case the device is attached using SCSI driver
// SCSI address is in the format SCSI-Id:LUN
SCSIAddr string

// VirtPath at which the device appears inside the VM, outside of the container mount namespace
VirtPath string
}

// VFIODrive represents a VFIO drive used for hotplugging
type VFIODrive struct {
// ID is used to identify this drive in the hypervisor options.
ID string
// BDF (Bus:Device.Function) of the PCI address
BDF string
}

// VhostUserDeviceAttrs represents data shared by most vhost-user devices
type VhostUserDeviceAttrs struct {
DevType DeviceType
Expand Down
106 changes: 42 additions & 64 deletions virtcontainers/device/drivers/block.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
package drivers

import (
"encoding/hex"
"path/filepath"

"github.com/kata-containers/runtime/virtcontainers/device/api"
Expand All @@ -17,62 +16,24 @@ import (

const maxDevIDSize = 31

// Drive represents a block storage drive which may be used in case the storage
// driver has an underlying block storage device.
type Drive struct {

// Path to the disk-image/device which will be used with this drive
File string

// Format of the drive
Format string

// ID is used to identify this drive in the hypervisor options.
ID string

// Index assigned to the drive. In case of virtio-scsi, this is used as SCSI LUN index
Index int

// PCIAddr is the PCI address used to identify the slot at which the drive is attached.
PCIAddr string
}

// BlockDevice refers to a block storage device implementation.
type BlockDevice struct {
DevType config.DeviceType
DeviceInfo config.DeviceInfo

// SCSI Address of the block device, in case the device is attached using SCSI driver
// SCSI address is in the format SCSI-Id:LUN
SCSIAddr string

// Path at which the device appears inside the VM, outside of the container mount namespace
VirtPath string

// PCI Slot of the block device
PCIAddr string

BlockDrive *Drive
ID string
DeviceInfo *config.DeviceInfo
BlockDrive *config.BlockDrive
}

// NewBlockDevice creates a new block device based on DeviceInfo
func NewBlockDevice(devInfo config.DeviceInfo) *BlockDevice {
func NewBlockDevice(devInfo *config.DeviceInfo) *BlockDevice {
return &BlockDevice{
DevType: config.DeviceBlock,
ID: devInfo.ID,
DeviceInfo: devInfo,
}
}

// Attach is standard interface of api.Device, it's used to add device to some
// DeviceReceiver
func (device *BlockDevice) Attach(devReceiver api.DeviceReceiver) (err error) {
randBytes, err := utils.GenerateRandomBytes(8)
if err != nil {
return err
}

device.DeviceInfo.ID = hex.EncodeToString(randBytes)

// Increment the block index for the sandbox. This is used to determine the name
// for the block device in the case where the block device is used as container
// rootfs and the predicted block device name needs to be provided to the agent.
Expand All @@ -88,58 +49,75 @@ func (device *BlockDevice) Attach(devReceiver api.DeviceReceiver) (err error) {
return err
}

drive := Drive{
drive := &config.BlockDrive{
File: device.DeviceInfo.HostPath,
Format: "raw",
ID: utils.MakeNameID("drive", device.DeviceInfo.ID, maxDevIDSize),
Index: index,
}

deviceLogger().WithField("device", device.DeviceInfo.HostPath).Info("Attaching block device")
device.BlockDrive = &drive
if err = devReceiver.HotplugAddDevice(device, config.DeviceBlock); err != nil {
return err
}

device.DeviceInfo.Hotplugged = true

driveName, err := utils.GetVirtDriveName(index)
if err != nil {
return err
}

customOptions := device.DeviceInfo.DriverOptions
if customOptions != nil && customOptions["block-driver"] == "virtio-blk" {
device.VirtPath = filepath.Join("/dev", driveName)
device.PCIAddr = drive.PCIAddr
drive.VirtPath = filepath.Join("/dev", driveName)
} else {
scsiAddr, err := utils.GetSCSIAddress(index)
if err != nil {
return err
}

device.SCSIAddr = scsiAddr
drive.SCSIAddr = scsiAddr
}

deviceLogger().WithField("device", device.DeviceInfo.HostPath).Info("Attaching block device")
device.BlockDrive = drive
if err = devReceiver.HotplugAddDevice(device, config.DeviceBlock); err != nil {
return err
}

device.DeviceInfo.Hotplugged = true

return nil
}

// Detach is standard interface of api.Device, it's used to remove device from some
// DeviceReceiver
func (device *BlockDevice) Detach(devReceiver api.DeviceReceiver) error {
if device.DeviceInfo.Hotplugged {
deviceLogger().WithField("device", device.DeviceInfo.HostPath).Info("Unplugging block device")

if err := devReceiver.HotplugRemoveDevice(device, config.DeviceBlock); err != nil {
deviceLogger().WithError(err).Error("Failed to unplug block device")
return err
}
deviceLogger().WithField("device", device.DeviceInfo.HostPath).Info("Unplugging block device")

if err := devReceiver.HotplugRemoveDevice(device, config.DeviceBlock); err != nil {
deviceLogger().WithError(err).Error("Failed to unplug block device")
return err
}
device.DeviceInfo.Hotplugged = false
return nil
}

// IsAttached checks if the device is attached
func (device *BlockDevice) IsAttached() bool {
return device.DeviceInfo.Hotplugged
}

// DeviceType is standard interface of api.Device, it returns device type
func (device *BlockDevice) DeviceType() config.DeviceType {
return device.DevType
return config.DeviceBlock
}

// DeviceID returns device ID
func (device *BlockDevice) DeviceID() string {
return device.ID
}

// GetDeviceInfo returns device information that the device is created based on
func (device *BlockDevice) GetDeviceInfo() *config.DeviceInfo {
return device.DeviceInfo
}

// GetDeviceDrive returns device information used for creating
func (device *BlockDevice) GetDeviceDrive() interface{} {
return device.BlockDrive
}
30 changes: 25 additions & 5 deletions virtcontainers/device/drivers/generic.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,14 @@ import (

// GenericDevice refers to a device that is neither a VFIO device or block device.
type GenericDevice struct {
DevType config.DeviceType
DeviceInfo config.DeviceInfo
ID string
DeviceInfo *config.DeviceInfo
}

// NewGenericDevice creates a new GenericDevice
func NewGenericDevice(devInfo config.DeviceInfo) *GenericDevice {
func NewGenericDevice(devInfo *config.DeviceInfo) *GenericDevice {
return &GenericDevice{
DevType: config.DeviceGeneric,
ID: devInfo.ID,
DeviceInfo: devInfo,
}
}
Expand All @@ -35,7 +35,27 @@ func (device *GenericDevice) Detach(devReceiver api.DeviceReceiver) error {
return nil
}

// IsAttached checks if the device is attached
func (device *GenericDevice) IsAttached() bool {
return device.DeviceInfo.Hotplugged
}

// DeviceID returns device ID
func (device *GenericDevice) DeviceID() string {
return device.ID
}

// DeviceType is standard interface of api.Device, it returns device type
func (device *GenericDevice) DeviceType() config.DeviceType {
return device.DevType
return config.DeviceGeneric
}

// GetDeviceInfo returns device information that the device is created based on
func (device *GenericDevice) GetDeviceInfo() *config.DeviceInfo {
return device.DeviceInfo
}

// GetDeviceDrive returns device information used for creating
func (device *GenericDevice) GetDeviceDrive() interface{} {
return device.DeviceInfo
}
Loading

0 comments on commit 1194154

Please sign in to comment.