Skip to content

Commit

Permalink
hypervisor: decouple hypervisor from sandbox
Browse files Browse the repository at this point in the history 
A hypervisor implementation does not need to depend on a sandbox
structure. Decouple them in preparation for vm factory.

Signed-off-by: Peng Tao <[email protected]>
  • Loading branch information
@bergwolf
bergwolf committed Jul 19, 2018
1 parent 4ac6754 commit 18e6a6e
Show file tree
Hide file tree
Showing 7 changed files with 107 additions and 72 deletions.
4 changes: 2 additions & 2 deletions virtcontainers/hypervisor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -499,8 +499,8 @@ func RunningOnVMM(cpuInfoPath string) (bool, error) {
// hypervisor is the virtcontainers hypervisor interface.
// The default hypervisor implementation is Qemu.
type hypervisor interface {
init(sandbox *Sandbox) error
createSandbox(sandboxConfig SandboxConfig) error
init(id string, hypervisorConfig *HypervisorConfig, vmConfig Resources, storage resourceStorage) error
createSandbox() error
startSandbox() error
waitSandbox(timeout int) error
stopSandbox() error
Expand Down
6 changes: 3 additions & 3 deletions virtcontainers/mock_hypervisor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@ type mockHypervisor struct {
vCPUs uint32
}

func (m *mockHypervisor) init(sandbox *Sandbox) error {
valid, err := sandbox.config.HypervisorConfig.valid()
func (m *mockHypervisor) init(id string, hypervisorConfig *HypervisorConfig, vmConfig Resources, storage resourceStorage) error {
valid, err := hypervisorConfig.valid()
if valid == false || err != nil {
return err
}
Expand All @@ -22,7 +22,7 @@ func (m *mockHypervisor) capabilities() capabilities {
return capabilities{}
}

func (m *mockHypervisor) createSandbox(sandboxConfig SandboxConfig) error {
func (m *mockHypervisor) createSandbox() error {
return nil
}

Expand Down
10 changes: 5 additions & 5 deletions virtcontainers/mock_hypervisor_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,16 +15,18 @@ func TestMockHypervisorInit(t *testing.T) {

sandbox := &Sandbox{
config: &SandboxConfig{
ID: "mock_sandbox",
HypervisorConfig: HypervisorConfig{
KernelPath: "",
ImagePath: "",
HypervisorPath: "",
},
},
storage: &filesystem{},
}

// wrong config
if err := m.init(sandbox); err == nil {
if err := m.init(sandbox.config.ID, &sandbox.config.HypervisorConfig, sandbox.config.VMConfig, sandbox.storage); err == nil {
t.Fatal()
}

Expand All @@ -35,17 +37,15 @@ func TestMockHypervisorInit(t *testing.T) {
}

// right config
if err := m.init(sandbox); err != nil {
if err := m.init(sandbox.config.ID, &sandbox.config.HypervisorConfig, sandbox.config.VMConfig, sandbox.storage); err != nil {
t.Fatal(err)
}
}

func TestMockHypervisorCreateSandbox(t *testing.T) {
var m *mockHypervisor

config := SandboxConfig{}

if err := m.createSandbox(config); err != nil {
if err := m.createSandbox(); err != nil {
t.Fatal(err)
}
}
Expand Down
147 changes: 91 additions & 56 deletions virtcontainers/qemu.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,16 +47,18 @@ type QemuState struct {

// qemu is an Hypervisor interface implementation for the Linux qemu hypervisor.
type qemu struct {
id string

vmConfig Resources

storage resourceStorage

config HypervisorConfig

qmpMonitorCh qmpChannel

qemuConfig govmmQemu.Config

sandbox *Sandbox

state QemuState

arch qemuArch
Expand All @@ -66,7 +68,7 @@ const qmpCapErrMsg = "Failed to negoatiate QMP capabilities"

const qmpSocket = "qmp.sock"

const defaultConsole = "console.sock"
const consoleSocket = "console.sock"

var qemuMajorVersion int
var qemuMinorVersion int
Expand Down Expand Up @@ -170,25 +172,26 @@ func (q *qemu) qemuPath() (string, error) {
}

// init intializes the Qemu structure.
func (q *qemu) init(sandbox *Sandbox) error {
valid, err := sandbox.config.HypervisorConfig.valid()
func (q *qemu) init(id string, hypervisorConfig *HypervisorConfig, vmConfig Resources, storage resourceStorage) error {
valid, err := hypervisorConfig.valid()
if valid == false || err != nil {
return err
}

q.vmConfig = sandbox.config.VMConfig
q.config = sandbox.config.HypervisorConfig
q.sandbox = sandbox
q.id = id
q.storage = storage
q.vmConfig = vmConfig
q.config = *hypervisorConfig
q.arch = newQemuArch(q.config)

if err = sandbox.storage.fetchHypervisorState(sandbox.id, &q.state); err != nil {
if err = q.storage.fetchHypervisorState(q.id, &q.state); err != nil {
q.Logger().Debug("Creating bridges")
q.state.Bridges = q.arch.bridges(q.config.DefaultBridges)

q.Logger().Debug("Creating UUID")
q.state.UUID = uuid.Generate().String()

if err = sandbox.storage.storeHypervisorState(sandbox.id, q.state); err != nil {
if err = q.storage.storeHypervisorState(q.id, q.state); err != nil {
return err
}
}
Expand Down Expand Up @@ -238,17 +241,17 @@ func (q *qemu) memoryTopology() (govmmQemu.Memory, error) {
return q.arch.memoryTopology(memMb, hostMemMb), nil
}

func (q *qemu) qmpSocketPath(sandboxID string) (string, error) {
return utils.BuildSocketPath(runStoragePath, sandboxID, qmpSocket)
func (q *qemu) qmpSocketPath(id string) (string, error) {
return utils.BuildSocketPath(runStoragePath, id, qmpSocket)
}

func (q *qemu) getQemuMachine(sandboxConfig SandboxConfig) (govmmQemu.Machine, error) {
func (q *qemu) getQemuMachine() (govmmQemu.Machine, error) {
machine, err := q.arch.machine()
if err != nil {
return govmmQemu.Machine{}, err
}

accelerators := sandboxConfig.HypervisorConfig.MachineAccelerators
accelerators := q.config.MachineAccelerators
if accelerators != "" {
if !strings.HasPrefix(accelerators, ",") {
accelerators = fmt.Sprintf(",%s", accelerators)
Expand All @@ -275,11 +278,65 @@ func (q *qemu) appendImage(devices []govmmQemu.Device) ([]govmmQemu.Device, erro
return devices, nil
}

// createSandbox is the Hypervisor sandbox creation implementation for govmmQemu.
func (q *qemu) createSandbox(sandboxConfig SandboxConfig) error {
func (q *qemu) createQmpSocket() ([]govmmQemu.QMPSocket, error) {
monitorSockPath, err := q.qmpSocketPath(q.id)
if err != nil {
return nil, err
}

q.qmpMonitorCh = qmpChannel{
ctx: context.Background(),
path: monitorSockPath,
}

err = os.MkdirAll(filepath.Dir(monitorSockPath), dirMode)
if err != nil {
return nil, err
}

return []govmmQemu.QMPSocket{
{
Type: "unix",
Name: q.qmpMonitorCh.path,
Server: true,
NoWait: true,
},
}, nil
}

func (q *qemu) buildDevices(initrdPath string) ([]govmmQemu.Device, *govmmQemu.IOThread, error) {
var devices []govmmQemu.Device

machine, err := q.getQemuMachine(sandboxConfig)
console, err := q.getSandboxConsole(q.id)
if err != nil {
return nil, nil, err
}

// Add bridges before any other devices. This way we make sure that
// bridge gets the first available PCI address i.e bridgePCIStartAddr
devices = q.arch.appendBridges(devices, q.state.Bridges)

devices = q.arch.appendConsole(devices, console)

if initrdPath == "" {
devices, err = q.appendImage(devices)
if err != nil {
return nil, nil, err
}
}

var ioThread *govmmQemu.IOThread
if q.config.BlockDeviceDriver == VirtioSCSI {
devices, ioThread = q.arch.appendSCSIController(devices, q.config.EnableIOThreads)
}

return devices, ioThread, nil

}

// createSandbox is the Hypervisor sandbox creation implementation for govmmQemu.
func (q *qemu) createSandbox() error {
machine, err := q.getQemuMachine()
if err != nil {
return err
}
Expand Down Expand Up @@ -314,7 +371,7 @@ func (q *qemu) createSandbox(sandboxConfig SandboxConfig) error {

// Pass the sandbox name to the agent via the kernel command-line to
// allow the agent to use it in log messages.
params := q.kernelParameters() + " " + "agent.sandbox=" + sandboxConfig.ID
params := q.kernelParameters() + " " + "agent.sandbox=" + q.id

kernel := govmmQemu.Kernel{
Path: kernelPath,
Expand All @@ -331,7 +388,7 @@ func (q *qemu) createSandbox(sandboxConfig SandboxConfig) error {
return fmt.Errorf("UUID should not be empty")
}

monitorSockPath, err := q.qmpSocketPath(sandboxConfig.ID)
monitorSockPath, err := q.qmpSocketPath(q.id)
if err != nil {
return err
}
Expand All @@ -346,41 +403,19 @@ func (q *qemu) createSandbox(sandboxConfig SandboxConfig) error {
return err
}

qmpSockets := []govmmQemu.QMPSocket{
{
Type: "unix",
Name: q.qmpMonitorCh.path,
Server: true,
NoWait: true,
},
}

// Add bridges before any other devices. This way we make sure that
// bridge gets the first available PCI address i.e bridgePCIStartAddr
devices = q.arch.appendBridges(devices, q.state.Bridges)

console, err := q.getSandboxConsole(sandboxConfig.ID)
qmpSockets, err := q.createQmpSocket()
if err != nil {
return err
}

devices = q.arch.appendConsole(devices, console)

if initrdPath == "" {
devices, err = q.appendImage(devices)
if err != nil {
return err
}
}

var ioThread *govmmQemu.IOThread
if q.config.BlockDeviceDriver == VirtioSCSI {
devices, ioThread = q.arch.appendSCSIController(devices, q.config.EnableIOThreads)
devices, ioThread, err := q.buildDevices(initrdPath)
if err != nil {
return err
}

cpuModel := q.arch.cpuModel()

firmwarePath, err := sandboxConfig.HypervisorConfig.FirmwareAssetPath()
firmwarePath, err := q.config.FirmwareAssetPath()
if err != nil {
return err
}
Expand All @@ -391,7 +426,7 @@ func (q *qemu) createSandbox(sandboxConfig SandboxConfig) error {
}

qemuConfig := govmmQemu.Config{
Name: fmt.Sprintf("sandbox-%s", sandboxConfig.ID),
Name: fmt.Sprintf("sandbox-%s", q.id),
UUID: q.state.UUID,
Path: qemuPath,
Ctx: q.qmpMonitorCh.ctx,
Expand Down Expand Up @@ -735,7 +770,7 @@ func (q *qemu) hotplugAddDevice(devInfo interface{}, devType deviceType) (interf
return data, err
}

return data, q.sandbox.storage.storeHypervisorState(q.sandbox.id, q.state)
return data, q.storage.storeHypervisorState(q.id, q.state)
}

func (q *qemu) hotplugRemoveDevice(devInfo interface{}, devType deviceType) (interface{}, error) {
Expand All @@ -744,7 +779,7 @@ func (q *qemu) hotplugRemoveDevice(devInfo interface{}, devType deviceType) (int
return data, err
}

return data, q.sandbox.storage.storeHypervisorState(q.sandbox.id, q.state)
return data, q.storage.storeHypervisorState(q.id, q.state)
}

func (q *qemu) hotplugCPUs(vcpus uint32, op operation) (uint32, error) {
Expand Down Expand Up @@ -820,12 +855,12 @@ func (q *qemu) hotplugAddCPUs(amount uint32) (uint32, error) {
hotpluggedVCPUs++
if hotpluggedVCPUs == amount {
// All vCPUs were hotplugged
return amount, q.sandbox.storage.storeHypervisorState(q.sandbox.id, q.state)
return amount, q.storage.storeHypervisorState(q.id, q.state)
}
}

// All vCPUs were NOT hotplugged
if err := q.sandbox.storage.storeHypervisorState(q.sandbox.id, q.state); err != nil {
if err := q.storage.storeHypervisorState(q.id, q.state); err != nil {
q.Logger().Errorf("failed to save hypervisor state after hotplug %d vCPUs: %v", hotpluggedVCPUs, err)
}

Expand All @@ -845,15 +880,15 @@ func (q *qemu) hotplugRemoveCPUs(amount uint32) (uint32, error) {
// get the last vCPUs and try to remove it
cpu := q.state.HotpluggedVCPUs[len(q.state.HotpluggedVCPUs)-1]
if err := q.qmpMonitorCh.qmp.ExecuteDeviceDel(q.qmpMonitorCh.ctx, cpu.ID); err != nil {
_ = q.sandbox.storage.storeHypervisorState(q.sandbox.id, q.state)
_ = q.storage.storeHypervisorState(q.id, q.state)
return i, fmt.Errorf("failed to hotunplug CPUs, only %d CPUs were hotunplugged: %v", i, err)
}

// remove from the list the vCPU hotunplugged
q.state.HotpluggedVCPUs = q.state.HotpluggedVCPUs[:len(q.state.HotpluggedVCPUs)-1]
}

return amount, q.sandbox.storage.storeHypervisorState(q.sandbox.id, q.state)
return amount, q.storage.storeHypervisorState(q.id, q.state)
}

func (q *qemu) hotplugMemory(memDev *memoryDevice, op operation) error {
Expand Down Expand Up @@ -910,7 +945,7 @@ func (q *qemu) hotplugAddMemory(memDev *memoryDevice) error {
}

q.state.HotpluggedMemory += memDev.sizeMB
return q.sandbox.storage.storeHypervisorState(q.sandbox.id, q.state)
return q.storage.storeHypervisorState(q.id, q.state)
}

func (q *qemu) pauseSandbox() error {
Expand Down Expand Up @@ -951,8 +986,8 @@ func (q *qemu) addDevice(devInfo interface{}, devType deviceType) error {

// getSandboxConsole builds the path of the console where we can read
// logs coming from the sandbox.
func (q *qemu) getSandboxConsole(sandboxID string) (string, error) {
return utils.BuildSocketPath(runStoragePath, sandboxID, defaultConsole)
func (q *qemu) getSandboxConsole(id string) (string, error) {
return utils.BuildSocketPath(runStoragePath, id, consoleSocket)
}

// genericAppendBridges appends to devices the given bridges
Expand Down
2 changes: 1 addition & 1 deletion virtcontainers/qemu_arch_base_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -222,7 +222,7 @@ func TestQemuArchBaseAppendConsoles(t *testing.T) {
assert := assert.New(t)
qemuArchBase := newQemuArchBase()

path := filepath.Join(runStoragePath, sandboxID, defaultConsole)
path := filepath.Join(runStoragePath, sandboxID, consoleSocket)

expectedOut := []govmmQemu.Device{
govmmQemu.SerialDevice{
Expand Down
Loading

0 comments on commit 18e6a6e

Please sign in to comment.