From f8146fb73bfe3e43d77331b0b69378fd4c5b9cb6 Mon Sep 17 00:00:00 2001 From: matthieudelaro Date: Sat, 28 May 2016 00:09:09 +0900 Subject: [PATCH] support docker volumes. upgrade mount->volumes in syntax --- config/config_base.go | 23 ++++----- config/config_interfaces.go | 23 ++++++--- config/config_test.go | 1 - config/config_v5.go | 4 +- config/config_v6.go | 4 +- config/config_v7.go | 74 +++++++++++++++++++---------- config/config_v7_test.go | 7 ++- config/functions_over_interfaces.go | 17 ++++--- config/utils.go | 14 ++++-- core.go | 50 ++++++++++++++----- nut.yml | 2 + 11 files changed, 143 insertions(+), 76 deletions(-) diff --git a/config/config_base.go b/config/config_base.go index 0718926e8..25b1e3a8a 100644 --- a/config/config_base.go +++ b/config/config_base.go @@ -1,27 +1,28 @@ package config import ( - "errors" containerFilepath "github.com/matthieudelaro/nut/container/filepath" - Utils "github.com/matthieudelaro/nut/utils" ) type VolumeBase struct { } - func (self *VolumeBase) getHostPath() string { + func (self *VolumeBase) getVolumeName() string { return "" } - func (self *VolumeBase) getContainerPath() string { + func (self *VolumeBase) getOptions() string { return "" } - func (self *VolumeBase) getOptions() string { + +type DeviceBase struct { +} + func (self *DeviceBase) getHostPath() string { return "" } - func (self *VolumeBase) fullHostPath(context Utils.Context) (string, error) { - return "", errors.New("VolumeBase.fullHostPath() must be overloaded.") + func (self *DeviceBase) getContainerPath() string { + return "" } - func (self *VolumeBase) fullContainerPath(context Utils.Context) (string, error) { - return "", errors.New("VolumeBase.fullContainerPath() must be overloaded.") + func (self *DeviceBase) getOptions() string { + return "" } type BaseEnvironmentBase struct { @@ -69,8 +70,8 @@ type ConfigBase struct { func (self *ConfigBase) getEnvironmentVariables() map[string]string { return make(map[string]string) } - func (self *ConfigBase) getDevices() map[string]Volume { - return make(map[string]Volume) + func (self *ConfigBase) getDevices() map[string]Device { + return make(map[string]Device) } func (self *ConfigBase) getPorts() []string { return []string{} diff --git a/config/config_interfaces.go b/config/config_interfaces.go index aa5c0b94f..06aa6d6d1 100644 --- a/config/config_interfaces.go +++ b/config/config_interfaces.go @@ -4,20 +4,29 @@ import ( Utils "github.com/matthieudelaro/nut/utils" ) - +type Bind interface { + getOptions() (string) +} type Volume interface { + getVolumeName() string + getFullHostPath(context Utils.Context) (string, error) + getFullContainerPath(context Utils.Context) (string, error) + + // implement Bind + getOptions() (string) +} +type Device interface { getHostPath() string getContainerPath() string + + // implement Bind getOptions() (string) - fullHostPath(context Utils.Context) (string, error) - fullContainerPath(context Utils.Context) (string, error) } type BaseEnvironment interface { getFilePath() string getGitHub() string } - type Config interface { getDockerImage() string getProjectName() string @@ -30,7 +39,7 @@ type Config interface { getVolumes() map[string]Volume getMacros() map[string]Macro getEnvironmentVariables() map[string]string - getDevices() map[string]Volume + getDevices() map[string]Device getPorts() []string getEnableGui() (bool, bool) getEnableNvidiaDevices() (bool, bool) @@ -56,7 +65,7 @@ type Project interface { // extends Config interface getVolumes() map[string]Volume getMacros() map[string]Macro getEnvironmentVariables() map[string]string - getDevices() map[string]Volume + getDevices() map[string]Device getPorts() []string getEnableGui() (bool, bool) getEnableNvidiaDevices() (bool, bool) @@ -86,7 +95,7 @@ type Macro interface { // extends Config interface getVolumes() map[string]Volume getMacros() map[string]Macro getEnvironmentVariables() map[string]string - getDevices() map[string]Volume + getDevices() map[string]Device getPorts() []string getEnableGui() (bool, bool) getEnableNvidiaDevices() (bool, bool) diff --git a/config/config_test.go b/config/config_test.go index e077184e3..8cf3b5cf5 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -18,7 +18,6 @@ func TestFromNutPackage(t *testing.T) { log.Debug("------Tests of main.go") var volume Volume - volume = &VolumeBase{} volume = &VolumeV6{} log.Debug("OK ", volume) diff --git a/config/config_v5.go b/config/config_v5.go index 73e8f0cd0..fb3d49ce3 100644 --- a/config/config_v5.go +++ b/config/config_v5.go @@ -12,7 +12,7 @@ type VolumeV5 struct { Host string `yaml:host_path` Container string `yaml:container_path` } - func (self *VolumeV5) fullHostPath(context Utils.Context) (string, error) { + func (self *VolumeV5) getFullHostPath(context Utils.Context) (string, error) { clean := filepath.Clean(self.Host) if filepath.IsAbs(clean) { return clean, nil @@ -20,7 +20,7 @@ type VolumeV5 struct { return filepath.Join(context.GetRootDirectory(), clean), nil } } - func (self *VolumeV5) fullContainerPath(context Utils.Context) (string, error) { + func (self *VolumeV5) getFullContainerPath(context Utils.Context) (string, error) { clean := containerFilepath.Clean(self.Container) if containerFilepath.IsAbs(clean) { return clean, nil diff --git a/config/config_v6.go b/config/config_v6.go index 0afa25e43..970210f21 100644 --- a/config/config_v6.go +++ b/config/config_v6.go @@ -12,7 +12,7 @@ type VolumeV6 struct { Host string `yaml:host_path` Container string `yaml:container_path` } - func (self *VolumeV6) fullHostPath(context Utils.Context) (string, error) { + func (self *VolumeV6) getFullHostPath(context Utils.Context) (string, error) { clean := filepath.Clean(self.Host) if filepath.IsAbs(clean) { return clean, nil @@ -20,7 +20,7 @@ type VolumeV6 struct { return filepath.Join(context.GetRootDirectory(), clean), nil } } - func (self *VolumeV6) fullContainerPath(context Utils.Context) (string, error) { + func (self *VolumeV6) getFullContainerPath(context Utils.Context) (string, error) { clean := containerFilepath.Clean(self.Container) if containerFilepath.IsAbs(clean) { diff --git a/config/config_v7.go b/config/config_v7.go index bd3887ca1..932926f02 100644 --- a/config/config_v7.go +++ b/config/config_v7.go @@ -1,6 +1,7 @@ package config import( + "errors" "path/filepath" Utils "github.com/matthieudelaro/nut/utils" containerFilepath "github.com/matthieudelaro/nut/container/filepath" @@ -9,37 +10,61 @@ import( type VolumeV7 struct { VolumeBase `yaml:"inheritedValues,inline"` + VolumeName string `yaml:"volume_name,omitempty"` Host string `yaml:"host_path,omitempty"` - Container string `yaml:"container_path,omitempty"` + Container string `yaml:"container_path"` Options string `yaml:"options,omitempty"` } - func (self *VolumeV7) getHostPath() string { - return self.Host - } - func (self *VolumeV7) getContainerPath() string { - return self.Container + func (self *VolumeV7) getVolumeName() string { + return self.VolumeName } func (self *VolumeV7) getOptions() string { return self.Options } - func (self *VolumeV7) fullHostPath(context Utils.Context) (string, error) { - clean := filepath.Clean(self.Host) - if filepath.IsAbs(clean) { - return clean, nil + func (self *VolumeV7) getFullHostPath(context Utils.Context) (string, error) { + if self.Host == "" { + return "", errors.New("Undefined host path") } else { - return filepath.Join(context.GetRootDirectory(), clean), nil + clean := filepath.Clean(self.Host) + if filepath.IsAbs(clean) { + return clean, nil + } else { + return filepath.Join(context.GetRootDirectory(), clean), nil + } } } - func (self *VolumeV7) fullContainerPath(context Utils.Context) (string, error) { - clean := containerFilepath.Clean(self.Container) - if containerFilepath.IsAbs(clean) { - return clean, nil + func (self *VolumeV7) getFullContainerPath(context Utils.Context) (string, error) { + if self.Container == "" { + return "", errors.New("Undefined container path") } else { - return containerFilepath.Join(context.GetRootDirectory(), clean), nil + clean := containerFilepath.Clean(self.Container) + if containerFilepath.IsAbs(clean) { + return clean, nil + } else { + return containerFilepath.Join(context.GetRootDirectory(), clean), nil + } } } +type DeviceV7 struct { + DeviceBase `yaml:"inheritedValues,inline"` + + Host string `yaml:"host_path"` + Container string `yaml:"container_path"` + Options string `yaml:"options,omitempty"` +} + func (self *DeviceV7) getHostPath() string { + return self.Host + } + func (self *DeviceV7) getContainerPath() string { + return self.Container + } + func (self *DeviceV7) getOptions() string { + return self.Options + } + + type BaseEnvironmentV7 struct { BaseEnvironmentBase `yaml:"inheritedValues,inline"` @@ -57,7 +82,7 @@ type ConfigV7 struct { ConfigBase `yaml:"inheritedValues,inline"` DockerImage string `yaml:"docker_image,omitempty"` - Mount map[string][]string `yaml:"mount,omitempty"` + Volume map[string]VolumeV7 `yaml:"volumes,omitempty"` WorkingDir string `yaml:"container_working_directory,omitempty"` EnvironmentVariables map[string]string `yaml:"environment,omitempty"` Ports []string `yaml:"ports,omitempty"` @@ -68,7 +93,7 @@ type ConfigV7 struct { Detached string `yaml:"detached,omitempty"` UTSMode string `yaml:"uts,omitempty"` NetworkMode string `yaml:"net,omitempty"` - Devices map[string]VolumeV7 `yaml:"devices,omitempty"` + Devices map[string]DeviceV7 `yaml:"devices,omitempty"` parent Config } func (self *ConfigV7) getDockerImage() string { @@ -88,19 +113,16 @@ type ConfigV7 struct { } func (self *ConfigV7) getVolumes() map[string]Volume { cacheVolumes := make(map[string]Volume) - for name, data := range(self.Mount) { - cacheVolumes[name] = &VolumeV7{ - Host: data[0], - Container: data[1], - } + for name, data := range(self.Volume) { + cacheVolumes[name] = &data } return cacheVolumes } func (self *ConfigV7) getEnvironmentVariables() map[string]string { return self.EnvironmentVariables } - func (self *ConfigV7) getDevices() map[string]Volume { - cacheVolumes := make(map[string]Volume) + func (self *ConfigV7) getDevices() map[string]Device { + cacheVolumes := make(map[string]Device) for name, data := range(self.Devices) { cacheVolumes[name] = &data } @@ -207,7 +229,7 @@ type MacroV7 struct { func NewConfigV7(parent Config) *ConfigV7 { return &ConfigV7{ - Mount: make(map[string][]string), + Volume: make(map[string]VolumeV7), parent: parent, } } diff --git a/config/config_v7_test.go b/config/config_v7_test.go index 7b3abbe62..4cd203a0c 100644 --- a/config/config_v7_test.go +++ b/config/config_v7_test.go @@ -17,7 +17,6 @@ func TestFromNutPackageV7(t *testing.T) { log.Debug("------Tests of main.go") var volume Volume - volume = &VolumeBase{} volume = &VolumeV7{} log.Debug("OK ", volume) @@ -304,7 +303,7 @@ func TestParsingV7(t *testing.T) { detached bool UTSMode string NetworkMode string - Devices map[string]Volume + Devices map[string]Device } nutFiles := []Tuple{} @@ -323,8 +322,8 @@ devices: ports: []string{}, docker_image: "golang:1.7", detached: false, -Devices: map[string]Volume{ - "first": &VolumeV7{ +Devices: map[string]Device{ + "first": &DeviceV7{ Host: "/dev/1", Container: "/dev/1", Options: "rw", diff --git a/config/functions_over_interfaces.go b/config/functions_over_interfaces.go index edbb71c13..319a44ebd 100644 --- a/config/functions_over_interfaces.go +++ b/config/functions_over_interfaces.go @@ -6,21 +6,24 @@ import ( ) // Define methods over interfaces -func GetHostPath(volume Volume) string { +func GetHostPath(volume Device) string { return volume.getHostPath() } -func GetContainerPath(volume Volume) string { +func GetContainerPath(volume Device) string { return volume.getContainerPath() } -func GetOptions(volume Volume) string { - return volume.getOptions() +func GetOptions(bind Bind) string { + return bind.getOptions() } +func GetVolumeName(volume Volume) string { + return volume.getVolumeName() +} func GetFullHostPath(volume Volume, context Utils.Context) (string, error) { - return volume.fullHostPath(context) + return volume.getFullHostPath(context) } func GetFullContainerPath(volume Volume, context Utils.Context) (string, error) { - return volume.fullContainerPath(context) + return volume.getFullContainerPath(context) } func SetParentProject(child Project, parent Project) { @@ -154,7 +157,7 @@ func GetEnvironmentVariables(config Config) map[string]string { return items } -func GetDevices(config Config) map[string]Volume { +func GetDevices(config Config) map[string]Device { items := config.getDevices() var parent = config.getParent() diff --git a/config/utils.go b/config/utils.go index a3fe0307e..263b9dff3 100644 --- a/config/utils.go +++ b/config/utils.go @@ -27,18 +27,22 @@ func TruthyString(s string) (bool, bool) { // Returns the first conflict element from the map, or nil if // there wasn't any conflict. func CheckConflict(context Utils.Context, key string, newPoint Volume, mountingPoints map[string]Volume) Volume { - h, errh := newPoint.fullHostPath(context) - c, errc := newPoint.fullContainerPath(context) + h, errh := newPoint.getFullHostPath(context) + c, errc := newPoint.getFullContainerPath(context) for key2, mountingPoint2 := range mountingPoints { // log.Debug("child point ", key) - h2, errh2 := mountingPoint2.fullHostPath(context) - c2, errc2 := mountingPoint2.fullContainerPath(context) + h2, errh2 := mountingPoint2.getFullHostPath(context) + c2, errc2 := mountingPoint2.getFullContainerPath(context) if key2 == key || h == h2 || c == c2 || - errh != nil || errc != nil || errh2 != nil || errc2 != nil { + (newPoint.getVolumeName() != "" && newPoint.getVolumeName() == mountingPoint2.getVolumeName()){ + // || errh != nil || errc != nil || errh2 != nil || errc2 != nil { // log.Debug("conflic between mounting points ", key, " and ", key2) + if errh != nil || errc != nil || errh2 != nil || errc2 != nil { + log.Debug("warning while checking conflic between volumes ", key, " and ", key2) + } return mountingPoint2 } } diff --git a/core.go b/core.go index 5518298c0..884826fbe 100644 --- a/core.go +++ b/core.go @@ -66,26 +66,54 @@ func execInContainer(commands []string, config Config.Config, context Utils.Cont // prepare names of directories to mount // inspired from https://github.com/fsouza/go-dockerclient/issues/220#issuecomment-77777365 - mountingPoints := Config.GetVolumes(config, context) - binds := make([]string, 0, len(mountingPoints)) + volumes := Config.GetVolumes(config, context) + binds := make([]string, 0, len(volumes)) portBindings := map[docker.Port][]docker.PortBinding{} exposedPorts := map[docker.Port]struct{}{} envVariables := []string{} volumeDriver := "" devices := []docker.Device{} - for _, directory := range(mountingPoints) { - hostPath, hostPathErr := Config.GetFullHostPath(directory, context) - containerPath, containerPathErr := Config.GetFullContainerPath(directory, context) - if hostPathErr != nil { - log.Error("Couldn't mount host directory: ", hostPathErr.Error()) - return - } + for key, volume := range(volumes) { + volumeName := Config.GetVolumeName(volume) + hostPath, hostPathErr := Config.GetFullHostPath(volume, context) + containerPath, containerPathErr := Config.GetFullContainerPath(volume, context) + log.Debug(key, " / ", len(volumes), " volume", volume) if containerPathErr != nil { - log.Error("Couldn't mount container directory: ", containerPathErr.Error()) + log.Error("Couldn't mount container directory (" + containerPath + "): ", containerPathErr.Error()) return } - binds = append(binds, hostPath + ":" + containerPath) + if hostPathErr != nil { // volume must have either a hostPath to a directory, ... + log.Debug(key, " unproper hostPath", hostPath) + if volumeName != "" { // ..., or a volumeName + log.Debug(key, " volume name", volumeName) + // prepare volume + var dockerVolume *docker.Volume + if dockerVolume, err = client.InspectVolume(volumeName); err != nil { + fmt.Println("Could not inspect volume", imageName, ":", err.Error()) + fmt.Println("Creating volume...") + + if dockerVolume, err = client.CreateVolume( + docker.CreateVolumeOptions{Name: volumeName}); err != nil { + log.Error("Could not create volume: ", err.Error()) + return + } + fmt.Println("Volume created.") + } + log.Debug(key, " dockerVolume ", dockerVolume) + log.Debug(key, " dockerVolume ", dockerVolume.Name) + log.Debug(key, " dockerVolume ", dockerVolume.Mountpoint) + log.Debug(key, " dockerVolume ", dockerVolume.Driver) + binds = append(binds, dockerVolume.Mountpoint + ":" + containerPath) + // TODO?: take dockerVolume.Driver into account? + } else { + log.Error("Couldn't mount host directory (" + hostPath + "): ", hostPathErr.Error()) + return + } + } else { + log.Debug(key, " proper host path", hostPath, hostPathErr) + binds = append(binds, hostPath + ":" + containerPath) + } } log.Debug("binds", binds) diff --git a/nut.yml b/nut.yml index 437df7de5..b2450aebb 100644 --- a/nut.yml +++ b/nut.yml @@ -54,6 +54,8 @@ macros: usage: test the project actions: - go test + - cd config + - go test code: usage: open this project in vscode docker_image: ctaggart/golang-vscode