Skip to content
This repository has been archived by the owner on Oct 22, 2021. It is now read-only.

Commit

Permalink
Merge pull request #449 from cloudfoundry-incubator/vladi/remote-dock…
Browse files Browse the repository at this point in the history 
…er-support

Add support for compiling on a remote docker
  • Loading branch information
@jandubois
jandubois authored Nov 23, 2018
2 parents b4a87d2 + f9325d1 commit 792f653
Show file tree
Hide file tree
Showing 355 changed files with 23,868 additions and 6,307 deletions.
166 changes: 99 additions & 67 deletions Gopkg.lock
View file

Large diffs are not rendered by default.

8 changes: 6 additions & 2 deletions Gopkg.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,10 @@
# unused-packages = true


[[constraint]]
branch = "master"
name = "code.cloudfoundry.org/archiver"

[[constraint]]
name = "github.com/Masterminds/sprig"
version = "2.16.0"
Expand All @@ -43,7 +47,7 @@

[[constraint]]
name = "github.com/fsouza/go-dockerclient"
version = "1.3.0"
version = "1.3.1"

[[constraint]]
name = "github.com/golang/mock"
Expand Down Expand Up @@ -87,7 +91,7 @@

[[constraint]]
name = "github.com/spf13/viper"
version = "1.2.0"
version = "1.2.1"

[[constraint]]
name = "github.com/stretchr/testify"
Expand Down
4 changes: 2 additions & 2 deletions app/fissile.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -314,7 +314,7 @@ func newPropertyInfo(maybeHash bool) *propertyInfo {
}

// Compile will compile a list of dev BOSH releases
func (f *Fissile) Compile(stemcellImageName string, targetPath, roleManifestPath, metricsPath string, instanceGroupNames, releaseNames []string, workerCount int, dockerNetworkMode string, withoutDocker, verbose bool, packageCacheConfigFilename string) error {
func (f *Fissile) Compile(stemcellImageName string, targetPath, roleManifestPath, metricsPath string, instanceGroupNames, releaseNames []string, workerCount int, dockerNetworkMode string, withoutDocker, verbose bool, packageCacheConfigFilename string, streamPackages bool) error {
if f.Manifest == nil || len(f.Manifest.LoadedReleases) == 0 {
return fmt.Errorf("Releases not loaded")
}
Expand Down Expand Up @@ -350,7 +350,7 @@ func (f *Fissile) Compile(stemcellImageName string, targetPath, roleManifestPath
return fmt.Errorf("Error creating a new compilator: %s", err.Error())
}
} else {
comp, err = compilator.NewDockerCompilator(dockerManager, targetPath, metricsPath, stemcellImageName, compilation.LinuxBase, f.Version, dockerNetworkMode, false, f.UI, f, packageStorage)
comp, err = compilator.NewDockerCompilator(dockerManager, targetPath, metricsPath, stemcellImageName, compilation.LinuxBase, f.Version, dockerNetworkMode, false, f.UI, f, packageStorage, streamPackages)
if err != nil {
return fmt.Errorf("Error creating a new compilator: %s", err.Error())
}
Expand Down
9 changes: 9 additions & 0 deletions cmd/build-packages.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ compiled once.
flagBuildPackagesStemcell := buildPackagesViper.GetString("stemcell")
flagBuildOutputGraph = buildViper.GetString("output-graph")
flagBuildCompilationCacheConfig := buildPackagesViper.GetString("compilation-cache-config")
flagBuildPackagesStreamPackages := buildPackagesViper.GetBool("stream-packages")

err := fissile.LoadManifest(
flagRoleManifest,
Expand Down Expand Up @@ -79,6 +80,7 @@ compiled once.
flagBuildPackagesWithoutDocker,
flagVerbose,
flagBuildCompilationCacheConfig,
flagBuildPackagesStreamPackages,
)
},
}
Expand Down Expand Up @@ -133,5 +135,12 @@ func init() {
"Points to a file containing configuration for a compiled package cache or contains the configuration as valid yaml",
)

buildPackagesCmd.PersistentFlags().BoolP(
"stream-packages",
"",
false,
"If true, fissile will stream packages to the docker daemon for compilation, instead of mounting volumes",
)

buildPackagesViper.BindPFlags(buildPackagesCmd.PersistentFlags())
}
29 changes: 28 additions & 1 deletion compilator/compilator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ type Compilator struct {
dockerNetworkMode string
compilePackage func(*Compilator, *model.Package) error
packageStorage *PackageStorage
streamPackages bool

// signalDependencies is a map of
// (package fingerprint) -> (channel to close when done)
Expand Down Expand Up @@ -89,6 +90,7 @@ func NewDockerCompilator(
ui *termui.UI,
grapher util.ModelGrapher,
packageStorage *PackageStorage,
streamPackages bool,
) (*Compilator, error) {

compilator := &Compilator{
Expand All @@ -99,6 +101,7 @@ func NewDockerCompilator(
baseType: baseType,
fissileVersion: fissileVersion,
compilePackage: (*Compilator).compilePackageInDocker,
streamPackages: streamPackages,
dockerNetworkMode: dockerNetworkMode,
keepContainer: keepContainer,
ui: ui,
Expand Down Expand Up @@ -560,17 +563,41 @@ func (c *Compilator) compilePackageInDocker(pkg *model.Package) (err error) {
// from, so it will be in some docker-maintained storage.
sourceMountName: ContainerSourceDir,
}

// If we stream the package, don't mount any local files.
// We assume the docker server is not local. So we stream the package to
// be compiled, and when done, we stream the compiled bits out.
volumes := map[string]map[string]string{sourceMountName: nil}
streamIn := map[string]string{}
streamOut := map[string]string{}
if c.streamPackages {
inVolume := fmt.Sprintf("in_volume-%s", uuid.New())
outVolume := fmt.Sprintf("out_volume-%s", uuid.New())
volumes[inVolume] = nil
volumes[outVolume] = nil

mounts = map[string]string{
inVolume: docker.ContainerInPath,
outVolume: docker.ContainerOutPath,
}

streamIn[pkg.GetTargetPackageSourcesDir(c.hostWorkDir)] = docker.ContainerInPath
streamOut[docker.ContainerOutPath] = pkg.GetPackageCompiledTempDir(c.hostWorkDir)
}

exitCode, container, err := c.dockerManager.RunInContainer(docker.RunInContainerOpts{
ContainerName: containerName,
ImageName: c.stemcellImageName,
EntryPoint: []string{},
Cmd: []string{"/bin/bash", containerScriptPath, pkg.Name, pkg.Version},
Mounts: mounts,
NetworkMode: c.dockerNetworkMode,
Volumes: map[string]map[string]string{sourceMountName: nil},
Volumes: volumes,
KeepContainer: c.keepContainer,
StdoutWriter: stdoutWriter,
StderrWriter: stderrWriter,
StreamIn: streamIn,
StreamOut: streamOut,
})

if container != nil && (!c.keepContainer || err == nil || exitCode == 0) {
Expand Down
36 changes: 19 additions & 17 deletions compilator/compilator_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ func TestMain(m *testing.M) {
func TestCompilationEmpty(t *testing.T) {
assert := assert.New(t)

c, err := NewDockerCompilator(nil, "", "", "", "", "", "", false, ui, nil, nil)
c, err := NewDockerCompilator(nil, "", "", "", "", "", "", false, ui, nil, nil, false)
assert.NoError(err)

waitCh := make(chan struct{})
Expand All @@ -70,7 +70,7 @@ func TestCompilationBasic(t *testing.T) {
metrics := file.Name()
defer os.Remove(metrics)

c, err := NewDockerCompilator(nil, "", metrics, "", "", "", "", false, ui, nil, nil)
c, err := NewDockerCompilator(nil, "", metrics, "", "", "", "", false, ui, nil, nil, false)
assert.NoError(err)

compileChan := make(chan string)
Expand Down Expand Up @@ -149,7 +149,7 @@ func TestCompilationSkipCompiled(t *testing.T) {

assert := assert.New(t)

c, err := NewDockerCompilator(nil, "", "", "", "", "", "", false, ui, nil, nil)
c, err := NewDockerCompilator(nil, "", "", "", "", "", "", false, ui, nil, nil, false)
assert.NoError(err)

compileChan := make(chan string)
Expand All @@ -173,7 +173,7 @@ func TestCompilationSkipCompiled(t *testing.T) {

func TestCompilationRoleManifest(t *testing.T) {

c, err := NewDockerCompilator(nil, "", "", "", "", "", "", false, ui, nil, nil)
c, err := NewDockerCompilator(nil, "", "", "", "", "", "", false, ui, nil, nil, false)
assert.NoError(t, err)

compileChan := make(chan string, 2)
Expand Down Expand Up @@ -268,7 +268,7 @@ func doTestContainerKeptAfterCompilationWithErrors(t *testing.T, keepContainer b

imageName := "splatform/fissile-stemcell-opensuse:42.2"

comp, err := NewDockerCompilator(dockerManager, compilationWorkDir, "", imageName, compilation.FakeBase, "3.14.15", "", keepContainer, ui, nil, nil)
comp, err := NewDockerCompilator(dockerManager, compilationWorkDir, "", imageName, compilation.FakeBase, "3.14.15", "", keepContainer, ui, nil, nil, false)
assert.NoError(err)

beforeCompileContainers, err := getContainerIDs(imageName)
Expand Down Expand Up @@ -356,7 +356,7 @@ func TestCompilationMultipleErrors(t *testing.T) {

assert := assert.New(t)

c, err := NewDockerCompilator(nil, "", "", "", "", "", "", false, ui, nil, nil)
c, err := NewDockerCompilator(nil, "", "", "", "", "", "", false, ui, nil, nil, false)
assert.NoError(err)

c.compilePackage = func(c *Compilator, pkg *model.Package) error {
Expand Down Expand Up @@ -386,7 +386,7 @@ func TestGetPackageStatusCompiled(t *testing.T) {
// For this test we assume that the release does not have multiple packages with a single fingerprint
assert.NoError(err)

compilator, err := NewDockerCompilator(dockerManager, compilationWorkDir, "", "fissile-test-compilator", compilation.FakeBase, "3.14.15", "", false, ui, nil, nil)
compilator, err := NewDockerCompilator(dockerManager, compilationWorkDir, "", "fissile-test-compilator", compilation.FakeBase, "3.14.15", "", false, ui, nil, nil, false)
assert.NoError(err)

compiledPackagePath := filepath.Join(compilationWorkDir, release.Packages[0].Fingerprint, "compiled")
Expand Down Expand Up @@ -445,7 +445,7 @@ func TestCompilationParallel(t *testing.T) {

assert := assert.New(t)

c, err := NewDockerCompilator(nil, "", "", "", "", "", "", false, ui, nil, nil)
c, err := NewDockerCompilator(nil, "", "", "", "", "", "", false, ui, nil, nil, false)
assert.NoError(err)
c.compilePackage = func(c *Compilator, pkg *model.Package) error {
mutex.Lock()
Expand Down Expand Up @@ -500,7 +500,7 @@ func TestGetPackageStatusNone(t *testing.T) {
// For this test we assume that the release does not have multiple packages with a single fingerprint
assert.NoError(err)

compilator, err := NewDockerCompilator(dockerManager, compilationWorkDir, "", "fissile-test-compilator", compilation.FakeBase, "3.14.15", "", false, ui, nil, nil)
compilator, err := NewDockerCompilator(dockerManager, compilationWorkDir, "", "fissile-test-compilator", compilation.FakeBase, "3.14.15", "", false, ui, nil, nil, false)
assert.NoError(err)

status, err := compilator.isPackageCompiled(release.Packages[0])
Expand All @@ -525,7 +525,7 @@ func TestPackageFolderStructure(t *testing.T) {
release, err := model.NewDevRelease(ntpReleasePath, "", "", ntpReleasePathBoshCache)
assert.NoError(err)

compilator, err := NewDockerCompilator(dockerManager, compilationWorkDir, "", "fissile-test-compilator", compilation.FakeBase, "3.14.15", "", false, ui, nil, nil)
compilator, err := NewDockerCompilator(dockerManager, compilationWorkDir, "", "fissile-test-compilator", compilation.FakeBase, "3.14.15", "", false, ui, nil, nil, false)
assert.NoError(err)

err = compilator.createCompilationDirStructure(release.Packages[0])
Expand Down Expand Up @@ -556,7 +556,7 @@ func TestPackageDependenciesPreparation(t *testing.T) {
release, err := model.NewDevRelease(torReleasePath, "", "", torReleasePathBoshCache)
assert.NoError(err)

compilator, err := NewDockerCompilator(dockerManager, compilationWorkDir, "", "fissile-test-compilator", compilation.FakeBase, "3.14.15", "", false, ui, nil, nil)
compilator, err := NewDockerCompilator(dockerManager, compilationWorkDir, "", "fissile-test-compilator", compilation.FakeBase, "3.14.15", "", false, ui, nil, nil, false)
assert.NoError(err)

pkg, err := release.LookupPackage("tor")
Expand All @@ -581,11 +581,13 @@ func TestPackageDependenciesPreparation(t *testing.T) {
}

func TestCompilePackageInDocker(t *testing.T) {
doTestCompilePackageInDocker(t, true)
doTestCompilePackageInDocker(t, false)
t.Run("KeepContainerWithVolumes", func(t *testing.T) { doTestCompilePackageInDocker(t, true, false) })
t.Run("RemoveContainerWithVolumes", func(t *testing.T) { doTestCompilePackageInDocker(t, false, false) })
t.Run("KeepContainerWithStreams", func(t *testing.T) { doTestCompilePackageInDocker(t, true, true) })
t.Run("RemoveContainerWithStreams", func(t *testing.T) { doTestCompilePackageInDocker(t, false, true) })
}

func doTestCompilePackageInDocker(t *testing.T, keepInContainer bool) {
func doTestCompilePackageInDocker(t *testing.T, keepInContainer bool, streamPackages bool) {
assert := assert.New(t)

dockerClient, err := dockerclient.NewClientFromEnv()
Expand All @@ -606,7 +608,7 @@ func doTestCompilePackageInDocker(t *testing.T, keepInContainer bool) {

imageName := "splatform/fissile-stemcell-opensuse:42.2"

comp, err := NewDockerCompilator(dockerManager, compilationWorkDir, "", imageName, compilation.FakeBase, "3.14.15", "", keepInContainer, ui, nil, nil)
comp, err := NewDockerCompilator(dockerManager, compilationWorkDir, "", imageName, compilation.FakeBase, "3.14.15", "", keepInContainer, ui, nil, nil, streamPackages)
assert.NoError(err)

containerName := comp.getPackageContainerName(release.Packages[0])
Expand Down Expand Up @@ -713,7 +715,7 @@ func TestCreateDepBucketsOnChain(t *testing.T) {
func TestGatherPackages(t *testing.T) {
assert := assert.New(t)

c, err := NewDockerCompilator(nil, "", "", "", "", "", "", false, ui, nil, nil)
c, err := NewDockerCompilator(nil, "", "", "", "", "", "", false, ui, nil, nil, false)
assert.NoError(err)

releases := genTestCase("ruby-2.5", "go-1.4.1:G", "go-1.4:G")
Expand All @@ -736,7 +738,7 @@ func TestRemoveCompiledPackages(t *testing.T) {

assert := assert.New(t)

c, err := NewDockerCompilator(nil, "", "", "", "", "", "", false, ui, nil, nil)
c, err := NewDockerCompilator(nil, "", "", "", "", "", "", false, ui, nil, nil, false)
assert.NoError(err)

releases := genTestCase("ruby-2.5", "consul>go-1.4", "go-1.4")
Expand Down
4 changes: 2 additions & 2 deletions compilator/storage_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ func TestStorePackageLocallyOK(t *testing.T) {
p, err := NewPackageStorage(packageCacheConfig["boshPackageCacheKind"].(string), false, configMap, compilationWorkDir, fullContainerPath, imageName)
assert.NoError(err)

c, err := NewDockerCompilator(dockerManager, compilationWorkDir, "", imageName, compilation.FakeBase, "3.14.15", "", false, ui, nil, p)
c, err := NewDockerCompilator(dockerManager, compilationWorkDir, "", imageName, compilation.FakeBase, "3.14.15", "", false, ui, nil, p, false)
assert.NoError(err)

releasePath := filepath.Join(workDir, "../test-assets/ntp-release")
Expand Down Expand Up @@ -131,7 +131,7 @@ func TestStorePackageExists(t *testing.T) {
p, err := NewPackageStorage(packageCacheConfig["boshPackageCacheKind"].(string), false, configMap, compilationWorkDir, fullContainerPath, imageName)
assert.NoError(err)

c, err := NewDockerCompilator(dockerManager, compilationWorkDir, "", imageName, compilation.FakeBase, "3.14.15", "", false, ui, nil, p)
c, err := NewDockerCompilator(dockerManager, compilationWorkDir, "", imageName, compilation.FakeBase, "3.14.15", "", false, ui, nil, p, false)
assert.NoError(err)

releasePath := filepath.Join(workDir, "../test-assets/ntp-release")
Expand Down
Loading

0 comments on commit 792f653

Please sign in to comment.