diff --git a/pkg/container/docker_network.go b/pkg/container/docker_network.go new file mode 100644 index 00000000000..bad329ba100 --- /dev/null +++ b/pkg/container/docker_network.go @@ -0,0 +1,71 @@ +package container + +import ( + "context" + + "github.com/docker/docker/api/types" + "github.com/nektos/act/pkg/common" +) + +func NewDockerNetworkCreateExecutor(name string, config types.NetworkCreate) common.Executor { + return func(ctx context.Context) error { + if common.Dryrun(ctx) { + return nil + } + + cli, err := GetDockerClient(ctx) + if err != nil { + return err + } + + if exists := DockerNetworkExists(ctx, name); exists { + return nil + } + + if _, err = cli.NetworkCreate(ctx, name, config); err != nil { + return err + } + + return nil + } +} + +func NewDockerNetworkRemoveExecutor(name string) common.Executor { + return func(ctx context.Context) error { + if common.Dryrun(ctx) { + return nil + } + + cli, err := GetDockerClient(ctx) + if err != nil { + return err + } + + if err = cli.NetworkRemove(ctx, name); err != nil { + return err + } + + return nil + } +} + +func DockerNetworkExists(ctx context.Context, name string) bool { + if _, exists, _ := GetDockerNetwork(ctx, name); !exists { + return false + } + return true +} + +func GetDockerNetwork(ctx context.Context, name string) (types.NetworkResource, bool, error) { + cli, err := GetDockerClient(ctx) + if err != nil { + return types.NetworkResource{}, false, err + } + + res, err := cli.NetworkInspect(ctx, name, types.NetworkInspectOptions{}) + if err != nil { + return types.NetworkResource{}, false, err + } + + return res, true, nil +} diff --git a/pkg/container/docker_run.go b/pkg/container/docker_run.go index bf6d6f378a4..044fdab32d1 100644 --- a/pkg/container/docker_run.go +++ b/pkg/container/docker_run.go @@ -65,7 +65,9 @@ type FileEntry struct { // Container for managing docker run containers type Container interface { + ID() string Create(capAdd []string, capDrop []string) common.Executor + ConnectToNetwork(name string) common.Executor Copy(destPath string, files ...*FileEntry) common.Executor CopyDir(destPath string, srcPath string, useGitIgnore bool) common.Executor GetContainerArchive(ctx context.Context, srcPath string) (io.ReadCloser, error) @@ -77,6 +79,7 @@ type Container interface { UpdateFromPath(env *map[string]string) common.Executor Remove() common.Executor Close() common.Executor + SetContainerNetworkMode(mode string) common.Executor } // NewContainer creates a reference to a container @@ -105,6 +108,10 @@ func supportsContainerImagePlatform(cli *client.Client) bool { return constraint.Check(sv) } +func (cr *containerReference) ID() string { + return cr.id +} + func (cr *containerReference) Create(capAdd []string, capDrop []string) common.Executor { return common. NewInfoExecutor("%sdocker create image=%s platform=%s entrypoint=%+q cmd=%+q", logPrefix, cr.input.Image, cr.input.Platform, cr.input.Entrypoint, cr.input.Cmd). @@ -146,19 +153,26 @@ func (cr *containerReference) Pull(forcePull bool) common.Executor { } func (cr *containerReference) Copy(destPath string, files ...*FileEntry) common.Executor { - return common.NewPipelineExecutor( - cr.connect(), - cr.find(), - cr.copyContent(destPath, files...), - ).IfNot(common.Dryrun) + return common. + NewInfoExecutor("%sdocker cp destination=%s", logPrefix, destPath). + Then( + common.NewPipelineExecutor( + cr.connect(), + cr.find(), + cr.copyContent(destPath, files...), + ).IfNot(common.Dryrun), + ) } func (cr *containerReference) CopyDir(destPath string, srcPath string, useGitIgnore bool) common.Executor { - return common.NewPipelineExecutor( - common.NewInfoExecutor("%sdocker cp src=%s dst=%s", logPrefix, srcPath, destPath), - cr.Exec([]string{"mkdir", "-p", destPath}, nil, "", ""), - cr.copyDir(destPath, srcPath, useGitIgnore), - ).IfNot(common.Dryrun) + return common. + NewInfoExecutor("%sdocker cp src=%s dst=%s", logPrefix, srcPath, destPath). + Then( + common.NewPipelineExecutor( + cr.Exec([]string{"mkdir", "-p", destPath}, nil, "", ""), + cr.copyDir(destPath, srcPath, useGitIgnore), + ).IfNot(common.Dryrun), + ) } func (cr *containerReference) GetContainerArchive(ctx context.Context, srcPath string) (io.ReadCloser, error) { @@ -178,22 +192,54 @@ func (cr *containerReference) UpdateFromPath(env *map[string]string) common.Exec return cr.extractPath(env).IfNot(common.Dryrun) } +func (cr *containerReference) SetContainerNetworkMode(mode string) common.Executor { + return common. + NewDebugExecutor("Changed network mode for container '%s' from '%s' to '%s'", cr.input.Name, cr.input.NetworkMode, mode). + Then( + common.NewPipelineExecutor( + func(ctx context.Context) error { + cr.input.NetworkMode = mode + return nil + }, + ).IfNot(common.Dryrun), + ) +} + +func (cr *containerReference) ConnectToNetwork(name string) common.Executor { + return common. + NewInfoExecutor("%sdocker network connect net=%s container=%s", logPrefix, name, cr.input.Name). + Then( + common.NewPipelineExecutor( + cr.connect(), + cr.find(), + cr.connectToNetwork(name), + ).IfNot(common.Dryrun), + ) +} + func (cr *containerReference) Exec(command []string, env map[string]string, user, workdir string) common.Executor { - return common.NewPipelineExecutor( - common.NewInfoExecutor("%sdocker exec cmd=[%s] user=%s workdir=%s", logPrefix, strings.Join(command, " "), user, workdir), - cr.connect(), - cr.find(), - cr.exec(command, env, user, workdir), - ).IfNot(common.Dryrun) + return common. + NewInfoExecutor("%sdocker exec cmd=%v user=%s workdir=%s", logPrefix, command, user, workdir). + Then( + common.NewPipelineExecutor( + cr.connect(), + cr.find(), + cr.exec(command, env, user, workdir), + ).IfNot(common.Dryrun), + ) } func (cr *containerReference) Remove() common.Executor { - return common.NewPipelineExecutor( - cr.connect(), - cr.find(), - ).Finally( - cr.remove(), - ).IfNot(common.Dryrun) + return common. + NewInfoExecutor("%sdocker rm %s", logPrefix, cr.id). + Then( + common.NewPipelineExecutor( + cr.connect(), + cr.find(), + ).Finally( + cr.remove(), + ).IfNot(common.Dryrun), + ).IfBool(cr.id != "") } type containerReference struct { @@ -233,6 +279,12 @@ func GetDockerClient(ctx context.Context) (*client.Client, error) { return cli, err } +func (cr *containerReference) connectToNetwork(name string) common.Executor { + return func(ctx context.Context) error { + return cr.cli.NetworkConnect(ctx, name, cr.id, nil) + } +} + func (cr *containerReference) connect() common.Executor { return func(ctx context.Context) error { if cr.cli != nil { @@ -315,14 +367,20 @@ func (cr *containerReference) create(capAdd []string, capDrop []string) common.E config := &container.Config{ Image: input.Image, - Cmd: input.Cmd, - Entrypoint: input.Entrypoint, WorkingDir: input.WorkingDir, Env: input.Env, Tty: isTerminal, Hostname: input.Hostname, } + if len(input.Cmd) > 0 { + config.Cmd = input.Cmd + } + + if len(input.Entrypoint) > 0 { + config.Entrypoint = input.Entrypoint + } + mounts := make([]mount.Mount, 0) for mountSource, mountTarget := range input.Mounts { mounts = append(mounts, mount.Mount{ @@ -357,6 +415,7 @@ func (cr *containerReference) create(capAdd []string, capDrop []string) common.E if err != nil { return errors.WithStack(err) } + logger.Debugf("Created container name=%s id=%v from image %v (platform: %s)", input.Name, resp.ID, input.Image, input.Platform) logger.Debugf("ENV ==> %v", input.Env) diff --git a/pkg/runner/run_context.go b/pkg/runner/run_context.go index cfe37ac3187..a6975c14409 100644 --- a/pkg/runner/run_context.go +++ b/pkg/runner/run_context.go @@ -10,6 +10,7 @@ import ( "runtime" "strings" + "github.com/docker/docker/api/types" "github.com/google/shlex" "github.com/spf13/pflag" @@ -27,19 +28,20 @@ const ActPath string = "/var/run/act" // RunContext contains info about current job type RunContext struct { - Name string - Config *Config - Matrix map[string]interface{} - Run *model.Run - EventJSON string - Env map[string]string - ExtraPath []string - CurrentStep string - StepResults map[string]*stepResult - ExprEval ExpressionEvaluator - JobContainer container.Container - OutputMappings map[MappableOutput]MappableOutput - JobName string + Name string + Config *Config + Matrix map[string]interface{} + Run *model.Run + EventJSON string + Env map[string]string + ExtraPath []string + CurrentStep string + StepResults map[string]*stepResult + ExprEval ExpressionEvaluator + JobContainer container.Container + OutputMappings map[MappableOutput]MappableOutput + JobName string + ServiceContainers map[string]container.Container } type MappableOutput struct { @@ -137,7 +139,13 @@ func (rc *RunContext) GetBindsAndMounts() ([]string, map[string]string) { return binds, mounts } +// JobNetworkName returns formatted network name used in main container and its services +func (rc *RunContext) JobNetworkName() string { + return createNetworkName("act", rc.Run.Workflow.Name, rc.Run.JobID, rc.Name) +} + func (rc *RunContext) startJobContainer() common.Executor { + job := rc.Run.Job() image := rc.platformImage() hostname := rc.hostname() @@ -159,6 +167,9 @@ func (rc *RunContext) startJobContainer() common.Executor { common.Logger(ctx).Infof("\U0001f680 Start image=%s", image) name := rc.jobContainerName() + networkName := rc.JobNetworkName() + workdir := rc.Config.ContainerWorkdir() + binds, mounts := rc.GetBindsAndMounts() envList := make([]string, 0) @@ -166,41 +177,81 @@ func (rc *RunContext) startJobContainer() common.Executor { envList = append(envList, fmt.Sprintf("%s=%s", "RUNNER_OS", "Linux")) envList = append(envList, fmt.Sprintf("%s=%s", "RUNNER_TEMP", "/tmp")) - binds, mounts := rc.GetBindsAndMounts() + if job.Services != nil { + rc.ServiceContainers = make(map[string]container.Container, len(job.Services)) + for name, spec := range job.Services { + mergedEnv := envList + for k, v := range spec.Env { + mergedEnv = append(mergedEnv, fmt.Sprintf("%s=%s", k, v)) + } + + mnt := mounts + mnt[name] = filepath.Dir(workdir) + + c := container.NewContainer(&container.NewContainerInput{ + Name: name, + Hostname: name, + WorkingDir: workdir, + Image: spec.Image, + Username: username, + Password: password, + Env: mergedEnv, + Mounts: mnt, + Binds: binds, + Stdout: logWriter, + Stderr: logWriter, + Privileged: rc.Config.Privileged, + UsernsMode: rc.Config.UsernsMode, + Platform: rc.Config.ContainerArchitecture, + }) + rc.ServiceContainers[name] = c + } + } rc.JobContainer = container.NewContainer(&container.NewContainerInput{ - Cmd: nil, - Entrypoint: []string{"/usr/bin/tail", "-f", "/dev/null"}, - WorkingDir: rc.Config.ContainerWorkdir(), - Image: image, - Username: username, - Password: password, - Name: name, - Env: envList, - Mounts: mounts, - NetworkMode: "host", - Binds: binds, - Stdout: logWriter, - Stderr: logWriter, - Privileged: rc.Config.Privileged, - UsernsMode: rc.Config.UsernsMode, - Platform: rc.Config.ContainerArchitecture, - Hostname: hostname, + Cmd: nil, + Entrypoint: []string{"/usr/bin/tail", "-f", "/dev/null"}, + WorkingDir: workdir, + Image: image, + Username: username, + Password: password, + Name: name, + Env: envList, + Mounts: mounts, + Binds: binds, + Stdout: logWriter, + Stderr: logWriter, + Privileged: rc.Config.Privileged, + UsernsMode: rc.Config.UsernsMode, + Platform: rc.Config.ContainerArchitecture, + Hostname: hostname, }) + if job.Services == nil { + rc.JobContainer.SetContainerNetworkMode("host") + } + var copyWorkspace bool var copyToPath string if !rc.Config.BindWorkdir { copyToPath, copyWorkspace = rc.localCheckoutPath() - copyToPath = filepath.Join(rc.Config.ContainerWorkdir(), copyToPath) + copyToPath = filepath.Join(workdir, copyToPath) } return common.NewPipelineExecutor( rc.JobContainer.Pull(rc.Config.ForcePull), + rc.stopServiceContainers(), rc.stopJobContainer(), + common.NewPipelineExecutor( + rc.createNetwork(networkName, types.NetworkCreate{CheckDuplicate: true}).IfBool(!container.DockerNetworkExists(ctx, networkName)), + rc.startServiceContainers(networkName), + ).IfBool(job.Services != nil || job.Container() != nil), rc.JobContainer.Create(rc.Config.ContainerCapAdd, rc.Config.ContainerCapDrop), rc.JobContainer.Start(false), rc.JobContainer.UpdateFromImageEnv(&rc.Env), + common.NewPipelineExecutor( + rc.JobContainer.ConnectToNetwork(networkName), + ).IfBool(job.Services != nil || job.Container() != nil), rc.JobContainer.UpdateFromEnv("/etc/environment", &rc.Env), rc.JobContainer.Exec([]string{"mkdir", "-m", "0777", "-p", ActPath}, rc.Env, "root", ""), rc.JobContainer.CopyDir(copyToPath, rc.Config.Workdir+string(filepath.Separator)+".", rc.Config.UseGitIgnore).IfBool(copyWorkspace), @@ -221,6 +272,18 @@ func (rc *RunContext) startJobContainer() common.Executor { } } +func (rc *RunContext) createNetwork(name string, config types.NetworkCreate) common.Executor { + return func(ctx context.Context) error { + return container.NewDockerNetworkCreateExecutor(name, config)(ctx) + } +} + +func (rc *RunContext) removeNetwork(name string) common.Executor { + return func(ctx context.Context) error { + return container.NewDockerNetworkRemoveExecutor(name)(ctx) + } +} + func (rc *RunContext) execJobContainer(cmd []string, env map[string]string, user, workdir string) common.Executor { return func(ctx context.Context) error { return rc.JobContainer.Exec(cmd, env, user, workdir)(ctx) @@ -231,14 +294,40 @@ func (rc *RunContext) execJobContainer(cmd []string, env map[string]string, user func (rc *RunContext) stopJobContainer() common.Executor { return func(ctx context.Context) error { if rc.JobContainer != nil && !rc.Config.ReuseContainers { - return rc.JobContainer.Remove(). - Then(container.NewDockerVolumeRemoveExecutor(rc.jobContainerName(), false))(ctx) + return common.NewPipelineExecutor( + rc.stopServiceContainers(), + rc.JobContainer.Remove().Then(container.NewDockerVolumeRemoveExecutor(rc.jobContainerName(), false)), + rc.removeNetwork(rc.JobNetworkName()).IfBool((rc.Run.Job().Services != nil || rc.Run.Job().Container() != nil) && !rc.Config.ReuseContainers && container.DockerNetworkExists(ctx, rc.JobNetworkName())), + )(ctx) } return nil } } -// Prepare the mounts and binds for the worker +func (rc *RunContext) startServiceContainers(networkName string) common.Executor { + return func(ctx context.Context) error { + execs := []common.Executor{} + for _, c := range rc.ServiceContainers { + execs = append(execs, common.NewPipelineExecutor( + c.Pull(rc.Config.ForcePull), + c.Create([]string{}, []string{}), + c.ConnectToNetwork(networkName), + c.Start(false), + )) + } + return common.NewParallelExecutor(execs...)(ctx) + } +} + +func (rc *RunContext) stopServiceContainers() common.Executor { + return func(ctx context.Context) error { + execs := []common.Executor{} + for _, c := range rc.ServiceContainers { + execs = append(execs, c.Remove()) + } + return common.NewParallelExecutor(execs...)(ctx) + } +} // ActionCacheDir is for rc func (rc *RunContext) ActionCacheDir() string { @@ -480,6 +569,10 @@ func createContainerName(parts ...string) string { return strings.ReplaceAll(strings.Trim(strings.Join(name, "-"), "-"), "--", "-") } +func createNetworkName(parts ...string) string { + return strings.ReplaceAll(strings.Trim(strings.Join(parts, "_"), "-"), "--", "-") +} + func trimToLen(s string, l int) string { if l < 0 { l = 0 @@ -491,17 +584,25 @@ func trimToLen(s string, l int) string { } type jobContext struct { - Status string `json:"status"` - Container struct { - ID string `json:"id"` - Network string `json:"network"` - } `json:"container"` - Services map[string]struct { - ID string `json:"id"` - } `json:"services"` + Status string `json:"status"` + Container jobContainerContext `json:"container"` + Services map[string]jobServiceContext `json:"services"` +} + +type jobContainerContext struct { + ID string `json:"id"` + Network string `json:"network"` +} + +type jobServiceContext struct { + ID string `json:"id"` + Network string `json:"network"` + Ports map[string]string `json:"ports"` } func (rc *RunContext) getJobContext() *jobContext { + job := rc.Run.Job() + jobStatus := "success" for _, stepStatus := range rc.StepResults { if stepStatus.Conclusion == stepStatusFailure { @@ -509,8 +610,36 @@ func (rc *RunContext) getJobContext() *jobContext { break } } + + container := jobContainerContext{} + if job.Container() != nil { + container = jobContainerContext{ + ID: rc.JobContainer.ID(), + Network: "host", + } + } + + services := make(map[string]jobServiceContext, len(job.Services)) + if rc.ServiceContainers != nil { + for name, container := range rc.ServiceContainers { + service := job.Services[name] + ports := make(map[string]string, len(service.Ports)) + for _, v := range service.Ports { + split := strings.Split(v, `:`) + ports[split[0]] = split[1] + } + services[name] = jobServiceContext{ + ID: container.ID(), + Network: rc.JobNetworkName(), + Ports: ports, + } + } + } + return &jobContext{ - Status: jobStatus, + Status: jobStatus, + Container: container, + Services: services, } } diff --git a/pkg/runner/runner_test.go b/pkg/runner/runner_test.go index 7ca000c32d7..79f154f6bff 100644 --- a/pkg/runner/runner_test.go +++ b/pkg/runner/runner_test.go @@ -181,6 +181,42 @@ func TestRunEventSecrets(t *testing.T) { assert.Nil(t, err, workflowPath) } +func TestRunWithService(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test") + } + + log.SetLevel(log.DebugLevel) + ctx := context.Background() + + platforms := map[string]string{ + "ubuntu-latest": baseImage, + } + + workflowPath := "services" + eventName := "push" + + workdir, err := filepath.Abs("testdata") + assert.Nil(t, err, workflowPath) + + runnerConfig := &Config{ + Workdir: workdir, + EventName: eventName, + Platforms: platforms, + ReuseContainers: false, + } + runner, err := New(runnerConfig) + assert.Nil(t, err, workflowPath) + + planner, err := model.NewWorkflowPlanner(fmt.Sprintf("testdata/%s", workflowPath), true) + assert.Nil(t, err, workflowPath) + + plan := planner.PlanEvent(eventName) + + err = runner.NewPlanExecutor(plan)(ctx) + assert.Nil(t, err, workflowPath) +} + func TestRunEventPullRequest(t *testing.T) { if testing.Short() { t.Skip("skipping integration test") diff --git a/pkg/runner/testdata/services/push.yaml b/pkg/runner/testdata/services/push.yaml new file mode 100644 index 00000000000..c519d9315e4 --- /dev/null +++ b/pkg/runner/testdata/services/push.yaml @@ -0,0 +1,41 @@ +name: services +on: push +jobs: + postgres: + runs-on: ubuntu-latest + services: + postgres: + image: postgres:12 + env: + POSTGRES_USER: runner + POSTGRES_PASSWORD: mysecretdbpass + POSTGRES_DB: mydb + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + steps: + - name: Echo the Postgres service ID / Network / Ports + run: | + echo "id: ${{ job.services.postgres.id }}" + echo "network: ${{ job.services.postgres.network }}" + echo "ports: ${{ toJSON(job.services.postgres.ports) }}" + mongodb: + runs-on: ubuntu-latest + services: + mongodb: + image: mongo:5 + env: + MONGO_INITDB_ROOT_USERNAME: root + MONGO_INITDB_ROOT_PASSWORD: example + ports: + - 27017:27017 + steps: + - name: Echo the Mongodb service ID / Network / Ports + run: | + echo "id: ${{ job.services.mongodb.id }}" + echo "network: ${{ job.services.mongodb.network }}" + echo "ports: ${{ toJSON(job.services.mongodb.ports) }}"