-
-
Notifications
You must be signed in to change notification settings - Fork 466
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #91 from rancher/feature/import-images-helper
[Feature] import images from docker daemon into k3d using a helper container
- Loading branch information
Showing
8 changed files
with
370 additions
and
23 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,209 @@ | ||
| package run | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "io/ioutil" | ||
| "log" | ||
| "strings" | ||
| "time" | ||
|
|
||
| "github.com/docker/docker/api/types" | ||
| "github.com/docker/docker/api/types/container" | ||
| "github.com/docker/docker/api/types/network" | ||
| "github.com/docker/docker/client" | ||
| ) | ||
|
|
||
| const ( | ||
| imageBasePathRemote = "/images" | ||
| k3dToolsImage = "docker.io/iwilltry42/k3d-tools:v0.0.1" | ||
| ) | ||
|
|
||
| func importImage(clusterName string, images []string, noRemove bool) error { | ||
| // get a docker client | ||
| ctx := context.Background() | ||
| docker, err := client.NewEnvClient() | ||
| if err != nil { | ||
| return fmt.Errorf("ERROR: couldn't create docker client\n%+v", err) | ||
| } | ||
|
|
||
| // get cluster directory to temporarily save the image tarball there | ||
| imageVolume, err := getImageVolume(clusterName) | ||
| if err != nil { | ||
| return fmt.Errorf("ERROR: couldn't get image volume for cluster [%s]\n%+v", clusterName, err) | ||
| } | ||
|
|
||
| //*** first, save the images using the local docker daemon | ||
| log.Printf("INFO: Saving images %s from local docker daemon...", images) | ||
| toolsContainerName := fmt.Sprintf("k3d-%s-tools", clusterName) | ||
| tarFileName := fmt.Sprintf("%s/k3d-%s-images-%s.tar", imageBasePathRemote, clusterName, time.Now().Format("20060102150405")) | ||
|
|
||
| // create a tools container to get the tarball into the named volume | ||
| containerConfig := container.Config{ | ||
| Hostname: toolsContainerName, | ||
| Image: k3dToolsImage, | ||
| Labels: map[string]string{ | ||
| "app": "k3d", | ||
| "cluster": clusterName, | ||
| "component": "tools", | ||
| }, | ||
| Cmd: append([]string{"save-image", "-d", tarFileName}, images...), | ||
| AttachStdout: true, | ||
| AttachStderr: true, | ||
| } | ||
| hostConfig := container.HostConfig{ | ||
| Binds: []string{ | ||
| "/var/run/docker.sock:/var/run/docker.sock", | ||
| fmt.Sprintf("%s:%s:rw", imageVolume.Name, imageBasePathRemote), | ||
| }, | ||
| } | ||
|
|
||
| toolsContainerID, err := startContainer(false, &containerConfig, &hostConfig, &network.NetworkingConfig{}, toolsContainerName) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| defer func() { | ||
| if err = docker.ContainerRemove(ctx, toolsContainerID, types.ContainerRemoveOptions{ | ||
| Force: true, | ||
| }); err != nil { | ||
| log.Println(fmt.Errorf("WARN: couldn't remove tools container\n%+v", err)) | ||
| } | ||
| }() | ||
|
|
||
| // loop to wait for tools container to exit (failed or successfully saved images) | ||
| for { | ||
| cont, err := docker.ContainerInspect(ctx, toolsContainerID) | ||
| if err != nil { | ||
| return fmt.Errorf("ERROR: couldn't get helper container's exit code\n%+v", err) | ||
| } | ||
| if !cont.State.Running { // container finished... | ||
| if cont.State.ExitCode == 0 { // ...successfully | ||
| log.Println("INFO: saved images to shared docker volume") | ||
| break | ||
| } else if cont.State.ExitCode != 0 { // ...failed | ||
| errTxt := "ERROR: helper container failed to save images" | ||
| logReader, err := docker.ContainerLogs(ctx, toolsContainerID, types.ContainerLogsOptions{ | ||
| ShowStdout: true, | ||
| ShowStderr: true, | ||
| }) | ||
| if err != nil { | ||
| return fmt.Errorf("%s\n> couldn't get logs from helper container\n%+v", errTxt, err) | ||
| } | ||
| logs, err := ioutil.ReadAll(logReader) // let's show somw logs indicating what happened | ||
| if err != nil { | ||
| return fmt.Errorf("%s\n> couldn't get logs from helper container\n%+v", errTxt, err) | ||
| } | ||
| return fmt.Errorf("%s -> Logs from [%s]:\n>>>>>>\n%s\n<<<<<<", errTxt, toolsContainerName, string(logs)) | ||
| } | ||
| } | ||
| time.Sleep(time.Second / 2) // wait for half a second so we don't spam the docker API too much | ||
| } | ||
|
|
||
| // Get the container IDs for all containers in the cluster | ||
| clusters, err := getClusters(false, clusterName) | ||
| if err != nil { | ||
| return fmt.Errorf("ERROR: couldn't get cluster by name [%s]\n%+v", clusterName, err) | ||
| } | ||
| containerList := []types.Container{clusters[clusterName].server} | ||
| containerList = append(containerList, clusters[clusterName].workers...) | ||
|
|
||
| // *** second, import the images using ctr in the k3d nodes | ||
|
|
||
| // create exec configuration | ||
| cmd := []string{"ctr", "image", "import", tarFileName} | ||
| execConfig := types.ExecConfig{ | ||
| AttachStderr: true, | ||
| AttachStdout: true, | ||
| Cmd: cmd, | ||
| Tty: true, | ||
| Detach: true, | ||
| } | ||
|
|
||
| execAttachConfig := types.ExecConfig{ | ||
| Tty: true, | ||
| } | ||
|
|
||
| execStartConfig := types.ExecStartCheck{ | ||
| Tty: true, | ||
| } | ||
|
|
||
| // import in each node separately | ||
| // TODO: import concurrently using goroutines or find a way to share the image cache | ||
| for _, container := range containerList { | ||
|
|
||
| containerName := container.Names[0][1:] // trimming the leading "/" from name | ||
| log.Printf("INFO: Importing images %s in container [%s]", images, containerName) | ||
|
|
||
| // create exec configuration | ||
| execResponse, err := docker.ContainerExecCreate(ctx, container.ID, execConfig) | ||
| if err != nil { | ||
| return fmt.Errorf("ERROR: Failed to create exec command for container [%s]\n%+v", containerName, err) | ||
| } | ||
|
|
||
| // attach to exec process in container | ||
| containerConnection, err := docker.ContainerExecAttach(ctx, execResponse.ID, execAttachConfig) | ||
| if err != nil { | ||
| return fmt.Errorf("ERROR: couldn't attach to container [%s]\n%+v", containerName, err) | ||
| } | ||
| defer containerConnection.Close() | ||
|
|
||
| // start exec | ||
| err = docker.ContainerExecStart(ctx, execResponse.ID, execStartConfig) | ||
| if err != nil { | ||
| return fmt.Errorf("ERROR: couldn't execute command in container [%s]\n%+v", containerName, err) | ||
| } | ||
|
|
||
| // get output from container | ||
| content, err := ioutil.ReadAll(containerConnection.Reader) | ||
| if err != nil { | ||
| return fmt.Errorf("ERROR: couldn't read output from container [%s]\n%+v", containerName, err) | ||
| } | ||
|
|
||
| // example output "unpacking image........ ...done" | ||
| if !strings.Contains(string(content), "done") { | ||
| return fmt.Errorf("ERROR: seems like something went wrong using `ctr image import` in container [%s]. Full output below:\n%s", containerName, string(content)) | ||
| } | ||
| } | ||
|
|
||
| log.Printf("INFO: Successfully imported images %s in all nodes of cluster [%s]", images, clusterName) | ||
|
|
||
| // remove tarball from inside the server container | ||
| if !noRemove { | ||
| log.Println("INFO: Cleaning up tarball") | ||
|
|
||
| execID, err := docker.ContainerExecCreate(ctx, clusters[clusterName].server.ID, types.ExecConfig{ | ||
| Cmd: []string{"rm", "-f", tarFileName}, | ||
| }) | ||
| if err != nil { | ||
| log.Printf("WARN: failed to delete tarball: couldn't create remove in container [%s]\n%+v", clusters[clusterName].server.ID, err) | ||
| } | ||
| err = docker.ContainerExecStart(ctx, execID.ID, types.ExecStartCheck{ | ||
| Detach: true, | ||
| }) | ||
| if err != nil { | ||
| log.Printf("WARN: couldn't start tarball deletion action\n%+v", err) | ||
| } | ||
|
|
||
| for { | ||
| execInspect, err := docker.ContainerExecInspect(ctx, execID.ID) | ||
| if err != nil { | ||
| log.Printf("WARN: couldn't verify deletion of tarball\n%+v", err) | ||
| } | ||
|
|
||
| if !execInspect.Running { | ||
| if execInspect.ExitCode == 0 { | ||
| log.Println("INFO: deleted tarball") | ||
| break | ||
| } else { | ||
| log.Println("WARN: failed to delete tarball") | ||
| break | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| log.Println("INFO: ...Done") | ||
|
|
||
| return nil | ||
| } |
Oops, something went wrong.