Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: support for loading images in the K3s module #1622

Merged
merged 12 commits into from
Sep 21, 2023
50 changes: 50 additions & 0 deletions docker.go
Original file line number Diff line number Diff line change
Expand Up @@ -1468,3 +1468,53 @@ func containerFromDockerResponse(ctx context.Context, response types.Container)

return &container, nil
}

// ListImages list images from the provider. If an image has multiple Tags, each tag is reported
// individually with the same ID and same labels
func (p *DockerProvider) ListImages(ctx context.Context) ([]ImageInfo, error) {
images := []ImageInfo{}

imageList, err := p.client.ImageList(ctx, types.ImageListOptions{})
if err != nil {
return images, fmt.Errorf("listing images %w", err)
}

for _, img := range imageList {
for _, tag := range img.RepoTags {
images = append(images, ImageInfo{ID: img.ID, Name: tag})
}
}

return images, nil
}

// SaveImages exports a list of images as an uncompressed tar
func (p *DockerProvider) SaveImages(ctx context.Context, output string, images ...string) error {
outputFile, err := os.Create(output)
if err != nil {
return fmt.Errorf("opening output file %w", err)
}
defer func() {
_ = outputFile.Close()
}()

imageReader, err := p.client.ImageSave(ctx, images)
if err != nil {
return fmt.Errorf("saving images %w", err)
}
defer func() {
_ = imageReader.Close()
}()

_, err = io.Copy(outputFile, imageReader)
if err != nil {
return fmt.Errorf("writing images to output %w", err)
}

return nil
}

// PullImage pulls image from registry
func (p *DockerProvider) PullImage(ctx context.Context, image string) error {
return p.attemptToPullImage(ctx, image, types.ImagePullOptions{})
}
8 changes: 8 additions & 0 deletions docs/modules/k3s.md
Original file line number Diff line number Diff line change
Expand Up @@ -79,3 +79,11 @@ to the Kubernetes Rest Client API using a Kubernetes client. It'll be returned i
<!--codeinclude-->
[Get KubeConifg](../../modules/k3s/k3s_test.go) inside_block:GetKubeConfig
<!--/codeinclude-->

#### LoadImages

The `LoadImages` method loads a list of images into the kubernetes cluster and makes them available to pods.

This is useful for testing images generated locally without having to push them to a public docker registry or having to configure `k3s` to [use a private registry](https://docs.k3s.io/installation/private-registry).

The images must be already present in the node running the test. [DockerProvider](https://pkg.go.dev/github.com/testcontainers/testcontainers-go#DockerProvider) offers a method for pulling images, which can be used from the test code to ensure the image is present locally before loading them to the cluster.
1 change: 1 addition & 0 deletions generic.go
Original file line number Diff line number Diff line change
Expand Up @@ -156,4 +156,5 @@ func GenericContainer(ctx context.Context, req GenericContainerRequest) (Contain
type GenericProvider interface {
ContainerProvider
NetworkProvider
ImageProvider
}
18 changes: 18 additions & 0 deletions image.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
package testcontainers

import (
"context"
)

// ImageInfo represents a summary information of an image
type ImageInfo struct {
ID string
Name string
}

// ImageProvider allows manipulating images
type ImageProvider interface {
ListImages(context.Context) ([]ImageInfo, error)
SaveImages(context.Context, string, ...string) error
PullImage(context.Context, string) error
}
95 changes: 95 additions & 0 deletions image_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
package testcontainers

import (
"context"
"os"
"path/filepath"
"testing"

"github.com/testcontainers/testcontainers-go/internal/testcontainersdocker"
)

func TestImageList(t *testing.T) {
t.Setenv("DOCKER_HOST", testcontainersdocker.ExtractDockerHost(context.Background()))

provider, err := ProviderDocker.GetProvider()
if err != nil {
t.Fatalf("failed to get provider %v", err)
}

defer func() {
_ = provider.Close()
}()

req := ContainerRequest{
Image: "redis:latest",
}

container, err := provider.CreateContainer(context.Background(), req)
if err != nil {
t.Fatalf("creating test container %v", err)
}

defer func() {
_ = container.Terminate(context.Background())
}()

images, err := provider.ListImages(context.Background())
if err != nil {
t.Fatalf("listing images %v", err)
}

if len(images) == 0 {
t.Fatal("no images retrieved")
}

// look if the list contains the container image
for _, img := range images {
if img.Name == req.Image {
return
}
}

t.Fatalf("expected image not found: %s", req.Image)
}

func TestSaveImages(t *testing.T) {
t.Setenv("DOCKER_HOST", testcontainersdocker.ExtractDockerHost(context.Background()))

provider, err := ProviderDocker.GetProvider()
if err != nil {
t.Fatalf("failed to get provider %v", err)
}

defer func() {
_ = provider.Close()
}()

req := ContainerRequest{
Image: "redis:latest",
}

container, err := provider.CreateContainer(context.Background(), req)
if err != nil {
t.Fatalf("creating test container %v", err)
}

defer func() {
_ = container.Terminate(context.Background())
}()

output := filepath.Join(t.TempDir(), "images.tar")
err = provider.SaveImages(context.Background(), output, req.Image)
if err != nil {
t.Fatalf("saving image %q: %v", req.Image, err)
}

info, err := os.Stat(output)
if err != nil {
t.Fatal(err)
}

if info.Size() == 0 {
t.Fatalf("output file is empty")
}
}
37 changes: 37 additions & 0 deletions modules/k3s/k3s.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import (
"context"
"fmt"
"io"
"os"
"path/filepath"

"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/mount"
Expand Down Expand Up @@ -163,3 +165,38 @@ func unmarshal(bytes []byte) (*KubeConfigValue, error) {
}
return &kubeConfig, nil
}

// LoadImages loads images into the k3s container.
func (c *K3sContainer) LoadImages(ctx context.Context, images ...string) error {
provider, err := testcontainers.ProviderDocker.GetProvider()
if err != nil {
return fmt.Errorf("getting docker provider %w", err)
}

// save image
imagesTar, err := os.CreateTemp(os.TempDir(), "images*.tar")
if err != nil {
return fmt.Errorf("creating temporary images file %w", err)
}
defer func() {
_ = os.Remove(imagesTar.Name())
}()

err = provider.SaveImages(context.Background(), imagesTar.Name(), images...)
if err != nil {
return fmt.Errorf("saving images %w", err)
}

containerPath := fmt.Sprintf("/tmp/%s", filepath.Base(imagesTar.Name()))
err = c.Container.CopyFileToContainer(ctx, imagesTar.Name(), containerPath, 0x644)
if err != nil {
return fmt.Errorf("copying image to container %w", err)
}

_, _, err = c.Container.Exec(ctx, []string{"ctr", "-n=k8s.io", "images", "import", containerPath})
if err != nil {
return fmt.Errorf("importing image %w", err)
}

return nil
}
109 changes: 109 additions & 0 deletions modules/k3s/k3s_image_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
package k3s_test

import (
"context"
"testing"
"time"

corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/clientcmd"

"github.com/testcontainers/testcontainers-go"
"github.com/testcontainers/testcontainers-go/modules/k3s"
"github.com/testcontainers/testcontainers-go/wait"
mdelapenya marked this conversation as resolved.
Show resolved Hide resolved
)

func Test_LoadImages(t *testing.T) {
ctx := context.Background()

k3sContainer, err := k3s.RunContainer(ctx,
testcontainers.WithImage("docker.io/rancher/k3s:v1.27.1-k3s1"),
testcontainers.WithWaitStrategy(wait.ForLog(".*Node controller sync successful.*").AsRegexp()),
)
if err != nil {
t.Fatal(err)
}

// Clean up the container
defer func() {
if err := k3sContainer.Terminate(ctx); err != nil {
t.Fatal(err)
}
}()

kubeConfigYaml, err := k3sContainer.GetKubeConfig(ctx)
if err != nil {
t.Fatal(err)
}

restcfg, err := clientcmd.RESTConfigFromKubeConfig(kubeConfigYaml)
if err != nil {
t.Fatal(err)
}

k8s, err := kubernetes.NewForConfig(restcfg)
if err != nil {
t.Fatal(err)
}

provider, err := testcontainers.ProviderDocker.GetProvider()
if err != nil {
t.Fatal(err)
}

// ensure nginx image is available locally
err = provider.PullImage(context.Background(), "nginx")
if err != nil {
t.Fatal(err)
}

t.Run("Test load image not available", func(t *testing.T) {
err := k3sContainer.LoadImages(context.Background(), "fake.registry/fake:non-existing")
if err == nil {
t.Fatal("should had failed")
}
})

t.Run("Test load image in cluster", func(t *testing.T) {
err := k3sContainer.LoadImages(context.Background(), "nginx")
if err != nil {
t.Fatal(err)
}

pod := &corev1.Pod{
TypeMeta: metav1.TypeMeta{
Kind: "Pod",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "test-pod",
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "nginx",
Image: "nginx",
ImagePullPolicy: corev1.PullNever, // use image only if already present
},
},
},
}

_, err = k8s.CoreV1().Pods("default").Create(context.Background(), pod, metav1.CreateOptions{})
if err != nil {
t.Fatal(err)
}

time.Sleep(1 * time.Second)
pod, err = k8s.CoreV1().Pods("default").Get(context.Background(), "test-pod", metav1.GetOptions{})
if err != nil {
t.Fatal(err)
}
waiting := pod.Status.ContainerStatuses[0].State.Waiting
if waiting != nil && waiting.Reason == "ErrImageNeverPull" {
mdelapenya marked this conversation as resolved.
Show resolved Hide resolved
t.Fatal("Image was not loaded")
}
})
}