Skip to content

Commit

Permalink
[content-service] Remove docker dependency
Browse files Browse the repository at this point in the history 
by moving archive creation back to ws-daemon
  • Loading branch information
Christian Weichel committed Feb 1, 2021
1 parent 3328fc6 commit 4d3947d
Show file tree
Hide file tree
Showing 9 changed files with 285 additions and 288 deletions.
15 changes: 0 additions & 15 deletions components/content-service/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,20 +2,8 @@ module github.com/gitpod-io/gitpod/content-service

go 1.14

// containerd, see https://github.com/containerd/containerd/issues/3031
replace github.com/docker/distribution v2.7.1+incompatible => github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible // leeway ignore

replace github.com/docker/docker v1.13.1 => github.com/docker/engine v0.0.0-20190822205725-ed20165a37b4 // leeway ignore

replace github.com/Sirupsen/logrus v1.6.0 => github.com/sirupsen/logrus v1.6.0 // leeway ignore

require (
cloud.google.com/go/storage v1.10.0
github.com/Microsoft/go-winio v0.4.16 // indirect
github.com/Microsoft/hcsshim v0.8.14 // indirect
github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7 // indirect
github.com/docker/docker v1.13.1
github.com/docker/go-units v0.4.0 // indirect
github.com/gitpod-io/gitpod/common-go v0.0.0-00010101000000-000000000000
github.com/gitpod-io/gitpod/content-service/api v0.0.0-00010101000000-000000000000
github.com/go-ozzo/ozzo-validation v3.5.0+incompatible
Expand All @@ -27,7 +15,6 @@ require (
github.com/minio/minio-go/v7 v7.0.7
github.com/opencontainers/go-digest v1.0.0
github.com/opencontainers/image-spec v1.0.1
github.com/opencontainers/runc v0.1.1 // indirect
github.com/opentracing/opentracing-go v1.1.0
github.com/prometheus/client_golang v1.1.0
github.com/spf13/cobra v1.1.1
Expand All @@ -38,9 +25,7 @@ require (
golang.org/x/text v0.3.5 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1
google.golang.org/api v0.32.0
google.golang.org/appengine v1.6.6
google.golang.org/grpc v1.34.0
google.golang.org/protobuf v1.25.0
gopkg.in/ini.v1 v1.62.0 // indirect
)

Expand Down
59 changes: 16 additions & 43 deletions components/content-service/go.sum
View file

Large diffs are not rendered by default.

136 changes: 12 additions & 124 deletions components/content-service/pkg/archive/tar.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,36 +6,33 @@ package archive

import (
"archive/tar"
"bufio"
"context"
"fmt"
"io"
"os"
"os/exec"
"path"
"sort"
"time"

"github.com/docker/docker/pkg/archive"
"github.com/docker/docker/pkg/idtools"
"github.com/gitpod-io/gitpod/common-go/log"
"github.com/gitpod-io/gitpod/common-go/tracing"
"github.com/opentracing/opentracing-go"
"golang.org/x/xerrors"
)

type tarConfig struct {
// TarConfig configures tarbal creation/extraction
type TarConfig struct {
MaxSizeBytes int64
UIDMaps []idtools.IDMap
GIDMaps []idtools.IDMap
UIDMaps []IDMapping
GIDMaps []IDMapping
}

// BuildTarbalOption configures the tarbal creation
type TarOption func(o *tarConfig)
type TarOption func(o *TarConfig)

// TarbalMaxSize limits the size of a tarbal
func TarbalMaxSize(n int64) TarOption {
return func(o *tarConfig) {
return func(o *TarConfig) {
o.MaxSizeBytes = n
}
}
Expand All @@ -49,35 +46,21 @@ type IDMapping struct {

// WithUIDMapping reverses the given user ID mapping during archive creation
func WithUIDMapping(mappings []IDMapping) TarOption {
return func(o *tarConfig) {
o.UIDMaps = make([]idtools.IDMap, len(mappings))
for i, m := range mappings {
o.UIDMaps[i] = idtools.IDMap{
ContainerID: m.ContainerID,
HostID: m.HostID,
Size: m.Size,
}
}
return func(o *TarConfig) {
o.UIDMaps = mappings
}
}

// WithGIDMapping reverses the given user ID mapping during archive creation
func WithGIDMapping(mappings []IDMapping) TarOption {
return func(o *tarConfig) {
o.GIDMaps = make([]idtools.IDMap, len(mappings))
for i, m := range mappings {
o.GIDMaps[i] = idtools.IDMap{
ContainerID: m.ContainerID,
HostID: m.HostID,
Size: m.Size,
}
}
return func(o *TarConfig) {
o.GIDMaps = mappings
}
}

// ExtractTarbal extracts an OCI compatible tar file src to the folder dst, expecting the overlay whiteout format
func ExtractTarbal(ctx context.Context, src io.Reader, dst string, opts ...TarOption) (err error) {
var cfg tarConfig
var cfg TarConfig
start := time.Now()
for _, opt := range opts {
opt(&cfg)
Expand Down Expand Up @@ -145,7 +128,7 @@ func ExtractTarbal(ctx context.Context, src io.Reader, dst string, opts ...TarOp
return nil
}

func toHostID(containerID int, idMap []idtools.IDMap) int {
func toHostID(containerID int, idMap []IDMapping) int {
for _, m := range idMap {
if (containerID >= m.ContainerID) && (containerID <= (m.ContainerID + m.Size - 1)) {
hostID := m.HostID + (containerID - m.ContainerID)
Expand All @@ -154,98 +137,3 @@ func toHostID(containerID int, idMap []idtools.IDMap) int {
}
return containerID
}

// BuildTarbal creates an OCI compatible tar file dst from the folder src, expecting the overlay whiteout format
func BuildTarbal(ctx context.Context, src string, dst string, opts ...TarOption) (err error) {
var cfg tarConfig
for _, opt := range opts {
opt(&cfg)
}

//nolint:staticcheck,ineffassign
span, ctx := opentracing.StartSpanFromContext(ctx, "buildTarbal")
span.LogKV("src", src, "dst", dst)
defer tracing.FinishSpan(span, &err)

// ensure the src actually exists before trying to tar it
if _, err := os.Stat(src); err != nil {
return fmt.Errorf("Unable to tar files: %v", err.Error())
}

tarout, err := archive.TarWithOptions(src, &archive.TarOptions{
Compression: archive.Uncompressed,
WhiteoutFormat: archive.OverlayWhiteoutFormat,
InUserNS: true,
UIDMaps: cfg.UIDMaps,
GIDMaps: cfg.GIDMaps,
})
if err != nil {
return xerrors.Errorf("cannot create tar: %w", err)
}

fout, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE, 0744)
if err != nil {
return xerrors.Errorf("cannot open archive for writing: %w", err)
}
defer fout.Close()
fbout := bufio.NewWriter(fout)
defer fbout.Flush()

targetOut := newLimitWriter(fbout, cfg.MaxSizeBytes)
defer func(e *error) {
if targetOut.DidMaxOut() {
*e = ErrMaxSizeExceeded
}
}(&err)

_, err = io.Copy(targetOut, tarout)
if err != nil {
return xerrors.Errorf("cannot write tar file: %w")
}
if err = fbout.Flush(); err != nil {
return xerrors.Errorf("cannot flush tar out stream: %w", err)
}

return nil
}

// ErrMaxSizeExceeded is emitted by LimitWriter when a write tries to write beyond the max number of bytes allowed
var ErrMaxSizeExceeded = fmt.Errorf("maximum size exceeded")

// newLimitWriter wraps a writer such that a maximum of N bytes can be written. Once that limit is exceeded
// the writer returns io.ErrClosedPipe
func newLimitWriter(out io.Writer, maxSizeBytes int64) *limitWriter {
return &limitWriter{
MaxSizeBytes: maxSizeBytes,
Out: out,
}
}

type limitWriter struct {
MaxSizeBytes int64
Out io.Writer
BytesWritten int64

didMaxOut bool
}

func (s *limitWriter) Write(b []byte) (n int, err error) {
if s.MaxSizeBytes == 0 {
return s.Out.Write(b)
}

bsize := int64(len(b))
if bsize+s.BytesWritten > s.MaxSizeBytes {
s.didMaxOut = true
return 0, ErrMaxSizeExceeded
}

n, err = s.Out.Write(b)
s.BytesWritten += int64(n)

return n, err
}

func (s *limitWriter) DidMaxOut() bool {
return s.didMaxOut
}
128 changes: 24 additions & 104 deletions components/content-service/pkg/archive/tar_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
package archive

import (
"archive/tar"
"bytes"
"context"
"io/ioutil"
"os"
Expand All @@ -13,91 +15,8 @@ import (
"testing"
)

func TestSizeLimitingWriter(t *testing.T) {
tests := []struct {
MaxSize int64
BlockSize int64
BlockCount int
ExpectWritten int
Err error
}{
{30, 10, 1, 10, nil},
{30, 10, 3, 30, nil},
{20, 10, 3, 20, ErrMaxSizeExceeded},
}

for _, test := range tests {
var (
written int
n int
err error
)
w := newLimitWriter(ioutil.Discard, test.MaxSize)

for i := 0; i < test.BlockCount; i++ {
n, err = w.Write(make([]byte, test.BlockSize))
written += n
if err != nil {
break
}
}

if err != test.Err {
t.Errorf("unexpected error: expected %v, actual %v", test.Err, err)
}
if written != test.ExpectWritten {
t.Errorf("wrote unexpected number of bytes: expected %v, actual %v", test.ExpectWritten, written)
}
}
}

func TestBuildTarbalMaxSize(t *testing.T) {
tests := []struct {
Name string
MaxSize int64
ContentSize int64
Err error
}{
{"positive", 1024 * 1024, 512, nil},
{"too-big", 512, 1024, ErrMaxSizeExceeded},
}

var cleanup []string
for _, test := range tests {
wd, err := ioutil.TempDir("", "")
if err != nil {
t.Errorf("cannot prepare test: %v", err)
continue
}
cleanup = append(cleanup, wd)

err = ioutil.WriteFile(filepath.Join(wd, "content.txt"), make([]byte, test.ContentSize), 0644)
if err != nil {
t.Errorf("cannot prepare test: %v", err)
continue
}

tgt, err := ioutil.TempFile("", "")
if err != nil {
t.Errorf("cannot prepare test: %v", err)
continue
}
tgt.Close()
cleanup = append(cleanup, tgt.Name())

err = BuildTarbal(context.Background(), wd, tgt.Name(), TarbalMaxSize(test.MaxSize))
if (err == nil && test.Err != nil) || (err != nil && test.Err == nil) || (err != nil && test.Err != nil && err.Error() != test.Err.Error()) {
t.Errorf("%s: unexpected error: expected \"%v\", actual \"%v\"", test.Name, test.Err, err)
}
}

for _, c := range cleanup {
os.RemoveAll(c)
}
}

func TestExtractTarbal(t *testing.T) {
type file = struct {
type file struct {
Name string
ContentSize int64
Uid int
Expand All @@ -121,37 +40,38 @@ func TestExtractTarbal(t *testing.T) {

for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
wd, err := ioutil.TempDir("", "")
defer os.RemoveAll(wd)
if err != nil {
t.Errorf("cannot prepare test: %v", err)
t.FailNow()
}
sourceFolder := filepath.Join(wd, "source")
os.MkdirAll(sourceFolder, 0777)

var (
buf = bytes.NewBuffer(nil)
tw = tar.NewWriter(buf)
)
for _, file := range test.Files {
fileName := filepath.Join(sourceFolder, file.Name)
err = ioutil.WriteFile(fileName, make([]byte, file.ContentSize), 0644)
err := tw.WriteHeader(&tar.Header{
Name: file.Name,
Size: file.ContentSize,
Uid: file.Uid,
Gid: file.Uid,
Mode: 0644,
Typeflag: tar.TypeReg,
})
if err != nil {
t.Errorf("cannot prepare test: %v", err)
continue
t.Fatalf("cannot prepare archive: %q", err)
}
err = os.Chown(fileName, file.Uid, file.Uid)
_, err = tw.Write(make([]byte, file.ContentSize))
if err != nil {
t.Errorf("Cannot chown %s to %d: %s", file.Name, file.Uid, err)
t.Fatalf("cannot prepare archive: %q", err)
}
}
tarFile := filepath.Join(wd, "my.tar")
BuildTarbal(context.Background(), sourceFolder, tarFile)
tw.Flush()
tw.Close()

reader, err := os.Open(tarFile)
wd, err := ioutil.TempDir("", "")
defer os.RemoveAll(wd)
if err != nil {
t.Errorf("Cannot open %s", tarFile)
t.Fatalf("cannot prepare test: %v", err)
}
targetFolder := filepath.Join(wd, "target")
os.MkdirAll(targetFolder, 0777)
ExtractTarbal(context.Background(), reader, targetFolder)
ExtractTarbal(context.Background(), buf, targetFolder)

for _, file := range test.Files {
stat, err := os.Stat(filepath.Join(targetFolder, file.Name))
Expand Down
1 change: 1 addition & 0 deletions components/ws-daemon/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,7 @@ github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.
github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
github.com/containerd/containerd v1.4.1 h1:pASeJT3R3YyVn+94qEPk0SnU1OQ20Jd/T+SPKy9xehY=
github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc h1:TP+534wVlf61smEIq1nwLLAjQVEK2EADoW3CX9AuT+8=
github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
github.com/containerd/continuity v0.0.0-20200928162600-f2cc35102c2a h1:jEIoR0aA5GogXZ8pP3DUzE+zrhaF6/1rYZy+7KkYEWM=
github.com/containerd/continuity v0.0.0-20200928162600-f2cc35102c2a/go.mod h1:W0qIOTD7mp2He++YVq+kgfXezRYqzP1uDuMVH1bITDY=
Expand Down
Loading

0 comments on commit 4d3947d

Please sign in to comment.