From 78ae01161edd840f90275750e9e17c9975b3c71e Mon Sep 17 00:00:00 2001 From: Bohdan Shulha Date: Mon, 14 Oct 2024 19:59:39 +0200 Subject: [PATCH] feat: #60 support removing files from s3 storages --- go.mod | 12 +++++++ go.sum | 24 +++++++++++++ internal/app/ptah-agent/parse_task.go | 2 ++ internal/app/ptah-agent/s3.go | 44 ++++++++++++++++++++++++ internal/app/ptah-agent/task_executor.go | 2 ++ internal/pkg/ptah-client/task_types.go | 8 +++++ 6 files changed, 92 insertions(+) diff --git a/go.mod b/go.mod index 87b5c77..f1da9e6 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,9 @@ module github.com/ptah-sh/ptah-agent go 1.22.4 require ( + github.com/aws/aws-sdk-go-v2 v1.32.2 + github.com/aws/aws-sdk-go-v2/credentials v1.17.41 + github.com/aws/aws-sdk-go-v2/service/s3 v1.65.3 github.com/docker/docker v27.0.0+incompatible github.com/mackerelio/go-osstat v0.2.5 github.com/ncruces/go-sqlite3 v0.18.4 @@ -15,6 +18,15 @@ require ( require ( github.com/Microsoft/go-winio v0.4.14 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.21 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.2 // indirect + github.com/aws/smithy-go v1.22.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/containerd/log v0.1.0 // indirect diff --git a/go.sum b/go.sum index c65eb2d..c704e3b 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,30 @@ github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOEl github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= +github.com/aws/aws-sdk-go-v2 v1.32.2 h1:AkNLZEyYMLnx/Q/mSKkcMqwNFXMAvFto9bNsHqcTduI= +github.com/aws/aws-sdk-go-v2 v1.32.2/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 h1:pT3hpW0cOHRJx8Y0DfJUEQuqPild8jRGmSFmBgvydr0= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6/go.mod h1:j/I2++U0xX+cr44QjHay4Cvxj6FUbnxrgmqN3H1jTZA= +github.com/aws/aws-sdk-go-v2/credentials v1.17.41 h1:7gXo+Axmp+R4Z+AK8YFQO0ZV3L0gizGINCOWxSLY9W8= +github.com/aws/aws-sdk-go-v2/credentials v1.17.41/go.mod h1:u4Eb8d3394YLubphT4jLEwN1rLNq2wFOlT6OuxFwPzU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 h1:UAsR3xA31QGf79WzpG/ixT9FZvQlh5HY1NRqSHBNOCk= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21/go.mod h1:JNr43NFf5L9YaG3eKTm7HQzls9J+A9YYcGI5Quh1r2Y= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 h1:6jZVETqmYCadGFvrYEQfC5fAQmlo80CeL5psbno6r0s= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21/go.mod h1:1SR0GbLlnN3QUmYaflZNiH1ql+1qrSiB2vwcJ+4UM60= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.21 h1:7edmS3VOBDhK00b/MwGtGglCm7hhwNYnjJs/PgFdMQE= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.21/go.mod h1:Q9o5h4HoIWG8XfzxqiuK/CGUbepCJ8uTlaE3bAbxytQ= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 h1:TToQNkvGguu209puTojY/ozlqy2d/SFNcoLIqTFi42g= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0/go.mod h1:0jp+ltwkf+SwG2fm/PKo8t4y8pJSgOCO4D8Lz3k0aHQ= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.2 h1:4FMHqLfk0efmTqhXVRL5xYRqlEBNBiRI7N6w4jsEdd4= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.2/go.mod h1:LWoqeWlK9OZeJxsROW2RqrSPvQHKTpp69r/iDjwsSaw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2 h1:s7NA1SOw8q/5c0wr8477yOPp0z+uBaXBnLE0XYb0POA= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2/go.mod h1:fnjjWyAW/Pj5HYOxl9LJqWtEwS7W2qgcRLWP+uWbss0= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.2 h1:t7iUP9+4wdc5lt3E41huP+GvQZJD38WLsgVp4iOtAjg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.2/go.mod h1:/niFCtmuQNxqx9v8WAPq5qh7EH25U4BF6tjoyq9bObM= +github.com/aws/aws-sdk-go-v2/service/s3 v1.65.3 h1:xxHGZ+wUgZNACQmxtdvP5tgzfsxGS3vPpTP5Hy3iToE= +github.com/aws/aws-sdk-go-v2/service/s3 v1.65.3/go.mod h1:cB6oAuus7YXRZhWCc1wIwPywwZ1XwweNp2TVAEGYeB8= +github.com/aws/smithy-go v1.22.0 h1:uunKnWlcoL3zO7q+gG2Pk53joueEOsnNB28QdMsmiMM= +github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= diff --git a/internal/app/ptah-agent/parse_task.go b/internal/app/ptah-agent/parse_task.go index 3e465bb..f608b42 100644 --- a/internal/app/ptah-agent/parse_task.go +++ b/internal/app/ptah-agent/parse_task.go @@ -56,6 +56,8 @@ func parseTask(taskType int, payload string) (interface{}, error) { return unmarshalTask(payload, &ptahClient.LaunchServiceReq{}) case 22: return unmarshalTask(payload, &ptahClient.S3DownloadReq{}) + case 23: + return unmarshalTask(payload, &ptahClient.S3RemoveReq{}) default: return nil, fmt.Errorf("parse task: unknown task type %d", taskType) } diff --git a/internal/app/ptah-agent/s3.go b/internal/app/ptah-agent/s3.go index ebd6e92..77e6570 100644 --- a/internal/app/ptah-agent/s3.go +++ b/internal/app/ptah-agent/s3.go @@ -10,6 +10,10 @@ import ( "strings" "time" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/credentials" + + "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/image" "github.com/docker/docker/api/types/mount" @@ -200,3 +204,43 @@ func (e *taskExecutor) runS3Cmd(ctx context.Context, mounts []mount.Mount, envVa return nil, nil } + +func (e *taskExecutor) removeS3Files(ctx context.Context, req *t.S3RemoveReq) (*t.S3RemoveRes, error) { + log := Logger(ctx) + + credentialsConfig, err := e.getConfigByName(ctx, req.S3StorageConfigName) + if err != nil { + return nil, fmt.Errorf("remove s3 file: get config: %w", err) + } + + var s3StorageSpec t.S3StorageSpec + err = json.Unmarshal(credentialsConfig.Spec.Data, &s3StorageSpec) + if err != nil { + return nil, fmt.Errorf("remove s3 file: unmarshal config: %w", err) + } + + s3client := s3.NewFromConfig(aws.Config{ + Credentials: credentials.NewStaticCredentialsProvider( + s3StorageSpec.AccessKey, + s3StorageSpec.SecretKey, + "", + ), + Region: s3StorageSpec.Region, + BaseEndpoint: aws.String("https://" + s3StorageSpec.Endpoint), + }) + + fullPath := strings.Trim(s3StorageSpec.PathPrefix, "/") + "/" + strings.TrimPrefix(req.FilePath, "/") + + log.Debug("remove s3 file", "bucket", s3StorageSpec.Bucket, "full_path", fullPath) + + _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: aws.String(s3StorageSpec.Bucket), + Key: aws.String(fullPath), + }) + + if err != nil { + return nil, fmt.Errorf("remove s3 file: delete object: %w", err) + } + + return &t.S3RemoveRes{}, nil +} diff --git a/internal/app/ptah-agent/task_executor.go b/internal/app/ptah-agent/task_executor.go index 6c8377b..5646d40 100644 --- a/internal/app/ptah-agent/task_executor.go +++ b/internal/app/ptah-agent/task_executor.go @@ -66,6 +66,8 @@ func (e *taskExecutor) executeTask(ctx context.Context, task interface{}) (inter return e.launchDockerService(ctx, task.(*t.LaunchServiceReq)) case *t.S3DownloadReq: return e.s3download(ctx, task.(*t.S3DownloadReq)) + case *t.S3RemoveReq: + return e.removeS3Files(ctx, task.(*t.S3RemoveReq)) default: return nil, fmt.Errorf("execute task: unknown task type %T", task) } diff --git a/internal/pkg/ptah-client/task_types.go b/internal/pkg/ptah-client/task_types.go index 94fd9c5..66e9057 100644 --- a/internal/pkg/ptah-client/task_types.go +++ b/internal/pkg/ptah-client/task_types.go @@ -216,6 +216,14 @@ type S3DownloadRes struct { Output []string `json:"output"` } +type S3RemoveReq struct { + S3StorageConfigName string + FilePath string +} + +type S3RemoveRes struct { +} + type JoinSwarmReq struct { JoinSpec swarm.JoinRequest }