From d69678b8fb5437cd4356e7ffb0d5467ddc3e5065 Mon Sep 17 00:00:00 2001 From: Adam Janikowski <12255597+ajanikow@users.noreply.github.com> Date: Fri, 1 Nov 2024 13:27:30 +0100 Subject: [PATCH] [Feature] AWS Client (#1755) --- CHANGELOG.md | 1 + go.mod | 2 + go.sum | 7 ++ pkg/util/aws/config.go | 89 +++++++++++++++++++++++++ pkg/util/aws/file.go | 86 ++++++++++++++++++++++++ pkg/util/aws/http.go | 44 +++++++++++++ pkg/util/aws/impersonate.go | 89 +++++++++++++++++++++++++ pkg/util/aws/provider.go | 127 ++++++++++++++++++++++++++++++++++++ pkg/util/aws/tls.go | 58 ++++++++++++++++ pkg/util/file.go | 26 ++++++++ 10 files changed, 529 insertions(+) create mode 100644 pkg/util/aws/config.go create mode 100644 pkg/util/aws/file.go create mode 100644 pkg/util/aws/http.go create mode 100644 pkg/util/aws/impersonate.go create mode 100644 pkg/util/aws/provider.go create mode 100644 pkg/util/aws/tls.go diff --git a/CHANGELOG.md b/CHANGELOG.md index e5a78b005..d35fbb8ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ - (Maintenance) Inspector Generics - (Bugfix) Fix Gateway Options - (Feature) StorageV2 Integration Service Definition +- (Feature) AWS Client ## [1.2.43](https://github.com/arangodb/kube-arangodb/tree/1.2.43) (2024-10-14) - (Feature) ArangoRoute CRD diff --git a/go.mod b/go.mod index a859e606c..7ef2eef63 100644 --- a/go.mod +++ b/go.mod @@ -88,6 +88,7 @@ require ( github.com/Microsoft/hcsshim v0.11.4 // indirect github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect + github.com/aws/aws-sdk-go v1.55.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/bytedance/sonic v1.9.1 // indirect @@ -146,6 +147,7 @@ require ( github.com/huandu/xstrings v1.5.0 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/jmoiron/sqlx v1.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect diff --git a/go.sum b/go.sum index 7e8ef8b29..dc872b988 100644 --- a/go.sum +++ b/go.sum @@ -50,6 +50,8 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= +github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -104,6 +106,7 @@ github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/cyphar/filepath-securejoin v0.3.1 h1:1V7cHiaW+C+39wEfpH6XlLBQo3j/PciWFrgfCLS8XrE= github.com/cyphar/filepath-securejoin v0.3.1/go.mod h1:F7i41x/9cBF7lzCrVsYs9fuzwRZm4NQsGTBdpp6mETc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -268,6 +271,9 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jessevdk/go-assets v0.0.0-20160921144138-4f4301a06e15 h1:cW/amwGEJK5MSKntPXRjX4dxs/nGxGT8gXKIsKFmHGc= github.com/jessevdk/go-assets v0.0.0-20160921144138-4f4301a06e15/go.mod h1:Fdm/oWRW+CH8PRbLntksCNtmcCBximKPkVQYvmMl80k= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= github.com/josephburnett/jd v1.6.1 h1:Uzqhcje4WqvVyp85F3Oj0ezISPTlnhnr/KaLZIy8qh0= @@ -432,6 +438,7 @@ github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= diff --git a/pkg/util/aws/config.go b/pkg/util/aws/config.go new file mode 100644 index 000000000..4edb2bd6d --- /dev/null +++ b/pkg/util/aws/config.go @@ -0,0 +1,89 @@ +// +// DISCLAIMER +// +// Copyright 2024 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package aws + +import ( + "net/http" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/client" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + + "github.com/arangodb/kube-arangodb/pkg/util" + "github.com/arangodb/kube-arangodb/pkg/util/errors" +) + +type Config struct { + Endpoint string + Region string + + DisableSSL bool + + HTTP HTTP + Provider Provider + TLS TLS +} + +func (c Config) GetRegion() string { + if c.Region == "" { + return "us-east-1" + } + + return c.Region +} + +func (c Config) GetProvider() (credentials.Provider, error) { + return c.Provider.Provider() +} + +func (c Config) GetHttpClient() (*http.Client, error) { + tls, err := c.TLS.configuration() + if err != nil { + return nil, errors.Wrapf(err, "Unable to create TLS") + } + + return &http.Client{ + Transport: c.HTTP.configuration(tls), + }, nil +} + +func (c Config) GetAWSSession() (client.ConfigProvider, error) { + prov, err := c.GetProvider() + if err != nil { + return nil, errors.Wrapf(err, "Unable to create Provider") + } + + cl, err := c.GetHttpClient() + if err != nil { + return nil, errors.Wrapf(err, "Unable to create HTTP Client") + } + + return session.NewSessionWithOptions(session.Options{ + Config: aws.Config{ + Credentials: credentials.NewCredentials(prov), + Endpoint: util.NewType(c.Endpoint), + S3ForcePathStyle: util.NewType(true), + DisableSSL: util.NewType(c.DisableSSL), + HTTPClient: cl, + }, + }) +} diff --git a/pkg/util/aws/file.go b/pkg/util/aws/file.go new file mode 100644 index 000000000..e8678ebc1 --- /dev/null +++ b/pkg/util/aws/file.go @@ -0,0 +1,86 @@ +// +// DISCLAIMER +// +// Copyright 2024 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package aws + +import ( + "os" + "sync" + "time" + + "github.com/aws/aws-sdk-go/aws/credentials" + + "github.com/arangodb/kube-arangodb/pkg/util" + "github.com/arangodb/kube-arangodb/pkg/util/errors" +) + +type fileProvider struct { + lock sync.Mutex + + accessKeyIDFile string + secretAccessKeyFile string + sessionTokenFile string + + recent time.Time +} + +func (f *fileProvider) Retrieve() (credentials.Value, error) { + if f == nil { + return credentials.Value{}, errors.Errorf("Object is nil") + } + + f.lock.Lock() + defer f.lock.Unlock() + + var v credentials.Value + + v.ProviderName = "dynamic-file-provider" + + if data, err := os.ReadFile(f.accessKeyIDFile); err != nil { + return credentials.Value{}, errors.Wrapf(err, "Unable to open AccessKeyID File") + } else { + v.AccessKeyID = string(data) + } + + if data, err := os.ReadFile(f.secretAccessKeyFile); err != nil { + return credentials.Value{}, errors.Wrapf(err, "Unable to open SecretAccessKey File") + } else { + v.SecretAccessKey = string(data) + } + + if f.sessionTokenFile != "" { + if data, err := os.ReadFile(f.sessionTokenFile); err != nil { + return credentials.Value{}, errors.Wrapf(err, "Unable to open SessionToken File") + } else { + v.SessionToken = string(data) + } + } + + f.recent = util.RecentFileModTime(f.accessKeyIDFile, f.secretAccessKeyFile, f.sessionTokenFile) + + return credentials.Value{}, nil +} + +func (f *fileProvider) IsExpired() bool { + f.lock.Lock() + defer f.lock.Unlock() + + return util.RecentFileModTime(f.accessKeyIDFile, f.secretAccessKeyFile, f.sessionTokenFile).After(f.recent) +} diff --git a/pkg/util/aws/http.go b/pkg/util/aws/http.go new file mode 100644 index 000000000..5b46114ed --- /dev/null +++ b/pkg/util/aws/http.go @@ -0,0 +1,44 @@ +// +// DISCLAIMER +// +// Copyright 2024 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package aws + +import ( + "crypto/tls" + "net/http" + "time" +) + +type HTTP struct { +} + +func (s HTTP) configuration(t *tls.Config) http.RoundTripper { + var c = http.Transport{ + Proxy: http.ProxyFromEnvironment, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + TLSClientConfig: t, + } + + return &c +} diff --git a/pkg/util/aws/impersonate.go b/pkg/util/aws/impersonate.go new file mode 100644 index 000000000..4a207ca76 --- /dev/null +++ b/pkg/util/aws/impersonate.go @@ -0,0 +1,89 @@ +// +// DISCLAIMER +// +// Copyright 2024 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package aws + +import ( + "context" + "sync" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/sts" + + "github.com/arangodb/kube-arangodb/pkg/util" +) + +type impersonate struct { + lock sync.Mutex + + credentials credentials.Value + expires time.Time + + config ProviderImpersonate + + creds credentials.Provider +} + +func (i *impersonate) Retrieve() (credentials.Value, error) { + i.lock.Lock() + defer i.lock.Unlock() + + awsSession, err := session.NewSessionWithOptions(session.Options{ + Config: aws.Config{ + Credentials: credentials.NewCredentials(i.creds), + S3ForcePathStyle: util.NewType(true), + }, + }) + if err != nil { + return credentials.Value{}, err + } + + s := sts.New(awsSession) + + resp, err := s.AssumeRoleWithContext(context.Background(), &sts.AssumeRoleInput{ + RoleArn: util.NewType(i.config.Role), + RoleSessionName: util.NewType(i.config.Name), + }) + if err != nil { + return credentials.Value{}, err + } + + if e := resp.Credentials.Expiration; e != nil { + i.expires = *e + } + + i.credentials = credentials.Value{ + AccessKeyID: util.WithDefault(resp.Credentials.AccessKeyId), + SecretAccessKey: util.WithDefault(resp.Credentials.SecretAccessKey), + SessionToken: util.WithDefault(resp.Credentials.SessionToken), + } + + return i.credentials, nil +} + +func (i *impersonate) IsExpired() bool { + i.lock.Lock() + defer i.lock.Unlock() + + return time.Now().After(i.expires) +} diff --git a/pkg/util/aws/provider.go b/pkg/util/aws/provider.go new file mode 100644 index 000000000..6dadabb06 --- /dev/null +++ b/pkg/util/aws/provider.go @@ -0,0 +1,127 @@ +// +// DISCLAIMER +// +// Copyright 2024 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package aws + +import ( + "github.com/aws/aws-sdk-go/aws/credentials" + + "github.com/arangodb/kube-arangodb/pkg/util/errors" +) + +type ProviderType string + +const ( + ProviderTypeConfig ProviderType = "config" + ProviderTypeStatic ProviderType = "static" + ProviderTypeFile ProviderType = "file" +) + +type Provider struct { + Type ProviderType + + Config ProviderConfig + Static ProviderConfigStatic + File ProviderConfigFile + + Impersonate ProviderImpersonate +} + +func (p Provider) provider() (credentials.Provider, error) { + switch p.Type { + case ProviderTypeConfig: + return p.Config.provider() + case ProviderTypeStatic: + return p.Static.provider() + case ProviderTypeFile: + return p.File.provider() + default: + return nil, errors.Errorf("Unknown provider: %s", p.Type) + } +} + +func (p Provider) Provider() (credentials.Provider, error) { + prov, err := p.provider() + if err != nil { + return nil, err + } + + return p.Impersonate.provider(prov) +} + +type ProviderImpersonate struct { + Impersonate bool + + Role string + Name string +} + +func (p ProviderImpersonate) provider(in credentials.Provider) (credentials.Provider, error) { + if !p.Impersonate { + return in, nil + } + + return &impersonate{ + config: p, + creds: in, + }, nil +} + +type ProviderConfigStatic struct { + AccessKeyID string + SecretAccessKey string + SessionToken string +} + +func (p ProviderConfigStatic) provider() (credentials.Provider, error) { + return &credentials.StaticProvider{ + Value: credentials.Value{ + AccessKeyID: p.AccessKeyID, + SecretAccessKey: p.SecretAccessKey, + SessionToken: p.SessionToken, + }, + }, nil +} + +type ProviderConfig struct { + Filename string + Profile string +} + +func (p ProviderConfig) provider() (credentials.Provider, error) { + return &credentials.SharedCredentialsProvider{ + Filename: p.Filename, + Profile: p.Profile, + }, nil +} + +type ProviderConfigFile struct { + AccessKeyIDFile string + SecretAccessKeyFile string + SessionTokenFile string +} + +func (p *ProviderConfigFile) provider() (credentials.Provider, error) { + return &fileProvider{ + accessKeyIDFile: p.AccessKeyIDFile, + secretAccessKeyFile: p.SecretAccessKeyFile, + sessionTokenFile: p.SessionTokenFile, + }, nil +} diff --git a/pkg/util/aws/tls.go b/pkg/util/aws/tls.go new file mode 100644 index 000000000..e2c6b7193 --- /dev/null +++ b/pkg/util/aws/tls.go @@ -0,0 +1,58 @@ +// +// DISCLAIMER +// +// Copyright 2024 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package aws + +import ( + "crypto/tls" + "crypto/x509" + "os" + + "github.com/arangodb/kube-arangodb/pkg/util/errors" +) + +type TLS struct { + Insecure bool + CAFiles []string +} + +func (s TLS) configuration() (*tls.Config, error) { + var r tls.Config + + if s.Insecure { + r.InsecureSkipVerify = true + } + + if len(s.CAFiles) > 0 { + caCertPool := x509.NewCertPool() + + for _, file := range s.CAFiles { + caCert, err := os.ReadFile(file) + if err != nil { + return nil, errors.Wrapf(err, "Unable to load CA from %s", file) + } + caCertPool.AppendCertsFromPEM(caCert) + } + + r.RootCAs = caCertPool + } + + return &r, nil +} diff --git a/pkg/util/file.go b/pkg/util/file.go index 154914f43..52eb0ac9e 100644 --- a/pkg/util/file.go +++ b/pkg/util/file.go @@ -23,6 +23,7 @@ package util import ( "io" "os" + "time" "github.com/pkg/errors" ) @@ -61,3 +62,28 @@ func Read(in io.Reader, buff []byte) (int, error) { } } } + +func RecentFileModTime(files ...string) time.Time { + var t time.Time + + for _, file := range files { + if z := FileModTime(file); !z.IsZero() && z.After(t) { + t = z + } + } + + return t +} + +func FileModTime(file string) time.Time { + if file == "" { + return time.Time{} + } + + stat, err := os.Stat(file) + if err != nil { + return time.Time{} + } + + return stat.ModTime() +}