From 1ed2700182b1b72dfedca1292c8478f6d612ed47 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Wed, 9 Aug 2023 09:10:16 +0200 Subject: [PATCH 01/34] Merge `broker` from sda-pipeline --- sda/internal/broker/broker.go | 63 +++++++++++++++--------------- sda/internal/broker/broker_test.go | 1 + 2 files changed, 33 insertions(+), 31 deletions(-) diff --git a/sda/internal/broker/broker.go b/sda/internal/broker/broker.go index 2186e0448..3c51110d0 100644 --- a/sda/internal/broker/broker.go +++ b/sda/internal/broker/broker.go @@ -22,23 +22,24 @@ type AMQPBroker struct { // MQConf stores information about the message broker type MQConf struct { - Host string - Port int - User string - Password string - Vhost string - Queue string - Exchange string - RoutingKey string - RoutingError string - Ssl bool - VerifyPeer bool - CACert string - ClientCert string - ClientKey string - ServerName string - Durable bool - SchemasPath string + Host string + Port int + User string + Password string + Vhost string + Queue string + Exchange string + RoutingKey string + RoutingError string + Ssl bool + VerifyPeer bool + CACert string + ClientCert string + ClientKey string + ServerName string + Durable bool + SchemasPath string + PrefetchCount int } // buildMQURI builds the MQ connection URI @@ -129,20 +130,6 @@ func NewMQ(config MQConf) (*AMQPBroker, error) { if err != nil { return nil, err } - if config.Queue != "" { - // The queues already exists so we can safely do a passive declaration - _, err = channel.QueueDeclarePassive( - config.Queue, // name - true, // durable - false, // auto-deleted - false, // internal - false, // noWait - nil, // arguments - ) - if err != nil { - return nil, err - } - } if e := channel.Confirm(false); e != nil { fmt.Printf("channel could not be put into confirm mode: %s", e) @@ -150,6 +137,14 @@ func NewMQ(config MQConf) (*AMQPBroker, error) { return nil, fmt.Errorf("channel could not be put into confirm mode: %s", e) } + if config.PrefetchCount > 0 { + // limit the number of messages retrieved from the queue + log.Debugf("prefetch count: %v", config.PrefetchCount) + if err := channel.Qos(config.PrefetchCount, 0, false); err != nil { + log.Errorf("failed to set Channel QoS to %d, reason: %v", config.PrefetchCount, err) + } + } + confirms := channel.NotifyPublish(make(chan amqp.Confirmation, 1)) return &AMQPBroker{connection, channel, config, confirms}, nil @@ -162,6 +157,12 @@ func (broker *AMQPBroker) ConnectionWatcher() *amqp.Error { return amqpError } +func (broker *AMQPBroker) ChannelWatcher() *amqp.Error { + amqpError := <-broker.Channel.NotifyClose(make(chan *amqp.Error)) + + return amqpError +} + // GetMessages reads messages from the queue func (broker *AMQPBroker) GetMessages(queue string) (<-chan amqp.Delivery, error) { ch := broker.Channel diff --git a/sda/internal/broker/broker_test.go b/sda/internal/broker/broker_test.go index 509a61525..7b20f1b62 100644 --- a/sda/internal/broker/broker_test.go +++ b/sda/internal/broker/broker_test.go @@ -129,6 +129,7 @@ func (suite *BrokerTestSuite) SetupTest() { "mq", true, "", + 2, } } From aba6047ce2876b83e762d1a4c5e252a52efa79c5 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Wed, 9 Aug 2023 09:10:38 +0200 Subject: [PATCH 02/34] Merge `storage` from sda-pipeline --- sda/internal/storage/storage.go | 515 ++++++++++++++++++++++ sda/internal/storage/storage_test.go | 635 +++++++++++++++++++++++++++ 2 files changed, 1150 insertions(+) create mode 100644 sda/internal/storage/storage.go create mode 100644 sda/internal/storage/storage_test.go diff --git a/sda/internal/storage/storage.go b/sda/internal/storage/storage.go new file mode 100644 index 000000000..98e6b29ba --- /dev/null +++ b/sda/internal/storage/storage.go @@ -0,0 +1,515 @@ +// Package storage provides interface for storage areas, e.g. s3 or POSIX file system. +package storage + +import ( + "crypto/tls" + "crypto/x509" + "encoding/base64" + "fmt" + "io" + "net" + "net/http" + "os" + "path/filepath" + "reflect" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/pkg/sftp" + "golang.org/x/crypto/ssh" + + log "github.com/sirupsen/logrus" +) + +// Backend defines methods to be implemented by PosixBackend, S3Backend and sftpBackend +type Backend interface { + GetFileSize(filePath string) (int64, error) + RemoveFile(filePath string) error + NewFileReader(filePath string) (io.ReadCloser, error) + NewFileWriter(filePath string) (io.WriteCloser, error) +} + +// Conf is a wrapper for the storage config +type Conf struct { + Type string + S3 S3Conf + Posix posixConf + SFTP SftpConf +} + +type posixBackend struct { + FileReader io.Reader + FileWriter io.Writer + Location string +} + +type posixConf struct { + Location string +} + +// NewBackend initiates a storage backend +func NewBackend(config Conf) (Backend, error) { + switch config.Type { + case "s3": + return newS3Backend(config.S3) + case "sftp": + return newSftpBackend(config.SFTP) + default: + return newPosixBackend(config.Posix) + } +} + +func newPosixBackend(config posixConf) (*posixBackend, error) { + fileInfo, err := os.Stat(config.Location) + + if err != nil { + return nil, err + } + + if !fileInfo.IsDir() { + return nil, fmt.Errorf("%s is not a directory", config.Location) + } + + return &posixBackend{Location: config.Location}, nil +} + +// NewFileReader returns an io.Reader instance +func (pb *posixBackend) NewFileReader(filePath string) (io.ReadCloser, error) { + if pb == nil { + return nil, fmt.Errorf("Invalid posixBackend") + } + + file, err := os.Open(filepath.Join(filepath.Clean(pb.Location), filePath)) + if err != nil { + log.Error(err) + + return nil, err + } + + return file, nil +} + +// NewFileWriter returns an io.Writer instance +func (pb *posixBackend) NewFileWriter(filePath string) (io.WriteCloser, error) { + if pb == nil { + return nil, fmt.Errorf("Invalid posixBackend") + } + + file, err := os.OpenFile(filepath.Join(filepath.Clean(pb.Location), filePath), os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0640) + if err != nil { + log.Error(err) + + return nil, err + } + + return file, nil +} + +// GetFileSize returns the size of the file +func (pb *posixBackend) GetFileSize(filePath string) (int64, error) { + if pb == nil { + return 0, fmt.Errorf("Invalid posixBackend") + } + + stat, err := os.Stat(filepath.Join(filepath.Clean(pb.Location), filePath)) + if err != nil { + log.Error(err) + + return 0, err + } + + return stat.Size(), nil +} + +// RemoveFile removes a file from a given path +func (pb *posixBackend) RemoveFile(filePath string) error { + if pb == nil { + return fmt.Errorf("Invalid posixBackend") + } + + err := os.Remove(filepath.Join(filepath.Clean(pb.Location), filePath)) + if err != nil { + log.Error(err) + + return err + } + + return nil +} + +type s3Backend struct { + Client *s3.S3 + Uploader *s3manager.Uploader + Bucket string + Conf *S3Conf +} + +// S3Conf stores information about the S3 storage backend +type S3Conf struct { + URL string + Port int + AccessKey string + SecretKey string + Bucket string + Region string + UploadConcurrency int + Chunksize int + CAcert string + NonExistRetryTime time.Duration + Readypath string +} + +func newS3Backend(config S3Conf) (*s3Backend, error) { + s3Transport := transportConfigS3(config) + client := http.Client{Transport: s3Transport} + s3Session := session.Must(session.NewSession( + &aws.Config{ + Endpoint: aws.String(fmt.Sprintf("%s:%d", config.URL, config.Port)), + Region: aws.String(config.Region), + HTTPClient: &client, + S3ForcePathStyle: aws.Bool(true), + DisableSSL: aws.Bool(strings.HasPrefix(config.URL, "http:")), + Credentials: credentials.NewStaticCredentials(config.AccessKey, config.SecretKey, ""), + }, + )) + + // Attempt to create a bucket, but we really expect an error here + // (BucketAlreadyOwnedByYou) + _, err := s3.New(s3Session).CreateBucket(&s3.CreateBucketInput{ + Bucket: aws.String(config.Bucket), + }) + + if err != nil { + if aerr, ok := err.(awserr.Error); ok { + + if aerr.Code() != s3.ErrCodeBucketAlreadyOwnedByYou && + aerr.Code() != s3.ErrCodeBucketAlreadyExists { + log.Error("Unexpected issue while creating bucket", err) + } + } + } + + sb := &s3Backend{ + Bucket: config.Bucket, + Uploader: s3manager.NewUploader(s3Session, func(u *s3manager.Uploader) { + u.PartSize = int64(config.Chunksize) + u.Concurrency = config.UploadConcurrency + u.LeavePartsOnError = false + }), + Client: s3.New(s3Session), + Conf: &config} + + _, err = sb.Client.ListObjectsV2(&s3.ListObjectsV2Input{Bucket: &config.Bucket}) + + if err != nil { + return nil, err + } + + return sb, nil +} + +// NewFileReader returns an io.Reader instance +func (sb *s3Backend) NewFileReader(filePath string) (io.ReadCloser, error) { + if sb == nil { + return nil, fmt.Errorf("Invalid s3Backend") + } + + r, err := sb.Client.GetObject(&s3.GetObjectInput{ + Bucket: aws.String(sb.Bucket), + Key: aws.String(filePath), + }) + + retryTime := 2 * time.Minute + if sb.Conf != nil { + retryTime = sb.Conf.NonExistRetryTime + } + + start := time.Now() + for err != nil && time.Since(start) < retryTime { + r, err = sb.Client.GetObject(&s3.GetObjectInput{ + Bucket: aws.String(sb.Bucket), + Key: aws.String(filePath), + }) + time.Sleep(1 * time.Second) + } + + if err != nil { + log.Error(err) + + return nil, err + } + + return r.Body, nil +} + +// NewFileWriter uploads the contents of an io.Reader to a S3 bucket +func (sb *s3Backend) NewFileWriter(filePath string) (io.WriteCloser, error) { + if sb == nil { + return nil, fmt.Errorf("Invalid s3Backend") + } + + reader, writer := io.Pipe() + go func() { + + _, err := sb.Uploader.Upload(&s3manager.UploadInput{ + Body: reader, + Bucket: aws.String(sb.Bucket), + Key: aws.String(filePath), + ContentEncoding: aws.String("application/octet-stream"), + }) + + if err != nil { + _ = reader.CloseWithError(err) + } + }() + + return writer, nil +} + +// GetFileSize returns the size of a specific object +func (sb *s3Backend) GetFileSize(filePath string) (int64, error) { + if sb == nil { + return 0, fmt.Errorf("Invalid s3Backend") + } + + r, err := sb.Client.HeadObject(&s3.HeadObjectInput{ + Bucket: aws.String(sb.Bucket), + Key: aws.String(filePath)}) + + start := time.Now() + + retryTime := 2 * time.Minute + if sb.Conf != nil { + retryTime = sb.Conf.NonExistRetryTime + } + + // Retry on error up to five minutes to allow for + // "slow writes' or s3 eventual consistency + for err != nil && time.Since(start) < retryTime { + r, err = sb.Client.HeadObject(&s3.HeadObjectInput{ + Bucket: aws.String(sb.Bucket), + Key: aws.String(filePath)}) + + time.Sleep(1 * time.Second) + + } + + if err != nil { + log.Errorln(err) + + return 0, err + } + + return *r.ContentLength, nil +} + +// RemoveFile removes an object from a bucket +func (sb *s3Backend) RemoveFile(filePath string) error { + if sb == nil { + return fmt.Errorf("Invalid s3Backend") + } + + _, err := sb.Client.DeleteObject(&s3.DeleteObjectInput{ + Bucket: aws.String(sb.Bucket), + Key: aws.String(filePath)}) + if err != nil { + log.Error(err) + + return err + } + + err = sb.Client.WaitUntilObjectNotExists(&s3.HeadObjectInput{ + Bucket: aws.String(sb.Bucket), + Key: aws.String(filePath)}) + if err != nil { + return err + } + + return nil +} + +// transportConfigS3 is a helper method to setup TLS for the S3 client. +func transportConfigS3(config S3Conf) http.RoundTripper { + cfg := new(tls.Config) + + // Enforce TLS1.2 or higher + cfg.MinVersion = 2 + + // Read system CAs + var systemCAs, _ = x509.SystemCertPool() + if reflect.DeepEqual(systemCAs, x509.NewCertPool()) { + log.Debug("creating new CApool") + systemCAs = x509.NewCertPool() + } + cfg.RootCAs = systemCAs + + if config.CAcert != "" { + cacert, e := os.ReadFile(config.CAcert) // #nosec this file comes from our config + if e != nil { + log.Fatalf("failed to append %q to RootCAs: %v", cacert, e) + } + if ok := cfg.RootCAs.AppendCertsFromPEM(cacert); !ok { + log.Debug("no certs appended, using system certs only") + } + } + + var trConfig http.RoundTripper = &http.Transport{ + TLSClientConfig: cfg, + ForceAttemptHTTP2: true} + + return trConfig +} + +type sftpBackend struct { + Connection *ssh.Client + Client *sftp.Client + Conf *SftpConf +} + +// sftpConf stores information about the sftp storage backend +type SftpConf struct { + Host string + Port string + UserName string + PemKeyPath string + PemKeyPass string + HostKey string +} + +func newSftpBackend(config SftpConf) (*sftpBackend, error) { + // read in and parse pem key + key, err := os.ReadFile(config.PemKeyPath) + if err != nil { + return nil, fmt.Errorf("Failed to read from key file, %v", err) + } + + var signer ssh.Signer + if config.PemKeyPass == "" { + signer, err = ssh.ParsePrivateKey(key) + } else { + signer, err = ssh.ParsePrivateKeyWithPassphrase(key, []byte(config.PemKeyPass)) + } + if err != nil { + return nil, fmt.Errorf("Failed to parse private key, %v", err) + } + + // connect + conn, err := ssh.Dial("tcp", config.Host+":"+config.Port, + &ssh.ClientConfig{ + User: config.UserName, + Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)}, + HostKeyCallback: TrustedHostKeyCallback(config.HostKey), + }, + ) + if err != nil { + return nil, fmt.Errorf("Failed to start ssh connection, %v", err) + } + + // create new SFTP client + client, err := sftp.NewClient(conn) + if err != nil { + return nil, fmt.Errorf("Failed to start sftp client, %v", err) + } + + sfb := &sftpBackend{ + Connection: conn, + Client: client, + Conf: &config, + } + + _, err = client.ReadDir("./") + + if err != nil { + return nil, fmt.Errorf("Failed to list files with sftp, %v", err) + } + + return sfb, nil +} + +// NewFileWriter returns an io.Writer instance for the sftp remote +func (sfb *sftpBackend) NewFileWriter(filePath string) (io.WriteCloser, error) { + if sfb == nil { + return nil, fmt.Errorf("Invalid sftpBackend") + } + // Make remote directories + parent := filepath.Dir(filePath) + err := sfb.Client.MkdirAll(parent) + if err != nil { + return nil, fmt.Errorf("Failed to create dir with sftp, %v", err) + } + + file, err := sfb.Client.OpenFile(filePath, os.O_CREATE|os.O_TRUNC|os.O_RDWR) + if err != nil { + return nil, fmt.Errorf("Failed to create file with sftp, %v", err) + } + + return file, nil +} + +// GetFileSize returns the size of the file +func (sfb *sftpBackend) GetFileSize(filePath string) (int64, error) { + if sfb == nil { + return 0, fmt.Errorf("Invalid sftpBackend") + } + + stat, err := sfb.Client.Lstat(filePath) + if err != nil { + return 0, fmt.Errorf("Failed to get file size with sftp, %v", err) + } + + return stat.Size(), nil +} + +// NewFileReader returns an io.Reader instance +func (sfb *sftpBackend) NewFileReader(filePath string) (io.ReadCloser, error) { + if sfb == nil { + return nil, fmt.Errorf("Invalid sftpBackend") + } + + file, err := sfb.Client.Open(filePath) + if err != nil { + return nil, fmt.Errorf("Failed to open file with sftp, %v", err) + } + + return file, nil +} + +// RemoveFile removes a file or an empty directory. +func (sfb *sftpBackend) RemoveFile(filePath string) error { + if sfb == nil { + return fmt.Errorf("Invalid sftpBackend") + } + + err := sfb.Client.Remove(filePath) + if err != nil { + return fmt.Errorf("Failed to remove file with sftp, %v", err) + } + + return nil +} + +func TrustedHostKeyCallback(key string) ssh.HostKeyCallback { + if key == "" { + return func(_ string, _ net.Addr, k ssh.PublicKey) error { + keyString := k.Type() + " " + base64.StdEncoding.EncodeToString(k.Marshal()) + log.Warningf("host key verification is not in effect (Fix by adding trustedKey: %q)", keyString) + + return nil + } + } + + return func(_ string, _ net.Addr, k ssh.PublicKey) error { + keyString := k.Type() + " " + base64.StdEncoding.EncodeToString(k.Marshal()) + if ks := keyString; key != ks { + return fmt.Errorf("host key verification expected %q but got %q", key, ks) + } + + return nil + } +} diff --git a/sda/internal/storage/storage_test.go b/sda/internal/storage/storage_test.go new file mode 100644 index 000000000..22192608c --- /dev/null +++ b/sda/internal/storage/storage_test.go @@ -0,0 +1,635 @@ +package storage + +import ( + "bytes" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "io" + "net/http/httptest" + "os" + "regexp" + "strconv" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/gliderlabs/ssh" + "github.com/johannesboyne/gofakes3/backend/s3mem" + "github.com/pkg/sftp" + cryptossh "golang.org/x/crypto/ssh" + + "github.com/johannesboyne/gofakes3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + log "github.com/sirupsen/logrus" +) + +// posixType is the configuration type used for posix backends +const posixType = "posix" + +// s3Type is the configuration type used for s3 backends +const s3Type = "s3" + +// sftpType is the configuration type used for sftp backends +const sftpType = "sftp" + +var testS3Conf = S3Conf{ + "http://127.0.0.1", + 9000, + "accesskey", + "secretkey", + "bucket", + "region", + 10, + 5 * 1024 * 1024, + "", + 2 * time.Second, + "", +} + +var testConf = Conf{posixType, testS3Conf, testPosixConf, testSftpConf} + +var posixDoesNotExist = "/this/does/not/exist" +var posixNotCreatable = posixDoesNotExist + +var ts *httptest.Server + +var s3DoesNotExist = "nothing such" +var s3Creatable = "somename" + +var writeData = []byte("this is a test") + +var cleanupFilesBack [1000]string +var cleanupFiles = cleanupFilesBack[0:0] + +var testPosixConf = posixConf{ + "/"} + +var testSftpConf = SftpConf{ + "localhost", + "6222", + "user", + "to/be/updated", + "test", + "", +} + +// HoneyPot encapsulates the initialized mock sftp server struct +type HoneyPot struct { + server *ssh.Server +} + +var hp *HoneyPot + +func writeName() (name string, err error) { + f, err := os.CreateTemp("", "writablefile") + + if err != nil { + return "", err + } + + name = f.Name() + + // Add to cleanup + cleanupFiles = append(cleanupFiles, name) + + return name, err +} + +func doCleanup() { + for _, name := range cleanupFiles { + os.Remove(name) + } + + cleanupFiles = cleanupFilesBack[0:0] +} +func TestNewBackend(t *testing.T) { + + testConf.Type = posixType + p, err := NewBackend(testConf) + assert.Nil(t, err, "Backend posix failed") + + var buf bytes.Buffer + log.SetOutput(&buf) + + testConf.Type = sftpType + sf, err := NewBackend(testConf) + assert.Nil(t, err, "Backend sftp failed") + assert.NotZero(t, buf.Len(), "Expected warning missing") + + // update host key from server for later use + rgx := regexp.MustCompile(`\\\"(.*?)\\"`) + testConf.SFTP.HostKey = strings.Trim(rgx.FindString(buf.String()), "\\\"") + buf.Reset() + + testConf.Type = s3Type + s, err := NewBackend(testConf) + assert.Nil(t, err, "Backend s3 failed") + + assert.IsType(t, p, &posixBackend{}, "Wrong type from NewBackend with posix") + assert.IsType(t, s, &s3Backend{}, "Wrong type from NewBackend with S3") + assert.IsType(t, sf, &sftpBackend{}, "Wrong type from NewBackend with SFTP") + + // test some extra ssl handling + testConf.S3.CAcert = "/dev/null" + s, err = NewBackend(testConf) + assert.Nil(t, err, "Backend s3 failed") + assert.IsType(t, s, &s3Backend{}, "Wrong type from NewBackend with S3") +} + +func TestMain(m *testing.M) { + + err := setupFakeS3() + + if err != nil { + log.Errorf("Setup of fake s3 failed, bailing out: %v", err) + os.Exit(1) + } + + err = setupMockSFTP() + + if err != nil { + log.Errorf("Setup of mock sftp failed, bailing out: %v", err) + os.Exit(1) + } + + ret := m.Run() + ts.Close() + hp.server.Close() + os.Remove(testConf.SFTP.PemKeyPath) + os.Exit(ret) +} + +func TestPosixBackend(t *testing.T) { + + defer doCleanup() + testConf.Type = posixType + backend, err := NewBackend(testConf) + assert.Nil(t, err, "POSIX backend failed unexpectedly") + + var buf bytes.Buffer + + assert.IsType(t, backend, &posixBackend{}, "Wrong type from NewBackend with posix") + + log.SetOutput(os.Stdout) + + writable, err := writeName() + if err != nil { + t.Error("could not find a writable name, bailing out from test") + + return + } + + writer, err := backend.NewFileWriter(writable) + + assert.NotNil(t, writer, "Got a nil reader for writer from posix") + assert.Nil(t, err, "posix NewFileWriter failed when it shouldn't") + + written, err := writer.Write(writeData) + + assert.Nil(t, err, "Failure when writing to posix writer") + assert.Equal(t, len(writeData), written, "Did not write all writeData") + writer.Close() + + log.SetOutput(&buf) + writer, err = backend.NewFileWriter(posixNotCreatable) + + assert.Nil(t, writer, "Got a non-nil reader for writer from posix") + assert.NotNil(t, err, "posix NewFileWriter worked when it shouldn't") + assert.NotZero(t, buf.Len(), "Expected warning missing") + + log.SetOutput(os.Stdout) + + reader, err := backend.NewFileReader(writable) + assert.Nil(t, err, "posix NewFileReader failed when it should work") + require.NotNil(t, reader, "Reader that should be usable is not, bailing out") + + var readBackBuffer [4096]byte + readBack, err := reader.Read(readBackBuffer[0:4096]) + + assert.Equal(t, len(writeData), readBack, "did not read back data as expected") + assert.Equal(t, writeData, readBackBuffer[:readBack], "did not read back data as expected") + assert.Nil(t, err, "unexpected error when reading back data") + + size, err := backend.GetFileSize(writable) + assert.Nil(t, err, "posix NewFileReader failed when it should work") + assert.NotNil(t, size, "Got a nil size for posix") + + err = backend.RemoveFile(writable) + assert.Nil(t, err, "posix RemoveFile failed when it should work") + + log.SetOutput(&buf) + + reader, err = backend.NewFileReader(posixDoesNotExist) + assert.NotNil(t, err, "posix NewFileReader worked when it should not") + assert.Nil(t, reader, "Got a non-nil reader for posix") + assert.NotZero(t, buf.Len(), "Expected warning missing") + + buf.Reset() + + _, err = backend.GetFileSize(posixDoesNotExist) // nolint + assert.NotNil(t, err, "posix GetFileSize worked when it should not") + assert.NotZero(t, buf.Len(), "Expected warning missing") + + buf.Reset() + +} + +func setupFakeS3() (err error) { + // fake s3 + + if ts != nil { + // Setup done already? + return + } + + backend := s3mem.New() + faker := gofakes3.New(backend) + ts = httptest.NewServer(faker.Server()) + + portAt := strings.LastIndex(ts.URL, ":") + + testConf.S3.URL = ts.URL[:portAt] + testConf.S3.Port, err = strconv.Atoi(ts.URL[portAt+1:]) + testConf.Type = s3Type + + if err != nil { + log.Error("Unexpected error while setting up fake s3") + + return err + } + + backEnd, err := NewBackend(testConf) + if err != nil { + return err + } + + s3back := backEnd.(*s3Backend) + + _, err = s3back.Client.CreateBucket(&s3.CreateBucketInput{ + Bucket: aws.String(testConf.S3.Bucket)}) + + if err != nil { + if aerr, ok := err.(awserr.Error); ok { + + if aerr.Code() != s3.ErrCodeBucketAlreadyOwnedByYou && + aerr.Code() != s3.ErrCodeBucketAlreadyExists { + log.Error("Unexpected issue while creating bucket: ", err) + } else { + // Do not flag an error for this + err = nil + } + } + } + + return err +} + +func TestS3Fail(t *testing.T) { + + testConf.Type = s3Type + + tmp := testConf.S3.URL + + defer func() { testConf.S3.URL = tmp }() + testConf.S3.URL = "file://tmp/" + _, err := NewBackend(testConf) + assert.NotNil(t, err, "Backend worked when it should not") + + var dummyBackend *s3Backend + reader, err := dummyBackend.NewFileReader("/") + assert.NotNil(t, err, "NewFileReader worked when it should not") + assert.Nil(t, reader, "Got a Reader when expected not to") + + writer, err := dummyBackend.NewFileWriter("/") + assert.NotNil(t, err, "NewFileWriter worked when it should not") + assert.Nil(t, writer, "Got a Writer when expected not to") + + _, err = dummyBackend.GetFileSize("/") + assert.NotNil(t, err, "GetFileSize worked when it should not") + + err = dummyBackend.RemoveFile("/") + assert.NotNil(t, err, "RemoveFile worked when it should not") +} + +func TestPOSIXFail(t *testing.T) { + testConf.Type = posixType + + tmp := testConf.Posix.Location + + defer func() { testConf.Posix.Location = tmp }() + + testConf.Posix.Location = "/thisdoesnotexist" + backEnd, err := NewBackend(testConf) + assert.NotNil(t, err, "Backend worked when it should not") + assert.Nil(t, backEnd, "Got a backend when expected not to") + + testConf.Posix.Location = "/etc/passwd" + + backEnd, err = NewBackend(testConf) + assert.NotNil(t, err, "Backend worked when it should not") + assert.Nil(t, backEnd, "Got a backend when expected not to") + + var dummyBackend *posixBackend + reader, err := dummyBackend.NewFileReader("/") + assert.NotNil(t, err, "NewFileReader worked when it should not") + assert.Nil(t, reader, "Got a Reader when expected not to") + + writer, err := dummyBackend.NewFileWriter("/") + assert.NotNil(t, err, "NewFileWriter worked when it should not") + assert.Nil(t, writer, "Got a Writer when expected not to") + + _, err = dummyBackend.GetFileSize("/") + assert.NotNil(t, err, "GetFileSize worked when it should not") + + err = dummyBackend.RemoveFile("/") + assert.NotNil(t, err, "RemoveFile worked when it should not") +} + +// Initializes a mock sftp server instance +func setupMockSFTP() error { + + password := testConf.SFTP.PemKeyPass + addr := testConf.SFTP.Host + ":" + testConf.SFTP.Port + + // Key-pair generation + privateKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return err + } + publicRsaKey, err := cryptossh.NewPublicKey(&privateKey.PublicKey) + if err != nil { + return err + } + // pem.Block + privBlock := pem.Block{ + Type: "RSA PRIVATE KEY", + Headers: nil, + Bytes: x509.MarshalPKCS1PrivateKey(privateKey), + } + block, err := x509.EncryptPEMBlock(rand.Reader, privBlock.Type, privBlock.Bytes, []byte(password), x509.PEMCipherAES256) //nolint:staticcheck + if err != nil { + return err + } + + // Private key file in PEM format + privateKeyBytes := pem.EncodeToMemory(block) + + // Create temp key file and update config + fi, err := os.CreateTemp("", "sftp-key") + testConf.SFTP.PemKeyPath = fi.Name() + if err != nil { + return err + } + err = os.WriteFile(testConf.SFTP.PemKeyPath, privateKeyBytes, 0600) + if err != nil { + return err + } + + // Initialize a sftp honeypot instance + hp = NewHoneyPot(addr, publicRsaKey) + + // Start the server in the background + go func() { + if err := hp.server.ListenAndServe(); err != nil { + log.Panic(err) + } + }() + + return err +} + +// NewHoneyPot takes in IP address to be used for sftp honeypot +func NewHoneyPot(addr string, key ssh.PublicKey) *HoneyPot { + return &HoneyPot{ + server: &ssh.Server{ + Addr: addr, + SubsystemHandlers: map[string]ssh.SubsystemHandler{ + "sftp": func(sess ssh.Session) { + debugStream := io.Discard + serverOptions := []sftp.ServerOption{ + sftp.WithDebug(debugStream), + } + server, err := sftp.NewServer( + sess, + serverOptions..., + ) + if err != nil { + log.Errorf("sftp server init error: %v\n", err) + + return + } + if err := server.Serve(); err != io.EOF { + log.Errorf("sftp server completed with error: %v\n", err) + } + }, + }, + PublicKeyHandler: func(ctx ssh.Context, key ssh.PublicKey) bool { + return true + }, + }, + } +} + +func TestSftpFail(t *testing.T) { + + testConf.Type = sftpType + + // test connection + tmpHost := testConf.SFTP.Host + + testConf.SFTP.Host = "nonexistenthost" + _, err := NewBackend(testConf) + assert.NotNil(t, err, "Backend worked when it should not") + + var dummyBackend *sftpBackend + reader, err := dummyBackend.NewFileReader("/") + assert.NotNil(t, err, "NewFileReader worked when it should not") + assert.Nil(t, reader, "Got a Reader when expected not to") + + writer, err := dummyBackend.NewFileWriter("/") + assert.NotNil(t, err, "NewFileWriter worked when it should not") + assert.Nil(t, writer, "Got a Writer when expected not to") + + _, err = dummyBackend.GetFileSize("/") + assert.NotNil(t, err, "GetFileSize worked when it should not") + + err = dummyBackend.RemoveFile("/") + assert.NotNil(t, err, "RemoveFile worked when it should not") + assert.EqualError(t, err, "Invalid sftpBackend") + testConf.SFTP.Host = tmpHost + + // wrong key password + tmpKeyPass := testConf.SFTP.PemKeyPass + testConf.SFTP.PemKeyPass = "wrongkey" + _, err = NewBackend(testConf) + assert.EqualError(t, err, "Failed to parse private key, x509: decryption password incorrect") + + // missing key password + testConf.SFTP.PemKeyPass = "" + _, err = NewBackend(testConf) + assert.EqualError(t, err, "Failed to parse private key, ssh: this private key is passphrase protected") + testConf.SFTP.PemKeyPass = tmpKeyPass + + // wrong key + tmpKeyPath := testConf.SFTP.PemKeyPath + testConf.SFTP.PemKeyPath = "nonexistentkey" + _, err = NewBackend(testConf) + testConf.SFTP.PemKeyPath = tmpKeyPath + assert.EqualError(t, err, "Failed to read from key file, open nonexistentkey: no such file or directory") + + defer doCleanup() + dummyKeyFile, err := writeName() + if err != nil { + t.Error("could not find a writable name, bailing out from test") + + return + } + testConf.SFTP.PemKeyPath = dummyKeyFile + _, err = NewBackend(testConf) + assert.EqualError(t, err, "Failed to parse private key, ssh: no key found") + testConf.SFTP.PemKeyPath = tmpKeyPath + + // wrong host key + tmpHostKey := testConf.SFTP.HostKey + testConf.SFTP.HostKey = "wronghostkey" + _, err = NewBackend(testConf) + assert.ErrorContains(t, err, "Failed to start ssh connection, ssh: handshake failed: host key verification expected") + testConf.SFTP.HostKey = tmpHostKey +} + +func TestS3Backend(t *testing.T) { + + testConf.Type = s3Type + backend, err := NewBackend(testConf) + assert.Nil(t, err, "Backend failed") + + s3back := backend.(*s3Backend) + + var buf bytes.Buffer + + assert.IsType(t, s3back, &s3Backend{}, "Wrong type from NewBackend with s3") + + writer, err := s3back.NewFileWriter(s3Creatable) + + assert.NotNil(t, writer, "Got a nil reader for writer from s3") + assert.Nil(t, err, "s3 NewFileWriter failed when it shouldn't") + + written, err := writer.Write(writeData) + + assert.Nil(t, err, "Failure when writing to s3 writer") + assert.Equal(t, len(writeData), written, "Did not write all writeData") + writer.Close() + + reader, err := s3back.NewFileReader(s3Creatable) + assert.Nil(t, err, "s3 NewFileReader failed when it should work") + require.NotNil(t, reader, "Reader that should be usable is not, bailing out") + + size, err := s3back.GetFileSize(s3Creatable) + assert.Nil(t, err, "s3 GetFileSize failed when it should work") + assert.NotNil(t, size, "Got a nil size for s3") + assert.Equal(t, int64(len(writeData)), size, "Got an incorrect file size") + + err = s3back.RemoveFile(s3Creatable) + assert.Nil(t, err, "s3 RemoveFile failed when it should work") + + var readBackBuffer [4096]byte + readBack, err := reader.Read(readBackBuffer[0:4096]) + + assert.Equal(t, len(writeData), readBack, "did not read back data as expected") + assert.Equal(t, writeData, readBackBuffer[:readBack], "did not read back data as expected") + + if err != nil && err != io.EOF { + assert.Nil(t, err, "unexpected error when reading back data") + } + + buf.Reset() + + log.SetOutput(&buf) + + if !testing.Short() { + _, err = backend.GetFileSize(s3DoesNotExist) + assert.NotNil(t, err, "s3 GetFileSize worked when it should not") + assert.NotZero(t, buf.Len(), "Expected warning missing") + + buf.Reset() + + reader, err = backend.NewFileReader(s3DoesNotExist) + assert.NotNil(t, err, "s3 NewFileReader worked when it should not") + assert.Nil(t, reader, "Got a non-nil reader for s3") + assert.NotZero(t, buf.Len(), "Expected warning missing") + } + + log.SetOutput(os.Stdout) + +} + +func TestSftpBackend(t *testing.T) { + + var buf bytes.Buffer + log.SetOutput(&buf) + + testConf.Type = sftpType + backend, err := NewBackend(testConf) + assert.Nil(t, err, "Backend failed") + + assert.Zero(t, buf.Len(), "Got warning when not expected") + buf.Reset() + + sftpBack := backend.(*sftpBackend) + + assert.IsType(t, sftpBack, &sftpBackend{}, "Wrong type from NewBackend with sftp") + + var sftpDoesNotExist = "nonexistent/file" + var sftpCreatable = os.TempDir() + "/this/file/exists" + + writer, err := sftpBack.NewFileWriter(sftpCreatable) + assert.NotNil(t, writer, "Got a nil reader for writer from sftp") + assert.Nil(t, err, "sftp NewFileWriter failed when it shouldn't") + + written, err := writer.Write(writeData) + assert.Nil(t, err, "Failure when writing to sftp writer") + assert.Equal(t, len(writeData), written, "Did not write all writeData") + writer.Close() + + reader, err := sftpBack.NewFileReader(sftpCreatable) + assert.Nil(t, err, "sftp NewFileReader failed when it should work") + require.NotNil(t, reader, "Reader that should be usable is not, bailing out") + + size, err := sftpBack.GetFileSize(sftpCreatable) + assert.Nil(t, err, "sftp GetFileSize failed when it should work") + assert.NotNil(t, size, "Got a nil size for sftp") + assert.Equal(t, int64(len(writeData)), size, "Got an incorrect file size") + + err = sftpBack.RemoveFile(sftpCreatable) + assert.Nil(t, err, "sftp RemoveFile failed when it should work") + + err = sftpBack.RemoveFile(sftpDoesNotExist) + assert.EqualError(t, err, "Failed to remove file with sftp, file does not exist") + + var readBackBuffer [4096]byte + readBack, err := reader.Read(readBackBuffer[0:4096]) + + assert.Equal(t, len(writeData), readBack, "did not read back data as expected") + assert.Equal(t, writeData, readBackBuffer[:readBack], "did not read back data as expected") + + if err != nil && err != io.EOF { + assert.Nil(t, err, "unexpected error when reading back data") + } + + if !testing.Short() { + _, err = backend.GetFileSize(sftpDoesNotExist) + assert.EqualError(t, err, "Failed to get file size with sftp, file does not exist") + + reader, err = backend.NewFileReader(sftpDoesNotExist) + assert.EqualError(t, err, "Failed to open file with sftp, file does not exist") + assert.Nil(t, reader, "Got a non-nil reader for sftp") + } + +} From 39548b4cc96c41f5b497385bc4877b0364c78828 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Wed, 9 Aug 2023 09:23:12 +0200 Subject: [PATCH 03/34] Merge `config` from sda-pipeline --- sda/internal/config/config.go | 785 +++++++++++++++++++++++++---- sda/internal/config/config_test.go | 145 ++++-- 2 files changed, 811 insertions(+), 119 deletions(-) diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index 6b8aeea5c..c838c2e50 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -6,34 +6,24 @@ import ( "fmt" "os" "reflect" - "strconv" "strings" + "time" "github.com/neicnordic/sensitive-data-archive/internal/broker" "github.com/neicnordic/sensitive-data-archive/internal/database" + "github.com/neicnordic/sensitive-data-archive/internal/storage" + "github.com/neicnordic/crypt4gh/keys" "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/viper" ) -var ( - requiredConfVars = []string{ - "aws.url", "aws.accessKey", "aws.secretKey", "aws.bucket", - "broker.host", "broker.port", "broker.user", "broker.password", "broker.vhost", "broker.exchange", "broker.routingKey", - } -) +const POSIX = "posix" +const S3 = "s3" +const SFTP = "sftp" -// S3Config stores information about the S3 backend -type S3Config struct { - URL string - Readypath string - AccessKey string - SecretKey string - Bucket string - Region string - CAcert string -} +var requiredConfVars []string // ServerConfig stores general server information type ServerConfig struct { @@ -45,40 +35,282 @@ type ServerConfig struct { // Config is a parent object for all the different configuration parts type Config struct { - S3 S3Config - Broker broker.MQConf - Server ServerConfig - DB database.DBConf + Archive storage.Conf + Broker broker.MQConf + Database database.DBConf + Inbox storage.Conf + Backup storage.Conf + Server ServerConfig + API APIConf + Notify SMTPConf + Orchestrator OrchestratorConf +} + +type APIConf struct { + CACert string + ServerCert string + ServerKey string + Host string + Port int + Session SessionConfig + DB *database.SDAdb + MQ *broker.AMQPBroker +} + +type SessionConfig struct { + Expiration time.Duration + Domain string + Secure bool + HTTPOnly bool + Name string +} + +type SMTPConf struct { + Password string + FromAddr string + Host string + Port int +} + +type OrchestratorConf struct { + ProjectFQDN string + QueueVerify string + QueueInbox string + QueueComplete string + QueueBackup string + QueueMapping string + QueueIngest string + QueueAccession string + ReleaseDelay time.Duration } // NewConfig initializes and parses the config file and/or environment using // the viper library. -func NewConfig() (*Config, error) { +func NewConfig(app string) (*Config, error) { viper.SetConfigName("config") viper.AddConfigPath(".") viper.AutomaticEnv() viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) viper.SetConfigType("yaml") - if viper.IsSet("server.confPath") { - cp := viper.GetString("server.confPath") + viper.SetDefault("schema.type", "federated") + + if viper.IsSet("configPath") { + cp := viper.GetString("configPath") + log.Infof("configPath: %s", cp) if !strings.HasSuffix(cp, "/") { cp += "/" } viper.AddConfigPath(cp) } - if viper.IsSet("server.confFile") { - viper.SetConfigFile(viper.GetString("server.confFile")) + if viper.IsSet("configFile") { + viper.SetConfigFile(viper.GetString("configFile")) } + log.Infoln("reading config") if err := viper.ReadInConfig(); err != nil { + log.Infoln(err.Error()) if _, ok := err.(viper.ConfigFileNotFoundError); ok { log.Infoln("No config file found, using ENVs only") } else { + log.Infoln("ReadInConfig Error") + return nil, err } } - requiredConfVars = []string{ - "broker.host", "broker.port", "broker.user", "broker.password", "broker.exchange", "broker.routingkey", "aws.url", "aws.accesskey", "aws.secretkey", "aws.bucket", + switch app { + case "api": + requiredConfVars = []string{ + "broker.host", + "broker.port", + "broker.user", + "broker.password", + "broker.routingkey", + "db.host", + "db.port", + "db.user", + "db.password", + "db.database", + } + case "backup": + requiredConfVars = []string{ + "broker.host", + "broker.port", + "broker.user", + "broker.password", + "broker.queue", + "broker.routingkey", + "db.host", + "db.port", + "db.user", + "db.password", + "db.database", + } + + switch viper.GetString("archive.type") { + case S3: + requiredConfVars = append(requiredConfVars, []string{"archive.url", "archive.accesskey", "archive.secretkey", "archive.bucket"}...) + case POSIX: + requiredConfVars = append(requiredConfVars, []string{"archive.location"}...) + default: + return nil, fmt.Errorf("archive.type not set") + } + + switch viper.GetString("backup.type") { + case S3: + requiredConfVars = append(requiredConfVars, []string{"backup.url", "backup.accesskey", "backup.secretkey", "backup.bucket"}...) + case POSIX: + requiredConfVars = append(requiredConfVars, []string{"backup.location"}...) + case SFTP: + requiredConfVars = append(requiredConfVars, []string{"backup.sftp.host", "backup.sftp.port", "backup.sftp.userName", "backup.sftp.pemKeyPath", "backup.sftp.pemKeyPass"}...) + default: + return nil, fmt.Errorf("backup.type not set") + } + case "ingest": + requiredConfVars = []string{ + "broker.host", + "broker.port", + "broker.user", + "broker.password", + "broker.queue", + "broker.routingkey", + "db.host", + "db.port", + "db.user", + "db.password", + "db.database", + } + + switch viper.GetString("archive.type") { + case S3: + requiredConfVars = append(requiredConfVars, []string{"archive.url", "archive.accesskey", "archive.secretkey", "archive.bucket"}...) + case POSIX: + requiredConfVars = append(requiredConfVars, []string{"archive.location"}...) + default: + return nil, fmt.Errorf("archive.type not set") + } + + switch viper.GetString("inbox.type") { + case S3: + requiredConfVars = append(requiredConfVars, []string{"inbox.url", "inbox.accesskey", "inbox.secretkey", "inbox.bucket"}...) + case POSIX: + requiredConfVars = append(requiredConfVars, []string{"inbox.location"}...) + default: + return nil, fmt.Errorf("inbox.type not set") + } + case "finalize": + requiredConfVars = []string{ + "broker.host", + "broker.port", + "broker.user", + "broker.password", + "broker.queue", + "broker.routingkey", + "db.host", + "db.port", + "db.user", + "db.password", + "db.database", + } + case "intercept": + // Intercept does not require these extra settings + requiredConfVars = []string{ + "broker.host", + "broker.port", + "broker.user", + "broker.password", + "broker.queue", + } + case "mapper": + // Mapper does not require broker.routingkey thus we remove it + requiredConfVars = []string{ + "broker.host", + "broker.port", + "broker.user", + "broker.password", + "broker.queue", + "db.host", + "db.port", + "db.user", + "db.password", + "db.database", + } + + switch viper.GetString("inbox.type") { + case S3: + requiredConfVars = append(requiredConfVars, []string{"inbox.url", "inbox.accesskey", "inbox.secretkey", "inbox.bucket"}...) + case POSIX: + requiredConfVars = append(requiredConfVars, []string{"inbox.location"}...) + } + case "notify": + requiredConfVars = []string{ + "broker.host", + "broker.port", + "broker.user", + "broker.password", + "broker.queue", + "smtp.host", + "smtp.port", + "smtp.password", + "smtp.from", + } + case "orchestrate": + // Orchestrate requires broker connection, a series of + // queues, and the project FQDN. + requiredConfVars = []string{ + "broker.host", + "broker.port", + "broker.user", + "broker.password", + "project.fqdn", + } + case "s3inbox": + requiredConfVars = []string{ + "broker.host", + "broker.port", + "broker.user", + "broker.password", + "broker.routingkey", + "inbox.url", + "inbox.accesskey", + "inbox.secretkey", + "inbox.bucket", + } + viper.Set("inbox.type", S3) + case "verify": + requiredConfVars = []string{ + "broker.host", + "broker.port", + "broker.user", + "broker.password", + "broker.queue", + "broker.routingkey", + "db.host", + "db.port", + "db.user", + "db.password", + "db.database", + } + + switch viper.GetString("archive.type") { + case S3: + requiredConfVars = append(requiredConfVars, []string{"archive.url", "archive.accesskey", "archive.secretkey", "archive.bucket"}...) + case POSIX: + requiredConfVars = append(requiredConfVars, []string{"archive.location"}...) + default: + return nil, fmt.Errorf("archive.type not set") + } + + switch viper.GetString("inbox.type") { + case S3: + requiredConfVars = append(requiredConfVars, []string{"inbox.url", "inbox.accesskey", "inbox.secretkey", "inbox.bucket"}...) + case POSIX: + requiredConfVars = append(requiredConfVars, []string{"inbox.location"}...) + default: + return nil, fmt.Errorf("inbox.type not set") + } + + default: + return nil, fmt.Errorf("application '%s' doesn't exist", app) } for _, s := range requiredConfVars { @@ -106,99 +338,468 @@ func NewConfig() (*Config, error) { } c := &Config{} - err := c.readConfig() - if err != nil { - return nil, err + switch app { + case "api": + err := c.configBroker() + if err != nil { + return nil, err + } + + err = c.configDatabase() + if err != nil { + return nil, err + } + + err = c.configAPI() + if err != nil { + return nil, err + } + case "backup": + c.configArchive() + c.configBackup() + + err := c.configBroker() + if err != nil { + return nil, err + } + + err = c.configDatabase() + if err != nil { + return nil, err + } + + c.configSchemas() + case "finalize": + err := c.configBroker() + if err != nil { + return nil, err + } + err = c.configDatabase() + if err != nil { + return nil, err + } + + c.configSchemas() + case "ingest": + c.configArchive() + err := c.configBroker() + if err != nil { + return nil, err + } + err = c.configDatabase() + if err != nil { + return nil, err + } + + c.configInbox() + c.configSchemas() + case "intercept": + err := c.configBroker() + if err != nil { + return nil, err + } + + c.configSchemas() + case "mapper": + err := c.configBroker() + if err != nil { + return nil, err + } + + err = c.configDatabase() + if err != nil { + return nil, err + } + + c.configInbox() + c.configSchemas() + case "notify": + c.configSMTP() + case "orchestrate": + err := c.configBroker() + if err != nil { + return nil, err + } + + c.configOrchestrator() + case "s3inbox": + err := c.configBroker() + if err != nil { + return nil, err + } + + err = c.configDatabase() + if err != nil { + return nil, err + } + + c.configInbox() + + err = c.configServer() + if err != nil { + return nil, err + } + case "verify": + c.configArchive() + + err := c.configBroker() + if err != nil { + return nil, err + } + + err = c.configDatabase() + if err != nil { + return nil, err + } + + c.configSchemas() } return c, nil } -func (c *Config) readConfig() error { - s3 := S3Config{} +// configDatabase provides configuration for the database +func (c *Config) configAPI() error { + c.apiDefaults() + api := APIConf{} - // All these are required - s3.URL = viper.GetString("aws.url") - s3.AccessKey = viper.GetString("aws.accessKey") - s3.SecretKey = viper.GetString("aws.secretKey") - s3.Bucket = viper.GetString("aws.bucket") + api.Session.Expiration = time.Duration(viper.GetInt("api.session.expiration")) * time.Second + api.Session.Domain = viper.GetString("api.session.domain") + api.Session.Secure = viper.GetBool("api.session.secure") + api.Session.HTTPOnly = viper.GetBool("api.session.httponly") + api.Session.Name = viper.GetString("api.session.name") - // Optional settings - if viper.IsSet("aws.readypath") { - s3.Readypath = viper.GetString("aws.readypath") - } - if viper.IsSet("aws.region") { - s3.Region = viper.GetString("aws.region") + api.Host = viper.GetString("api.host") + api.Port = viper.GetInt("api.port") + api.ServerKey = viper.GetString("api.serverKey") + api.ServerCert = viper.GetString("api.serverCert") + api.CACert = viper.GetString("api.CACert") + + c.API = api + + return nil +} + +// apiDefaults set default values for web server and session +func (c *Config) apiDefaults() { + viper.SetDefault("api.host", "0.0.0.0") + viper.SetDefault("api.port", 8080) + viper.SetDefault("api.session.expiration", -1) + viper.SetDefault("api.session.secure", true) + viper.SetDefault("api.session.httponly", true) + viper.SetDefault("api.session.name", "api_session_key") +} + +// configArchive provides configuration for the archive storage +func (c *Config) configArchive() { + if viper.GetString("archive.type") == S3 { + c.Archive.Type = S3 + c.Archive.S3 = configS3Storage("archive") } else { - s3.Region = "us-east-1" - } - if viper.IsSet("aws.cacert") { - s3.CAcert = viper.GetString("aws.cacert") + c.Archive.Type = POSIX + c.Archive.Posix.Location = viper.GetString("archive.location") } +} - c.S3 = s3 +// configBackup provides configuration for the backup storage +func (c *Config) configBackup() { + switch viper.GetString("backup.type") { + case S3: + c.Backup.Type = S3 + c.Backup.S3 = configS3Storage("backup") + case SFTP: + c.Backup.Type = SFTP + c.Backup.SFTP = configSFTP("backup") + default: + c.Backup.Type = POSIX + c.Backup.Posix.Location = viper.GetString("backup.location") + } +} +// configBroker provides configuration for the message broker +func (c *Config) configBroker() error { // Setup broker - b := broker.MQConf{} + broker := broker.MQConf{} + + broker.Host = viper.GetString("broker.host") + broker.Port = viper.GetInt("broker.port") + broker.User = viper.GetString("broker.user") + broker.Password = viper.GetString("broker.password") - b.Host = viper.GetString("broker.host") - b.Port, _ = strconv.Atoi(viper.GetString("broker.port")) - b.User = viper.GetString("broker.user") - b.Password = viper.GetString("broker.password") - b.Exchange = viper.GetString("broker.exchange") - b.RoutingKey = viper.GetString("broker.routingKey") - b.ServerName = viper.GetString("broker.serverName") + broker.Queue = viper.GetString("broker.queue") + + if viper.IsSet("broker.serverName") { + broker.ServerName = viper.GetString("broker.serverName") + } + if viper.IsSet("broker.routingkey") { + broker.RoutingKey = viper.GetString("broker.routingkey") + } + + if viper.IsSet("broker.exchange") { + broker.Exchange = viper.GetString("broker.exchange") + } + + if viper.IsSet("broker.durable") { + broker.Durable = viper.GetBool("broker.durable") + } + if viper.IsSet("broker.routingerror") { + broker.RoutingError = viper.GetString("broker.routingerror") + } if viper.IsSet("broker.vhost") { if strings.HasPrefix(viper.GetString("broker.vhost"), "/") { - b.Vhost = viper.GetString("broker.vhost") + broker.Vhost = viper.GetString("broker.vhost") } else { - b.Vhost = "/" + viper.GetString("broker.vhost") + broker.Vhost = "/" + viper.GetString("broker.vhost") } } else { - b.Vhost = "/" + broker.Vhost = "/" } if viper.IsSet("broker.ssl") { - b.Ssl = viper.GetBool("broker.ssl") + broker.Ssl = viper.GetBool("broker.ssl") } + if viper.IsSet("broker.verifyPeer") { - b.VerifyPeer = viper.GetBool("broker.verifyPeer") - if b.VerifyPeer { + broker.VerifyPeer = viper.GetBool("broker.verifyPeer") + if broker.VerifyPeer { // Since verifyPeer is specified, these are required. if !(viper.IsSet("broker.clientCert") && viper.IsSet("broker.clientKey")) { return errors.New("when broker.verifyPeer is set both broker.clientCert and broker.clientKey is needed") } - b.ClientCert = viper.GetString("broker.clientCert") - b.ClientKey = viper.GetString("broker.clientKey") + broker.ClientCert = viper.GetString("broker.clientCert") + broker.ClientKey = viper.GetString("broker.clientKey") } } if viper.IsSet("broker.cacert") { - b.CACert = viper.GetString("broker.cacert") + broker.CACert = viper.GetString("broker.cacert") } - c.Broker = b - - // Setup psql db - c.DB.Host = viper.GetString("db.host") - c.DB.Port = viper.GetInt("db.port") - c.DB.User = viper.GetString("db.user") - c.DB.Password = viper.GetString("db.password") - c.DB.Database = viper.GetString("db.database") - if viper.IsSet("db.cacert") { - c.DB.CACert = viper.GetString("db.cacert") + broker.PrefetchCount = 2 + if viper.IsSet("broker.prefetchCount") { + broker.PrefetchCount = viper.GetInt("broker.prefetchCount") } - c.DB.SslMode = viper.GetString("db.sslmode") - if c.DB.SslMode == "verify-full" { + + c.Broker = broker + + return nil +} + +// configDatabase provides configuration for the database +func (c *Config) configDatabase() error { + db := database.DBConf{} + + // All these are required + db.Host = viper.GetString("db.host") + db.Port = viper.GetInt("db.port") + db.User = viper.GetString("db.user") + db.Password = viper.GetString("db.password") + db.Database = viper.GetString("db.database") + db.SslMode = viper.GetString("db.sslmode") + + // Optional settings + if db.SslMode == "verify-full" { // Since verify-full is specified, these are required. if !(viper.IsSet("db.clientCert") && viper.IsSet("db.clientKey")) { return errors.New("when db.sslMode is set to verify-full both db.clientCert and db.clientKey are needed") } - c.DB.ClientCert = viper.GetString("db.clientcert") - c.DB.ClientKey = viper.GetString("db.clientkey") + } + if viper.IsSet("db.clientKey") { + db.ClientKey = viper.GetString("db.clientKey") + } + if viper.IsSet("db.clientCert") { + db.ClientCert = viper.GetString("db.clientCert") + } + if viper.IsSet("db.cacert") { + db.CACert = viper.GetString("db.cacert") + } + + c.Database = db + + return nil +} + +// configInbox provides configuration for the inbox storage +func (c *Config) configInbox() { + if viper.GetString("inbox.type") == S3 { + c.Inbox.Type = S3 + c.Inbox.S3 = configS3Storage("inbox") + } else { + c.Inbox.Type = POSIX + c.Inbox.Posix.Location = viper.GetString("inbox.location") + } +} + +// configOrchestrator provides the configuration for the standalone orchestator. +func (c *Config) configOrchestrator() { + c.Orchestrator = OrchestratorConf{} + if viper.IsSet("broker.dataset.releasedelay") { + c.Orchestrator.ReleaseDelay = time.Duration(viper.GetInt("broker.dataset.releasedelay")) + } else { + c.Orchestrator.ReleaseDelay = 1 + } + c.Orchestrator.ProjectFQDN = viper.GetString("project.fqdn") + if viper.IsSet("broker.queue.verified") { + c.Orchestrator.QueueVerify = viper.GetString("broker.queue.verified") + } else { + c.Orchestrator.QueueVerify = "verified" + } + + if viper.IsSet("broker.queue.inbox") { + c.Orchestrator.QueueInbox = viper.GetString("broker.queue.inbox") + } else { + c.Orchestrator.QueueInbox = "inbox" + } + + if viper.IsSet("broker.queue.completed") { + c.Orchestrator.QueueComplete = viper.GetString("broker.queue.completed") + } else { + c.Orchestrator.QueueComplete = "completed" + } + + if viper.IsSet("broker.queue.backup") { + c.Orchestrator.QueueBackup = viper.GetString("broker.queue.backup") + } else { + c.Orchestrator.QueueBackup = "backup" + } + + if viper.IsSet("broker.queue.mappings") { + c.Orchestrator.QueueMapping = viper.GetString("broker.queue.mappings") + } else { + c.Orchestrator.QueueMapping = "mappings" + } + + if viper.IsSet("broker.queue.ingest") { + c.Orchestrator.QueueIngest = viper.GetString("broker.queue.ingest") + } else { + c.Orchestrator.QueueIngest = "ingest" + } + + if viper.IsSet("broker.queue.accessionIDs") { + c.Orchestrator.QueueAccession = viper.GetString("broker.queue.accessionIDs") + } else { + c.Orchestrator.QueueAccession = "accessionIDs" + } +} + +// configSchemas configures the schemas to load depending on +// the type IDs of connection Federated EGA or isolate (stand-alone) +func (c *Config) configSchemas() { + if viper.GetString("schema.type") == "federated" { + c.Broker.SchemasPath = "/schemas/federated/" + } else { + c.Broker.SchemasPath = "/schemas/isolated/" + } +} + +// configS3Storage populates and returns a S3Conf from the +// configuration +func configS3Storage(prefix string) storage.S3Conf { + s3 := storage.S3Conf{} + // All these are required + s3.URL = viper.GetString(prefix + ".url") + s3.AccessKey = viper.GetString(prefix + ".accesskey") + s3.SecretKey = viper.GetString(prefix + ".secretkey") + s3.Bucket = viper.GetString(prefix + ".bucket") + + // Defaults (move to viper?) + + s3.Port = 443 + s3.Region = "us-east-1" + s3.NonExistRetryTime = 2 * time.Minute + + if viper.IsSet(prefix + ".port") { + s3.Port = viper.GetInt(prefix + ".port") + } + + if viper.IsSet(prefix + ".region") { + s3.Region = viper.GetString(prefix + ".region") + } + + if viper.IsSet(prefix + ".readypath") { + s3.Readypath = viper.GetString(prefix + ".readypath") + } + + if viper.IsSet(prefix + ".chunksize") { + s3.Chunksize = viper.GetInt(prefix+".chunksize") * 1024 * 1024 + } + + if viper.IsSet(prefix + ".cacert") { + s3.CAcert = viper.GetString(prefix + ".cacert") + } + + return s3 +} + +// configSFTP populates and returns a sftpConf with sftp backend configuration +func configSFTP(prefix string) storage.SftpConf { + sftpConf := storage.SftpConf{} + if viper.IsSet(prefix + ".sftp.hostKey") { + sftpConf.HostKey = viper.GetString(prefix + ".sftp.hostKey") + } else { + sftpConf.HostKey = "" + } + // All these are required + sftpConf.Host = viper.GetString(prefix + ".sftp.host") + sftpConf.Port = viper.GetString(prefix + ".sftp.port") + sftpConf.UserName = viper.GetString(prefix + ".sftp.userName") + sftpConf.PemKeyPath = viper.GetString(prefix + ".sftp.pemKeyPath") + sftpConf.PemKeyPass = viper.GetString(prefix + ".sftp.pemKeyPass") + + return sftpConf +} + +// configNotify provides configuration for the backup storage +func (c *Config) configSMTP() { + c.Notify = SMTPConf{} + c.Notify.Host = viper.GetString("smtp.host") + c.Notify.Port = viper.GetInt("smtp.port") + c.Notify.Password = viper.GetString("smtp.password") + c.Notify.FromAddr = viper.GetString("smtp.from") +} + +// GetC4GHKey reads and decrypts and returns the c4gh key +func GetC4GHKey() (*[32]byte, error) { + keyPath := viper.GetString("c4gh.filepath") + passphrase := viper.GetString("c4gh.passphrase") + + // Make sure the key path and passphrase is valid + keyFile, err := os.Open(keyPath) + if err != nil { + return nil, err + } + + key, err := keys.ReadPrivateKey(keyFile, []byte(passphrase)) + if err != nil { + return nil, err + } + + keyFile.Close() + + return &key, nil +} + +// GetC4GHPublicKey reads the c4gh public key +func GetC4GHPublicKey() (*[32]byte, error) { + keyPath := viper.GetString("c4gh.backupPubKey") + + // Make sure the key path and passphrase is valid + keyFile, err := os.Open(keyPath) + if err != nil { + return nil, err + } + + key, err := keys.ReadPublicKey(keyFile) + if err != nil { + return nil, err } - // Setup server + keyFile.Close() + + return &key, nil +} + +func (c *Config) configServer() error { s := ServerConfig{} if !(viper.IsSet("server.jwtpubkeypath") || viper.IsSet("server.jwtpubkeyurl")) { @@ -243,13 +844,8 @@ func TLSConfigBroker(c *Config) (*tls.Config, error) { } cfg.RootCAs = systemCAs - // Add CAs for broker and s3 - for _, cacert := range []string{c.Broker.CACert, c.S3.CAcert} { - if cacert == "" { - continue - } - - cacert, e := os.ReadFile(cacert) // #nosec this file comes from our configuration + if c.Broker.CACert != "" { + cacert, e := os.ReadFile(c.Broker.CACert) // #nosec this file comes from our configuration if e != nil { return nil, fmt.Errorf("failed to append %q to RootCAs: %v", cacert, e) } @@ -297,8 +893,8 @@ func TLSConfigProxy(c *Config) (*tls.Config, error) { } cfg.RootCAs = systemCAs - if c.S3.CAcert != "" { - cacert, e := os.ReadFile(c.S3.CAcert) // #nosec this file comes from our configuration + if c.Inbox.S3.CAcert != "" { + cacert, e := os.ReadFile(c.Inbox.S3.CAcert) // #nosec this file comes from our configuration if e != nil { return nil, fmt.Errorf("failed to append %q to RootCAs: %v", cacert, e) } @@ -309,3 +905,12 @@ func TLSConfigProxy(c *Config) (*tls.Config, error) { return cfg, nil } + +// CopyHeader reads the config and returns if the header will be copied +func CopyHeader() bool { + if viper.IsSet("backup.copyHeader") { + return viper.GetBool("backup.copyHeader") + } + + return false +} diff --git a/sda/internal/config/config_test.go b/sda/internal/config/config_test.go index 7bfb8be2a..08c841f59 100644 --- a/sda/internal/config/config_test.go +++ b/sda/internal/config/config_test.go @@ -1,12 +1,14 @@ package config import ( + "errors" "fmt" "os" "path" "path/filepath" "runtime" "testing" + "time" helper "github.com/neicnordic/sensitive-data-archive/internal/helper" @@ -37,11 +39,18 @@ func (suite *ConfigTestSuite) SetupTest() { viper.Set("broker.routingkey", "routingtest") viper.Set("broker.exchange", "testexchange") viper.Set("broker.vhost", "testvhost") - viper.Set("aws.url", "testurl") - viper.Set("aws.accesskey", "testaccess") - viper.Set("aws.secretkey", "testsecret") - viper.Set("aws.bucket", "testbucket") + viper.Set("broker.queue", "testqueue") + viper.Set("db.host", "test") + viper.Set("db.port", 123) + viper.Set("db.user", "test") + viper.Set("db.password", "test") + viper.Set("db.database", "test") + viper.Set("inbox.url", "testurl") + viper.Set("inbox.accesskey", "testaccess") + viper.Set("inbox.secretkey", "testsecret") + viper.Set("inbox.bucket", "testbucket") viper.Set("server.jwtpubkeypath", "testpath") + viper.Set("log.level", "debug") } func (suite *ConfigTestSuite) TearDownTest() { @@ -53,9 +62,18 @@ func TestConfigTestSuite(t *testing.T) { suite.Run(t, new(ConfigTestSuite)) } +func (suite *ConfigTestSuite) TestNonExistingApplication() { + expectedError := errors.New("application 'test' doesn't exist") + config, err := NewConfig("test") + assert.Nil(suite.T(), config) + if assert.Error(suite.T(), err) { + assert.Equal(suite.T(), expectedError, err) + } +} + func (suite *ConfigTestSuite) TestConfigFile() { - viper.Set("server.confFile", rootDir+"/.github/integration/sda/config.yaml") - config, err := NewConfig() + viper.Set("configFile", rootDir+"/.github/integration/sda/config.yaml") + config, err := NewConfig("s3inbox") assert.NotNil(suite.T(), config) assert.NoError(suite.T(), err) absPath, _ := filepath.Abs(rootDir + "/.github/integration/sda/config.yaml") @@ -63,8 +81,8 @@ func (suite *ConfigTestSuite) TestConfigFile() { } func (suite *ConfigTestSuite) TestWrongConfigFile() { - viper.Set("server.confFile", rootDir+"/.github/integration/rabbitmq/cega.conf") - config, err := NewConfig() + viper.Set("configFile", rootDir+"/.github/integration/rabbitmq/cega.conf") + config, err := NewConfig("s3inbox") assert.Nil(suite.T(), config) assert.Error(suite.T(), err) absPath, _ := filepath.Abs(rootDir + "/.github/integration/rabbitmq/cega.conf") @@ -73,8 +91,8 @@ func (suite *ConfigTestSuite) TestWrongConfigFile() { func (suite *ConfigTestSuite) TestConfigPath() { viper.Reset() - viper.Set("server.confPath", rootDir+"/.github/integration/sda/") - config, err := NewConfig() + viper.Set("configPath", rootDir+"/.github/integration/sda/") + config, err := NewConfig("s3inbox") assert.NotNil(suite.T(), config) assert.NoError(suite.T(), err) absPath, _ := filepath.Abs(rootDir + "/.github/integration/sda/config.yaml") @@ -83,7 +101,7 @@ func (suite *ConfigTestSuite) TestConfigPath() { func (suite *ConfigTestSuite) TestNoConfig() { viper.Reset() - config, err := NewConfig() + config, err := NewConfig("s3inbox") assert.Nil(suite.T(), config) assert.Error(suite.T(), err) } @@ -93,7 +111,7 @@ func (suite *ConfigTestSuite) TestMissingRequiredConfVar() { requiredConfVarValue := viper.Get(requiredConfVar) viper.Set(requiredConfVar, nil) expectedError := fmt.Errorf("%s not set", requiredConfVar) - config, err := NewConfig() + config, err := NewConfig("s3inbox") assert.Nil(suite.T(), config) if assert.Error(suite.T(), err) { assert.Equal(suite.T(), expectedError, err) @@ -103,35 +121,35 @@ func (suite *ConfigTestSuite) TestMissingRequiredConfVar() { } func (suite *ConfigTestSuite) TestConfigS3Storage() { - config, err := NewConfig() + config, err := NewConfig("s3inbox") assert.NotNil(suite.T(), config) assert.NoError(suite.T(), err) - assert.NotNil(suite.T(), config.S3) - assert.Equal(suite.T(), "testurl", config.S3.URL) - assert.Equal(suite.T(), "testaccess", config.S3.AccessKey) - assert.Equal(suite.T(), "testsecret", config.S3.SecretKey) - assert.Equal(suite.T(), "testbucket", config.S3.Bucket) + assert.NotNil(suite.T(), config.Inbox.S3) + assert.Equal(suite.T(), "testurl", config.Inbox.S3.URL) + assert.Equal(suite.T(), "testaccess", config.Inbox.S3.AccessKey) + assert.Equal(suite.T(), "testsecret", config.Inbox.S3.SecretKey) + assert.Equal(suite.T(), "testbucket", config.Inbox.S3.Bucket) } func (suite *ConfigTestSuite) TestConfigBroker() { - config, err := NewConfig() + config, err := NewConfig("s3inbox") assert.NotNil(suite.T(), config) assert.NoError(suite.T(), err) - assert.NotNil(suite.T(), config.S3) + assert.NotNil(suite.T(), config.Inbox.S3) assert.Equal(suite.T(), "/testvhost", config.Broker.Vhost) assert.Equal(suite.T(), false, config.Broker.Ssl) viper.Set("broker.ssl", true) viper.Set("broker.verifyPeer", true) - _, err = NewConfig() + _, err = NewConfig("s3inbox") assert.Error(suite.T(), err, "Error expected") viper.Set("broker.clientCert", "dummy-value") viper.Set("broker.clientKey", "dummy-value") - _, err = NewConfig() + _, err = NewConfig("s3inbox") assert.NoError(suite.T(), err) viper.Set("broker.vhost", nil) - config, err = NewConfig() + config, err = NewConfig("s3inbox") assert.NotNil(suite.T(), config) assert.NoError(suite.T(), err) assert.Equal(suite.T(), "/", config.Broker.Vhost) @@ -141,7 +159,7 @@ func (suite *ConfigTestSuite) TestTLSConfigBroker() { viper.Set("broker.serverName", "broker") viper.Set("broker.ssl", true) viper.Set("broker.cacert", certPath+"/ca.crt") - config, err := NewConfig() + config, err := NewConfig("s3inbox") assert.NotNil(suite.T(), config) assert.NoError(suite.T(), err) tlsBroker, err := TLSConfigBroker(config) @@ -151,7 +169,7 @@ func (suite *ConfigTestSuite) TestTLSConfigBroker() { viper.Set("broker.verifyPeer", true) viper.Set("broker.clientCert", certPath+"/tls.crt") viper.Set("broker.clientKey", certPath+"/tls.key") - config, err = NewConfig() + config, err = NewConfig("s3inbox") assert.NotNil(suite.T(), config) assert.NoError(suite.T(), err) tlsBroker, err = TLSConfigBroker(config) @@ -160,7 +178,7 @@ func (suite *ConfigTestSuite) TestTLSConfigBroker() { viper.Set("broker.clientCert", certPath+"tls.crt") viper.Set("broker.clientKey", certPath+"/tls.key") - config, err = NewConfig() + config, err = NewConfig("s3inbox") assert.NotNil(suite.T(), config) assert.NoError(suite.T(), err) tlsBroker, err = TLSConfigBroker(config) @@ -169,8 +187,8 @@ func (suite *ConfigTestSuite) TestTLSConfigBroker() { } func (suite *ConfigTestSuite) TestTLSConfigProxy() { - viper.Set("aws.cacert", certPath+"/ca.crt") - config, err := NewConfig() + viper.Set("inbox.cacert", certPath+"/ca.crt") + config, err := NewConfig("s3inbox") assert.NotNil(suite.T(), config) assert.NoError(suite.T(), err) tlsProxy, err := TLSConfigProxy(config) @@ -180,8 +198,77 @@ func (suite *ConfigTestSuite) TestTLSConfigProxy() { func (suite *ConfigTestSuite) TestDefaultLogLevel() { viper.Set("log.level", "test") - config, err := NewConfig() + config, err := NewConfig("s3inbox") assert.NotNil(suite.T(), config) assert.NoError(suite.T(), err) assert.Equal(suite.T(), log.TraceLevel, log.GetLevel()) } + +func (suite *ConfigTestSuite) TestAPIConfiguration() { + // At this point we should fail because we lack configuration + viper.Reset() + config, err := NewConfig("api") + assert.Error(suite.T(), err) + assert.Nil(suite.T(), config) + + // testing deafult values + suite.SetupTest() + config, err = NewConfig("api") + assert.NotNil(suite.T(), config) + assert.NoError(suite.T(), err) + assert.NotNil(suite.T(), config.API) + assert.Equal(suite.T(), "0.0.0.0", config.API.Host) + assert.Equal(suite.T(), 8080, config.API.Port) + assert.Equal(suite.T(), true, config.API.Session.Secure) + assert.Equal(suite.T(), true, config.API.Session.HTTPOnly) + assert.Equal(suite.T(), "api_session_key", config.API.Session.Name) + assert.Equal(suite.T(), -1*time.Second, config.API.Session.Expiration) + + viper.Reset() + suite.SetupTest() + // over write defaults + viper.Set("api.port", 8443) + viper.Set("api.session.secure", false) + viper.Set("api.session.domain", "test") + viper.Set("api.session.expiration", 60) + + config, err = NewConfig("api") + assert.NotNil(suite.T(), config) + assert.NoError(suite.T(), err) + assert.NotNil(suite.T(), config.API) + assert.Equal(suite.T(), "0.0.0.0", config.API.Host) + assert.Equal(suite.T(), 8443, config.API.Port) + assert.Equal(suite.T(), false, config.API.Session.Secure) + assert.Equal(suite.T(), "test", config.API.Session.Domain) + assert.Equal(suite.T(), 60*time.Second, config.API.Session.Expiration) +} + +func (suite *ConfigTestSuite) TestNotifyConfiguration() { + // At this point we should fail because we lack configuration + config, err := NewConfig("notify") + assert.Error(suite.T(), err) + assert.Nil(suite.T(), config) + + viper.Set("broker.host", "test") + viper.Set("broker.port", 123) + viper.Set("broker.user", "test") + viper.Set("broker.password", "test") + viper.Set("broker.queue", "test") + viper.Set("broker.routingkey", "test") + viper.Set("broker.exchange", "test") + + viper.Set("smtp.host", "test") + viper.Set("smtp.port", 456) + viper.Set("smtp.password", "test") + viper.Set("smtp.from", "noreply") + + config, err = NewConfig("notify") + assert.NoError(suite.T(), err) + assert.NotNil(suite.T(), config) +} + +func (suite *ConfigTestSuite) TestCopyHeader() { + viper.Set("backup.copyHeader", "true") + cHeader := CopyHeader() + assert.Equal(suite.T(), cHeader, true, "The CopyHeader does not work") +} From a39aa50267536f2ac0a80abbf6f55ad89a378bb1 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Wed, 9 Aug 2023 09:30:17 +0200 Subject: [PATCH 04/34] Update s3inbox to use the merged config & proxy code --- .github/integration/sda/config.yaml | 5 +++-- sda/cmd/s3inbox/bucket.go | 6 +++--- sda/cmd/s3inbox/bucket_test.go | 19 ++++++++++--------- sda/cmd/s3inbox/healthchecks.go | 6 +++--- sda/cmd/s3inbox/healthchecks_test.go | 12 ++++++------ sda/cmd/s3inbox/main.go | 8 ++++---- sda/cmd/s3inbox/proxy.go | 14 +++++++------- sda/cmd/s3inbox/proxy_test.go | 11 ++++++----- sda/go.mod | 13 ++++++++++--- sda/go.sum | 28 +++++++++++++++++++++++++--- 10 files changed, 77 insertions(+), 45 deletions(-) diff --git a/.github/integration/sda/config.yaml b/.github/integration/sda/config.yaml index a1da0ea7d..9669ca001 100644 --- a/.github/integration/sda/config.yaml +++ b/.github/integration/sda/config.yaml @@ -1,7 +1,8 @@ log: format: "json" -aws: - url: "http://s3:9000" +inbox: + url: "http://s3" + port: 9000 readypath: "/minio/health/ready" accessKey: "access" secretKey: "secretKey" diff --git a/sda/cmd/s3inbox/bucket.go b/sda/cmd/s3inbox/bucket.go index c18bcb680..1fe696e59 100644 --- a/sda/cmd/s3inbox/bucket.go +++ b/sda/cmd/s3inbox/bucket.go @@ -8,7 +8,7 @@ import ( "reflect" "strings" - "github.com/neicnordic/sensitive-data-archive/internal/config" + "github.com/neicnordic/sensitive-data-archive/internal/storage" "github.com/pkg/errors" log "github.com/sirupsen/logrus" @@ -20,7 +20,7 @@ import ( "github.com/aws/aws-sdk-go/service/s3" ) -func checkS3Bucket(config config.S3Config) error { +func checkS3Bucket(config storage.S3Conf) error { s3Transport := transportConfigS3(config) client := http.Client{Transport: s3Transport} s3Session := session.Must(session.NewSession( @@ -54,7 +54,7 @@ func checkS3Bucket(config config.S3Config) error { } // transportConfigS3 is a helper method to setup TLS for the S3 client. -func transportConfigS3(config config.S3Config) http.RoundTripper { +func transportConfigS3(config storage.S3Conf) http.RoundTripper { cfg := new(tls.Config) // Enforce TLS1.2 or higher diff --git a/sda/cmd/s3inbox/bucket_test.go b/sda/cmd/s3inbox/bucket_test.go index 4a9b4c769..13ea1a56d 100644 --- a/sda/cmd/s3inbox/bucket_test.go +++ b/sda/cmd/s3inbox/bucket_test.go @@ -29,6 +29,7 @@ func (suite *BucketTestSuite) SetupTest() { os.Exit(1) } + viper.Set("log.level", "debug") viper.Set("broker.host", "localhost") viper.Set("broker.port", "1234") viper.Set("broker.user", "guest") @@ -36,10 +37,10 @@ func (suite *BucketTestSuite) SetupTest() { viper.Set("broker.routingkey", "ingest") viper.Set("broker.exchange", "amq.topic") viper.Set("broker.vhost", "/") - viper.Set("aws.url", ts.URL) - viper.Set("aws.accesskey", "testaccess") - viper.Set("aws.secretkey", "testsecret") - viper.Set("aws.bucket", "testbucket") + viper.Set("inbox.url", ts.URL) + viper.Set("inbox.accesskey", "testaccess") + viper.Set("inbox.secretkey", "testsecret") + viper.Set("inbox.bucket", "testbucket") viper.Set("server.jwtpubkeypath", "testpath") } @@ -69,20 +70,20 @@ func TestBucketTestSuite(t *testing.T) { } func (suite *BucketTestSuite) TestBucketPass() { - config, err := config.NewConfig() + config, err := config.NewConfig("s3inbox") assert.NotNil(suite.T(), config) assert.NoError(suite.T(), err) - err = checkS3Bucket(config.S3) + err = checkS3Bucket(config.Inbox.S3) assert.NoError(suite.T(), err) } func (suite *BucketTestSuite) TestBucketFail() { - viper.Set("aws.url", "http://localhost:12345") - config, err := config.NewConfig() + viper.Set("inbox.url", "http://localhost:12345") + config, err := config.NewConfig("s3inbox") assert.NotNil(suite.T(), config) assert.NoError(suite.T(), err) - err = checkS3Bucket(config.S3) + err = checkS3Bucket(config.Inbox.S3) assert.Error(suite.T(), err) } diff --git a/sda/cmd/s3inbox/healthchecks.go b/sda/cmd/s3inbox/healthchecks.go index e61247d76..ad57b7bb3 100644 --- a/sda/cmd/s3inbox/healthchecks.go +++ b/sda/cmd/s3inbox/healthchecks.go @@ -25,9 +25,9 @@ type HealthCheck struct { // NewHealthCheck creates a new healthchecker. It needs to know where to find // the backend S3 storage and the Message Broker so it can report readiness. func NewHealthCheck(port int, db *sql.DB, conf *config.Config, tlsConfig *tls.Config) *HealthCheck { - s3URL := conf.S3.URL - if conf.S3.Readypath != "" { - s3URL = conf.S3.URL + conf.S3.Readypath + s3URL := conf.Inbox.S3.URL + if conf.Inbox.S3.Readypath != "" { + s3URL = conf.Inbox.S3.URL + conf.Inbox.S3.Readypath } brokerURL := fmt.Sprintf("%s:%d", conf.Broker.Host, conf.Broker.Port) diff --git a/sda/cmd/s3inbox/healthchecks_test.go b/sda/cmd/s3inbox/healthchecks_test.go index 80b38d156..85633d274 100644 --- a/sda/cmd/s3inbox/healthchecks_test.go +++ b/sda/cmd/s3inbox/healthchecks_test.go @@ -34,16 +34,16 @@ func (suite *HealthcheckTestSuite) SetupTest() { viper.Set("broker.routingkey", "ingest") viper.Set("broker.exchange", "sda") viper.Set("broker.vhost", "sda") - viper.Set("aws.url", "http://localhost:8080") - viper.Set("aws.accesskey", "testaccess") - viper.Set("aws.secretkey", "testsecret") - viper.Set("aws.bucket", "testbucket") + viper.Set("inbox.url", "http://localhost:8080") + viper.Set("inbox.accesskey", "testaccess") + viper.Set("inbox.secretkey", "testsecret") + viper.Set("inbox.bucket", "testbucket") viper.Set("server.jwtpubkeypath", "testpath") } func (suite *HealthcheckTestSuite) TestHttpsGetCheck() { db, _, _ := sqlmock.New() - conf, err := config.NewConfig() + conf, err := config.NewConfig("s3inbox") assert.NoError(suite.T(), err) assert.NotNil(suite.T(), conf) h := NewHealthCheck(8888, @@ -68,7 +68,7 @@ func (suite *HealthcheckTestSuite) TestHealthchecks() { db, mock, _ := sqlmock.New(sqlmock.MonitorPingsOption(true)) mock.ExpectPing() - conf, err := config.NewConfig() + conf, err := config.NewConfig("s3inbox") assert.NoError(suite.T(), err) assert.NotNil(suite.T(), conf) h := NewHealthCheck(8888, diff --git a/sda/cmd/s3inbox/main.go b/sda/cmd/s3inbox/main.go index 9daf013fe..e3bf0c77d 100644 --- a/sda/cmd/s3inbox/main.go +++ b/sda/cmd/s3inbox/main.go @@ -28,7 +28,7 @@ func main() { } }() - c, err := config.NewConfig() + c, err := config.NewConfig("s3inbox") if err != nil { log.Error(err) sigc <- syscall.SIGINT @@ -43,7 +43,7 @@ func main() { panic(err) } - sdaDB, err := database.NewSDAdb(Conf.DB) + sdaDB, err := database.NewSDAdb(Conf.Database) if err != nil { log.Error(err) sigc <- syscall.SIGINT @@ -57,7 +57,7 @@ func main() { log.Debugf("Connected to sda-db (v%v)", sdaDB.Version) - err = checkS3Bucket(Conf.S3) + err = checkS3Bucket(Conf.Inbox.S3) if err != nil { log.Error(err) sigc <- syscall.SIGINT @@ -94,7 +94,7 @@ func main() { log.Panicf("Error while getting key %s: %v", Conf.Server.Jwtpubkeypath, err) } } - proxy := NewProxy(Conf.S3, auth, messenger, sdaDB, tlsProxy) + proxy := NewProxy(Conf.Inbox.S3, auth, messenger, sdaDB, tlsProxy) log.Debug("got the proxy ", proxy) diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index 1af44b742..53ae6c28b 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -15,9 +15,9 @@ import ( "time" "github.com/neicnordic/sensitive-data-archive/internal/broker" - "github.com/neicnordic/sensitive-data-archive/internal/config" "github.com/neicnordic/sensitive-data-archive/internal/database" "github.com/neicnordic/sensitive-data-archive/internal/helper" + "github.com/neicnordic/sensitive-data-archive/internal/storage" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" @@ -31,7 +31,7 @@ import ( // Proxy represents the toplevel object in this application type Proxy struct { - s3 config.S3Config + s3 storage.S3Conf auth Authenticator messenger *broker.AMQPBroker database *database.SDAdb @@ -72,7 +72,7 @@ const ( ) // NewProxy creates a new S3Proxy. This implements the ServerHTTP interface. -func NewProxy(s3conf config.S3Config, auth Authenticator, messenger *broker.AMQPBroker, database *database.SDAdb, tls *tls.Config) *Proxy { +func NewProxy(s3conf storage.S3Conf, auth Authenticator, messenger *broker.AMQPBroker, database *database.SDAdb, tls *tls.Config) *Proxy { tr := &http.Transport{TLSClientConfig: tls} client := &http.Client{Transport: tr, Timeout: 30 * time.Second} @@ -245,10 +245,10 @@ func (p *Proxy) uploadFinishedSuccessfully(req *http.Request, response *http.Res func (p *Proxy) forwardToBackend(r *http.Request) (*http.Response, error) { - p.resignHeader(r, p.s3.AccessKey, p.s3.SecretKey, p.s3.URL) + p.resignHeader(r, p.s3.AccessKey, p.s3.SecretKey, fmt.Sprintf("%s:%d", p.s3.URL, p.s3.Port)) // Redirect request - nr, err := http.NewRequest(r.Method, p.s3.URL+r.URL.String(), r.Body) + nr, err := http.NewRequest(r.Method, fmt.Sprintf("%s:%d", p.s3.URL, p.s3.Port)+r.URL.String(), r.Body) if err != nil { log.Debug("error when redirecting the request") log.Debug(err) @@ -454,7 +454,7 @@ func (p *Proxy) newSession() (*session.Session, error) { CustomCABundle: cacert, Config: aws.Config{ Region: aws.String(p.s3.Region), - Endpoint: aws.String(p.s3.URL), + Endpoint: aws.String(fmt.Sprintf("%s:%d", p.s3.URL, p.s3.Port)), DisableSSL: aws.Bool(strings.HasPrefix(p.s3.URL, "http:")), S3ForcePathStyle: aws.Bool(true), Credentials: credentials.NewStaticCredentials(p.s3.AccessKey, p.s3.SecretKey, ""), @@ -465,7 +465,7 @@ func (p *Proxy) newSession() (*session.Session, error) { } else { mySession, err = session.NewSession(&aws.Config{ Region: aws.String(p.s3.Region), - Endpoint: aws.String(p.s3.URL), + Endpoint: aws.String(fmt.Sprintf("%s:%d", p.s3.URL, p.s3.Port)), DisableSSL: aws.Bool(strings.HasPrefix(p.s3.URL, "http:")), S3ForcePathStyle: aws.Bool(true), Credentials: credentials.NewStaticCredentials(p.s3.AccessKey, p.s3.SecretKey, ""), diff --git a/sda/cmd/s3inbox/proxy_test.go b/sda/cmd/s3inbox/proxy_test.go index fb995b0ed..7abe4847c 100644 --- a/sda/cmd/s3inbox/proxy_test.go +++ b/sda/cmd/s3inbox/proxy_test.go @@ -12,8 +12,8 @@ import ( "testing" "github.com/neicnordic/sensitive-data-archive/internal/broker" - "github.com/neicnordic/sensitive-data-archive/internal/config" "github.com/neicnordic/sensitive-data-archive/internal/database" + "github.com/neicnordic/sensitive-data-archive/internal/storage" "github.com/golang-jwt/jwt/v4" "github.com/stretchr/testify/assert" @@ -22,7 +22,7 @@ import ( type ProxyTests struct { suite.Suite - S3conf config.S3Config + S3conf storage.S3Conf DBConf database.DBConf fakeServer *FakeServer MQConf broker.MQConf @@ -40,8 +40,9 @@ func (suite *ProxyTests) SetupTest() { suite.fakeServer = startFakeServer("9024") // Create an s3config for the fake server - suite.S3conf = config.S3Config{ - URL: "http://127.0.0.1:9024", + suite.S3conf = storage.S3Conf{ + URL: "http://127.0.0.1", + Port: 9024, AccessKey: "someAccess", SecretKey: "someSecret", Bucket: "buckbuck", @@ -210,7 +211,7 @@ func (suite *ProxyTests) TestServeHTTP_disallowed() { } func (suite *ProxyTests) TestServeHTTPS3Unresponsive() { - s3conf := config.S3Config{ + s3conf := storage.S3Conf{ URL: "http://localhost:40211", AccessKey: "someAccess", SecretKey: "someSecret", diff --git a/sda/go.mod b/sda/go.mod index 49b0d081f..5d3e0830b 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -5,30 +5,37 @@ go 1.20 require ( github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/aws/aws-sdk-go v1.44.253 + github.com/gliderlabs/ssh v0.3.5 github.com/golang-jwt/jwt/v4 v4.5.0 github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb github.com/johannesboyne/gofakes3 v0.0.0-20230310080033-c0edf658332b github.com/lestrrat-go/jwx v1.2.26 github.com/lib/pq v1.10.9 github.com/minio/minio-go/v6 v6.0.57 + github.com/neicnordic/crypt4gh v1.7.6 github.com/ory/dockertest/v3 v3.10.0 github.com/pkg/errors v0.9.1 + github.com/pkg/sftp v1.13.1 github.com/rabbitmq/amqp091-go v1.8.0 github.com/santhosh-tekuri/jsonschema/v5 v5.3.0 github.com/sirupsen/logrus v1.9.0 github.com/spf13/viper v1.15.0 github.com/stretchr/testify v1.8.4 + golang.org/x/crypto v0.11.0 ) require ( + filippo.io/edwards25519 v1.0.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect + github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.1.3 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/containerd/continuity v0.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect github.com/docker/cli v20.10.17+incompatible // indirect github.com/docker/docker v20.10.24+incompatible // indirect @@ -42,6 +49,7 @@ require ( github.com/hashicorp/hcl v1.0.0 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/kr/fs v0.1.0 // indirect github.com/lestrrat-go/backoff/v2 v2.0.8 // indirect github.com/lestrrat-go/blackmagic v1.0.1 // indirect github.com/lestrrat-go/httpcc v1.0.1 // indirect @@ -71,10 +79,9 @@ require ( github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect - golang.org/x/crypto v0.9.0 // indirect golang.org/x/mod v0.9.0 // indirect - golang.org/x/sys v0.8.0 // indirect - golang.org/x/text v0.9.0 // indirect + golang.org/x/sys v0.10.0 // indirect + golang.org/x/text v0.11.0 // indirect golang.org/x/tools v0.7.0 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 // indirect diff --git a/sda/go.sum b/sda/go.sum index c94ed9fa4..950a8f9cd 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -36,6 +36,8 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +filippo.io/edwards25519 v1.0.0 h1:0wAIcmJUqRdI8IJ/3eGi5/HwXZWPujYXXlkrQogz0Ek= +filippo.io/edwards25519 v1.0.0/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5EwJns= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= @@ -52,6 +54,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/aws/aws-sdk-go v1.33.0/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-sdk-go v1.44.253 h1:iqDd0okcH4ShfFexz2zzf4VmeDFf6NOMm07pHnEb8iY= github.com/aws/aws-sdk-go v1.44.253/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= @@ -79,6 +83,8 @@ github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a h1:saTgr5tMLFnmy/yg3qDTft4rE5DY2uJ/cCxCe3q0XTU= +github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a/go.mod h1:Bw9BbhOJVNR+t0jCqx2GC6zv0TGBsShs56Y3gfSCvl0= github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= @@ -100,6 +106,8 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= +github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -217,6 +225,7 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -260,6 +269,8 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/neicnordic/crypt4gh v1.7.6 h1:Vqcb8Yb950oaBBJFepDK1oLeu9rZzpywYWVHLmO0oI8= +github.com/neicnordic/crypt4gh v1.7.6/go.mod h1:rqmVXsprDFBRRLJkm1cK9kLETBPGEZmft9lHD/V40wk= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= @@ -274,6 +285,7 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.13.1 h1:I2qBYMChEhIjOgazfJmV3/mZM256btk6wkCDRmW7JYs= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -379,8 +391,10 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= +golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= +golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= +golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -453,9 +467,11 @@ golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= @@ -534,16 +550,21 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -554,8 +575,9 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From 4e49d13cb43c6a9a2c95dec4098795b15d760aa5 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Wed, 9 Aug 2023 19:17:15 +0200 Subject: [PATCH 05/34] Rework storage_test to run as a suite. Also removes all mocked backends in favor for live containers. --- sda/internal/helper/helper.go | 45 ++ sda/internal/storage/storage_test.go | 794 +++++++++++---------------- 2 files changed, 353 insertions(+), 486 deletions(-) diff --git a/sda/internal/helper/helper.go b/sda/internal/helper/helper.go index eb67299cf..45e435fe6 100644 --- a/sda/internal/helper/helper.go +++ b/sda/internal/helper/helper.go @@ -19,6 +19,7 @@ import ( "time" "github.com/golang-jwt/jwt/v4" + "golang.org/x/crypto/ssh" ) // Global variables for test token creation @@ -153,6 +154,50 @@ func CreateRSAkeys(prPath, pubPath string) error { return nil } +func CreateSSHKey(path string) error { + privatekey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return err + } + pk := &privatekey.PublicKey + + // dump private key to file + privateKeyBytes := x509.MarshalPKCS1PrivateKey(privatekey) + privateKeyBlock := &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: privateKeyBytes, + } + encPrivateKeyBlock, err := x509.EncryptPEMBlock(rand.Reader, privateKeyBlock.Type, privateKeyBlock.Bytes, []byte("password"), x509.PEMCipherAES256) //nolint:staticcheck + if err != nil { + return err + } + privatePem, err := os.Create(path + "/id_rsa") + if err != nil { + return err + } + err = pem.Encode(privatePem, encPrivateKeyBlock) + if err != nil { + return err + } + + err = os.Chmod(path+"/id_rsa", 0600) + if err != nil { + return err + } + + publicKey, err := ssh.NewPublicKey(pk) + if err != nil { + return err + } + pubKeyBytes := ssh.MarshalAuthorizedKey(publicKey) + err = os.WriteFile(path+"/id_rsa.pub", pubKeyBytes, 0600) + if err != nil { + return err + } + + return nil +} + // CreateRSAToken creates an RSA token func CreateRSAToken(key *rsa.PrivateKey, headerAlg, headerType string, tokenClaims map[string]interface{}) (string, error) { token := jwt.New(jwt.SigningMethodRS256) diff --git a/sda/internal/storage/storage_test.go b/sda/internal/storage/storage_test.go index 22192608c..97e8a91df 100644 --- a/sda/internal/storage/storage_test.go +++ b/sda/internal/storage/storage_test.go @@ -2,634 +2,456 @@ package storage import ( "bytes" - "crypto/rand" - "crypto/rsa" - "crypto/x509" - "encoding/pem" + "fmt" "io" - "net/http/httptest" + "net/http" "os" - "regexp" "strconv" - "strings" "testing" "time" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/gliderlabs/ssh" - "github.com/johannesboyne/gofakes3/backend/s3mem" - "github.com/pkg/sftp" - cryptossh "golang.org/x/crypto/ssh" - - "github.com/johannesboyne/gofakes3" + "github.com/neicnordic/sensitive-data-archive/internal/helper" + "github.com/ory/dockertest/v3" + "github.com/ory/dockertest/v3/docker" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" log "github.com/sirupsen/logrus" ) -// posixType is the configuration type used for posix backends -const posixType = "posix" +type StorageTestSuite struct { + suite.Suite +} -// s3Type is the configuration type used for s3 backends -const s3Type = "s3" +var testConf = Conf{} +var sshPath string +var s3Port, sftpPort int +var writeData = []byte("this is a test") -// sftpType is the configuration type used for sftp backends +const posixType = "posix" +const s3Type = "s3" const sftpType = "sftp" -var testS3Conf = S3Conf{ - "http://127.0.0.1", - 9000, - "accesskey", - "secretkey", - "bucket", - "region", - 10, - 5 * 1024 * 1024, - "", - 2 * time.Second, - "", -} - -var testConf = Conf{posixType, testS3Conf, testPosixConf, testSftpConf} - -var posixDoesNotExist = "/this/does/not/exist" -var posixNotCreatable = posixDoesNotExist +func TestMain(m *testing.M) { + sshPath, _ = os.MkdirTemp("", "ssh") + defer os.RemoveAll(sshPath) + if err := helper.CreateSSHKey(sshPath); err != nil { + log.Panicf("Failed to create SSH keys, reason: %v", err.Error()) + } -var ts *httptest.Server + defer func() { + if r := recover(); r != nil { + log.Infoln("Recovered") + } + }() + // uses a sensible default on windows (tcp/http) and linux/osx (socket) + pool, err := dockertest.NewPool("") + if err != nil { + log.Panicf("Could not construct pool: %s", err) + } -var s3DoesNotExist = "nothing such" -var s3Creatable = "somename" + // uses pool to try to connect to Docker + err = pool.Client.Ping() + if err != nil { + log.Panicf("Could not connect to Docker: %s", err) + } -var writeData = []byte("this is a test") + // pulls an image, creates a container based on it and runs it + sftp, err := pool.RunWithOptions(&dockertest.RunOptions{ + Name: "sftp", + Repository: "atmoz/sftp", + Tag: "latest", + Cmd: []string{"user:test:1001::share"}, + Mounts: []string{ + fmt.Sprintf("%s/id_rsa.pub:/home/user/.ssh/keys/id_rsa.pub", sshPath), + fmt.Sprintf("%s/id_rsa:/etc/ssh/ssh_host_rsa_key", sshPath), + }, + }, func(config *docker.HostConfig) { + // set AutoRemove to true so that stopped container goes away by itself + config.AutoRemove = true + config.RestartPolicy = docker.RestartPolicy{ + Name: "no", + } + }) + if err != nil { + log.Panicf("Could not start resource: %s", err) + } -var cleanupFilesBack [1000]string -var cleanupFiles = cleanupFilesBack[0:0] + // sftpHostAndPort := sftp.GetHostPort("22/tcp") + sftpPort, _ = strconv.Atoi(sftp.GetPort("22/tcp")) + + // pulls an image, creates a container based on it and runs it + minio, err := pool.RunWithOptions(&dockertest.RunOptions{ + Name: "s3", + Repository: "minio/minio", + Tag: "RELEASE.2023-05-18T00-05-36Z", + Cmd: []string{"server", "/data"}, + Env: []string{ + "MINIO_ROOT_USER=access", + "MINIO_ROOT_PASSWORD=secretKey", + "MINIO_SERVER_URL=http://127.0.0.1:9000", + }, + }, func(config *docker.HostConfig) { + // set AutoRemove to true so that stopped container goes away by itself + config.AutoRemove = true + config.RestartPolicy = docker.RestartPolicy{ + Name: "no", + } + }) + if err != nil { + log.Panicf("Could not start resource: %s", err) + } -var testPosixConf = posixConf{ - "/"} + s3HostAndPort := minio.GetHostPort("9000/tcp") + s3Port, _ = strconv.Atoi(minio.GetPort("9000/tcp")) -var testSftpConf = SftpConf{ - "localhost", - "6222", - "user", - "to/be/updated", - "test", - "", -} + client := http.Client{Timeout: 5 * time.Second} + req, err := http.NewRequest(http.MethodGet, "http://"+s3HostAndPort+"/minio/health/live", http.NoBody) + if err != nil { + log.Panic(err) + } -// HoneyPot encapsulates the initialized mock sftp server struct -type HoneyPot struct { - server *ssh.Server -} + // exponential backoff-retry, because the application in the container might not be ready to accept connections yet + if err := pool.Retry(func() error { + res, err := client.Do(req) + if err != nil { + return err + } + res.Body.Close() -var hp *HoneyPot + return nil + }); err != nil { + if err := pool.Purge(minio); err != nil { + log.Panicf("Could not purge resource: %s", err) + } + log.Panicf("Could not connect to minio: %s", err) + } -func writeName() (name string, err error) { - f, err := os.CreateTemp("", "writablefile") + _ = m.Run() - if err != nil { - return "", err + log.Println("tests completed") + if err := pool.Purge(minio); err != nil { + log.Panicf("Could not purge resource: %s", err) + } + if err := pool.Purge(sftp); err != nil { + log.Panicf("Could not purge resource: %s", err) } +} - name = f.Name() +func TestStorageTestSuite(t *testing.T) { + suite.Run(t, new(StorageTestSuite)) +} - // Add to cleanup - cleanupFiles = append(cleanupFiles, name) +func (suite *StorageTestSuite) SetupTest() { + testS3Conf := S3Conf{ + "http://127.0.0.1", + s3Port, + "access", + "secretKey", + "bucket", + "region", + 10, + 5 * 1024 * 1024, + "", + 2 * time.Second, + "", + } - return name, err -} + testSftpConf := SftpConf{ + "localhost", + strconv.Itoa(sftpPort), + "user", + fmt.Sprintf("%s/id_rsa", sshPath), + "password", + "", + } -func doCleanup() { - for _, name := range cleanupFiles { - os.Remove(name) + testPosixConf := posixConf{ + os.TempDir(), } - cleanupFiles = cleanupFilesBack[0:0] + testConf = Conf{posixType, testS3Conf, testPosixConf, testSftpConf} } -func TestNewBackend(t *testing.T) { +func (suite *StorageTestSuite) TestNewBackend() { testConf.Type = posixType p, err := NewBackend(testConf) - assert.Nil(t, err, "Backend posix failed") + assert.NoError(suite.T(), err, "Backend posix failed") + assert.IsType(suite.T(), p, &posixBackend{}, "Wrong type from NewBackend with posix") var buf bytes.Buffer log.SetOutput(&buf) testConf.Type = sftpType sf, err := NewBackend(testConf) - assert.Nil(t, err, "Backend sftp failed") - assert.NotZero(t, buf.Len(), "Expected warning missing") - - // update host key from server for later use - rgx := regexp.MustCompile(`\\\"(.*?)\\"`) - testConf.SFTP.HostKey = strings.Trim(rgx.FindString(buf.String()), "\\\"") + assert.NoError(suite.T(), err, "Backend sftp failed") + assert.NotZero(suite.T(), buf.Len(), "Expected warning missing") + assert.IsType(suite.T(), sf, &sftpBackend{}, "Wrong type from NewBackend with SFTP") buf.Reset() testConf.Type = s3Type s, err := NewBackend(testConf) - assert.Nil(t, err, "Backend s3 failed") - - assert.IsType(t, p, &posixBackend{}, "Wrong type from NewBackend with posix") - assert.IsType(t, s, &s3Backend{}, "Wrong type from NewBackend with S3") - assert.IsType(t, sf, &sftpBackend{}, "Wrong type from NewBackend with SFTP") + assert.NoError(suite.T(), err, "Backend s3 failed") + assert.IsType(suite.T(), s, &s3Backend{}, "Wrong type from NewBackend with S3") // test some extra ssl handling testConf.S3.CAcert = "/dev/null" s, err = NewBackend(testConf) - assert.Nil(t, err, "Backend s3 failed") - assert.IsType(t, s, &s3Backend{}, "Wrong type from NewBackend with S3") + assert.NoError(suite.T(), err, "Backend s3 failed") + assert.IsType(suite.T(), s, &s3Backend{}, "Wrong type from NewBackend with S3") } -func TestMain(m *testing.M) { - - err := setupFakeS3() - - if err != nil { - log.Errorf("Setup of fake s3 failed, bailing out: %v", err) - os.Exit(1) - } - - err = setupMockSFTP() - - if err != nil { - log.Errorf("Setup of mock sftp failed, bailing out: %v", err) - os.Exit(1) - } - - ret := m.Run() - ts.Close() - hp.server.Close() - os.Remove(testConf.SFTP.PemKeyPath) - os.Exit(ret) -} - -func TestPosixBackend(t *testing.T) { - - defer doCleanup() +func (suite *StorageTestSuite) TestPosixBackend() { + posixPath, _ := os.MkdirTemp("", "posix") + defer os.RemoveAll(posixPath) testConf.Type = posixType + testConf.Posix = posixConf{posixPath} backend, err := NewBackend(testConf) - assert.Nil(t, err, "POSIX backend failed unexpectedly") - - var buf bytes.Buffer - - assert.IsType(t, backend, &posixBackend{}, "Wrong type from NewBackend with posix") + assert.Nil(suite.T(), err, "POSIX backend failed unexpectedly") log.SetOutput(os.Stdout) - writable, err := writeName() - if err != nil { - t.Error("could not find a writable name, bailing out from test") - - return - } - - writer, err := backend.NewFileWriter(writable) - - assert.NotNil(t, writer, "Got a nil reader for writer from posix") - assert.Nil(t, err, "posix NewFileWriter failed when it shouldn't") + writer, err := backend.NewFileWriter("testFile") + assert.NotNil(suite.T(), writer, "Got a nil reader for writer from posix") + assert.NoError(suite.T(), err, "posix NewFileWriter failed when it shouldn't") written, err := writer.Write(writeData) - - assert.Nil(t, err, "Failure when writing to posix writer") - assert.Equal(t, len(writeData), written, "Did not write all writeData") + assert.NoError(suite.T(), err, "Failure when writing to posix writer") + assert.Equal(suite.T(), len(writeData), written, "Did not write all writeData") writer.Close() - log.SetOutput(&buf) - writer, err = backend.NewFileWriter(posixNotCreatable) - - assert.Nil(t, writer, "Got a non-nil reader for writer from posix") - assert.NotNil(t, err, "posix NewFileWriter worked when it shouldn't") - assert.NotZero(t, buf.Len(), "Expected warning missing") + reader, err := backend.NewFileReader("testFile") + assert.Nil(suite.T(), err, "posix NewFileReader failed when it should work") + assert.NotNil(suite.T(), reader, "Reader that should be usable is nosuite.T(), bailing out") + var buf bytes.Buffer + log.SetOutput(&buf) + writer, err = backend.NewFileWriter("posix/Not/Creatable") + assert.Nil(suite.T(), writer, "Got a non-nil reader for writer from posix") + assert.Error(suite.T(), err, "posix NewFileWriter worked when it shouldn't") + assert.NotZero(suite.T(), buf.Len(), "Expected warning missing") + buf.Reset() log.SetOutput(os.Stdout) - reader, err := backend.NewFileReader(writable) - assert.Nil(t, err, "posix NewFileReader failed when it should work") - require.NotNil(t, reader, "Reader that should be usable is not, bailing out") - var readBackBuffer [4096]byte readBack, err := reader.Read(readBackBuffer[0:4096]) - assert.Equal(t, len(writeData), readBack, "did not read back data as expected") - assert.Equal(t, writeData, readBackBuffer[:readBack], "did not read back data as expected") - assert.Nil(t, err, "unexpected error when reading back data") + assert.Equal(suite.T(), len(writeData), readBack, "did not read back data as expected") + assert.Equal(suite.T(), writeData, readBackBuffer[:readBack], "did not read back data as expected") + assert.Nil(suite.T(), err, "unexpected error when reading back data") - size, err := backend.GetFileSize(writable) - assert.Nil(t, err, "posix NewFileReader failed when it should work") - assert.NotNil(t, size, "Got a nil size for posix") + size, err := backend.GetFileSize("testFile") + assert.Nil(suite.T(), err, "posix NewFileReader failed when it should work") + assert.NotNil(suite.T(), size, "Got a nil size for posix") - err = backend.RemoveFile(writable) - assert.Nil(t, err, "posix RemoveFile failed when it should work") + err = backend.RemoveFile("testFile") + assert.Nil(suite.T(), err, "posix RemoveFile failed when it should work") log.SetOutput(&buf) - - reader, err = backend.NewFileReader(posixDoesNotExist) - assert.NotNil(t, err, "posix NewFileReader worked when it should not") - assert.Nil(t, reader, "Got a non-nil reader for posix") - assert.NotZero(t, buf.Len(), "Expected warning missing") + reader, err = backend.NewFileReader("posixDoesNotExist") + assert.Error(suite.T(), err, "posix NewFileReader worked when it should not") + assert.Nil(suite.T(), reader, "Got a non-nil reader for posix") + assert.NotZero(suite.T(), buf.Len(), "Expected warning missing") buf.Reset() + _, err = backend.GetFileSize("posixDoesNotExist") + assert.Error(suite.T(), err, "posix GetFileSize worked when it should not") + assert.NotZero(suite.T(), buf.Len(), "Expected warning missing") - _, err = backend.GetFileSize(posixDoesNotExist) // nolint - assert.NotNil(t, err, "posix GetFileSize worked when it should not") - assert.NotZero(t, buf.Len(), "Expected warning missing") - - buf.Reset() - -} - -func setupFakeS3() (err error) { - // fake s3 - - if ts != nil { - // Setup done already? - return - } - - backend := s3mem.New() - faker := gofakes3.New(backend) - ts = httptest.NewServer(faker.Server()) - - portAt := strings.LastIndex(ts.URL, ":") - - testConf.S3.URL = ts.URL[:portAt] - testConf.S3.Port, err = strconv.Atoi(ts.URL[portAt+1:]) - testConf.Type = s3Type - - if err != nil { - log.Error("Unexpected error while setting up fake s3") - - return err - } - - backEnd, err := NewBackend(testConf) - if err != nil { - return err - } - - s3back := backEnd.(*s3Backend) - - _, err = s3back.Client.CreateBucket(&s3.CreateBucketInput{ - Bucket: aws.String(testConf.S3.Bucket)}) - - if err != nil { - if aerr, ok := err.(awserr.Error); ok { - - if aerr.Code() != s3.ErrCodeBucketAlreadyOwnedByYou && - aerr.Code() != s3.ErrCodeBucketAlreadyExists { - log.Error("Unexpected issue while creating bucket: ", err) - } else { - // Do not flag an error for this - err = nil - } - } - } - - return err -} - -func TestS3Fail(t *testing.T) { - - testConf.Type = s3Type - - tmp := testConf.S3.URL - - defer func() { testConf.S3.URL = tmp }() - testConf.S3.URL = "file://tmp/" - _, err := NewBackend(testConf) - assert.NotNil(t, err, "Backend worked when it should not") - - var dummyBackend *s3Backend - reader, err := dummyBackend.NewFileReader("/") - assert.NotNil(t, err, "NewFileReader worked when it should not") - assert.Nil(t, reader, "Got a Reader when expected not to") - - writer, err := dummyBackend.NewFileWriter("/") - assert.NotNil(t, err, "NewFileWriter worked when it should not") - assert.Nil(t, writer, "Got a Writer when expected not to") - - _, err = dummyBackend.GetFileSize("/") - assert.NotNil(t, err, "GetFileSize worked when it should not") - - err = dummyBackend.RemoveFile("/") - assert.NotNil(t, err, "RemoveFile worked when it should not") -} - -func TestPOSIXFail(t *testing.T) { - testConf.Type = posixType - - tmp := testConf.Posix.Location - - defer func() { testConf.Posix.Location = tmp }() + log.SetOutput(os.Stdout) testConf.Posix.Location = "/thisdoesnotexist" backEnd, err := NewBackend(testConf) - assert.NotNil(t, err, "Backend worked when it should not") - assert.Nil(t, backEnd, "Got a backend when expected not to") + assert.NotNil(suite.T(), err, "Backend worked when it should not") + assert.Nil(suite.T(), backEnd, "Got a backend when expected not to") testConf.Posix.Location = "/etc/passwd" backEnd, err = NewBackend(testConf) - assert.NotNil(t, err, "Backend worked when it should not") - assert.Nil(t, backEnd, "Got a backend when expected not to") + assert.NotNil(suite.T(), err, "Backend worked when it should not") + assert.Nil(suite.T(), backEnd, "Got a backend when expected not to") var dummyBackend *posixBackend - reader, err := dummyBackend.NewFileReader("/") - assert.NotNil(t, err, "NewFileReader worked when it should not") - assert.Nil(t, reader, "Got a Reader when expected not to") + failReader, err := dummyBackend.NewFileReader("/") + assert.NotNil(suite.T(), err, "NewFileReader worked when it should not") + assert.Nil(suite.T(), failReader, "Got a Reader when expected not to") - writer, err := dummyBackend.NewFileWriter("/") - assert.NotNil(t, err, "NewFileWriter worked when it should not") - assert.Nil(t, writer, "Got a Writer when expected not to") + failWriter, err := dummyBackend.NewFileWriter("/") + assert.NotNil(suite.T(), err, "NewFileWriter worked when it should not") + assert.Nil(suite.T(), failWriter, "Got a Writer when expected not to") _, err = dummyBackend.GetFileSize("/") - assert.NotNil(t, err, "GetFileSize worked when it should not") + assert.NotNil(suite.T(), err, "GetFileSize worked when it should not") err = dummyBackend.RemoveFile("/") - assert.NotNil(t, err, "RemoveFile worked when it should not") -} - -// Initializes a mock sftp server instance -func setupMockSFTP() error { - - password := testConf.SFTP.PemKeyPass - addr := testConf.SFTP.Host + ":" + testConf.SFTP.Port - - // Key-pair generation - privateKey, err := rsa.GenerateKey(rand.Reader, 2048) - if err != nil { - return err - } - publicRsaKey, err := cryptossh.NewPublicKey(&privateKey.PublicKey) - if err != nil { - return err - } - // pem.Block - privBlock := pem.Block{ - Type: "RSA PRIVATE KEY", - Headers: nil, - Bytes: x509.MarshalPKCS1PrivateKey(privateKey), - } - block, err := x509.EncryptPEMBlock(rand.Reader, privBlock.Type, privBlock.Bytes, []byte(password), x509.PEMCipherAES256) //nolint:staticcheck - if err != nil { - return err - } - - // Private key file in PEM format - privateKeyBytes := pem.EncodeToMemory(block) - - // Create temp key file and update config - fi, err := os.CreateTemp("", "sftp-key") - testConf.SFTP.PemKeyPath = fi.Name() - if err != nil { - return err - } - err = os.WriteFile(testConf.SFTP.PemKeyPath, privateKeyBytes, 0600) - if err != nil { - return err - } - - // Initialize a sftp honeypot instance - hp = NewHoneyPot(addr, publicRsaKey) - - // Start the server in the background - go func() { - if err := hp.server.ListenAndServe(); err != nil { - log.Panic(err) - } - }() - - return err + assert.NotNil(suite.T(), err, "RemoveFile worked when it should not") } -// NewHoneyPot takes in IP address to be used for sftp honeypot -func NewHoneyPot(addr string, key ssh.PublicKey) *HoneyPot { - return &HoneyPot{ - server: &ssh.Server{ - Addr: addr, - SubsystemHandlers: map[string]ssh.SubsystemHandler{ - "sftp": func(sess ssh.Session) { - debugStream := io.Discard - serverOptions := []sftp.ServerOption{ - sftp.WithDebug(debugStream), - } - server, err := sftp.NewServer( - sess, - serverOptions..., - ) - if err != nil { - log.Errorf("sftp server init error: %v\n", err) - - return - } - if err := server.Serve(); err != io.EOF { - log.Errorf("sftp server completed with error: %v\n", err) - } - }, - }, - PublicKeyHandler: func(ctx ssh.Context, key ssh.PublicKey) bool { - return true - }, - }, - } -} - -func TestSftpFail(t *testing.T) { - - testConf.Type = sftpType - - // test connection - tmpHost := testConf.SFTP.Host - - testConf.SFTP.Host = "nonexistenthost" - _, err := NewBackend(testConf) - assert.NotNil(t, err, "Backend worked when it should not") - - var dummyBackend *sftpBackend - reader, err := dummyBackend.NewFileReader("/") - assert.NotNil(t, err, "NewFileReader worked when it should not") - assert.Nil(t, reader, "Got a Reader when expected not to") - - writer, err := dummyBackend.NewFileWriter("/") - assert.NotNil(t, err, "NewFileWriter worked when it should not") - assert.Nil(t, writer, "Got a Writer when expected not to") - - _, err = dummyBackend.GetFileSize("/") - assert.NotNil(t, err, "GetFileSize worked when it should not") - - err = dummyBackend.RemoveFile("/") - assert.NotNil(t, err, "RemoveFile worked when it should not") - assert.EqualError(t, err, "Invalid sftpBackend") - testConf.SFTP.Host = tmpHost - - // wrong key password - tmpKeyPass := testConf.SFTP.PemKeyPass - testConf.SFTP.PemKeyPass = "wrongkey" - _, err = NewBackend(testConf) - assert.EqualError(t, err, "Failed to parse private key, x509: decryption password incorrect") - - // missing key password - testConf.SFTP.PemKeyPass = "" - _, err = NewBackend(testConf) - assert.EqualError(t, err, "Failed to parse private key, ssh: this private key is passphrase protected") - testConf.SFTP.PemKeyPass = tmpKeyPass - - // wrong key - tmpKeyPath := testConf.SFTP.PemKeyPath - testConf.SFTP.PemKeyPath = "nonexistentkey" - _, err = NewBackend(testConf) - testConf.SFTP.PemKeyPath = tmpKeyPath - assert.EqualError(t, err, "Failed to read from key file, open nonexistentkey: no such file or directory") - - defer doCleanup() - dummyKeyFile, err := writeName() - if err != nil { - t.Error("could not find a writable name, bailing out from test") - - return - } - testConf.SFTP.PemKeyPath = dummyKeyFile - _, err = NewBackend(testConf) - assert.EqualError(t, err, "Failed to parse private key, ssh: no key found") - testConf.SFTP.PemKeyPath = tmpKeyPath - - // wrong host key - tmpHostKey := testConf.SFTP.HostKey - testConf.SFTP.HostKey = "wronghostkey" - _, err = NewBackend(testConf) - assert.ErrorContains(t, err, "Failed to start ssh connection, ssh: handshake failed: host key verification expected") - testConf.SFTP.HostKey = tmpHostKey -} - -func TestS3Backend(t *testing.T) { - +func (suite *StorageTestSuite) TestS3Backend() { testConf.Type = s3Type - backend, err := NewBackend(testConf) - assert.Nil(t, err, "Backend failed") - - s3back := backend.(*s3Backend) - - var buf bytes.Buffer + s3back, err := NewBackend(testConf) + assert.NoError(suite.T(), err, "Backend failed") - assert.IsType(t, s3back, &s3Backend{}, "Wrong type from NewBackend with s3") - - writer, err := s3back.NewFileWriter(s3Creatable) - - assert.NotNil(t, writer, "Got a nil reader for writer from s3") - assert.Nil(t, err, "s3 NewFileWriter failed when it shouldn't") + writer, err := s3back.NewFileWriter("s3Creatable") + assert.NotNil(suite.T(), writer, "Got a nil reader for writer from s3") + assert.Nil(suite.T(), err, "s3 NewFileWriter failed when it shouldn't") written, err := writer.Write(writeData) - - assert.Nil(t, err, "Failure when writing to s3 writer") - assert.Equal(t, len(writeData), written, "Did not write all writeData") + assert.Nil(suite.T(), err, "Failure when writing to s3 writer") + assert.Equal(suite.T(), len(writeData), written, "Did not write all writeData") writer.Close() - reader, err := s3back.NewFileReader(s3Creatable) - assert.Nil(t, err, "s3 NewFileReader failed when it should work") - require.NotNil(t, reader, "Reader that should be usable is not, bailing out") + reader, err := s3back.NewFileReader("s3Creatable") + assert.Nil(suite.T(), err, "s3 NewFileReader failed when it should work") + assert.NotNil(suite.T(), reader, "Reader that should be usable is not, bailing out") - size, err := s3back.GetFileSize(s3Creatable) - assert.Nil(t, err, "s3 GetFileSize failed when it should work") - assert.NotNil(t, size, "Got a nil size for s3") - assert.Equal(t, int64(len(writeData)), size, "Got an incorrect file size") + size, err := s3back.GetFileSize("s3Creatable") + assert.Nil(suite.T(), err, "s3 GetFileSize failed when it should work") + assert.NotNil(suite.T(), size, "Got a nil size for s3") + assert.Equal(suite.T(), int64(len(writeData)), size, "Got an incorrect file size") - err = s3back.RemoveFile(s3Creatable) - assert.Nil(t, err, "s3 RemoveFile failed when it should work") + err = s3back.RemoveFile("s3Creatable") + assert.Nil(suite.T(), err, "s3 RemoveFile failed when it should work") var readBackBuffer [4096]byte readBack, err := reader.Read(readBackBuffer[0:4096]) - assert.Equal(t, len(writeData), readBack, "did not read back data as expected") - assert.Equal(t, writeData, readBackBuffer[:readBack], "did not read back data as expected") + assert.Equal(suite.T(), len(writeData), readBack, "did not read back data as expected") + assert.Equal(suite.T(), writeData, readBackBuffer[:readBack], "did not read back data as expected") if err != nil && err != io.EOF { - assert.Nil(t, err, "unexpected error when reading back data") + assert.Nil(suite.T(), err, "unexpected error when reading back data") } + var buf bytes.Buffer + log.SetOutput(&buf) + + _, err = s3back.GetFileSize("s3DoesNotExist") + assert.NotNil(suite.T(), err, "s3 GetFileSize worked when it should not") + assert.NotZero(suite.T(), buf.Len(), "Expected warning missing") buf.Reset() - log.SetOutput(&buf) + reader, err = s3back.NewFileReader("s3DoesNotExist") + assert.NotNil(suite.T(), err, "s3 NewFileReader worked when it should not") + assert.Nil(suite.T(), reader, "Got a non-nil reader for s3") + assert.NotZero(suite.T(), buf.Len(), "Expected warning missing") - if !testing.Short() { - _, err = backend.GetFileSize(s3DoesNotExist) - assert.NotNil(t, err, "s3 GetFileSize worked when it should not") - assert.NotZero(t, buf.Len(), "Expected warning missing") + log.SetOutput(os.Stdout) - buf.Reset() + testConf.S3.URL = "file://tmp/" + _, err = NewBackend(testConf) + assert.Error(suite.T(), err, "Backend worked when it should not") - reader, err = backend.NewFileReader(s3DoesNotExist) - assert.NotNil(t, err, "s3 NewFileReader worked when it should not") - assert.Nil(t, reader, "Got a non-nil reader for s3") - assert.NotZero(t, buf.Len(), "Expected warning missing") - } + var dummyBackend *s3Backend + failReader, err := dummyBackend.NewFileReader("/") + assert.NotNil(suite.T(), err, "NewFileReader worked when it should not") + assert.Nil(suite.T(), failReader, "Got a Reader when expected not to") - log.SetOutput(os.Stdout) + failWriter, err := dummyBackend.NewFileWriter("/") + assert.NotNil(suite.T(), err, "NewFileWriter worked when it should not") + assert.Nil(suite.T(), failWriter, "Got a Writer when expected not to") -} + _, err = dummyBackend.GetFileSize("/") + assert.NotNil(suite.T(), err, "GetFileSize worked when it should not") + + err = dummyBackend.RemoveFile("/") + assert.NotNil(suite.T(), err, "RemoveFile worked when it should not") -func TestSftpBackend(t *testing.T) { +} +func (suite *StorageTestSuite) TestSftpBackend() { var buf bytes.Buffer log.SetOutput(&buf) - testConf.Type = sftpType - backend, err := NewBackend(testConf) - assert.Nil(t, err, "Backend failed") - - assert.Zero(t, buf.Len(), "Got warning when not expected") + sftpBack, err := NewBackend(testConf) + assert.NoError(suite.T(), err, "Backend failed") buf.Reset() - sftpBack := backend.(*sftpBackend) - - assert.IsType(t, sftpBack, &sftpBackend{}, "Wrong type from NewBackend with sftp") - var sftpDoesNotExist = "nonexistent/file" - var sftpCreatable = os.TempDir() + "/this/file/exists" + var sftpCreatable = "/share/file/exists" writer, err := sftpBack.NewFileWriter(sftpCreatable) - assert.NotNil(t, writer, "Got a nil reader for writer from sftp") - assert.Nil(t, err, "sftp NewFileWriter failed when it shouldn't") + assert.NotNil(suite.T(), writer, "Got a nil reader for writer from sftp") + assert.Nil(suite.T(), err, "sftp NewFileWriter failed when it shouldn't") written, err := writer.Write(writeData) - assert.Nil(t, err, "Failure when writing to sftp writer") - assert.Equal(t, len(writeData), written, "Did not write all writeData") + assert.Nil(suite.T(), err, "Failure when writing to sftp writer") + assert.Equal(suite.T(), len(writeData), written, "Did not write all writeData") writer.Close() reader, err := sftpBack.NewFileReader(sftpCreatable) - assert.Nil(t, err, "sftp NewFileReader failed when it should work") - require.NotNil(t, reader, "Reader that should be usable is not, bailing out") + assert.Nil(suite.T(), err, "sftp NewFileReader failed when it should work") + assert.NotNil(suite.T(), reader, "Reader that should be usable is not, bailing out") size, err := sftpBack.GetFileSize(sftpCreatable) - assert.Nil(t, err, "sftp GetFileSize failed when it should work") - assert.NotNil(t, size, "Got a nil size for sftp") - assert.Equal(t, int64(len(writeData)), size, "Got an incorrect file size") + assert.Nil(suite.T(), err, "sftp GetFileSize failed when it should work") + assert.NotNil(suite.T(), size, "Got a nil size for sftp") + assert.Equal(suite.T(), int64(len(writeData)), size, "Got an incorrect file size") err = sftpBack.RemoveFile(sftpCreatable) - assert.Nil(t, err, "sftp RemoveFile failed when it should work") + assert.Nil(suite.T(), err, "sftp RemoveFile failed when it should work") err = sftpBack.RemoveFile(sftpDoesNotExist) - assert.EqualError(t, err, "Failed to remove file with sftp, file does not exist") + assert.EqualError(suite.T(), err, "Failed to remove file with sftp, file does not exist") var readBackBuffer [4096]byte readBack, err := reader.Read(readBackBuffer[0:4096]) - assert.Equal(t, len(writeData), readBack, "did not read back data as expected") - assert.Equal(t, writeData, readBackBuffer[:readBack], "did not read back data as expected") + assert.Equal(suite.T(), len(writeData), readBack, "did not read back data as expected") + assert.Equal(suite.T(), writeData, readBackBuffer[:readBack], "did not read back data as expected") if err != nil && err != io.EOF { - assert.Nil(t, err, "unexpected error when reading back data") + assert.Nil(suite.T(), err, "unexpected error when reading back data") } - if !testing.Short() { - _, err = backend.GetFileSize(sftpDoesNotExist) - assert.EqualError(t, err, "Failed to get file size with sftp, file does not exist") + _, err = sftpBack.GetFileSize(sftpDoesNotExist) + assert.EqualError(suite.T(), err, "Failed to get file size with sftp, file does not exist") + reader, err = sftpBack.NewFileReader(sftpDoesNotExist) + assert.EqualError(suite.T(), err, "Failed to open file with sftp, file does not exist") + assert.Nil(suite.T(), reader, "Got a non-nil reader for sftp") - reader, err = backend.NewFileReader(sftpDoesNotExist) - assert.EqualError(t, err, "Failed to open file with sftp, file does not exist") - assert.Nil(t, reader, "Got a non-nil reader for sftp") - } + // wrong host key + testConf.SFTP.HostKey = "wronghostkey" + _, err = NewBackend(testConf) + assert.ErrorContains(suite.T(), err, "Failed to start ssh connection, ssh: handshake failed: host key verification expected") + // wrong key password + testConf.SFTP.PemKeyPass = "wrongkey" + _, err = NewBackend(testConf) + assert.EqualError(suite.T(), err, "Failed to parse private key, x509: decryption password incorrect") + + // missing key password + testConf.SFTP.PemKeyPass = "" + _, err = NewBackend(testConf) + assert.EqualError(suite.T(), err, "Failed to parse private key, ssh: this private key is passphrase protected") + + // wrong key + testConf.SFTP.PemKeyPath = "nonexistentkey" + _, err = NewBackend(testConf) + assert.EqualError(suite.T(), err, "Failed to read from key file, open nonexistentkey: no such file or directory") + + f, _ := os.CreateTemp(sshPath, "dummy") + testConf.SFTP.PemKeyPath = f.Name() + _, err = NewBackend(testConf) + assert.EqualError(suite.T(), err, "Failed to parse private key, ssh: no key found") + + testConf.SFTP.Host = "nonexistenthost" + _, err = NewBackend(testConf) + assert.NotNil(suite.T(), err, "Backend worked when it should not") + + var dummyBackend *sftpBackend + failReader, err := dummyBackend.NewFileReader("/") + assert.NotNil(suite.T(), err, "NewFileReader worked when it should not") + assert.Nil(suite.T(), failReader, "Got a Reader when expected not to") + + failWriter, err := dummyBackend.NewFileWriter("/") + assert.NotNil(suite.T(), err, "NewFileWriter worked when it should not") + assert.Nil(suite.T(), failWriter, "Got a Writer when expected not to") + + _, err = dummyBackend.GetFileSize("/") + assert.NotNil(suite.T(), err, "GetFileSize worked when it should not") + + err = dummyBackend.RemoveFile("/") + assert.NotNil(suite.T(), err, "RemoveFile worked when it should not") + assert.EqualError(suite.T(), err, "Invalid sftpBackend") } From 09e00d332b2990f2408c39cf6ea1864b397bd219 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 10 Aug 2023 08:21:59 +0200 Subject: [PATCH 06/34] Merge `bucket` from s3inbox into `internal/storage` --- sda/cmd/s3inbox/bucket.go | 86 --------------------------- sda/cmd/s3inbox/bucket_test.go | 89 ---------------------------- sda/cmd/s3inbox/main.go | 3 +- sda/internal/storage/storage.go | 79 +++++++++++++++++------- sda/internal/storage/storage_test.go | 27 ++++++--- 5 files changed, 76 insertions(+), 208 deletions(-) delete mode 100644 sda/cmd/s3inbox/bucket.go delete mode 100644 sda/cmd/s3inbox/bucket_test.go diff --git a/sda/cmd/s3inbox/bucket.go b/sda/cmd/s3inbox/bucket.go deleted file mode 100644 index 1fe696e59..000000000 --- a/sda/cmd/s3inbox/bucket.go +++ /dev/null @@ -1,86 +0,0 @@ -package main - -import ( - "crypto/tls" - "crypto/x509" - "net/http" - "os" - "reflect" - "strings" - - "github.com/neicnordic/sensitive-data-archive/internal/storage" - - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3" -) - -func checkS3Bucket(config storage.S3Conf) error { - s3Transport := transportConfigS3(config) - client := http.Client{Transport: s3Transport} - s3Session := session.Must(session.NewSession( - &aws.Config{ - Endpoint: aws.String(config.URL), - Region: aws.String(config.Region), - HTTPClient: &client, - S3ForcePathStyle: aws.Bool(true), - DisableSSL: aws.Bool(strings.HasPrefix(config.URL, "http:")), - Credentials: credentials.NewStaticCredentials(config.AccessKey, config.SecretKey, ""), - }, - )) - - _, err := s3.New(s3Session).CreateBucket(&s3.CreateBucketInput{ - Bucket: aws.String(config.Bucket), - }) - if err != nil { - if aerr, ok := err.(awserr.Error); ok { - if aerr.Code() != s3.ErrCodeBucketAlreadyOwnedByYou && - aerr.Code() != s3.ErrCodeBucketAlreadyExists { - return errors.Errorf("Unexpected issue while creating bucket: %v", err) - } - - return nil - } - - return errors.New("Verifying bucket failed, check S3 configuration") - } - - return nil -} - -// transportConfigS3 is a helper method to setup TLS for the S3 client. -func transportConfigS3(config storage.S3Conf) http.RoundTripper { - cfg := new(tls.Config) - - // Enforce TLS1.2 or higher - cfg.MinVersion = 2 - - // Read system CAs - var systemCAs, _ = x509.SystemCertPool() - if reflect.DeepEqual(systemCAs, x509.NewCertPool()) { - log.Debug("creating new CApool") - systemCAs = x509.NewCertPool() - } - cfg.RootCAs = systemCAs - - if config.CAcert != "" { - cacert, e := os.ReadFile(config.CAcert) // #nosec this file comes from our config - if e != nil { - log.Fatalf("failed to append %q to RootCAs: %v", cacert, e) - } - if ok := cfg.RootCAs.AppendCertsFromPEM(cacert); !ok { - log.Debug("no certs appended, using system certs only") - } - } - - var trConfig http.RoundTripper = &http.Transport{ - TLSClientConfig: cfg, - ForceAttemptHTTP2: true} - - return trConfig -} diff --git a/sda/cmd/s3inbox/bucket_test.go b/sda/cmd/s3inbox/bucket_test.go deleted file mode 100644 index 13ea1a56d..000000000 --- a/sda/cmd/s3inbox/bucket_test.go +++ /dev/null @@ -1,89 +0,0 @@ -package main - -import ( - "net/http/httptest" - "os" - "testing" - - "github.com/neicnordic/sensitive-data-archive/internal/config" - - log "github.com/sirupsen/logrus" - - "github.com/johannesboyne/gofakes3" - "github.com/johannesboyne/gofakes3/backend/s3mem" - "github.com/spf13/viper" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/suite" -) - -var ts *httptest.Server - -type BucketTestSuite struct { - suite.Suite -} - -func (suite *BucketTestSuite) SetupTest() { - err := setupFakeS3() - if err != nil { - log.Error("Setup of fake s3 failed, bailing out") - os.Exit(1) - } - - viper.Set("log.level", "debug") - viper.Set("broker.host", "localhost") - viper.Set("broker.port", "1234") - viper.Set("broker.user", "guest") - viper.Set("broker.password", "guest") - viper.Set("broker.routingkey", "ingest") - viper.Set("broker.exchange", "amq.topic") - viper.Set("broker.vhost", "/") - viper.Set("inbox.url", ts.URL) - viper.Set("inbox.accesskey", "testaccess") - viper.Set("inbox.secretkey", "testsecret") - viper.Set("inbox.bucket", "testbucket") - viper.Set("server.jwtpubkeypath", "testpath") -} - -func setupFakeS3() (err error) { - // fake s3 - - if ts != nil { - // Setup done already? - return - } - - backend := s3mem.New() - faker := gofakes3.New(backend) - ts = httptest.NewServer(faker.Server()) - - if err != nil { - log.Error("Unexpected error while setting up fake s3") - - return err - } - - return err -} - -func TestBucketTestSuite(t *testing.T) { - suite.Run(t, new(BucketTestSuite)) -} - -func (suite *BucketTestSuite) TestBucketPass() { - config, err := config.NewConfig("s3inbox") - assert.NotNil(suite.T(), config) - assert.NoError(suite.T(), err) - - err = checkS3Bucket(config.Inbox.S3) - assert.NoError(suite.T(), err) -} - -func (suite *BucketTestSuite) TestBucketFail() { - viper.Set("inbox.url", "http://localhost:12345") - config, err := config.NewConfig("s3inbox") - assert.NotNil(suite.T(), config) - assert.NoError(suite.T(), err) - - err = checkS3Bucket(config.Inbox.S3) - assert.Error(suite.T(), err) -} diff --git a/sda/cmd/s3inbox/main.go b/sda/cmd/s3inbox/main.go index e3bf0c77d..de6acb8c2 100644 --- a/sda/cmd/s3inbox/main.go +++ b/sda/cmd/s3inbox/main.go @@ -10,6 +10,7 @@ import ( "github.com/neicnordic/sensitive-data-archive/internal/broker" "github.com/neicnordic/sensitive-data-archive/internal/config" "github.com/neicnordic/sensitive-data-archive/internal/database" + "github.com/neicnordic/sensitive-data-archive/internal/storage" log "github.com/sirupsen/logrus" ) @@ -57,7 +58,7 @@ func main() { log.Debugf("Connected to sda-db (v%v)", sdaDB.Version) - err = checkS3Bucket(Conf.Inbox.S3) + err = storage.CheckS3Bucket(Conf.Inbox.S3) if err != nil { log.Error(err) sigc <- syscall.SIGINT diff --git a/sda/internal/storage/storage.go b/sda/internal/storage/storage.go index 98e6b29ba..f44bbf7bf 100644 --- a/sda/internal/storage/storage.go +++ b/sda/internal/storage/storage.go @@ -82,7 +82,7 @@ func newPosixBackend(config posixConf) (*posixBackend, error) { // NewFileReader returns an io.Reader instance func (pb *posixBackend) NewFileReader(filePath string) (io.ReadCloser, error) { if pb == nil { - return nil, fmt.Errorf("Invalid posixBackend") + return nil, fmt.Errorf("invalid posixBackend") } file, err := os.Open(filepath.Join(filepath.Clean(pb.Location), filePath)) @@ -98,7 +98,7 @@ func (pb *posixBackend) NewFileReader(filePath string) (io.ReadCloser, error) { // NewFileWriter returns an io.Writer instance func (pb *posixBackend) NewFileWriter(filePath string) (io.WriteCloser, error) { if pb == nil { - return nil, fmt.Errorf("Invalid posixBackend") + return nil, fmt.Errorf("invalid posixBackend") } file, err := os.OpenFile(filepath.Join(filepath.Clean(pb.Location), filePath), os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0640) @@ -114,7 +114,7 @@ func (pb *posixBackend) NewFileWriter(filePath string) (io.WriteCloser, error) { // GetFileSize returns the size of the file func (pb *posixBackend) GetFileSize(filePath string) (int64, error) { if pb == nil { - return 0, fmt.Errorf("Invalid posixBackend") + return 0, fmt.Errorf("invalid posixBackend") } stat, err := os.Stat(filepath.Join(filepath.Clean(pb.Location), filePath)) @@ -130,7 +130,7 @@ func (pb *posixBackend) GetFileSize(filePath string) (int64, error) { // RemoveFile removes a file from a given path func (pb *posixBackend) RemoveFile(filePath string) error { if pb == nil { - return fmt.Errorf("Invalid posixBackend") + return fmt.Errorf("invalid posixBackend") } err := os.Remove(filepath.Join(filepath.Clean(pb.Location), filePath)) @@ -190,7 +190,7 @@ func newS3Backend(config S3Conf) (*s3Backend, error) { if aerr.Code() != s3.ErrCodeBucketAlreadyOwnedByYou && aerr.Code() != s3.ErrCodeBucketAlreadyExists { - log.Error("Unexpected issue while creating bucket", err) + log.Error("unexpected issue while creating bucket", err) } } } @@ -214,10 +214,43 @@ func newS3Backend(config S3Conf) (*s3Backend, error) { return sb, nil } +func CheckS3Bucket(config S3Conf) error { + s3Transport := transportConfigS3(config) + client := http.Client{Transport: s3Transport} + s3Session := session.Must(session.NewSession( + &aws.Config{ + Endpoint: aws.String(fmt.Sprintf("%s:%d", config.URL, config.Port)), + Region: aws.String(config.Region), + HTTPClient: &client, + S3ForcePathStyle: aws.Bool(true), + DisableSSL: aws.Bool(strings.HasPrefix(config.URL, "http:")), + Credentials: credentials.NewStaticCredentials(config.AccessKey, config.SecretKey, ""), + }, + )) + + _, err := s3.New(s3Session).CreateBucket(&s3.CreateBucketInput{ + Bucket: aws.String(config.Bucket), + }) + if err != nil { + if aerr, ok := err.(awserr.Error); ok { + if aerr.Code() != s3.ErrCodeBucketAlreadyOwnedByYou && + aerr.Code() != s3.ErrCodeBucketAlreadyExists { + return fmt.Errorf("unexpected issue while creating bucket: %v", err) + } + + return nil + } + + return fmt.Errorf("verifying bucket failed, check S3 configuration") + } + + return nil +} + // NewFileReader returns an io.Reader instance func (sb *s3Backend) NewFileReader(filePath string) (io.ReadCloser, error) { if sb == nil { - return nil, fmt.Errorf("Invalid s3Backend") + return nil, fmt.Errorf("invalid s3Backend") } r, err := sb.Client.GetObject(&s3.GetObjectInput{ @@ -251,7 +284,7 @@ func (sb *s3Backend) NewFileReader(filePath string) (io.ReadCloser, error) { // NewFileWriter uploads the contents of an io.Reader to a S3 bucket func (sb *s3Backend) NewFileWriter(filePath string) (io.WriteCloser, error) { if sb == nil { - return nil, fmt.Errorf("Invalid s3Backend") + return nil, fmt.Errorf("invalid s3Backend") } reader, writer := io.Pipe() @@ -275,7 +308,7 @@ func (sb *s3Backend) NewFileWriter(filePath string) (io.WriteCloser, error) { // GetFileSize returns the size of a specific object func (sb *s3Backend) GetFileSize(filePath string) (int64, error) { if sb == nil { - return 0, fmt.Errorf("Invalid s3Backend") + return 0, fmt.Errorf("invalid s3Backend") } r, err := sb.Client.HeadObject(&s3.HeadObjectInput{ @@ -312,7 +345,7 @@ func (sb *s3Backend) GetFileSize(filePath string) (int64, error) { // RemoveFile removes an object from a bucket func (sb *s3Backend) RemoveFile(filePath string) error { if sb == nil { - return fmt.Errorf("Invalid s3Backend") + return fmt.Errorf("invalid s3Backend") } _, err := sb.Client.DeleteObject(&s3.DeleteObjectInput{ @@ -386,7 +419,7 @@ func newSftpBackend(config SftpConf) (*sftpBackend, error) { // read in and parse pem key key, err := os.ReadFile(config.PemKeyPath) if err != nil { - return nil, fmt.Errorf("Failed to read from key file, %v", err) + return nil, fmt.Errorf("failed to read from key file, %v", err) } var signer ssh.Signer @@ -396,7 +429,7 @@ func newSftpBackend(config SftpConf) (*sftpBackend, error) { signer, err = ssh.ParsePrivateKeyWithPassphrase(key, []byte(config.PemKeyPass)) } if err != nil { - return nil, fmt.Errorf("Failed to parse private key, %v", err) + return nil, fmt.Errorf("failed to parse private key, %v", err) } // connect @@ -408,13 +441,13 @@ func newSftpBackend(config SftpConf) (*sftpBackend, error) { }, ) if err != nil { - return nil, fmt.Errorf("Failed to start ssh connection, %v", err) + return nil, fmt.Errorf("failed to start ssh connection, %v", err) } // create new SFTP client client, err := sftp.NewClient(conn) if err != nil { - return nil, fmt.Errorf("Failed to start sftp client, %v", err) + return nil, fmt.Errorf("failed to start sftp client, %v", err) } sfb := &sftpBackend{ @@ -426,7 +459,7 @@ func newSftpBackend(config SftpConf) (*sftpBackend, error) { _, err = client.ReadDir("./") if err != nil { - return nil, fmt.Errorf("Failed to list files with sftp, %v", err) + return nil, fmt.Errorf("failed to list files with sftp, %v", err) } return sfb, nil @@ -435,18 +468,18 @@ func newSftpBackend(config SftpConf) (*sftpBackend, error) { // NewFileWriter returns an io.Writer instance for the sftp remote func (sfb *sftpBackend) NewFileWriter(filePath string) (io.WriteCloser, error) { if sfb == nil { - return nil, fmt.Errorf("Invalid sftpBackend") + return nil, fmt.Errorf("invalid sftpBackend") } // Make remote directories parent := filepath.Dir(filePath) err := sfb.Client.MkdirAll(parent) if err != nil { - return nil, fmt.Errorf("Failed to create dir with sftp, %v", err) + return nil, fmt.Errorf("failed to create dir with sftp, %v", err) } file, err := sfb.Client.OpenFile(filePath, os.O_CREATE|os.O_TRUNC|os.O_RDWR) if err != nil { - return nil, fmt.Errorf("Failed to create file with sftp, %v", err) + return nil, fmt.Errorf("failed to create file with sftp, %v", err) } return file, nil @@ -455,12 +488,12 @@ func (sfb *sftpBackend) NewFileWriter(filePath string) (io.WriteCloser, error) { // GetFileSize returns the size of the file func (sfb *sftpBackend) GetFileSize(filePath string) (int64, error) { if sfb == nil { - return 0, fmt.Errorf("Invalid sftpBackend") + return 0, fmt.Errorf("invalid sftpBackend") } stat, err := sfb.Client.Lstat(filePath) if err != nil { - return 0, fmt.Errorf("Failed to get file size with sftp, %v", err) + return 0, fmt.Errorf("failed to get file size with sftp, %v", err) } return stat.Size(), nil @@ -469,12 +502,12 @@ func (sfb *sftpBackend) GetFileSize(filePath string) (int64, error) { // NewFileReader returns an io.Reader instance func (sfb *sftpBackend) NewFileReader(filePath string) (io.ReadCloser, error) { if sfb == nil { - return nil, fmt.Errorf("Invalid sftpBackend") + return nil, fmt.Errorf("invalid sftpBackend") } file, err := sfb.Client.Open(filePath) if err != nil { - return nil, fmt.Errorf("Failed to open file with sftp, %v", err) + return nil, fmt.Errorf("failed to open file with sftp, %v", err) } return file, nil @@ -483,12 +516,12 @@ func (sfb *sftpBackend) NewFileReader(filePath string) (io.ReadCloser, error) { // RemoveFile removes a file or an empty directory. func (sfb *sftpBackend) RemoveFile(filePath string) error { if sfb == nil { - return fmt.Errorf("Invalid sftpBackend") + return fmt.Errorf("invalid sftpBackend") } err := sfb.Client.Remove(filePath) if err != nil { - return fmt.Errorf("Failed to remove file with sftp, %v", err) + return fmt.Errorf("failed to remove file with sftp, %v", err) } return nil diff --git a/sda/internal/storage/storage_test.go b/sda/internal/storage/storage_test.go index 97e8a91df..2b7b83d20 100644 --- a/sda/internal/storage/storage_test.go +++ b/sda/internal/storage/storage_test.go @@ -201,6 +201,15 @@ func (suite *StorageTestSuite) TestNewBackend() { assert.IsType(suite.T(), s, &s3Backend{}, "Wrong type from NewBackend with S3") } +func (suite *StorageTestSuite) TestCheckS3Bucket() { + err := CheckS3Bucket(testConf.S3) + assert.NoError(suite.T(), err) + + testConf.S3.URL = "file://tmp/" + err = CheckS3Bucket(testConf.S3) + assert.Error(suite.T(), err) +} + func (suite *StorageTestSuite) TestPosixBackend() { posixPath, _ := os.MkdirTemp("", "posix") defer os.RemoveAll(posixPath) @@ -392,7 +401,7 @@ func (suite *StorageTestSuite) TestSftpBackend() { assert.Nil(suite.T(), err, "sftp RemoveFile failed when it should work") err = sftpBack.RemoveFile(sftpDoesNotExist) - assert.EqualError(suite.T(), err, "Failed to remove file with sftp, file does not exist") + assert.EqualError(suite.T(), err, "failed to remove file with sftp, file does not exist") var readBackBuffer [4096]byte readBack, err := reader.Read(readBackBuffer[0:4096]) @@ -405,35 +414,35 @@ func (suite *StorageTestSuite) TestSftpBackend() { } _, err = sftpBack.GetFileSize(sftpDoesNotExist) - assert.EqualError(suite.T(), err, "Failed to get file size with sftp, file does not exist") + assert.EqualError(suite.T(), err, "failed to get file size with sftp, file does not exist") reader, err = sftpBack.NewFileReader(sftpDoesNotExist) - assert.EqualError(suite.T(), err, "Failed to open file with sftp, file does not exist") + assert.EqualError(suite.T(), err, "failed to open file with sftp, file does not exist") assert.Nil(suite.T(), reader, "Got a non-nil reader for sftp") // wrong host key testConf.SFTP.HostKey = "wronghostkey" _, err = NewBackend(testConf) - assert.ErrorContains(suite.T(), err, "Failed to start ssh connection, ssh: handshake failed: host key verification expected") + assert.ErrorContains(suite.T(), err, "failed to start ssh connection, ssh: handshake failed: host key verification expected") // wrong key password testConf.SFTP.PemKeyPass = "wrongkey" _, err = NewBackend(testConf) - assert.EqualError(suite.T(), err, "Failed to parse private key, x509: decryption password incorrect") + assert.EqualError(suite.T(), err, "failed to parse private key, x509: decryption password incorrect") // missing key password testConf.SFTP.PemKeyPass = "" _, err = NewBackend(testConf) - assert.EqualError(suite.T(), err, "Failed to parse private key, ssh: this private key is passphrase protected") + assert.EqualError(suite.T(), err, "failed to parse private key, ssh: this private key is passphrase protected") // wrong key testConf.SFTP.PemKeyPath = "nonexistentkey" _, err = NewBackend(testConf) - assert.EqualError(suite.T(), err, "Failed to read from key file, open nonexistentkey: no such file or directory") + assert.EqualError(suite.T(), err, "failed to read from key file, open nonexistentkey: no such file or directory") f, _ := os.CreateTemp(sshPath, "dummy") testConf.SFTP.PemKeyPath = f.Name() _, err = NewBackend(testConf) - assert.EqualError(suite.T(), err, "Failed to parse private key, ssh: no key found") + assert.EqualError(suite.T(), err, "failed to parse private key, ssh: no key found") testConf.SFTP.Host = "nonexistenthost" _, err = NewBackend(testConf) @@ -453,5 +462,5 @@ func (suite *StorageTestSuite) TestSftpBackend() { err = dummyBackend.RemoveFile("/") assert.NotNil(suite.T(), err, "RemoveFile worked when it should not") - assert.EqualError(suite.T(), err, "Invalid sftpBackend") + assert.EqualError(suite.T(), err, "invalid sftpBackend") } From e80bdac3aa4c4ab4a37b5865d26f86f5268eda96 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 10 Aug 2023 08:55:09 +0200 Subject: [PATCH 07/34] Deduplicate code `newS3Backend` and `CheckS3Bucket` had some overlap that where put as a separate function. --- sda/cmd/s3inbox/main.go | 5 ++-- sda/internal/storage/storage.go | 41 +++++++--------------------- sda/internal/storage/storage_test.go | 4 +-- 3 files changed, 14 insertions(+), 36 deletions(-) diff --git a/sda/cmd/s3inbox/main.go b/sda/cmd/s3inbox/main.go index de6acb8c2..c6e62f692 100644 --- a/sda/cmd/s3inbox/main.go +++ b/sda/cmd/s3inbox/main.go @@ -29,13 +29,12 @@ func main() { } }() - c, err := config.NewConfig("s3inbox") + Conf, err := config.NewConfig("s3inbox") if err != nil { log.Error(err) sigc <- syscall.SIGINT panic(err) } - Conf = c tlsProxy, err := config.TLSConfigProxy(Conf) if err != nil { @@ -58,7 +57,7 @@ func main() { log.Debugf("Connected to sda-db (v%v)", sdaDB.Version) - err = storage.CheckS3Bucket(Conf.Inbox.S3) + err = storage.CheckS3Bucket(Conf.Inbox.S3.Bucket, storage.CreateS3Session(Conf.Inbox.S3)) if err != nil { log.Error(err) sigc <- syscall.SIGINT diff --git a/sda/internal/storage/storage.go b/sda/internal/storage/storage.go index f44bbf7bf..05a354dcb 100644 --- a/sda/internal/storage/storage.go +++ b/sda/internal/storage/storage.go @@ -166,33 +166,9 @@ type S3Conf struct { } func newS3Backend(config S3Conf) (*s3Backend, error) { - s3Transport := transportConfigS3(config) - client := http.Client{Transport: s3Transport} - s3Session := session.Must(session.NewSession( - &aws.Config{ - Endpoint: aws.String(fmt.Sprintf("%s:%d", config.URL, config.Port)), - Region: aws.String(config.Region), - HTTPClient: &client, - S3ForcePathStyle: aws.Bool(true), - DisableSSL: aws.Bool(strings.HasPrefix(config.URL, "http:")), - Credentials: credentials.NewStaticCredentials(config.AccessKey, config.SecretKey, ""), - }, - )) - - // Attempt to create a bucket, but we really expect an error here - // (BucketAlreadyOwnedByYou) - _, err := s3.New(s3Session).CreateBucket(&s3.CreateBucketInput{ - Bucket: aws.String(config.Bucket), - }) - - if err != nil { - if aerr, ok := err.(awserr.Error); ok { - - if aerr.Code() != s3.ErrCodeBucketAlreadyOwnedByYou && - aerr.Code() != s3.ErrCodeBucketAlreadyExists { - log.Error("unexpected issue while creating bucket", err) - } - } + s3Session := CreateS3Session(config) + if err := CheckS3Bucket(config.Bucket, s3Session); err != nil { + return nil, err } sb := &s3Backend{ @@ -205,8 +181,7 @@ func newS3Backend(config S3Conf) (*s3Backend, error) { Client: s3.New(s3Session), Conf: &config} - _, err = sb.Client.ListObjectsV2(&s3.ListObjectsV2Input{Bucket: &config.Bucket}) - + _, err := sb.Client.ListObjectsV2(&s3.ListObjectsV2Input{Bucket: &config.Bucket}) if err != nil { return nil, err } @@ -214,7 +189,7 @@ func newS3Backend(config S3Conf) (*s3Backend, error) { return sb, nil } -func CheckS3Bucket(config S3Conf) error { +func CreateS3Session(config S3Conf) *session.Session { s3Transport := transportConfigS3(config) client := http.Client{Transport: s3Transport} s3Session := session.Must(session.NewSession( @@ -228,8 +203,12 @@ func CheckS3Bucket(config S3Conf) error { }, )) + return s3Session +} + +func CheckS3Bucket(bucket string, s3Session *session.Session) error { _, err := s3.New(s3Session).CreateBucket(&s3.CreateBucketInput{ - Bucket: aws.String(config.Bucket), + Bucket: aws.String(bucket), }) if err != nil { if aerr, ok := err.(awserr.Error); ok { diff --git a/sda/internal/storage/storage_test.go b/sda/internal/storage/storage_test.go index 2b7b83d20..a249b2f36 100644 --- a/sda/internal/storage/storage_test.go +++ b/sda/internal/storage/storage_test.go @@ -202,11 +202,11 @@ func (suite *StorageTestSuite) TestNewBackend() { } func (suite *StorageTestSuite) TestCheckS3Bucket() { - err := CheckS3Bucket(testConf.S3) + err := CheckS3Bucket(testConf.S3.Bucket, CreateS3Session(testConf.S3)) assert.NoError(suite.T(), err) testConf.S3.URL = "file://tmp/" - err = CheckS3Bucket(testConf.S3) + err = CheckS3Bucket(testConf.S3.Bucket, CreateS3Session(testConf.S3)) assert.Error(suite.T(), err) } From c1ea2a967960e4a83e6b3cc25dce290f14709fc6 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 10 Aug 2023 09:30:28 +0200 Subject: [PATCH 08/34] [integration tests] Change timings of sda healthchecks --- .github/integration/sda-integration.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/integration/sda-integration.yml b/.github/integration/sda-integration.yml index 60f3bf445..11406c866 100644 --- a/.github/integration/sda-integration.yml +++ b/.github/integration/sda-integration.yml @@ -24,9 +24,9 @@ services: - POSTGRES_PASSWORD=rootpasswd healthcheck: test: ["CMD-SHELL", "pg_isready -U postgres"] - interval: 5s - timeout: 20s - retries: 3 + interval: 10s + timeout: 2s + retries: 6 image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-postgres ports: - "15432:5432" @@ -46,9 +46,9 @@ services: "-c", "rabbitmq-diagnostics -q check_running && rabbitmq-diagnostics -q check_local_alarms", ] - interval: 5s - timeout: 20s - retries: 3 + interval: 10s + timeout: 2s + retries: 6 image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-rabbitmq ports: - "15672:15672" @@ -66,9 +66,9 @@ services: - MINIO_SERVER_URL=http://127.0.0.1:9000 healthcheck: test: ["CMD", "curl", "-fkq", "http://localhost:9000/minio/health/live"] - interval: 5s - timeout: 20s - retries: 3 + interval: 10s + timeout: 2s + retries: 6 ports: - "19000:9000" - "19001:9001" @@ -122,9 +122,9 @@ services: condition: service_completed_successfully healthcheck: test: ["CMD", "python3", "-c", 'import requests; print(requests.get(url = "http://localhost:8080/jwk").text)'] - interval: 5s - timeout: 20s - retries: 3 + interval: 10s + timeout: 2s + retries: 6 image: python:3.10-slim ports: - "8080:8080" From b09cb00ad9a52b22ddbf101d809d30a6c933e97a Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 10 Aug 2023 11:04:33 +0200 Subject: [PATCH 09/34] Remove unused variable from `SendMessage` --- sda/cmd/s3inbox/proxy.go | 2 +- sda/internal/broker/broker.go | 3 +-- sda/internal/broker/broker_test.go | 3 +-- sda/internal/config/config.go | 3 --- 4 files changed, 3 insertions(+), 8 deletions(-) diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index 53ae6c28b..b775d354a 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -177,7 +177,7 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request) { } case false: - if err = p.messenger.SendMessage(p.fileIds[r.URL.Path], p.messenger.Conf.Exchange, p.messenger.Conf.RoutingKey, true, jsonMessage); err != nil { + if err = p.messenger.SendMessage(p.fileIds[r.URL.Path], p.messenger.Conf.Exchange, p.messenger.Conf.RoutingKey, jsonMessage); err != nil { log.Debug("error when sending message") log.Error(err) } diff --git a/sda/internal/broker/broker.go b/sda/internal/broker/broker.go index 3c51110d0..ac6b98a34 100644 --- a/sda/internal/broker/broker.go +++ b/sda/internal/broker/broker.go @@ -37,7 +37,6 @@ type MQConf struct { ClientCert string ClientKey string ServerName string - Durable bool SchemasPath string PrefetchCount int } @@ -179,7 +178,7 @@ func (broker *AMQPBroker) GetMessages(queue string) (<-chan amqp.Delivery, error } // SendMessage sends a message to RabbitMQ -func (broker *AMQPBroker) SendMessage(corrID, exchange, routingKey string, _ bool, body []byte) error { +func (broker *AMQPBroker) SendMessage(corrID, exchange, routingKey string, body []byte) error { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() diff --git a/sda/internal/broker/broker_test.go b/sda/internal/broker/broker_test.go index 7b20f1b62..d5a95da70 100644 --- a/sda/internal/broker/broker_test.go +++ b/sda/internal/broker/broker_test.go @@ -127,7 +127,6 @@ func (suite *BrokerTestSuite) SetupTest() { certPath + "/tls.crt", certPath + "/tls.key", "mq", - true, "", 2, } @@ -232,7 +231,7 @@ func (suite *BrokerTestSuite) TestSendMessage() { assert.NotNil(suite.T(), b, "NewMQ without ssl did not return a broker") assert.False(suite.T(), b.Connection.IsClosed()) - err = b.SendMessage("1", "", "ingest", true, []byte("test message")) + err = b.SendMessage("1", "", "ingest", []byte("test message")) assert.NoError(suite.T(), err) b.Channel.Close() diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index c838c2e50..be4404f1c 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -540,9 +540,6 @@ func (c *Config) configBroker() error { broker.Exchange = viper.GetString("broker.exchange") } - if viper.IsSet("broker.durable") { - broker.Durable = viper.GetBool("broker.durable") - } if viper.IsSet("broker.routingerror") { broker.RoutingError = viper.GetString("broker.routingerror") } From 95911f5545b4749936627c24581ef8c1f4bb5ad2 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 10 Aug 2023 11:05:13 +0200 Subject: [PATCH 10/34] Rename `main` to `s3inbox` --- sda/cmd/s3inbox/{main.go => s3inbox.go} | 0 sda/cmd/s3inbox/{main_test.go => s3inbox_test.go} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename sda/cmd/s3inbox/{main.go => s3inbox.go} (100%) rename sda/cmd/s3inbox/{main_test.go => s3inbox_test.go} (100%) diff --git a/sda/cmd/s3inbox/main.go b/sda/cmd/s3inbox/s3inbox.go similarity index 100% rename from sda/cmd/s3inbox/main.go rename to sda/cmd/s3inbox/s3inbox.go diff --git a/sda/cmd/s3inbox/main_test.go b/sda/cmd/s3inbox/s3inbox_test.go similarity index 100% rename from sda/cmd/s3inbox/main_test.go rename to sda/cmd/s3inbox/s3inbox_test.go From f6f3a99090048ceab6d21d741057f3b75a7a304f Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 10 Aug 2023 16:28:58 +0200 Subject: [PATCH 11/34] Remove routingkey option for error messages They will be sent to `error` --- sda/internal/broker/broker.go | 1 - sda/internal/broker/broker_test.go | 1 - sda/internal/config/config.go | 3 --- 3 files changed, 5 deletions(-) diff --git a/sda/internal/broker/broker.go b/sda/internal/broker/broker.go index ac6b98a34..4145fdf92 100644 --- a/sda/internal/broker/broker.go +++ b/sda/internal/broker/broker.go @@ -30,7 +30,6 @@ type MQConf struct { Queue string Exchange string RoutingKey string - RoutingError string Ssl bool VerifyPeer bool CACert string diff --git a/sda/internal/broker/broker_test.go b/sda/internal/broker/broker_test.go index d5a95da70..890e64a19 100644 --- a/sda/internal/broker/broker_test.go +++ b/sda/internal/broker/broker_test.go @@ -120,7 +120,6 @@ func (suite *BrokerTestSuite) SetupTest() { "ingest", "amq.default", "ingest", - "error", false, false, certPath + "/ca.crt", diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index be4404f1c..2e7781b04 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -540,9 +540,6 @@ func (c *Config) configBroker() error { broker.Exchange = viper.GetString("broker.exchange") } - if viper.IsSet("broker.routingerror") { - broker.RoutingError = viper.GetString("broker.routingerror") - } if viper.IsSet("broker.vhost") { if strings.HasPrefix(viper.GetString("broker.vhost"), "/") { broker.Vhost = viper.GetString("broker.vhost") From 0fdba22ba55e8e7716320891c4ba15d8d7af6377 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 15 Aug 2023 10:16:55 +0200 Subject: [PATCH 12/34] Merge in `database` from sda-pipeline --- sda/internal/database/database.go | 12 +- sda/internal/database/db_functions.go | 247 +++++++++++++++++++++ sda/internal/database/db_functions_test.go | 177 +++++++++++++-- 3 files changed, 414 insertions(+), 22 deletions(-) diff --git a/sda/internal/database/database.go b/sda/internal/database/database.go index 4a0c12f0e..26348cf19 100644 --- a/sda/internal/database/database.go +++ b/sda/internal/database/database.go @@ -62,6 +62,15 @@ var FastConnectRate = 5 * time.Second // database during the after FastConnectTimeout. var SlowConnectRate = 1 * time.Minute +// dbRetryTimes is the number of times to retry the same function if it fails +var RetryTimes = 5 + +// hashType returns the identification string for the hash type +func hashType(_ hash.Hash) string { + // TODO: Support/check type + return "SHA256" +} + // NewSDAdb creates a new DB connection from the given DBConf variables. // Currently, only postgresql connections are supported. func NewSDAdb(config DBConf) (*SDAdb, error) { @@ -101,8 +110,7 @@ func (dbs *SDAdb) Connect() error { err := fmt.Errorf("failed to connect within reconnect time") log.Infoln("Connecting to database") - log.Debugf("host: %s:%d, database: %s, user: %s", dbs.Config.Host, - dbs.Config.Port, dbs.Config.Database, dbs.Config.User) + log.Debugf("host: %s:%d, database: %s, user: %s", dbs.Config.Host, dbs.Config.Port, dbs.Config.Database, dbs.Config.User) for ConnectTimeout <= 0 || ConnectTimeout > time.Since(start) { dbs.DB, err = sql.Open(dbs.Config.PgDataSource()) diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 2298f9ed0..7a450e614 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -3,7 +3,9 @@ package database import ( + "encoding/hex" "errors" + "fmt" ) // RegisterFile inserts a file in the database, along with a "registered" log @@ -41,3 +43,248 @@ func (dbs *SDAdb) UpdateFileEventLog(fileID, event, userID, message string) erro return err } + +func (dbs *SDAdb) GetFileID(corrID string) (string, error) { + var ( + err error + count int + ID string + ) + + for count == 0 || (err != nil && count < RetryTimes) { + ID, err = dbs.getFileID(corrID) + count++ + } + + return ID, err +} +func (dbs *SDAdb) getFileID(corrID string) (string, error) { + dbs.checkAndReconnectIfNeeded() + db := dbs.DB + const getFileID = "SELECT DISTINCT file_id FROM sda.file_event_log where correlation_id = $1;" + + var fileID string + err := db.QueryRow(getFileID, corrID).Scan(&fileID) + if err != nil { + return "", err + } + + return fileID, nil +} + +func (dbs *SDAdb) UpdateFileStatus(fileUUID, event, corrID, user, message string) error { + var ( + err error + count int + ) + + for count == 0 || (err != nil && count < RetryTimes) { + err = dbs.updateFileStatus(fileUUID, event, corrID, user, message) + count++ + } + + return err +} +func (dbs *SDAdb) updateFileStatus(fileUUID, event, corrID, user, message string) error { + dbs.checkAndReconnectIfNeeded() + + db := dbs.DB + const query = "INSERT INTO sda.file_event_log(file_id, event, correlation_id, user_id, message) VALUES($1, $2, $3, $4, $5);" + + result, err := db.Exec(query, fileUUID, event, corrID, user, message) + if err != nil { + return err + } + if rowsAffected, _ := result.RowsAffected(); rowsAffected == 0 { + return errors.New("something went wrong with the query zero rows were changed") + } + + return nil +} + +// StoreHeader stores the file header in the database +func (dbs *SDAdb) StoreHeader(header []byte, id string) error { + var ( + err error + count int + ) + + for count == 0 || (err != nil && count < RetryTimes) { + err = dbs.storeHeader(header, id) + count++ + } + + return err +} +func (dbs *SDAdb) storeHeader(header []byte, id string) error { + dbs.checkAndReconnectIfNeeded() + + db := dbs.DB + const query = "UPDATE sda.files SET header = $1 WHERE id = $2;" + result, err := db.Exec(query, hex.EncodeToString(header), id) + if err != nil { + return err + } + if rowsAffected, _ := result.RowsAffected(); rowsAffected == 0 { + return errors.New("something went wrong with the query zero rows were changed") + } + + return nil +} + +// SetArchived marks the file as 'ARCHIVED' +func (dbs *SDAdb) SetArchived(file FileInfo, fileID, corrID string) error { + var ( + err error + count int + ) + + for count == 0 || (err != nil && count < RetryTimes) { + err = dbs.setArchived(file, fileID, corrID) + count++ + } + + return err +} +func (dbs *SDAdb) setArchived(file FileInfo, fileID, corrID string) error { + dbs.checkAndReconnectIfNeeded() + + db := dbs.DB + const query = "SELECT sda.set_archived($1, $2, $3, $4, $5, $6);" + _, err := db.Exec(query, + fileID, + corrID, + file.Path, + file.Size, + fmt.Sprintf("%x", file.Checksum.Sum(nil)), + hashType(file.Checksum), + ) + + return err +} + +func (dbs *SDAdb) GetFileStatus(corrID string) (string, error) { + var ( + err error + count int + status string + ) + + for count == 0 || (err != nil && count < RetryTimes) { + status, err = dbs.getFileStatus(corrID) + count++ + } + + return status, err +} +func (dbs *SDAdb) getFileStatus(corrID string) (string, error) { + dbs.checkAndReconnectIfNeeded() + db := dbs.DB + const getFileID = "SELECT event from sda.file_event_log WHERE correlation_id = $1 ORDER BY id DESC LIMIT 1;" + + var status string + err := db.QueryRow(getFileID, corrID).Scan(&status) + if err != nil { + return "", err + } + + return status, nil +} + +// GetHeader retrieves the file header +func (dbs *SDAdb) GetHeader(fileID string) ([]byte, error) { + var ( + r []byte + err error + count int + ) + + for count == 0 || (err != nil && count < RetryTimes) { + r, err = dbs.getHeader(fileID) + count++ + } + + return r, err +} +func (dbs *SDAdb) getHeader(fileID string) ([]byte, error) { + dbs.checkAndReconnectIfNeeded() + + db := dbs.DB + const query = "SELECT header from sda.files WHERE id = $1" + + var hexString string + if err := db.QueryRow(query, fileID).Scan(&hexString); err != nil { + return nil, err + } + + header, err := hex.DecodeString(hexString) + if err != nil { + return nil, err + } + + return header, nil +} + +// MarkCompleted marks the file as "COMPLETED" +func (dbs *SDAdb) MarkCompleted(file FileInfo, fileID, corrID string) error { + var ( + err error + count int + ) + + for count == 0 || (err != nil && count < RetryTimes) { + err = dbs.markCompleted(file, fileID, corrID) + count++ + } + + return err +} +func (dbs *SDAdb) markCompleted(file FileInfo, fileID, corrID string) error { + dbs.checkAndReconnectIfNeeded() + + db := dbs.DB + const completed = "SELECT sda.set_verified($1, $2, $3, $4, $5, $6, $7);" + _, err := db.Exec(completed, + fileID, + corrID, + fmt.Sprintf("%x", file.Checksum.Sum(nil)), + hashType(file.Checksum), + file.DecryptedSize, + fmt.Sprintf("%x", file.DecryptedChecksum.Sum(nil)), + hashType(file.DecryptedChecksum), + ) + + return err +} + +// GetArchived retrieves the location and size of archive +func (dbs *SDAdb) GetArchived(user, filepath, checksum string) (string, int, error) { + var ( + filePath string + fileSize int + err error + count int + ) + + for count == 0 || (err != nil && count < RetryTimes) { + filePath, fileSize, err = dbs.getArchived(user, filepath, checksum) + count++ + } + + return filePath, fileSize, err +} +func (dbs *SDAdb) getArchived(user, filepath, checksum string) (string, int, error) { + dbs.checkAndReconnectIfNeeded() + + db := dbs.DB + const query = "SELECT archive_path, archive_filesize from local_ega.files WHERE " + + "elixir_id = $1 and inbox_path = $2 and decrypted_file_checksum = $3 and status in ('COMPLETED', 'READY');" + + var filePath string + var fileSize int + if err := db.QueryRow(query, user, filepath, checksum).Scan(&filePath, &fileSize); err != nil { + return "", 0, err + } + + return filePath, fileSize, nil +} diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index b307a4b95..3095a10a2 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -1,8 +1,11 @@ package database import ( + "crypto/sha256" + "fmt" "regexp" + "github.com/google/uuid" "github.com/stretchr/testify/assert" ) @@ -11,16 +14,11 @@ func (suite *DatabaseTests) TestRegisterFile() { // create database connection db, err := NewSDAdb(suite.dbConf) - assert.Nil(suite.T(), err, "got %v when creating new connection", err) + assert.NoError(suite.T(), err, "got %v when creating new connection", err) // register a file in the database fileID, err := db.RegisterFile("/testuser/file1.c4gh", "testuser") - if db.Version < 4 { - assert.NotNil(suite.T(), err, "RegisterFile() should not work in db version %v", db.Version) - - return - } - assert.Nil(suite.T(), err, "failed to register file in database") + assert.NoError(suite.T(), err, "failed to register file in database") // check that the returning fileID is a uuid uuidPattern := "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$" @@ -30,31 +28,25 @@ func (suite *DatabaseTests) TestRegisterFile() { // check that the file is in the database exists := false err = db.DB.QueryRow("SELECT EXISTS(SELECT 1 FROM sda.files WHERE id=$1)", fileID).Scan(&exists) - assert.Nil(suite.T(), err, "Failed to check if registered file exists") + assert.NoError(suite.T(), err, "Failed to check if registered file exists") assert.True(suite.T(), exists, "RegisterFile() did not insert a row into sda.files with id: "+fileID) // check that there is a "registered" file event connected to the file err = db.DB.QueryRow("SELECT EXISTS(SELECT 1 FROM sda.file_event_log WHERE file_id=$1 AND event='registered')", fileID).Scan(&exists) - assert.Nil(suite.T(), err, "Failed to check if registered file event exists") + assert.NoError(suite.T(), err, "Failed to check if registered file event exists") assert.True(suite.T(), exists, "RegisterFile() did not insert a row into sda.file_event_log with id: "+fileID) } // TestMarkFileAsUploaded tests that MarkFileAsUploaded() behaves as intended -func (suite *DatabaseTests) UpdateFileEventLog() { - +func (suite *DatabaseTests) TestUpdateFileEventLog() { // create database connection db, err := NewSDAdb(suite.dbConf) - assert.Nil(suite.T(), err, "got %v when creating new connection", err) + assert.NoError(suite.T(), err, "got %v when creating new connection", err) // register a file in the database fileID, err := db.RegisterFile("/testuser/file2.c4gh", "testuser") - if db.Version < 4 { - assert.NotNil(suite.T(), err, "MarkFileAsUploaded() should not work in db version %v", db.Version) - - return - } - assert.Nil(suite.T(), err, "failed to register file in database") + assert.NoError(suite.T(), err, "failed to register file in database") // Attempt to mark a file that doesn't exist as uploaded err = db.UpdateFileEventLog("00000000-0000-0000-0000-000000000000", "uploaded", "testuser", "{}") @@ -62,11 +54,156 @@ func (suite *DatabaseTests) UpdateFileEventLog() { // mark file as uploaded err = db.UpdateFileEventLog(fileID, "uploaded", "testuser", "{}") - assert.Nil(suite.T(), err, "failed to set file as uploaded in database") + assert.NoError(suite.T(), err, "failed to set file as uploaded in database") + + exists := false + // check that there is an "uploaded" file event connected to the file + err = db.DB.QueryRow("SELECT EXISTS(SELECT 1 FROM sda.file_event_log WHERE file_id=$1 AND event='uploaded')", fileID).Scan(&exists) + assert.NoError(suite.T(), err, "Failed to check if uploaded file event exists") + assert.True(suite.T(), exists, "UpdateFileEventLog() did not insert a row into sda.file_event_log with id: "+fileID) +} + +func (suite *DatabaseTests) TestGetFileID() { + db, err := NewSDAdb(suite.dbConf) + assert.NoError(suite.T(), err, "got %v when creating new connection", err) + + fileID, err := db.RegisterFile("/testuser/file3.c4gh", "testuser") + assert.NoError(suite.T(), err, "failed to register file in database") + + corrID := uuid.New().String() + err = db.UpdateFileStatus(fileID, "uploaded", corrID, "testuser", "{}") + assert.NoError(suite.T(), err, "failed to update file status") + + fID, err := db.GetFileID(corrID) + assert.NoError(suite.T(), err, "GetFileId failed") + assert.Equal(suite.T(), fileID, fID) +} + +func (suite *DatabaseTests) TestUpdateFileStatus() { + db, err := NewSDAdb(suite.dbConf) + assert.NoError(suite.T(), err, "got %v when creating new connection", err) + + // register a file in the database + fileID, err := db.RegisterFile("/testuser/file4.c4gh", "testuser") + assert.Nil(suite.T(), err, "failed to register file in database") + + corrID := uuid.New().String() + // Attempt to mark a file that doesn't exist as uploaded + err = db.UpdateFileStatus("00000000-0000-0000-0000-000000000000", "uploaded", corrID, "testuser", "{}") + assert.NotNil(suite.T(), err, "Unknown file could be marked as uploaded in database") + + // mark file as uploaded + err = db.UpdateFileStatus(fileID, "uploaded", corrID, "testuser", "{}") + assert.NoError(suite.T(), err, "failed to set file as uploaded in database") exists := false // check that there is an "uploaded" file event connected to the file err = db.DB.QueryRow("SELECT EXISTS(SELECT 1 FROM sda.file_event_log WHERE file_id=$1 AND event='uploaded')", fileID).Scan(&exists) - assert.Nil(suite.T(), err, "Failed to check if uploaded file event exists") + assert.NoError(suite.T(), err, "Failed to check if uploaded file event exists") assert.True(suite.T(), exists, "UpdateFileEventLog() did not insert a row into sda.file_event_log with id: "+fileID) } + +func (suite *DatabaseTests) TestStoreHeader() { + db, err := NewSDAdb(suite.dbConf) + assert.NoError(suite.T(), err, "got %v when creating new connection", err) + + // register a file in the database + fileID, err := db.RegisterFile("/testuser/TestStoreHeader.c4gh", "testuser") + assert.NoError(suite.T(), err, "failed to register file in database") + + err = db.StoreHeader([]byte{15, 45, 20, 40, 48}, fileID) + assert.NoError(suite.T(), err, "failed to store file header") + + // store header for non existing entry + err = db.StoreHeader([]byte{15, 45, 20, 40, 48}, "00000000-0000-0000-0000-000000000000") + assert.EqualError(suite.T(), err, "something went wrong with the query zero rows were changed") +} + +func (suite *DatabaseTests) TestSetArchived() { + db, err := NewSDAdb(suite.dbConf) + assert.NoError(suite.T(), err, "got %v when creating new connection", err) + + // register a file in the database + fileID, err := db.RegisterFile("/testuser/TestSetArchived.c4gh", "testuser") + assert.NoError(suite.T(), err, "failed to register file in database") + + fileInfo := FileInfo{sha256.New(), 1000, "/tmp/TestSetArchived.c4gh", sha256.New(), -1} + corrID := uuid.New().String() + err = db.SetArchived(fileInfo, fileID, corrID) + assert.NoError(suite.T(), err, "failed to mark file as Archived") + + err = db.SetArchived(fileInfo, "00000000-0000-0000-0000-000000000000", corrID) + assert.ErrorContains(suite.T(), err, "violates foreign key constraint") + + err = db.SetArchived(fileInfo, fileID, "00000000-0000-0000-0000-000000000000") + assert.ErrorContains(suite.T(), err, "duplicate key value violates unique constraint") +} + +func (suite *DatabaseTests) TestGetFileStatus() { + db, err := NewSDAdb(suite.dbConf) + assert.NoError(suite.T(), err, "got %v when creating new connection", err) + + // register a file in the database + fileID, err := db.RegisterFile("/testuser/TestGetFileStatus.c4gh", "testuser") + assert.NoError(suite.T(), err, "failed to register file in database") + + corrID := uuid.New().String() + err = db.UpdateFileStatus(fileID, "downloaded", corrID, "testuser", "{}") + assert.NoError(suite.T(), err, "failed to set file as downloaded in database") + + status, err := db.GetFileStatus(corrID) + assert.NoError(suite.T(), err, "failed to get file status") + assert.Equal(suite.T(), "downloaded", status) +} + +func (suite *DatabaseTests) TestGetHeader() { + db, err := NewSDAdb(suite.dbConf) + assert.NoError(suite.T(), err, "got %v when creating new connection", err) + + // register a file in the database + fileID, err := db.RegisterFile("/testuser/TestGetHeader.c4gh", "testuser") + assert.NoError(suite.T(), err, "failed to register file in database") + + err = db.StoreHeader([]byte{15, 45, 20, 40, 48}, fileID) + assert.NoError(suite.T(), err, "failed to store file header") + + header, err := db.GetHeader(fileID) + assert.NoError(suite.T(), err, "failed to get file header") + assert.Equal(suite.T(), []byte{15, 45, 20, 40, 48}, header) +} + +func (suite *DatabaseTests) TestMarkCompleted() { + db, err := NewSDAdb(suite.dbConf) + assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) + + // register a file in the database + fileID, err := db.RegisterFile("/testuser/TestMarkCompleted.c4gh", "testuser") + assert.NoError(suite.T(), err, "failed to register file in database") + + corrID := uuid.New().String() + fileInfo := FileInfo{sha256.New(), 1000, "/testuser/TestMarkCompleted.c4gh", sha256.New(), 948} + err = db.MarkCompleted(fileInfo, fileID, corrID) + assert.NoError(suite.T(), err, "got (%v) when marking file as completed", err) +} + +func (suite *DatabaseTests) TestGetArchived() { + db, err := NewSDAdb(suite.dbConf) + assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) + + // register a file in the database + fileID, err := db.RegisterFile("/testuser/TestGetArchived.c4gh", "testuser") + assert.NoError(suite.T(), err, "failed to register file in database") + + decSha := sha256.New() + fileInfo := FileInfo{sha256.New(), 1000, "/tmp/TestGetArchived.c4gh", decSha, 987} + corrID := uuid.New().String() + err = db.SetArchived(fileInfo, fileID, corrID) + assert.NoError(suite.T(), err, "got (%v) when marking file as Archived") + err = db.MarkCompleted(fileInfo, fileID, corrID) + assert.NoError(suite.T(), err, "got (%v) when marking file as completed", err) + + filePath, fileSize, err := db.GetArchived("testuser", "/testuser/TestGetArchived.c4gh", fmt.Sprintf("%x", decSha.Sum(nil))) + assert.NoError(suite.T(), err, "got (%v) when getting file archive information", err) + assert.Equal(suite.T(), 1000, fileSize) + assert.Equal(suite.T(), "/tmp/TestGetArchived.c4gh", filePath) +} From 7287d7120bf10e70f4eeff76e716e281e81b1104 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 15 Aug 2023 10:17:50 +0200 Subject: [PATCH 13/34] Merge in `ingest` & `verify` from sda-pipeline --- .../scripts/make_sda_credentials.sh | 4 +- .github/integration/sda-integration.yml | 57 ++- .github/integration/sda/config.yaml | 16 +- .github/integration/tests/run_scripts.sh | 2 +- .../integration/tests/sda/10_upload_test.sh | 3 +- .../tests/sda/20_ingest-verify_test.sh | 70 +++ sda/cmd/ingest/ingest.go | 444 ++++++++++++++++++ sda/cmd/ingest/ingest.md | 168 +++++++ sda/cmd/ingest/ingest_test.go | 110 +++++ sda/cmd/verify/verify.go | 303 ++++++++++++ sda/cmd/verify/verify.md | 154 ++++++ sda/cmd/verify/verify_test.go | 20 + sda/internal/broker/broker.go | 9 + sda/internal/schema/schema.go | 2 +- sda/internal/schema/schema_test.go | 4 +- .../federated/ingestion-verification.json | 2 +- .../isolated/ingestion-verification.json | 1 + 17 files changed, 1360 insertions(+), 9 deletions(-) create mode 100644 .github/integration/tests/sda/20_ingest-verify_test.sh create mode 100644 sda/cmd/ingest/ingest.go create mode 100644 sda/cmd/ingest/ingest.md create mode 100644 sda/cmd/ingest/ingest_test.go create mode 100644 sda/cmd/verify/verify.go create mode 100644 sda/cmd/verify/verify.md create mode 100644 sda/cmd/verify/verify_test.go create mode 120000 sda/schemas/isolated/ingestion-verification.json diff --git a/.github/integration/scripts/make_sda_credentials.sh b/.github/integration/scripts/make_sda_credentials.sh index cb741287e..40b496f4f 100644 --- a/.github/integration/scripts/make_sda_credentials.sh +++ b/.github/integration/scripts/make_sda_credentials.sh @@ -10,13 +10,15 @@ for n in download finalize inbox ingest mapper sync verify; do if [ "$n" = inbox ]; then psql -U postgres -h postgres -d sda -c "DROP ROLE IF EXISTS inbox;" psql -U postgres -h postgres -d sda -c "CREATE ROLE inbox;" - psql -U postgres -h postgres -d sda -c "GRANT base, ingest TO inbox;" + psql -U postgres -h postgres -d sda -c "GRANT ingest TO inbox;" fi if [ "$n" = ingest ]; then psql -U postgres -h postgres -d sda -c "GRANT UPDATE ON local_ega.main TO ingest;" fi + psql -U postgres -h postgres -d sda -c "GRANT base TO $n;" + psql -U postgres -h postgres -d sda -c "ALTER ROLE $n LOGIN PASSWORD '$n';" ## password and permissions for MQ diff --git a/.github/integration/sda-integration.yml b/.github/integration/sda-integration.yml index 11406c866..d18e9e701 100644 --- a/.github/integration/sda-integration.yml +++ b/.github/integration/sda-integration.yml @@ -47,7 +47,7 @@ services: "rabbitmq-diagnostics -q check_running && rabbitmq-diagnostics -q check_local_alarms", ] interval: 10s - timeout: 2s + timeout: 5s retries: 6 image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-rabbitmq ports: @@ -98,6 +98,7 @@ services: environment: - BROKER_PASSWORD=inbox - BROKER_USER=inbox + - BROKER_ROUTINGKEY=inbox - DB_PASSWORD=inbox - DB_USER=inbox restart: always @@ -108,6 +109,56 @@ services: - "18000:8000" - "18001:8001" + ingest: + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + command: [ sda-ingest ] + container_name: ingest + depends_on: + credentials: + condition: service_completed_successfully + minio: + condition: service_healthy + postgres: + condition: service_healthy + rabbitmq: + condition: service_healthy + environment: + - BROKER_PASSWORD=ingest + - BROKER_USER=ingest + - BROKER_QUEUE=ingest + - BROKER_ROUTINGKEY=archived + - DB_PASSWORD=ingest + - DB_USER=ingest + restart: always + volumes: + - ./sda/config.yaml:/config.yaml + - shared:/shared + + verify: + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + command: [ sda-verify ] + container_name: verify + depends_on: + credentials: + condition: service_completed_successfully + minio: + condition: service_healthy + postgres: + condition: service_healthy + rabbitmq: + condition: service_healthy + environment: + - BROKER_PASSWORD=verify + - BROKER_USER=verify + - BROKER_QUEUE=archived + - BROKER_ROUTINGKEY=verified + - DB_PASSWORD=verify + - DB_USER=verify + restart: always + volumes: + - ./sda/config.yaml:/config.yaml + - shared:/shared + oidc: container_name: oidc command: @@ -142,8 +193,12 @@ services: depends_on: credentials: condition: service_completed_successfully + ingest: + condition: service_started s3inbox: condition: service_started + verify: + condition: service_started environment: - PGPASSWORD=rootpasswd image: python:3.10-slim-bullseye diff --git a/.github/integration/sda/config.yaml b/.github/integration/sda/config.yaml index 9669ca001..b75ca9ffc 100644 --- a/.github/integration/sda/config.yaml +++ b/.github/integration/sda/config.yaml @@ -1,6 +1,17 @@ log: format: "json" + level: "debug" +archive: + type: s3 + url: "http://s3" + port: 9000 + readypath: "/minio/health/ready" + accessKey: "access" + secretKey: "secretKey" + bucket: "archive" + region: "us-east-1" inbox: + type: s3 url: "http://s3" port: 9000 readypath: "/minio/health/ready" @@ -16,7 +27,7 @@ broker: password: "" vhost: "/sda" exchange: "sda" - routingKey: "inbox" + routingKey: "" ssl: "false" db: @@ -27,6 +38,9 @@ db: database: "sda" sslmode: "disable" +c4gh: + filePath: /shared/c4gh.sec.pem + passphrase: "c4ghpass" server: cert: "" diff --git a/.github/integration/tests/run_scripts.sh b/.github/integration/tests/run_scripts.sh index d7552d615..788fd8610 100644 --- a/.github/integration/tests/run_scripts.sh +++ b/.github/integration/tests/run_scripts.sh @@ -4,7 +4,7 @@ set -e apt-get -o DPkg::Lock::Timeout=60 update > /dev/null apt-get -o DPkg::Lock::Timeout=60 install -y postgresql-client > /dev/null -find "$1"/*.sh 2>/dev/null | sort -t/ -k3 -n | while read -r runscript; do +for runscript in "$1"/*.sh; do echo "Executing test script $runscript" bash -x "$runscript" done diff --git a/.github/integration/tests/sda/10_upload_test.sh b/.github/integration/tests/sda/10_upload_test.sh index df70f78a3..bdbc01ef3 100644 --- a/.github/integration/tests/sda/10_upload_test.sh +++ b/.github/integration/tests/sda/10_upload_test.sh @@ -14,6 +14,7 @@ for t in curl jq postgresql-client; do fi done +pip -q install --upgrade pip pip -q install s3cmd cd shared || true @@ -21,7 +22,7 @@ cd shared || true for file in NA12878.bam NA12878_20k_b37.bam; do curl -s -L -o /shared/$file "https://github.com/ga4gh/htsget-refserver/raw/main/data/gcp/gatk-test-data/wgs_bam/$file" if [ ! -f "$file.c4gh" ]; then - /shared/crypt4gh encrypt -p c4gh.pub.pem -f "$file" + yes | /shared/crypt4gh encrypt -p c4gh.pub.pem -f "$file" fi s3cmd -c s3cfg put "$file.c4gh" s3://test_dummy.org/ done diff --git a/.github/integration/tests/sda/20_ingest-verify_test.sh b/.github/integration/tests/sda/20_ingest-verify_test.sh new file mode 100644 index 000000000..2935a817e --- /dev/null +++ b/.github/integration/tests/sda/20_ingest-verify_test.sh @@ -0,0 +1,70 @@ +#!/bin/sh +set -e + +cd shared || true + +for file in NA12878.bam NA12878_20k_b37.bam; do + ENC_SHA=$(sha256sum "$file.c4gh" | cut -d' ' -f 1) + ENC_MD5=$(md5sum "$file.c4gh" | cut -d' ' -f 1) + + ## get correlation id from upload message + CORRID=$( + curl -s -X POST \ + -H "content-type:application/json" \ + -u guest:guest http://rabbitmq:15672/api/queues/sda/inbox/get \ + -d '{"count":1,"encoding":"auto","ackmode":"ack_requeue_false"}' | jq -r .[0].properties.correlation_id + ) + + ## publish message to trigger ingestion + properties=$( + jq -c -n \ + --argjson delivery_mode 2 \ + --arg correlation_id "$CORRID" \ + --arg content_encoding UTF-8 \ + --arg content_type application/json \ + '$ARGS.named' + ) + + encrypted_checksums=$( + jq -c -n \ + --arg sha256 "$ENC_SHA" \ + --arg md5 "$ENC_MD5" \ + '$ARGS.named|to_entries|map(with_entries(select(.key=="key").key="type"))' + ) + + ingest_payload=$( + jq -r -c -n \ + --arg type ingest \ + --arg user test@dummy.org \ + --arg filepath test_dummy.org/"$file.c4gh" \ + --argjson encrypted_checksums "$encrypted_checksums" \ + '$ARGS.named|@base64' + ) + + ingest_body=$( + jq -c -n \ + --arg vhost sda \ + --arg name sda \ + --argjson properties "$properties" \ + --arg routing_key "ingest" \ + --arg payload_encoding base64 \ + --arg payload "$ingest_payload" \ + '$ARGS.named' + ) + + curl -s -u guest:guest "http://rabbitmq:15672/api/exchanges/sda/sda/publish" \ + -H 'Content-Type: application/json;charset=UTF-8' \ + -d "$ingest_body" +done + +echo "waiting for verify to complete" +RETRY_TIMES=0 +until [ "$(curl -su guest:guest http://rabbitmq:15672/api/queues/sda/verified/ | jq -r '.messages_ready')" -eq 2 ]; do + echo "waiting for verify to complete" + RETRY_TIMES=$((RETRY_TIMES + 1)) + if [ "$RETRY_TIMES" -eq 30 ]; then + echo "::error::Time out while waiting for verify to complete" + exit 1 + fi + sleep 2 +done diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go new file mode 100644 index 000000000..618a32e1b --- /dev/null +++ b/sda/cmd/ingest/ingest.go @@ -0,0 +1,444 @@ +// The ingest service accepts messages for files uploaded to the inbox, +// registers the files in the database with their headers, and stores them +// header-stripped in the archive storage. +package main + +import ( + "bytes" + "crypto/sha256" + "encoding/json" + "fmt" + "io" + "os" + "os/signal" + "syscall" + + "github.com/neicnordic/crypt4gh/model/headers" + "github.com/neicnordic/crypt4gh/streaming" + "github.com/neicnordic/sensitive-data-archive/internal/broker" + "github.com/neicnordic/sensitive-data-archive/internal/config" + "github.com/neicnordic/sensitive-data-archive/internal/database" + "github.com/neicnordic/sensitive-data-archive/internal/schema" + "github.com/neicnordic/sensitive-data-archive/internal/storage" + + log "github.com/sirupsen/logrus" +) + +func main() { + sigc := make(chan os.Signal, 5) + signal.Notify(sigc, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) + // Create a function to handle panic and exit gracefully + defer func() { + if err := recover(); err != nil { + log.Fatal("Could not recover, exiting") + } + }() + + forever := make(chan bool) + conf, err := config.NewConfig("ingest") + if err != nil { + log.Error(err) + sigc <- syscall.SIGINT + panic(err) + } + mq, err := broker.NewMQ(conf.Broker) + if err != nil { + log.Error(err) + sigc <- syscall.SIGINT + panic(err) + } + db, err := database.NewSDAdb(conf.Database) + if err != nil { + log.Error(err) + sigc <- syscall.SIGINT + panic(err) + } + if db.Version < 8 { + log.Error("database schema v8 is required") + sigc <- syscall.SIGINT + panic(err) + } + key, err := config.GetC4GHKey() + if err != nil { + log.Error(err) + sigc <- syscall.SIGINT + panic(err) + } + archive, err := storage.NewBackend(conf.Archive) + if err != nil { + log.Error(err) + sigc <- syscall.SIGINT + panic(err) + } + inbox, err := storage.NewBackend(conf.Inbox) + if err != nil { + log.Error(err) + sigc <- syscall.SIGINT + panic(err) + } + + defer mq.Channel.Close() + defer mq.Connection.Close() + defer db.Close() + + go func() { + connError := mq.ConnectionWatcher() + log.Error(connError) + forever <- false + }() + + go func() { + connError := mq.ChannelWatcher() + log.Error(connError) + forever <- false + }() + + log.Info("starting ingest service") + var message schema.IngestionTrigger + + go func() { + messages, err := mq.GetMessages(conf.Broker.Queue) + if err != nil { + log.Fatal(err) + } + mainWorkLoop: + for delivered := range messages { + log.Debugf("received a message (corr-id: %s, message: %s)", delivered.CorrelationId, delivered.Body) + err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-trigger.json", conf.Broker.SchemasPath), delivered.Body) + if err != nil { + log.Errorf("validation of incoming message (ingestion-trigger) failed, reason: %v", err.Error()) + // Send the message to an error queue so it can be analyzed. + infoErrorMessage := broker.InfoError{ + Error: "Message validation failed", + Reason: err.Error(), + OriginalMessage: message, + } + + body, _ := json.Marshal(infoErrorMessage) + if e := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); e != nil { + log.Errorf("failed so publish message, reason: (%v)", err.Error()) + } + if err := delivered.Ack(false); err != nil { + log.Errorf("Failed acking canceled work, reason: (%v)", err.Error()) + } + + // Restart on new message + continue + } + + // we unmarshal the message in the validation step so this is safe to do + _ = json.Unmarshal(delivered.Body, &message) + + log.Infof( + "Received work (corr-id: %s, filepath: %s, user: %s)", + delivered.CorrelationId, message.FilePath, message.User, + ) + + switch message.Type { + case "cancel": + fileUUID, err := db.GetFileID(delivered.CorrelationId) + if err != nil || fileUUID == "" { + log.Errorf("failed to get ID for file from message: %v", delivered.CorrelationId) + + if err = delivered.Nack(false, false); err != nil { + log.Errorf("Failed to Nack message, reason: (%v)", err.Error()) + } + + continue + } + + if err := db.UpdateFileStatus(fileUUID, "disabled", delivered.CorrelationId, message.User, string(delivered.Body)); err != nil { + log.Errorf("failed to set ingestion status for file from message: %v", delivered.CorrelationId) + if err = delivered.Nack(false, false); err != nil { + log.Errorf("Failed to Nack message, reason: (%v)", err.Error()) + } + + continue + } + + if err := delivered.Ack(false); err != nil { + log.Errorf("failed to ack message for reason: (%v)", err.Error()) + } + + continue + case "ingest": + file, err := inbox.NewFileReader(message.FilePath) + if err != nil { + log.Errorf("Failed to open file to ingest reason: (%v)", err.Error()) + // Send the message to an error queue so it can be analyzed. + fileError := broker.InfoError{ + Error: "Failed to open file to ingest", + Reason: err.Error(), + OriginalMessage: message, + } + body, _ := json.Marshal(fileError) + if e := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); e != nil { + log.Errorf("failed so publish message, reason: (%v)", err.Error()) + } + if err = delivered.Ack(false); err != nil { + log.Errorf("Failed to Ack message, reason: (%v)", err.Error()) + } + + // Restart on new message + continue + } + + fileSize, err := inbox.GetFileSize(message.FilePath) + if err != nil { + log.Errorf("Failed to get file size of file to ingest, reason: %v", err.Error()) + // Nack message so the server gets notified that something is wrong and requeue the message. + // Since reading the file worked, this should eventually succeed so it is ok to requeue. + if err = delivered.Nack(false, true); err != nil { + log.Errorf("Failed to Nack message, reason: (%v)", err.Error()) + } + // Send the message to an error queue so it can be analyzed. + fileError := broker.InfoError{ + Error: "Failed to get file size of file to ingest", + Reason: err.Error(), + OriginalMessage: message, + } + body, _ := json.Marshal(fileError) + if err = mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); err != nil { + log.Errorf("failed so publish message, reason: (%v)", err.Error()) + } + + // Restart on new message + continue + } + + fileID, err := db.RegisterFile(message.FilePath, message.User) + if err != nil { + log.Errorf("InsertFile failed, reason: (%v)", err.Error()) + } + err = db.UpdateFileStatus(fileID, "submitted", delivered.CorrelationId, message.User, string(delivered.Body)) + if err != nil { + log.Errorf("failed to set ingestion status for file from message: %v", delivered.CorrelationId) + } + + dest, err := archive.NewFileWriter(fileID) + if err != nil { + log.Errorf("Failed to create archive file, reason: (%v)", err.Error()) + // Nack message so the server gets notified that something is wrong and requeue the message. + // NewFileWriter returns an error when the backend itself fails so this is reasonable to requeue. + if err = delivered.Nack(false, true); err != nil { + log.Errorf("Failed to Nack message, reason: (%v)", err.Error()) + } + + continue + } + + // 4MiB readbuffer, this must be large enough that we get the entire header and the first 64KiB datablock + var bufSize int + if bufSize = 4 * 1024 * 1024; conf.Inbox.S3.Chunksize > 4*1024*1024 { + bufSize = conf.Inbox.S3.Chunksize + } + readBuffer := make([]byte, bufSize) + hash := sha256.New() + var bytesRead int64 + var byteBuf bytes.Buffer + + for bytesRead < fileSize { + i, _ := io.ReadFull(file, readBuffer) + if i == 0 { + return + } + // truncate the readbuffer if the file is smaller than the buffer size + if i < len(readBuffer) { + readBuffer = readBuffer[:i] + } + + bytesRead += int64(i) + + h := bytes.NewReader(readBuffer) + if _, err = io.Copy(hash, h); err != nil { + log.Errorf("Copy to hash failed while reading file, reason: (%v)", err.Error()) + if err = delivered.Nack(false, true); err != nil { + log.Errorf("Failed to Nack message, reason: (%v)", err.Error()) + } + + continue mainWorkLoop + } + + //nolint:nestif + if bytesRead <= int64(len(readBuffer)) { + header, err := tryDecrypt(key, readBuffer) + if err != nil { + log.Errorf("Trying to decrypt start of file failed, reason: (%v)", err.Error()) + + // Nack message so the server gets notified that something is wrong. Do not requeue the message. + if err = delivered.Nack(false, false); err != nil { + log.Errorf("Failed to Nack message, reason: (%v)", err.Error()) + } + + // Send the message to an error queue so it can be analyzed. + fileError := broker.InfoError{ + Error: "Trying to decrypt start of file failed", + Reason: err.Error(), + OriginalMessage: message, + } + body, _ := json.Marshal(fileError) + if e := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); e != nil { + log.Errorf("failed so publish message, reason: (%v)", err.Error()) + } + + continue mainWorkLoop + } + + log.Debugln("store header") + if err := db.StoreHeader(header, fileID); err != nil { + log.Errorf("StoreHeader failed, reason: (%v)", err.Error()) + if err = delivered.Nack(false, true); err != nil { + log.Errorf("Failed to Nack message, reason: (%v)", err.Error()) + } + + continue mainWorkLoop + } + + if _, err = byteBuf.Write(readBuffer); err != nil { + log.Errorf("Failed to write to read buffer for header read, reason: %v)", err.Error()) + if err = delivered.Nack(false, true); err != nil { + log.Errorf("Failed to Nack message, reason: (%v)", err.Error()) + } + + continue mainWorkLoop + } + + // Strip header from buffer + h := make([]byte, len(header)) + if _, err = byteBuf.Read(h); err != nil { + log.Errorf("Failed to strip header from buffer, reason: (%v)", err.Error()) + if err = delivered.Nack(false, true); err != nil { + log.Errorf("Failed to Nack message, reason: (%v)", err.Error()) + } + + continue mainWorkLoop + } + } else { + if i < len(readBuffer) { + readBuffer = readBuffer[:i] + } + if _, err = byteBuf.Write(readBuffer); err != nil { + log.Errorf("Failed to write to read buffer for full read, reason: (%v)", err.Error()) + if err = delivered.Nack(false, true); err != nil { + log.Errorf("Failed to Nack message, reason: (%v)", err.Error()) + } + + continue mainWorkLoop + } + } + + // Write data to file + if _, err = byteBuf.WriteTo(dest); err != nil { + log.Errorf("Failed to write to archive file, reason: (%v)", err.Error()) + + continue mainWorkLoop + } + } + + file.Close() + dest.Close() + + fileInfo := database.FileInfo{} + fileInfo.Path = fileID + fileInfo.Checksum = hash + fileInfo.Size, err = archive.GetFileSize(fileID) + if err != nil { + log.Errorf("Couldn't get file size from archive, reason: %v)", err.Error()) + if err = delivered.Nack(false, true); err != nil { + log.Errorf("Failed to Nack message, reason: (%v)", err.Error()) + } + + continue + } + + log.Debugf("Wrote archived file (corr-id: %s, user: %s, filepath: %s, archivepath: %s, archivedsize: %d)", + delivered.CorrelationId, message.User, message.FilePath, fileID, fileInfo.Size) + + status, err := db.GetFileStatus(delivered.CorrelationId) + if err != nil { + log.Errorf("failed to get file status, reason: %v", err.Error()) + if err = delivered.Nack(false, true); err != nil { + log.Errorf("Failed to Nack message, reason: (%v)", err.Error()) + } + } + if status == "disabled" { + log.Infof("file with correlation ID: %s is disabled, stopping ingestion", delivered.CorrelationId) + if err := delivered.Ack(false); err != nil { + log.Errorf("Failed acking canceled work, reason: %v", err.Error()) + } + + continue + } + + if err := db.SetArchived(fileInfo, fileID, delivered.CorrelationId); err != nil { + log.Errorf("SetArchived failed, reason: (%v)", err.Error()) + } + + log.Debugf("File marked as archived (corr-id: %s, user: %s, filepath: %s, archivepath: %s)", + delivered.CorrelationId, message.User, message.FilePath, fileID) + + // Send message to archived + msg := schema.IngestionVerification{ + User: message.User, + FilePath: message.FilePath, + FileID: fileID, + ArchivePath: fileID, + EncryptedChecksums: []schema.Checksums{ + {Type: "sha256", Value: fmt.Sprintf("%x", hash.Sum(nil))}, + }, + } + archivedMsg, _ := json.Marshal(&msg) + + err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-verification.json", conf.Broker.SchemasPath), archivedMsg) + if err != nil { + log.Errorf("Validation of outgoing message failed, reason: (%v)", err.Error()) + + continue + } + + if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, conf.Broker.RoutingKey, archivedMsg); err != nil { + // TODO fix resend mechanism + log.Errorf("failed to publish message, reason: (%v)", err.Error()) + + // Do not try to ACK message to make sure we have another go + continue + } + if err := delivered.Ack(false); err != nil { + log.Errorf("failed to Ack message, reason: (%v)", err.Error()) + } + } + } + }() + + <-forever +} + +// tryDecrypt tries to decrypt the start of buf. +func tryDecrypt(key *[32]byte, buf []byte) ([]byte, error) { + + log.Debugln("Try decrypting the first data block") + a := bytes.NewReader(buf) + b, err := streaming.NewCrypt4GHReader(a, *key, nil) + if err != nil { + log.Error(err) + + return nil, err + + } + _, err = b.ReadByte() + if err != nil { + log.Error(err) + + return nil, err + } + + f := bytes.NewReader(buf) + header, err := headers.ReadHeader(f) + if err != nil { + log.Error(err) + + return nil, err + } + + return header, nil +} diff --git a/sda/cmd/ingest/ingest.md b/sda/cmd/ingest/ingest.md new file mode 100644 index 000000000..3575469e5 --- /dev/null +++ b/sda/cmd/ingest/ingest.md @@ -0,0 +1,168 @@ +# sda-pipeline: ingest + +Splits the Crypt4GH header and moves it to database. The remainder of the file +is sent to the storage backend (archive). No cryptographic tasks are done. + +## Configuration + +There are a number of options that can be set for the ingest service. +These settings can be set by mounting a yaml-file at `/config.yaml` with settings. + +ex. +```yaml +log: + level: "debug" + format: "json" +``` +They may also be set using environment variables like: +```bash +export LOG_LEVEL="debug" +export LOG_FORMAT="json" +``` + +### Keyfile settings + +These settings control which crypt4gh keyfile is loaded. + + - `C4GH_FILEPATH`: filepath to the crypt4gh keyfile + - `C4GH_PASSPHRASE`: pass phrase to unlock the keyfile + +### RabbitMQ broker settings + +These settings control how ingest connects to the RabbitMQ message broker. + + - `BROKER_HOST`: hostname of the rabbitmq server + + - `BROKER_PORT`: rabbitmq broker port (commonly `5671` with TLS and `5672` without) + + - `BROKER_QUEUE`: message queue to read messages from (commonly `ingest`) + + - `BROKER_ROUTINGKEY`: message queue to write success messages to (commonly `archived`) + + - `BROKER_USER`: username to connect to rabbitmq + + - `BROKER_PASSWORD`: password to connect to rabbitmq + + - `BROKER_PREFETCHCOUNT`: Number of messages to pull from the message server at the time (default to 2) + +### PostgreSQL Database settings: + + - `DB_HOST`: hostname for the postgresql database + + - `DB_PORT`: database port (commonly 5432) + + - `DB_USER`: username for the database + + - `DB_PASSWORD`: password for the database + + - `DB_DATABASE`: database name + + - `DB_SSLMODE`: The TLS encryption policy to use for database connections. + Valid options are: + - `disable` + - `allow` + - `prefer` + - `require` + - `verify-ca` + - `verify-full` + + More information is available + [in the postgresql documentation](https://www.postgresql.org/docs/current/libpq-ssl.html#LIBPQ-SSL-PROTECTION) + + Note that if `DB_SSLMODE` is set to anything but `disable`, then `DB_CACERT` needs to be set, + and if set to `verify-full`, then `DB_CLIENTCERT`, and `DB_CLIENTKEY` must also be set + + - `DB_CLIENTKEY`: key-file for the database client certificate + + - `DB_CLIENTCERT`: database client certificate file + + - `DB_CACERT`: Certificate Authority (CA) certificate for the database to use + +### Storage settings + +Storage backend is defined by the `ARCHIVE_TYPE`, and `INBOX_TYPE` variables. +Valid values for these options are `S3` or `POSIX` +(Defaults to `POSIX` on unknown values). + +The value of these variables define what other variables are read. +The same variables are available for all storage types, differing by prefix (`ARCHIVE_`, or `INBOX_`) + +if `*_TYPE` is `S3` then the following variables are available: + - `*_URL`: URL to the S3 system + - `*_ACCESSKEY`: The S3 access and secret key are used to authenticate to S3, + [more info at AWS](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) + - `*_SECRETKEY`: The S3 access and secret key are used to authenticate to S3, + [more info at AWS](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) + - `*_BUCKET`: The S3 bucket to use as the storage root + - `*_PORT`: S3 connection port (default: `443`) + - `*_REGION`: S3 region (default: `us-east-1`) + - `*_CHUNKSIZE`: S3 chunk size for multipart uploads. +# CA certificate is only needed if the S3 server has a certificate signed by a private entity + - `*_CACERT`: Certificate Authority (CA) certificate for the storage system + +and if `*_TYPE` is `POSIX`: + - `*_LOCATION`: POSIX path to use as storage root + +### Logging settings: + + - `LOG_FORMAT` can be set to “json” to get logs in json format. + All other values result in text logging + + - `LOG_LEVEL` can be set to one of the following, in increasing order of severity: + - `trace` + - `debug` + - `info` + - `warn` (or `warning`) + - `error` + - `fatal` + - `panic` + +## Service Description +The ingest service copies files from the file inbox to the archive, and registers them in the database. + +When running, ingest reads messages from the configured RabbitMQ queue (default: "ingest"). +For each message, these steps are taken (if not otherwise noted, errors halt progress and the service moves on to the next message): + +1. The message is validated as valid JSON that matches the "ingestion-trigger" schema (defined in sda-common). +If the message can’t be validated it is discarded with an error message in the logs. + +1. A file reader is created for the filepath in the message. +If the file reader can’t be created an error is written to the logs, the message is Nacked and forwarded to the error queue. + +1. The file size is read from the file reader. +On error, the error is written to the logs, the message is Nacked and forwarded to the error queue. + +1. A uuid is generated, and a file writer is created in the archive using the uuid as filename. +On error the error is written to the logs and the message is Nacked and then re-queued. + +1. The filename is inserted into the database along with the user id of the uploading user. In case the file is already existing in the database, the status is updated. +Errors are written to the error log. +Errors writing the filename to the database do not halt ingestion progress. + +1. The header is read from the file, and decrypted to ensure that it’s encrypted with the correct key. +If the decryption fails, an error is written to the error log, the message is Nacked, and the message is forwarded to the error queue. + +1. The header is written to the database. +Errors are written to the error log. + +1. The header is stripped from the file data, and the remaining file data is written to the archive. +Errors are written to the error log. + +1. The size of the archived file is read. +Errors are written to the error log. + +1. The database is updated with the file size, archive path, and archive checksum, and the file is set as “archived”. +Errors are written to the error log. +This error does not halt ingestion. + +1. A message is sent back to the original RabbitMQ broker containing the upload user, upload file path, database file id, archive file path and checksum of the archived file. + +## Communication + + - Ingest reads messages from one rabbitmq queue (commonly `ingest`). + + - Ingest writes messages to one rabbitmq queue (commonly `archived`). + + - Ingest inserts file information in the database using three database functions, `InsertFile`, `StoreHeader`, and `SetArchived`. + + - Ingest reads file data from inbox storage and writes data to archive storage. diff --git a/sda/cmd/ingest/ingest_test.go b/sda/cmd/ingest/ingest_test.go new file mode 100644 index 000000000..fcb0244d5 --- /dev/null +++ b/sda/cmd/ingest/ingest_test.go @@ -0,0 +1,110 @@ +package main + +import ( + "fmt" + "io" + "os" + "testing" + + "github.com/neicnordic/crypt4gh/keys" + "github.com/neicnordic/crypt4gh/streaming" + "github.com/neicnordic/sensitive-data-archive/internal/config" + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type TestSuite struct { + suite.Suite +} + +var pubKeyList = [][32]byte{} + +func TestConfigTestSuite(t *testing.T) { + suite.Run(t, new(TestSuite)) +} + +func (suite *TestSuite) SetupTest() { + viper.Set("log.level", "debug") + archive := suite.T().TempDir() + defer os.RemoveAll(archive) + viper.Set("archive.location", archive) + + // Generate a crypth4gh keypair + publicKey, privateKey, err := keys.GenerateKeyPair() + assert.NoError(suite.T(), err) + pubKeyList = append(pubKeyList, publicKey) + + tempDir := suite.T().TempDir() + privateKeyFile, err := os.Create(fmt.Sprintf("%s/c4fg.key", tempDir)) + assert.NoError(suite.T(), err) + err = keys.WriteCrypt4GHX25519PrivateKey(privateKeyFile, privateKey, []byte("password")) + assert.NoError(suite.T(), err) + viper.Set("c4gh.filepath", fmt.Sprintf("%s/c4fg.key", tempDir)) + viper.Set("c4gh.passphrase", "password") + + viper.Set("broker.host", "test") + viper.Set("broker.port", 123) + viper.Set("broker.user", "test") + viper.Set("broker.password", "test") + viper.Set("broker.queue", "test") + viper.Set("broker.routingkey", "test") + viper.Set("db.host", "test") + viper.Set("db.port", 123) + viper.Set("db.user", "test") + viper.Set("db.password", "test") + viper.Set("db.database", "test") +} + +func (suite *TestSuite) TestTryDecrypt_wrongFile() { + tempDir := suite.T().TempDir() + err := os.WriteFile(fmt.Sprintf("%s/dummy.file", tempDir), []byte("hello\ngo\n"), 0600) + assert.NoError(suite.T(), err) + + file, err := os.Open(fmt.Sprintf("%s/dummy.file", tempDir)) + assert.NoError(suite.T(), err) + defer file.Close() + buf, err := io.ReadAll(file) + assert.NoError(suite.T(), err) + + key, err := config.GetC4GHKey() + assert.Nil(suite.T(), err) + b, err := tryDecrypt(key, buf) + assert.Nil(suite.T(), b) + assert.EqualError(suite.T(), err, "not a Crypt4GH file") +} + +func (suite *TestSuite) TestTryDecrypt() { + _, signingKey, err := keys.GenerateKeyPair() + assert.NoError(suite.T(), err) + + // encrypt test file + tempDir := suite.T().TempDir() + unencryptedFile, err := os.CreateTemp(tempDir, "unencryptedFile-") + assert.NoError(suite.T(), err) + + err = os.WriteFile(unencryptedFile.Name(), []byte("content"), 0600) + assert.NoError(suite.T(), err) + + encryptedFile, err := os.CreateTemp(tempDir, "encryptedFile-") + assert.NoError(suite.T(), err) + + crypt4GHWriter, err := streaming.NewCrypt4GHWriter(encryptedFile, signingKey, pubKeyList, nil) + assert.NoError(suite.T(), err) + + _, err = io.Copy(crypt4GHWriter, unencryptedFile) + assert.NoError(suite.T(), err) + crypt4GHWriter.Close() + + file, err := os.Open(encryptedFile.Name()) + assert.NoError(suite.T(), err) + defer file.Close() + buf, err := io.ReadAll(file) + assert.NoError(suite.T(), err) + + key, err := config.GetC4GHKey() + assert.NoError(suite.T(), err) + header, err := tryDecrypt(key, buf) + assert.NoError(suite.T(), err) + assert.NotNil(suite.T(), header) +} diff --git a/sda/cmd/verify/verify.go b/sda/cmd/verify/verify.go new file mode 100644 index 000000000..89b046472 --- /dev/null +++ b/sda/cmd/verify/verify.go @@ -0,0 +1,303 @@ +// The verify service reads and decrypts ingested files from the archive +// storage and sends accession requests. +package main + +import ( + "bytes" + "crypto/md5" // #nosec + "crypto/sha256" + "encoding/json" + "fmt" + "io" + + "github.com/neicnordic/crypt4gh/streaming" + "github.com/neicnordic/sensitive-data-archive/internal/broker" + "github.com/neicnordic/sensitive-data-archive/internal/config" + "github.com/neicnordic/sensitive-data-archive/internal/database" + "github.com/neicnordic/sensitive-data-archive/internal/schema" + "github.com/neicnordic/sensitive-data-archive/internal/storage" + + log "github.com/sirupsen/logrus" +) + +func main() { + forever := make(chan bool) + conf, err := config.NewConfig("verify") + if err != nil { + log.Fatal(err) + } + mq, err := broker.NewMQ(conf.Broker) + if err != nil { + log.Fatal(err) + } + db, err := database.NewSDAdb(conf.Database) + if err != nil { + log.Fatal(err) + } + archive, err := storage.NewBackend(conf.Archive) + if err != nil { + log.Fatal(err) + } + key, err := config.GetC4GHKey() + if err != nil { + log.Fatal(err) + } + + defer mq.Channel.Close() + defer mq.Connection.Close() + defer db.Close() + + go func() { + connError := mq.ConnectionWatcher() + log.Error(connError) + forever <- false + }() + + go func() { + connError := mq.ChannelWatcher() + log.Error(connError) + forever <- false + }() + + log.Info("starting verify service") + var message schema.IngestionVerification + + go func() { + messages, err := mq.GetMessages(conf.Broker.Queue) + if err != nil { + log.Fatalf("Failed to get messages (error: %v) ", + err) + } + for delivered := range messages { + log.Debugf("received a message (corr-id: %s, message: %s)", delivered.CorrelationId, delivered.Body) + err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-verification.json", conf.Broker.SchemasPath), delivered.Body) + if err != nil { + log.Errorf("validation of incoming message (ingestion-verifiation) failed, reason: %v", err.Error()) + // Send the message to an error queue so it can be analyzed. + infoErrorMessage := broker.InfoError{ + Error: "Message validation failed", + Reason: err.Error(), + OriginalMessage: message, + } + + body, _ := json.Marshal(infoErrorMessage) + if e := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); e != nil { + log.Errorf("failed so publish message, reason: %v", err.Error()) + } + if err := delivered.Ack(false); err != nil { + log.Errorf("Failed to Ack message, reason: %v", err.Error()) + } + + // Restart on new message + continue + } + + // we unmarshal the message in the validation step so this is safe to do + _ = json.Unmarshal(delivered.Body, &message) + + // If the file has been canceled by the uploader, don't spend time working on it. + status, err := db.GetFileStatus(delivered.CorrelationId) + if err != nil { + log.Errorf("failed to get file status, reason: %v", err.Error()) + // Send the message to an error queue so it can be analyzed. + infoErrorMessage := broker.InfoError{ + Error: "Getheader failed", + Reason: err.Error(), + OriginalMessage: message, + } + + body, _ := json.Marshal(infoErrorMessage) + if e := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); e != nil { + log.Errorf("failed so publish message, reason: %v", err.Error()) + } + + if err := delivered.Ack(false); err != nil { + log.Errorf("Failed acking canceled work, reason: %v", err.Error()) + } + + continue + } + if status == "disabled" { + log.Infof("file with correlation ID: %s is disabled, stopping verification", delivered.CorrelationId) + if err := delivered.Ack(false); err != nil { + log.Errorf("Failed acking canceled work, reason: %v", err.Error()) + } + + continue + } + + header, err := db.GetHeader(message.FileID) + if err != nil { + log.Errorf("GetHeader failed for file with ID: %v, readon: %v", message.FileID, err.Error()) + + // Nack message so the server gets notified that something is wrong but don't requeue the message + if e := delivered.Nack(false, false); e != nil { + log.Errorf("Failed to nack following getheader error message") + + } + // store full message info in case we want to fix the db entry and retry + infoErrorMessage := broker.InfoError{ + Error: "Getheader failed", + Reason: err.Error(), + OriginalMessage: message, + } + + body, _ := json.Marshal(infoErrorMessage) + + // Send the message to an error queue so it can be analyzed. + if e := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); e != nil { + log.Errorf("failed so publish message, reason: %v", err.Error()) + } + + continue + } + + var file database.FileInfo + + file.Size, err = archive.GetFileSize(message.ArchivePath) + + if err != nil { + log.Errorf("Failed to get archived file size, reson: %v", err.Error()) + + continue + } + + archiveFileHash := sha256.New() + f, err := archive.NewFileReader(message.ArchivePath) + if err != nil { + log.Errorf("Failed to open archived file, reson: %v ", err.Error()) + + // Send the message to an error queue so it can be analyzed. + infoErrorMessage := broker.InfoError{ + Error: "Failed to open archived file", + Reason: err.Error(), + OriginalMessage: message, + } + + body, _ := json.Marshal(infoErrorMessage) + if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); err != nil { + log.Errorf("failed to publish message, reason: (%v)", err.Error()) + } + + // Restart on new message + continue + } + + hr := bytes.NewReader(header) + // Feed everything read from the archive file to archiveFileHash + mr := io.MultiReader(hr, io.TeeReader(f, archiveFileHash)) + + c4ghr, err := streaming.NewCrypt4GHReader(mr, *key, nil) + if err != nil { + log.Errorf("Failed to open c4gh decryptor stream, reson: %v", err.Error()) + + continue + } + + md5hash := md5.New() // #nosec + sha256hash := sha256.New() + + stream := io.TeeReader(c4ghr, md5hash) + + if file.DecryptedSize, err = io.Copy(sha256hash, stream); err != nil { + log.Errorf("failed to copy decrypted data, reson: %v", err.Error()) + + // Send the message to an error queue so it can be analyzed. + infoErrorMessage := broker.InfoError{ + Error: "Failed to verify archived file", + Reason: err.Error(), + OriginalMessage: message, + } + + body, _ := json.Marshal(infoErrorMessage) + if e := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); e != nil { + log.Errorf("Failed to publish error message: %v", e) + } + + if err := delivered.Ack(false); err != nil { + log.Errorf("Failed to ack message: %v", err.Error()) + } + + continue + } + + file.Checksum = archiveFileHash + file.DecryptedChecksum = sha256hash + + //nolint:nestif + if !message.ReVerify { + + c := schema.IngestionAccessionRequest{ + User: message.User, + FilePath: message.FilePath, + DecryptedChecksums: []schema.Checksums{ + {Type: "sha256", Value: fmt.Sprintf("%x", sha256hash.Sum(nil))}, + {Type: "md5", Value: fmt.Sprintf("%x", md5hash.Sum(nil))}, + }, + } + + verifiedMessage, _ := json.Marshal(&c) + + err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-accession-request.json", conf.Broker.SchemasPath), verifiedMessage) + + if err != nil { + log.Errorf("Validation of outgoing (ingestion-accession-request) failed, reason: %v", err.Error()) + + // Logging is in ValidateJSON so just restart on new message + continue + } + status, err := db.GetFileStatus(delivered.CorrelationId) + if err != nil { + log.Errorf("failed to get file status, reason: %v", err.Error()) + // Send the message to an error queue so it can be analyzed. + infoErrorMessage := broker.InfoError{ + Error: "Getheader failed", + Reason: err.Error(), + OriginalMessage: message, + } + + body, _ := json.Marshal(infoErrorMessage) + if e := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); e != nil { + log.Errorf("failed so publish message, reason: %v", err.Error()) + } + + if err := delivered.Ack(false); err != nil { + log.Errorf("Failed acking canceled work, reason: %v", err.Error()) + } + + continue + } + if status == "disabled" { + log.Infof("file with correlation ID: %s is disabled, stopping verification", delivered.CorrelationId) + if err := delivered.Ack(false); err != nil { + log.Errorf("Failed acking canceled work, reason: %v", err.Error()) + } + + continue + } + + // Mark file as "COMPLETED" + if err := db.MarkCompleted(file, message.FileID, delivered.CorrelationId); err != nil { + log.Errorf("MarkCompleted failed, reason: (%v)", err.Error()) + + continue + // this should really be hadled by the DB retry mechanism + } + + // Send message to verified queue + if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, conf.Broker.RoutingKey, verifiedMessage); err != nil { + // TODO fix resend mechanism + log.Errorf("failed to publish message, reason: (%v)", err.Error()) + + continue + } + + if err := delivered.Ack(false); err != nil { + log.Errorf("failed to Ack message, reason: (%v)", err.Error()) + } + } + } + }() + + <-forever +} diff --git a/sda/cmd/verify/verify.md b/sda/cmd/verify/verify.md new file mode 100644 index 000000000..d15553621 --- /dev/null +++ b/sda/cmd/verify/verify.md @@ -0,0 +1,154 @@ +# sda-pipeline: verify + +Uses a crypt4gh secret key, this service can decrypt the stored files and checksum them against the embedded checksum for the unencrypted file. + +## Configuration + +There are a number of options that can be set for the verify service. +These settings can be set by mounting a yaml-file at `/config.yaml` with settings. + +ex. + +```yaml +log: + level: "debug" + format: "json" +``` + +They may also be set using environment variables like: + +```bash +export LOG_LEVEL="debug" +export LOG_FORMAT="json" +``` + +### Keyfile settings + +These settings control which crypt4gh keyfile is loaded. + +- `C4GH_FILEPATH`: filepath to the crypt4gh keyfile +- `C4GH_PASSPHRASE`: pass phrase to unlock the keyfile + +### RabbitMQ broker settings + +These settings control how verify connects to the RabbitMQ message broker. + +- `BROKER_HOST`: hostname of the rabbitmq server +- `BROKER_PORT`: rabbitmq broker port (commonly `5671` with TLS and `5672` without) +- `BROKER_QUEUE`: message queue to read messages from (commonly `archived`) +- `BROKER_ROUTINGKEY`: message queue to write success messages to (commonly `verified`) +- `BROKER_USER`: username to connect to rabbitmq +- `BROKER_PASSWORD`: password to connect to rabbitmq +- `BROKER_PREFETCHCOUNT`: Number of messages to pull from the message server at the time (default to 2) + +### PostgreSQL Database settings + +- `DB_HOST`: hostname for the postgresql database +- `DB_PORT`: database port (commonly 5432) +- `DB_USER`: username for the database +- `DB_PASSWORD`: password for the database +- `DB_DATABASE`: database name +- `DB_SSLMODE`: The TLS encryption policy to use for database connections, valid options are: + - `disable` + - `allow` + - `prefer` + - `require` + - `verify-ca` + - `verify-full` + + More information is available + [in the postgresql documentation](https://www.postgresql.org/docs/current/libpq-ssl.html#LIBPQ-SSL-PROTECTION) + + Note that if `DB_SSLMODE` is set to anything but `disable`, then `DB_CACERT` needs to be set, + and if set to `verify-full`, then `DB_CLIENTCERT`, and `DB_CLIENTKEY` must also be set + +- `DB_CLIENTKEY`: key-file for the database client certificate +- `DB_CLIENTCERT`: database client certificate file +- `DB_CACERT`: Certificate Authority (CA) certificate for the database to use + +### Storage settings + +Storage backend is defined by the `ARCHIVE_TYPE`, and `INBOX_TYPE` variables. +Valid values for these options are `S3` or `POSIX` +(Defaults to `POSIX` on unknown values). + +The value of these variables define what other variables are read. +The same variables are available for all storage types, differing by prefix (`ARCHIVE_`, or `INBOX_`) + +if `*_TYPE` is `S3` then the following variables are available: + +- `*_URL`: URL to the S3 system +- `*_ACCESSKEY`: The S3 access and secret key are used to authenticate to S3, + [more info at AWS](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) +- `*_SECRETKEY`: The S3 access and secret key are used to authenticate to S3, + [more info at AWS](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) +- `*_BUCKET`: The S3 bucket to use as the storage root +- `*_PORT`: S3 connection port (default: `443`) +- `*_REGION`: S3 region (default: `us-east-1`) +- `*_CHUNKSIZE`: S3 chunk size for multipart uploads. +- `*_CACERT`: Certificate Authority (CA) certificate for the storage system, this is only needed if the S3 server has a certificate signed by a private entity + +and if `*_TYPE` is `POSIX`: + +- `*_LOCATION`: POSIX path to use as storage root + +### Logging settings + +- `LOG_FORMAT` can be set to “json” to get logs in json format. All other values result in text logging +- `LOG_LEVEL` can be set to one of the following, in increasing order of severity: + - `trace` + - `debug` + - `info` + - `warn` (or `warning`) + - `error` + - `fatal` + - `panic` + +## Service Description + +The verify service ensures that ingested files are encrypted with the correct key, and that the provided checksums match those of the ingested files. + +When running, verify reads messages from the configured RabbitMQ queue (default: "archived"). +For each message, these steps are taken (if not otherwise noted, errors halt progress and the service moves on to the next message. +Unless explicitly stated, error messages are *not* written to the RabbitMQ error queue, and messages are not NACK or ACKed.): + +1. The message is validated as valid JSON that matches the "ingestion-verification" schema (defined in sda-common). +If the message can’t be validated it is discarded with an error message in the logs. + +1. The service attempts to fetch the header for the file id in the message from the database. +If this fails a NACK will be sent for the RabbitMQ message, the error will be written to the logs, and sent to the RabbitMQ error queue. + +1. The file size of the encrypted file is fetched from the archive storage system. +If this fails an error will be written to the logs. + +1. The archive file is then opened for reading. +If this fails an error will be written to the logs and to the RabbitMQ error queue. + +1. A decryptor is opened with the archive file. +If this fails an error will be written to the logs. + +1. The file size, md5 and sha256 checksum will be read from the decryptor. +If this fails an error will be written to the logs. + +1. If the `re_verify` boolean is not set in the RabbitMQ message, the message processing ends here, and continues with the next message. +Otherwise the processing continues with verification: + + 1. A verification message is created, and validated against the "ingestion-accession-request" schema. + If this fails an error will be written to the logs. + + 1. The file is marked as *verified* in the database (*COMPLETED* if you are using database schema <= 3). + If this fails an error will be written to the logs. + + 1. The verification message created in step 7.1 is sent to the "verified" queue. + If this fails an error will be written to the logs. + + 1. The original RabbitMQ message is ACKed. + If this fails an error is written to the logs, but processing continues to the next step. + +## Communication + +- Verify reads messages from one rabbitmq queue (commonly `archived`). +- Verify writes messages to one rabbitmq queue (commonly `verified`). +- Verify gets the file encryption header from the database using `GetHeader`, +and marks the files as `verified` (`COMPLETED` in db version <= 2.0) using `MarkCompleted`. +- Verify reads file data from archive storage and removes data from inbox storage. diff --git a/sda/cmd/verify/verify_test.go b/sda/cmd/verify/verify_test.go new file mode 100644 index 000000000..0f8a2819d --- /dev/null +++ b/sda/cmd/verify/verify_test.go @@ -0,0 +1,20 @@ +package main + +import ( + "testing" + + "github.com/spf13/viper" + "github.com/stretchr/testify/suite" +) + +type TestSuite struct { + suite.Suite +} + +func TestConfigTestSuite(t *testing.T) { + suite.Run(t, new(TestSuite)) +} + +func (suite *TestSuite) SetupTest() { + viper.Set("log.level", "debug") +} diff --git a/sda/internal/broker/broker.go b/sda/internal/broker/broker.go index 4145fdf92..72ced0a90 100644 --- a/sda/internal/broker/broker.go +++ b/sda/internal/broker/broker.go @@ -40,6 +40,15 @@ type MQConf struct { PrefetchCount int } +// InfoError struct for sending detailed error messages to analysis. +// The empty interface allows for appending various json msgs but also broken json msgs as strings. +// It is ok as long as we do not need to access fields in the msg, which we don't. +type InfoError struct { + Error string `json:"error"` + Reason string `json:"reason"` + OriginalMessage interface{} `json:"original-message"` +} + // buildMQURI builds the MQ connection URI func buildMQURI(mqHost, mqUser, mqPassword, mqVhost string, mqPort int, ssl bool) string { protocol := "amqp" diff --git a/sda/internal/schema/schema.go b/sda/internal/schema/schema.go index 4cc8ce91a..5244c9776 100644 --- a/sda/internal/schema/schema.go +++ b/sda/internal/schema/schema.go @@ -136,7 +136,7 @@ type IngestionUserError struct { type IngestionVerification struct { User string `json:"user"` FilePath string `json:"filepath"` - FileID int64 `json:"file_id"` + FileID string `json:"file_id"` ArchivePath string `json:"archive_path"` EncryptedChecksums []Checksums `json:"encrypted_checksums"` ReVerify bool `json:"re_verify"` diff --git a/sda/internal/schema/schema_test.go b/sda/internal/schema/schema_test.go index 2180f5eb7..15793a720 100644 --- a/sda/internal/schema/schema_test.go +++ b/sda/internal/schema/schema_test.go @@ -281,7 +281,7 @@ func TestValidateJSONIngestionVerification(t *testing.T) { okMsg := IngestionVerification{ User: "JohnDoe", FilePath: "path/to/file", - FileID: 123456789, + FileID: "074803cc-718e-4dc4-a48d-a4770aa9f93b", ArchivePath: "filename", EncryptedChecksums: []Checksums{ {Type: "sha256", Value: "da886a89637d125ef9f15f6d676357f3a9e5e10306929f0bad246375af89c2e2"}, @@ -297,7 +297,7 @@ func TestValidateJSONIngestionVerification(t *testing.T) { badMsg := IngestionVerification{ User: "JohnDoe", FilePath: "path/to/file", - FileID: 123456789, + FileID: "074803cc-718e-4dc4-a48d-a4770aa9f93b", ArchivePath: "filename", EncryptedChecksums: []Checksums{ {Type: "sha256", Value: "68b329da9893e34099c7d8ad5cb9c940"}, diff --git a/sda/schemas/federated/ingestion-verification.json b/sda/schemas/federated/ingestion-verification.json index 246259d43..fd63dcbf5 100644 --- a/sda/schemas/federated/ingestion-verification.json +++ b/sda/schemas/federated/ingestion-verification.json @@ -107,7 +107,7 @@ }, "file_id": { "$id": "#/properties/file_id", - "type": "number", + "type": "string", "title": "The Accession identifier", "description": "The Accession identifier", "examples": [ diff --git a/sda/schemas/isolated/ingestion-verification.json b/sda/schemas/isolated/ingestion-verification.json new file mode 120000 index 000000000..80cd7082d --- /dev/null +++ b/sda/schemas/isolated/ingestion-verification.json @@ -0,0 +1 @@ +../federated/ingestion-verification.json \ No newline at end of file From 6c0192888d69c2fb2a59704e5981ee3cf5936f96 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 17 Aug 2023 12:21:34 +0200 Subject: [PATCH 14/34] Merge in `backup` from sda-pipeline --- .../scripts/make_sda_credentials.sh | 10 +- .github/integration/sda-integration.yml | 27 ++ .github/integration/sda/config.yaml | 9 + .../integration/tests/sda/30_backup_test.sh | 98 +++++++ sda/cmd/backup/backup.go | 249 ++++++++++++++++++ sda/cmd/backup/backup.md | 159 +++++++++++ sda/cmd/backup/backup_test.go | 34 +++ 7 files changed, 581 insertions(+), 5 deletions(-) create mode 100644 .github/integration/tests/sda/30_backup_test.sh create mode 100644 sda/cmd/backup/backup.go create mode 100644 sda/cmd/backup/backup.md create mode 100644 sda/cmd/backup/backup_test.go diff --git a/.github/integration/scripts/make_sda_credentials.sh b/.github/integration/scripts/make_sda_credentials.sh index 40b496f4f..5dee949ff 100644 --- a/.github/integration/scripts/make_sda_credentials.sh +++ b/.github/integration/scripts/make_sda_credentials.sh @@ -4,13 +4,13 @@ set -e apt-get -o DPkg::Lock::Timeout=60 update > /dev/null apt-get -o DPkg::Lock::Timeout=60 install -y curl jq openssl postgresql-client >/dev/null -for n in download finalize inbox ingest mapper sync verify; do +for n in backup download finalize inbox ingest mapper sync verify; do echo "creating credentials for: $n" - if [ "$n" = inbox ]; then - psql -U postgres -h postgres -d sda -c "DROP ROLE IF EXISTS inbox;" - psql -U postgres -h postgres -d sda -c "CREATE ROLE inbox;" - psql -U postgres -h postgres -d sda -c "GRANT ingest TO inbox;" + if [ "$n" = inbox ] || [ "$n" = backup ]; then + psql -U postgres -h postgres -d sda -c "DROP ROLE IF EXISTS $n;" + psql -U postgres -h postgres -d sda -c "CREATE ROLE $n;" + psql -U postgres -h postgres -d sda -c "GRANT ingest TO $n;" fi if [ "$n" = ingest ]; then diff --git a/.github/integration/sda-integration.yml b/.github/integration/sda-integration.yml index d18e9e701..135aeb9a1 100644 --- a/.github/integration/sda-integration.yml +++ b/.github/integration/sda-integration.yml @@ -159,6 +159,31 @@ services: - ./sda/config.yaml:/config.yaml - shared:/shared + backup: + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + command: [ sda-backup ] + container_name: backup + depends_on: + credentials: + condition: service_completed_successfully + minio: + condition: service_healthy + postgres: + condition: service_healthy + rabbitmq: + condition: service_healthy + environment: + - BROKER_PASSWORD=backup + - BROKER_USER=backup + - BROKER_QUEUE=accession + - BROKER_ROUTINGKEY=completed + - DB_PASSWORD=backup + - DB_USER=backup + restart: always + volumes: + - ./sda/config.yaml:/config.yaml + - shared:/shared + oidc: container_name: oidc command: @@ -193,6 +218,8 @@ services: depends_on: credentials: condition: service_completed_successfully + backup: + condition: service_started ingest: condition: service_started s3inbox: diff --git a/.github/integration/sda/config.yaml b/.github/integration/sda/config.yaml index b75ca9ffc..15e949f6e 100644 --- a/.github/integration/sda/config.yaml +++ b/.github/integration/sda/config.yaml @@ -10,6 +10,15 @@ archive: secretKey: "secretKey" bucket: "archive" region: "us-east-1" +backup: + type: s3 + url: "http://s3" + port: 9000 + readypath: "/minio/health/ready" + accessKey: "access" + secretKey: "secretKey" + bucket: "backup" + region: "us-east-1" inbox: type: s3 url: "http://s3" diff --git a/.github/integration/tests/sda/30_backup_test.sh b/.github/integration/tests/sda/30_backup_test.sh new file mode 100644 index 000000000..c07e4b949 --- /dev/null +++ b/.github/integration/tests/sda/30_backup_test.sh @@ -0,0 +1,98 @@ +#!/bin/bash +set -e + +cd shared || true + +i=1 +while [ $i -le "$(curl -su guest:guest http://rabbitmq:15672/api/queues/sda/verified/ | jq -r '.messages_ready')" ]; do + ## get correlation id from upload message + MSG=$( + curl -s -X POST \ + -H "content-type:application/json" \ + -u guest:guest http://rabbitmq:15672/api/queues/sda/verified/get \ + -d '{"count":1,"encoding":"auto","ackmode":"ack_requeue_false"}' + ) + + corrid=$(jq -r '.[0].properties.correlation_id' <<< "$MSG") + user=$(jq -r '.[0].payload|fromjson|.user' <<< "$MSG") + filepath=$(jq -r '.[0].payload|fromjson|.filepath' <<< "$MSG") + decrypted_checksums=$(jq -r '.[0].payload|fromjson|.decrypted_checksums|tostring' <<< "$MSG") + + ## publish message to trigger backup + properties=$( + jq -c -n \ + --argjson delivery_mode 2 \ + --arg correlation_id "$corrid" \ + --arg content_encoding UTF-8 \ + --arg content_type application/json \ + '$ARGS.named' + ) + + accession_payload=$( + jq -r -c -n \ + --arg type accession \ + --arg user "$user" \ + --arg filepath "$filepath" \ + --arg accession_id "EGAF7490000000$I" \ + --argjson decrypted_checksums "$decrypted_checksums" \ + '$ARGS.named|@base64' + ) + + accession_body=$( + jq -c -n \ + --arg vhost sda \ + --arg name sda \ + --argjson properties "$properties" \ + --arg routing_key "accession" \ + --arg payload_encoding base64 \ + --arg payload "$accession_payload" \ + '$ARGS.named' + ) + + curl -s -u guest:guest "http://rabbitmq:15672/api/exchanges/sda/sda/publish" \ + -H 'Content-Type: application/json;charset=UTF-8' \ + -d "$accession_body" + + i=$(( i + 1 )) +done + +echo "waiting for backup to complete" +RETRY_TIMES=0 +until [ "$(curl -su guest:guest http://rabbitmq:15672/api/queues/sda/completed/ | jq -r '.messages_ready')" -eq 2 ]; do + echo "waiting for backup to complete" + RETRY_TIMES=$((RETRY_TIMES + 1)) + if [ "$RETRY_TIMES" -eq 30 ]; then + echo "::error::Time out while waiting for backup to complete" + exit 1 + fi + sleep 2 +done + +cat >/shared/direct < Date: Thu, 17 Aug 2023 12:29:20 +0200 Subject: [PATCH 15/34] Update GO mod files --- sda/go.mod | 51 +++++++------- sda/go.sum | 202 +++++++++++++++-------------------------------------- 2 files changed, 79 insertions(+), 174 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index 5d3e0830b..a3655481f 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -4,24 +4,23 @@ go 1.20 require ( github.com/DATA-DOG/go-sqlmock v1.5.0 - github.com/aws/aws-sdk-go v1.44.253 - github.com/gliderlabs/ssh v0.3.5 + github.com/aws/aws-sdk-go v1.44.325 github.com/golang-jwt/jwt/v4 v4.5.0 + github.com/google/uuid v1.1.2 github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb - github.com/johannesboyne/gofakes3 v0.0.0-20230310080033-c0edf658332b github.com/lestrrat-go/jwx v1.2.26 github.com/lib/pq v1.10.9 github.com/minio/minio-go/v6 v6.0.57 github.com/neicnordic/crypt4gh v1.7.6 github.com/ory/dockertest/v3 v3.10.0 github.com/pkg/errors v0.9.1 - github.com/pkg/sftp v1.13.1 - github.com/rabbitmq/amqp091-go v1.8.0 - github.com/santhosh-tekuri/jsonschema/v5 v5.3.0 - github.com/sirupsen/logrus v1.9.0 - github.com/spf13/viper v1.15.0 + github.com/pkg/sftp v1.13.6 + github.com/rabbitmq/amqp091-go v1.8.1 + github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 + github.com/sirupsen/logrus v1.9.3 + github.com/spf13/viper v1.16.0 github.com/stretchr/testify v1.8.4 - golang.org/x/crypto v0.11.0 + golang.org/x/crypto v0.12.0 ) require ( @@ -29,10 +28,9 @@ require ( github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect - github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.1.3 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/containerd/continuity v0.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a // indirect @@ -44,11 +42,12 @@ require ( github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/kr/fs v0.1.0 // indirect github.com/lestrrat-go/backoff/v2 v2.0.8 // indirect github.com/lestrrat-go/blackmagic v1.0.1 // indirect @@ -56,34 +55,32 @@ require ( github.com/lestrrat-go/iter v1.0.2 // indirect github.com/lestrrat-go/option v1.0.1 // indirect github.com/magiconair/properties v1.8.7 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect - github.com/minio/sha256-simd v0.1.1 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.2 // indirect github.com/opencontainers/runc v1.1.7 // indirect - github.com/pelletier/go-toml/v2 v2.0.6 // indirect + github.com/pelletier/go-toml/v2 v2.0.9 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_golang v1.13.1 // indirect - github.com/prometheus/client_model v0.2.0 // indirect - github.com/prometheus/common v0.37.0 // indirect - github.com/prometheus/procfs v0.8.0 // indirect - github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect - github.com/shabbyrobe/gocovmerge v0.0.0-20180507124511-f6ea450bfb63 // indirect - github.com/spf13/afero v1.9.3 // indirect - github.com/spf13/cast v1.5.0 // indirect + github.com/prometheus/client_golang v1.16.0 // indirect + github.com/prometheus/client_model v0.4.0 // indirect + github.com/prometheus/common v0.44.0 // indirect + github.com/prometheus/procfs v0.11.1 // indirect + github.com/spf13/afero v1.9.5 // indirect + github.com/spf13/cast v1.5.1 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/subosito/gotenv v1.4.2 // indirect + github.com/subosito/gotenv v1.6.0 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect golang.org/x/mod v0.9.0 // indirect - golang.org/x/sys v0.10.0 // indirect - golang.org/x/text v0.11.0 // indirect + golang.org/x/sys v0.11.0 // indirect + golang.org/x/text v0.12.0 // indirect golang.org/x/tools v0.7.0 // indirect - google.golang.org/protobuf v1.28.1 // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/sda/go.sum b/sda/go.sum index 950a8f9cd..84d6186cb 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -49,26 +49,15 @@ github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migc github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= -github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= -github.com/aws/aws-sdk-go v1.33.0/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= -github.com/aws/aws-sdk-go v1.44.253 h1:iqDd0okcH4ShfFexz2zzf4VmeDFf6NOMm07pHnEb8iY= -github.com/aws/aws-sdk-go v1.44.253/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/aws/aws-sdk-go v1.44.325 h1:jF/L99fJSq/BfiLmUOflO/aM+LwcqBm0Fe/qTK5xxuI= +github.com/aws/aws-sdk-go v1.44.325/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -103,28 +92,15 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= +github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= -github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= -github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= @@ -155,8 +131,8 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -187,6 +163,7 @@ github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLe github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= +github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= @@ -202,34 +179,24 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/johannesboyne/gofakes3 v0.0.0-20230310080033-c0edf658332b h1:dRMf9/2xfp4tky4wnvFxsMQz78n92VeqDIxR27uass4= -github.com/johannesboyne/gofakes3 v0.0.0-20230310080033-c0edf658332b/go.mod h1:Cnosl0cRZIfKjTMuH49sQog2LeNsU5Hf4WnPIDWIDV0= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= +github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -250,25 +217,21 @@ github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= -github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= github.com/minio/minio-go/v6 v6.0.57 h1:ixPkbKkyD7IhnluRgQpGSpHdpvNVaW6OD5R9IAO/9Tw= github.com/minio/minio-go/v6 v6.0.57/go.mod h1:5+R/nM9Pwrh0vqF+HbYYDQ84wdUFPyXHkrdT4AIkifM= -github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 h1:rzf0wL0CHVc8CEsgyygG0Mn9CNCCPZqOPaz8RiiHYQk= github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/neicnordic/crypt4gh v1.7.6 h1:Vqcb8Yb950oaBBJFepDK1oLeu9rZzpywYWVHLmO0oI8= github.com/neicnordic/crypt4gh v1.7.6/go.mod h1:rqmVXsprDFBRRLJkm1cK9kLETBPGEZmft9lHD/V40wk= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= @@ -279,73 +242,48 @@ github.com/opencontainers/runc v1.1.7 h1:y2EZDS8sNng4Ksf0GUYNhKbTShZJPJg1FiXJNH/ github.com/opencontainers/runc v1.1.7/go.mod h1:CbUumNnWCuTGFukNXahoo/RFBZvDAgRh/smNYNOhA50= github.com/ory/dockertest/v3 v3.10.0 h1:4K3z2VMe8Woe++invjaTB7VRyQXQy5UY+loujO4aNE4= github.com/ory/dockertest/v3 v3.10.0/go.mod h1:nr57ZbRWMqfsdGdFNLHz5jjNdDb7VVFnzAeW1n5N1Lg= -github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= -github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pelletier/go-toml/v2 v2.0.9 h1:uH2qQXheeefCCkuBBSLi7jCiSmj3VRh2+Goq2N7Xxu0= +github.com/pelletier/go-toml/v2 v2.0.9/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.13.1 h1:I2qBYMChEhIjOgazfJmV3/mZM256btk6wkCDRmW7JYs= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= +github.com/pkg/sftp v1.13.6 h1:JFZT4XbOU7l77xGSpOdW+pwIMqP044IyjXX6FGyEKFo= +github.com/pkg/sftp v1.13.6/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.13.1 h1:3gMjIY2+/hzmqhtUC/aQNYldJA6DtH3CgQvwS+02K1c= -github.com/prometheus/client_golang v1.13.1/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= +github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= -github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= -github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= -github.com/rabbitmq/amqp091-go v1.8.0 h1:GBFy5PpLQ5jSVVSYv8ecHGqeX7UTLYR4ItQbDCss9MM= -github.com/rabbitmq/amqp091-go v1.8.0/go.mod h1:+jPrT9iY2eLjRaMSRHUhc3z14E/l85kv/f+6luSD3pc= +github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= +github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= +github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= +github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= +github.com/rabbitmq/amqp091-go v1.8.1 h1:RejT1SBUim5doqcL6s7iN6SBmsQqyTgXb1xMlH0h1hA= +github.com/rabbitmq/amqp091-go v1.8.1/go.mod h1:+jPrT9iY2eLjRaMSRHUhc3z14E/l85kv/f+6luSD3pc= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= -github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= -github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= -github.com/santhosh-tekuri/jsonschema/v5 v5.3.0 h1:uIkTLo0AGRc8l7h5l9r+GcYi9qfVPt6lD4/bhmzfiKo= -github.com/santhosh-tekuri/jsonschema/v5 v5.3.0/go.mod h1:FKdcjfQW6rpZSnxxUvEA5H/cDPdvJ/SZJQLWWXWGrZ0= -github.com/shabbyrobe/gocovmerge v0.0.0-20180507124511-f6ea450bfb63 h1:J6qvD6rbmOil46orKqJaRPG+zTpoGlBTUdyv8ki63L0= -github.com/shabbyrobe/gocovmerge v0.0.0-20180507124511-f6ea450bfb63/go.mod h1:n+VKSARF5y/tS9XFSP7vWDfS+GUC5vs/YT7M5XDTUEM= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4= +github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY= github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/afero v1.9.3 h1:41FoI0fD7OR7mGcKE/aOiLkGreyf8ifIOQmJANWogMk= -github.com/spf13/afero v1.9.3/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= -github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= -github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= +github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= +github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= +github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= +github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.15.0 h1:js3yy885G8xwJa6iOISGFwd+qlUo5AvyXb7CiihdtiU= -github.com/spf13/viper v1.15.0/go.mod h1:fFcTBJxvhhzSJiZy8n+PeW6t8l+KeT/uTARa0jHOQLA= +github.com/spf13/viper v1.16.0 h1:rGGH0XDZhdUOryiDWjmIvUSWpbNqisK8Wk0Vyefw8hc= +github.com/spf13/viper v1.16.0/go.mod h1:yg78JgCJcbrQOvV9YLXgkLaZqUidkY9K+Dd1FofRzQg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -356,11 +294,10 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= -github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= @@ -372,7 +309,6 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -381,7 +317,6 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -390,11 +325,11 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= -golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= -golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -434,7 +369,6 @@ golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -443,7 +377,6 @@ golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -466,12 +399,8 @@ golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= @@ -485,8 +414,6 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -498,11 +425,9 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -515,7 +440,6 @@ golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -528,8 +452,6 @@ golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -538,33 +460,26 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= -golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= +golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -576,15 +491,14 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= -golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190308174544-00c44ba9c14f/go.mod h1:25r3+/G6/xytQM8iWZKq3Hn0kr0rgFKPUNVEL/dr3z4= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -731,24 +645,18 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 h1:FVCohIoYO7IJoDDVpV2pdq7SgrMH6wHnuTyrdrxJNoY= gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0/go.mod h1:OdE7CF6DbADk7lN8LIKRzRJTTZXIjtWgA5THM5lhBAw= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= From 6b3c2ec8cdca0d2c63a368faf29c99a9fc58c6a3 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 17 Aug 2023 14:04:20 +0200 Subject: [PATCH 16/34] Merge in `finalize` from sda-pipeline --- sda/cmd/finalize/finalize.go | 186 +++++++++++++++++++++ sda/cmd/finalize/finalize.md | 114 +++++++++++++ sda/cmd/finalize/finalize_test.go | 20 +++ sda/internal/database/db_functions.go | 64 +++++++ sda/internal/database/db_functions_test.go | 42 +++++ 5 files changed, 426 insertions(+) create mode 100644 sda/cmd/finalize/finalize.go create mode 100644 sda/cmd/finalize/finalize.md create mode 100644 sda/cmd/finalize/finalize_test.go diff --git a/sda/cmd/finalize/finalize.go b/sda/cmd/finalize/finalize.go new file mode 100644 index 000000000..6b11cef61 --- /dev/null +++ b/sda/cmd/finalize/finalize.go @@ -0,0 +1,186 @@ +// The finalize command accepts messages with accessionIDs for +// ingested files and registers them in the database. +package main + +import ( + "encoding/json" + "fmt" + + "github.com/neicnordic/sensitive-data-archive/internal/broker" + "github.com/neicnordic/sensitive-data-archive/internal/config" + "github.com/neicnordic/sensitive-data-archive/internal/database" + "github.com/neicnordic/sensitive-data-archive/internal/schema" + + log "github.com/sirupsen/logrus" +) + +func main() { + forever := make(chan bool) + conf, err := config.NewConfig("finalize") + if err != nil { + log.Fatal(err) + } + mq, err := broker.NewMQ(conf.Broker) + if err != nil { + log.Fatal(err) + } + db, err := database.NewSDAdb(conf.Database) + if err != nil { + log.Fatal(err) + } + + defer mq.Channel.Close() + defer mq.Connection.Close() + defer db.Close() + + go func() { + connError := mq.ConnectionWatcher() + log.Error(connError) + forever <- false + }() + + go func() { + connError := mq.ChannelWatcher() + log.Error(connError) + forever <- false + }() + + log.Info("Starting finalize service") + var message schema.IngestionAccession + + go func() { + messages, err := mq.GetMessages(conf.Broker.Queue) + if err != nil { + log.Fatal(err) + } + for delivered := range messages { + log.Debugf("Received a message (corr-id: %s, message: %s)", delivered.CorrelationId, delivered.Body) + err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-accession.json", conf.Broker.SchemasPath), delivered.Body) + if err != nil { + log.Errorf("validation of incoming message (ingestion-accession) failed, reason: %v ", err) + if err := delivered.Ack(false); err != nil { + log.Errorf("Failed acking canceled work, reason: %v", err) + } + + continue + } + + // we unmarshal the message in the validation step so this is safe to do + _ = json.Unmarshal(delivered.Body, &message) + // If the file has been canceled by the uploader, don't spend time working on it. + status, err := db.GetFileStatus(delivered.CorrelationId) + if err != nil { + log.Errorf("failed to get file status, reason: %v", err) + } + if status == "disabled" { + log.Infof("file with correlation ID: %s is disabled, stopping work", delivered.CorrelationId) + if err := delivered.Ack(false); err != nil { + log.Errorf("Failed acking canceled work, reason: %v", err) + } + + continue + } + + // Extract the sha256 from the message and use it for the database + var checksumSha256 string + for _, checksum := range message.DecryptedChecksums { + if checksum.Type == "sha256" { + checksumSha256 = checksum.Value + } + } + + c := schema.IngestionCompletion{ + User: message.User, + FilePath: message.FilePath, + AccessionID: message.AccessionID, + DecryptedChecksums: message.DecryptedChecksums, + } + completeMsg, _ := json.Marshal(&c) + err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-completion.json", conf.Broker.SchemasPath), completeMsg) + if err != nil { + log.Errorf("Validation of outgoing message failed, reason: (%v)", err) + + continue + } + + accessionIDExists, err := db.CheckAccessionIDExists(message.AccessionID) + if err != nil { + log.Errorf("CheckAccessionIdExists failed, reason: %v ", err) + if err := delivered.Nack(false, true); err != nil { + log.Errorf("failed to Nack message, reason: (%v)", err) + } + + continue + } + + if accessionIDExists { + log.Debugf("Seems accession ID already exists (corr-id: %s, accessionid: %s", delivered.CorrelationId, message.AccessionID) + // Send the message to an error queue so it can be analyzed. + fileError := broker.InfoError{ + Error: "There is a conflict regarding the file accessionID", + Reason: "The Accession ID already exists in the database, skipping marking it ready.", + OriginalMessage: message, + } + body, _ := json.Marshal(fileError) + + // Send the message to an error queue so it can be analyzed. + if e := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); e != nil { + log.Errorf("failed to publish message, reason: (%v)", err) + } + + if err := delivered.Ack(false); err != nil { + log.Errorf("failed to Ack message, reason: (%v)", err) + } + + continue + + } + + if err := db.SetAccessionID(message.AccessionID, message.User, message.FilePath, checksumSha256); err != nil { + log.Errorf("Failed to set accessionID for file with corrID: %v, reason: %v", delivered.CorrelationId, err) + if err := delivered.Nack(false, true); err != nil { + log.Errorf("failed to Nack message, reason: (%v)", err) + } + + continue + } + + // Mark file as "ready" + if err := db.UpdateFileStatus(fileID, "ready", delivered.CorrelationId, "finalize", string(delivered.Body)); err != nil { + log.Errorf("set status ready failed, reason: (%v)", err) + if err := delivered.Nack(false, true); err != nil { + log.Errorf("failed to Nack message, reason: (%v)", err) + } + + continue + } + + c := schema.IngestionCompletion{ + User: message.User, + FilePath: message.FilePath, + AccessionID: message.AccessionID, + DecryptedChecksums: message.DecryptedChecksums, + } + completeMsg, _ := json.Marshal(&c) + err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-completion.json", conf.Broker.SchemasPath), completeMsg) + if err != nil { + log.Errorf("Validation of outgoing message failed, reason: (%v)", err) + + continue + } + + if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, conf.Broker.RoutingKey, completeMsg); err != nil { + // TODO fix resend mechanism + log.Errorf("failed to publish message, reason: (%v)", err) + + continue + } + + if err := delivered.Ack(false); err != nil { + log.Errorf("failed to Ack message, reason: (%v)", err) + } + } + }() + + <-forever +} diff --git a/sda/cmd/finalize/finalize.md b/sda/cmd/finalize/finalize.md new file mode 100644 index 000000000..d2af7d3a2 --- /dev/null +++ b/sda/cmd/finalize/finalize.md @@ -0,0 +1,114 @@ +# sda-pipeline: finalize + +Handles the so-called _Accession ID (stable ID)_ to filename mappings from Central EGA. + + +## Configuration + +There are a number of options that can be set for the finalize service. +These settings can be set by mounting a yaml-file at `/config.yaml` with settings. + +ex. +```yaml +log: + level: "debug" + format: "json" +``` +They may also be set using environment variables like: +```bash +export LOG_LEVEL="debug" +export LOG_FORMAT="json" +``` + +### RabbitMQ broker settings + +These settings control how finalize connects to the RabbitMQ message broker. + + - `BROKER_HOST`: hostname of the rabbitmq server + + - `BROKER_PORT`: rabbitmq broker port (commonly `5671` with TLS and `5672` without) + + - `BROKER_QUEUE`: message queue to read messages from (commonly `accessionIDs`) + + - `BROKER_ROUTINGKEY`: message queue to write success messages to (commonly `backup`) + + - `BROKER_USER`: username to connect to rabbitmq + + - `BROKER_PASSWORD`: password to connect to rabbitmq + + - `BROKER_PREFETCHCOUNT`: Number of messages to pull from the message server at the time (default to 2) + +### PostgreSQL Database settings: + + - `DB_HOST`: hostname for the postgresql database + + - `DB_PORT`: database port (commonly 5432) + + - `DB_USER`: username for the database + + - `DB_PASSWORD`: password for the database + + - `DB_DATABASE`: database name + + - `DB_SSLMODE`: The TLS encryption policy to use for database connections. + Valid options are: + - `disable` + - `allow` + - `prefer` + - `require` + - `verify-ca` + - `verify-full` + + More information is available + [in the postgresql documentation](https://www.postgresql.org/docs/current/libpq-ssl.html#LIBPQ-SSL-PROTECTION) + + Note that if `DB_SSLMODE` is set to anything but `disable`, then `DB_CACERT` needs to be set, + and if set to `verify-full`, then `DB_CLIENTCERT`, and `DB_CLIENTKEY` must also be set + + - `DB_CLIENTKEY`: key-file for the database client certificate + + - `DB_CLIENTCERT`: database client certificate file + + - `DB_CACERT`: Certificate Authority (CA) certificate for the database to use + +### Logging settings: + + - `LOG_FORMAT` can be set to “json” to get logs in json format. + All other values result in text logging + + - `LOG_LEVEL` can be set to one of the following, in increasing order of severity: + - `trace` + - `debug` + - `info` + - `warn` (or `warning`) + - `error` + - `fatal` + - `panic` + +## Service Description +Finalize adds stable, shareable _Accession ID_'s to archive files. +When running, finalize reads messages from the configured RabbitMQ queue (default "accessionIDs"). +For each message, these steps are taken (if not otherwise noted, errors halt progress and the service moves on to the next message): + +1. The message is validated as valid JSON that matches the "ingestion-accession" schema (defined in sda-common). +If the message can’t be validated it is discarded with an error message in the logs. + +1. if the type of the `DecryptedChecksums` field in the message is `sha256`, the value is stored. + +1. A new RabbitMQ "complete" message is created and validated against the "ingestion-completion" schema. +If the validation fails, an error message is written to the logs. + +1. The file accession ID in the message is marked as "ready" in the database. +On error the service sleeps for up to 5 minutes to allow for database recovery, after 5 minutes the message is Nacked, re-queued and an error message is written to the logs. + +1. The complete message is sent to RabbitMQ. On error, a message is written to the logs. + +1. The original RabbitMQ message is Ack'ed. + +## Communication + + - Finalize reads messages from one rabbitmq queue (default `accessionIDs`). + + - Finalize writes messages to one rabbitmq queue (default `backup`). + + - Finalize assigns the accession ID to a file in the database using the `SetAccessionID` function. diff --git a/sda/cmd/finalize/finalize_test.go b/sda/cmd/finalize/finalize_test.go new file mode 100644 index 000000000..0f8a2819d --- /dev/null +++ b/sda/cmd/finalize/finalize_test.go @@ -0,0 +1,20 @@ +package main + +import ( + "testing" + + "github.com/spf13/viper" + "github.com/stretchr/testify/suite" +) + +type TestSuite struct { + suite.Suite +} + +func TestConfigTestSuite(t *testing.T) { + suite.Run(t, new(TestSuite)) +} + +func (suite *TestSuite) SetupTest() { + viper.Set("log.level", "debug") +} diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 7a450e614..f694ebfe7 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -6,6 +6,8 @@ import ( "encoding/hex" "errors" "fmt" + "math" + "time" ) // RegisterFile inserts a file in the database, along with a "registered" log @@ -288,3 +290,65 @@ func (dbs *SDAdb) getArchived(user, filepath, checksum string) (string, int, err return filePath, fileSize, nil } + +// CheckAccessionIdExists validates if an accessionID exists in the db +func (dbs *SDAdb) CheckAccessionIDExists(accessionID string) (bool, error) { + var err error + var exists bool + // 2, 4, 8, 16, 32 seconds between each retry event. + for count := 1; count <= RetryTimes; count++ { + exists, err = dbs.checkAccessionIDExists(accessionID) + if err == nil { + break + } + time.Sleep(time.Duration(math.Pow(2, float64(count))) * time.Second) + } + + return exists, err +} +func (dbs *SDAdb) checkAccessionIDExists(accessionID string) (bool, error) { + dbs.checkAndReconnectIfNeeded() + db := dbs.DB + const checkIDExist = "SELECT COUNT(*) FROM sda.files WHERE stable_id = $1;" + var stableIDCount int + if err := db.QueryRow(checkIDExist, accessionID).Scan(&stableIDCount); err != nil { + return false, err + } + + if stableIDCount >= 1 { + return true, nil + } + + return false, nil +} + +// SetAccessionID adds a stable id to a file +// identified by the user submitting it, inbox path and decrypted checksum +func (dbs *SDAdb) SetAccessionID(accessionID, user, filepath, checksum string) error { + var err error + // 2, 4, 8, 16, 32 seconds between each retry event. + for count := 1; count <= RetryTimes; count++ { + err = dbs.setAccessionID(accessionID, user, filepath, checksum) + if err == nil { + break + } + time.Sleep(time.Duration(math.Pow(2, float64(count))) * time.Second) + } + + return err +} +func (dbs *SDAdb) setAccessionID(accessionID, user, filepath, checksum string) error { + dbs.checkAndReconnectIfNeeded() + db := dbs.DB + const ready = "UPDATE local_ega.files SET stable_id = $1 WHERE elixir_id = $2 and inbox_path = $3 and decrypted_file_checksum = $4 and status = 'COMPLETED';" + result, err := db.Exec(ready, accessionID, user, filepath, checksum) + if err != nil { + return err + } + + if rowsAffected, _ := result.RowsAffected(); rowsAffected == 0 { + return errors.New("something went wrong with the query zero rows were changed") + } + + return nil +} diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index 3095a10a2..6e199538f 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -207,3 +207,45 @@ func (suite *DatabaseTests) TestGetArchived() { assert.Equal(suite.T(), 1000, fileSize) assert.Equal(suite.T(), "/tmp/TestGetArchived.c4gh", filePath) } + +func (suite *DatabaseTests) TestSetAccessionID() { + db, err := NewSDAdb(suite.dbConf) + assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) + + // register a file in the database + fileID, err := db.RegisterFile("/testuser/TestSetAccessionID.c4gh", "testuser") + assert.NoError(suite.T(), err, "failed to register file in database") + decSha := sha256.New() + fileInfo := FileInfo{sha256.New(), 1000, "/tmp/TestSetAccessionID.c4gh", decSha, 987} + corrID := uuid.New().String() + err = db.SetArchived(fileInfo, fileID, corrID) + assert.NoError(suite.T(), err, "got (%v) when marking file as Archived") + err = db.MarkCompleted(fileInfo, fileID, corrID) + assert.NoError(suite.T(), err, "got (%v) when marking file as completed", err) + stableID := "TEST:000-1234-4567" + err = db.SetAccessionID(stableID, "testuser", "/testuser/TestSetAccessionID.c4gh", fmt.Sprintf("%x", decSha.Sum(nil))) + assert.NoError(suite.T(), err, "got (%v) when getting file archive information", err) +} + +func (suite *DatabaseTests) TestCheckAccessionIDExists() { + db, err := NewSDAdb(suite.dbConf) + assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) + + // register a file in the database + fileID, err := db.RegisterFile("/testuser/TestCheckAccessionIDExists.c4gh", "testuser") + assert.NoError(suite.T(), err, "failed to register file in database") + decSha := sha256.New() + fileInfo := FileInfo{sha256.New(), 1000, "/tmp/TestCheckAccessionIDExists.c4gh", decSha, 987} + corrID := uuid.New().String() + err = db.SetArchived(fileInfo, fileID, corrID) + assert.NoError(suite.T(), err, "got (%v) when marking file as Archived") + err = db.MarkCompleted(fileInfo, fileID, corrID) + assert.NoError(suite.T(), err, "got (%v) when marking file as completed", err) + stableID := "TEST:111-1234-4567" + err = db.SetAccessionID(stableID, "testuser", "/testuser/TestCheckAccessionIDExists.c4gh", fmt.Sprintf("%x", decSha.Sum(nil))) + assert.NoError(suite.T(), err, "got (%v) when getting file archive information", err) + + ok, err := db.CheckAccessionIDExists(stableID) + assert.NoError(suite.T(), err, "got (%v) when getting file archive information", err) + assert.True(suite.T(), ok, "CheckAccessionIDExists returned false when true was expected") +} From 57de69a7ef3476570cfb3e744b42d633f4d8f0df Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Fri, 18 Aug 2023 10:51:22 +0200 Subject: [PATCH 17/34] Simplify `SetAccessionID` database call --- sda/cmd/finalize/finalize.go | 19 ++++++++++--------- sda/internal/database/db_functions.go | 10 +++++----- sda/internal/database/db_functions_test.go | 4 ++-- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/sda/cmd/finalize/finalize.go b/sda/cmd/finalize/finalize.go index 6b11cef61..7447ac438 100644 --- a/sda/cmd/finalize/finalize.go +++ b/sda/cmd/finalize/finalize.go @@ -81,14 +81,6 @@ func main() { continue } - // Extract the sha256 from the message and use it for the database - var checksumSha256 string - for _, checksum := range message.DecryptedChecksums { - if checksum.Type == "sha256" { - checksumSha256 = checksum.Value - } - } - c := schema.IngestionCompletion{ User: message.User, FilePath: message.FilePath, @@ -133,10 +125,19 @@ func main() { } continue + } + + fileID, err := db.GetFileID(delivered.CorrelationId) + if err != nil { + log.Errorf("failed to get ID for file, reason: %v", err) + if err := delivered.Nack(false, true); err != nil { + log.Errorf("failed to Nack message, reason: (%v)", err) + } + continue } - if err := db.SetAccessionID(message.AccessionID, message.User, message.FilePath, checksumSha256); err != nil { + if err := db.SetAccessionID(message.AccessionID, fileID); err != nil { log.Errorf("Failed to set accessionID for file with corrID: %v, reason: %v", delivered.CorrelationId, err) if err := delivered.Nack(false, true); err != nil { log.Errorf("failed to Nack message, reason: (%v)", err) diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index f694ebfe7..3d43d96ec 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -324,11 +324,11 @@ func (dbs *SDAdb) checkAccessionIDExists(accessionID string) (bool, error) { // SetAccessionID adds a stable id to a file // identified by the user submitting it, inbox path and decrypted checksum -func (dbs *SDAdb) SetAccessionID(accessionID, user, filepath, checksum string) error { +func (dbs *SDAdb) SetAccessionID(accessionID, fileID string) error { var err error // 2, 4, 8, 16, 32 seconds between each retry event. for count := 1; count <= RetryTimes; count++ { - err = dbs.setAccessionID(accessionID, user, filepath, checksum) + err = dbs.setAccessionID(accessionID, fileID) if err == nil { break } @@ -337,11 +337,11 @@ func (dbs *SDAdb) SetAccessionID(accessionID, user, filepath, checksum string) e return err } -func (dbs *SDAdb) setAccessionID(accessionID, user, filepath, checksum string) error { +func (dbs *SDAdb) setAccessionID(accessionID, fileID string) error { dbs.checkAndReconnectIfNeeded() db := dbs.DB - const ready = "UPDATE local_ega.files SET stable_id = $1 WHERE elixir_id = $2 and inbox_path = $3 and decrypted_file_checksum = $4 and status = 'COMPLETED';" - result, err := db.Exec(ready, accessionID, user, filepath, checksum) + const setStableID = "UPDATE sda.files SET stable_id = $1 WHERE id = $2;" + result, err := db.Exec(setStableID, accessionID, fileID) if err != nil { return err } diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index 6e199538f..8586996b4 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -223,7 +223,7 @@ func (suite *DatabaseTests) TestSetAccessionID() { err = db.MarkCompleted(fileInfo, fileID, corrID) assert.NoError(suite.T(), err, "got (%v) when marking file as completed", err) stableID := "TEST:000-1234-4567" - err = db.SetAccessionID(stableID, "testuser", "/testuser/TestSetAccessionID.c4gh", fmt.Sprintf("%x", decSha.Sum(nil))) + err = db.SetAccessionID(stableID, fileID) assert.NoError(suite.T(), err, "got (%v) when getting file archive information", err) } @@ -242,7 +242,7 @@ func (suite *DatabaseTests) TestCheckAccessionIDExists() { err = db.MarkCompleted(fileInfo, fileID, corrID) assert.NoError(suite.T(), err, "got (%v) when marking file as completed", err) stableID := "TEST:111-1234-4567" - err = db.SetAccessionID(stableID, "testuser", "/testuser/TestCheckAccessionIDExists.c4gh", fmt.Sprintf("%x", decSha.Sum(nil))) + err = db.SetAccessionID(stableID, fileID) assert.NoError(suite.T(), err, "got (%v) when getting file archive information", err) ok, err := db.CheckAccessionIDExists(stableID) From 124068dfcdc3525d6527f907166df5a4b4629e51 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Fri, 18 Aug 2023 13:04:49 +0200 Subject: [PATCH 18/34] Merge `backup` into `finalize` --- .../scripts/make_sda_credentials.sh | 4 +- .github/integration/sda-integration.yml | 16 +- ...kup_test.sh => 30_backup-finalize_test.sh} | 2 +- sda/cmd/backup/backup.go | 249 ------------------ sda/cmd/backup/backup.md | 159 ----------- sda/cmd/backup/backup_test.go | 34 --- sda/cmd/finalize/finalize.go | 156 +++++++++-- sda/internal/config/config.go | 59 ++--- 8 files changed, 164 insertions(+), 515 deletions(-) rename .github/integration/tests/sda/{30_backup_test.sh => 30_backup-finalize_test.sh} (98%) delete mode 100644 sda/cmd/backup/backup.go delete mode 100644 sda/cmd/backup/backup.md delete mode 100644 sda/cmd/backup/backup_test.go diff --git a/.github/integration/scripts/make_sda_credentials.sh b/.github/integration/scripts/make_sda_credentials.sh index 5dee949ff..78c21c417 100644 --- a/.github/integration/scripts/make_sda_credentials.sh +++ b/.github/integration/scripts/make_sda_credentials.sh @@ -4,10 +4,10 @@ set -e apt-get -o DPkg::Lock::Timeout=60 update > /dev/null apt-get -o DPkg::Lock::Timeout=60 install -y curl jq openssl postgresql-client >/dev/null -for n in backup download finalize inbox ingest mapper sync verify; do +for n in download finalize inbox ingest mapper sync verify; do echo "creating credentials for: $n" - if [ "$n" = inbox ] || [ "$n" = backup ]; then + if [ "$n" = inbox ]; then psql -U postgres -h postgres -d sda -c "DROP ROLE IF EXISTS $n;" psql -U postgres -h postgres -d sda -c "CREATE ROLE $n;" psql -U postgres -h postgres -d sda -c "GRANT ingest TO $n;" diff --git a/.github/integration/sda-integration.yml b/.github/integration/sda-integration.yml index 135aeb9a1..ead597eb3 100644 --- a/.github/integration/sda-integration.yml +++ b/.github/integration/sda-integration.yml @@ -159,10 +159,10 @@ services: - ./sda/config.yaml:/config.yaml - shared:/shared - backup: + finalize: image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} - command: [ sda-backup ] - container_name: backup + command: [ sda-finalize ] + container_name: finalize depends_on: credentials: condition: service_completed_successfully @@ -173,12 +173,12 @@ services: rabbitmq: condition: service_healthy environment: - - BROKER_PASSWORD=backup - - BROKER_USER=backup + - BROKER_PASSWORD=finalize + - BROKER_USER=finalize - BROKER_QUEUE=accession - BROKER_ROUTINGKEY=completed - - DB_PASSWORD=backup - - DB_USER=backup + - DB_PASSWORD=finalize + - DB_USER=finalize restart: always volumes: - ./sda/config.yaml:/config.yaml @@ -218,7 +218,7 @@ services: depends_on: credentials: condition: service_completed_successfully - backup: + finalize: condition: service_started ingest: condition: service_started diff --git a/.github/integration/tests/sda/30_backup_test.sh b/.github/integration/tests/sda/30_backup-finalize_test.sh similarity index 98% rename from .github/integration/tests/sda/30_backup_test.sh rename to .github/integration/tests/sda/30_backup-finalize_test.sh index c07e4b949..37c3684da 100644 --- a/.github/integration/tests/sda/30_backup_test.sh +++ b/.github/integration/tests/sda/30_backup-finalize_test.sh @@ -33,7 +33,7 @@ while [ $i -le "$(curl -su guest:guest http://rabbitmq:15672/api/queues/sda/veri --arg type accession \ --arg user "$user" \ --arg filepath "$filepath" \ - --arg accession_id "EGAF7490000000$I" \ + --arg accession_id "EGAF7490000000$i" \ --argjson decrypted_checksums "$decrypted_checksums" \ '$ARGS.named|@base64' ) diff --git a/sda/cmd/backup/backup.go b/sda/cmd/backup/backup.go deleted file mode 100644 index cf08f7873..000000000 --- a/sda/cmd/backup/backup.go +++ /dev/null @@ -1,249 +0,0 @@ -// The backup command accepts messages with accessionIDs for -// ingested files and copies them to the second storage. -package main - -import ( - "encoding/json" - "fmt" - "io" - - "github.com/neicnordic/crypt4gh/model/headers" - "github.com/neicnordic/sensitive-data-archive/internal/broker" - "github.com/neicnordic/sensitive-data-archive/internal/config" - "github.com/neicnordic/sensitive-data-archive/internal/database" - "github.com/neicnordic/sensitive-data-archive/internal/schema" - "github.com/neicnordic/sensitive-data-archive/internal/storage" - "golang.org/x/crypto/chacha20poly1305" - - log "github.com/sirupsen/logrus" -) - -func main() { - forever := make(chan bool) - conf, err := config.NewConfig("backup") - if err != nil { - log.Fatal(err) - } - mq, err := broker.NewMQ(conf.Broker) - if err != nil { - log.Fatal(err) - } - db, err := database.NewSDAdb(conf.Database) - if err != nil { - log.Fatal(err) - } - backupStorage, err := storage.NewBackend(conf.Backup) - if err != nil { - log.Fatal(err) - } - archive, err := storage.NewBackend(conf.Archive) - if err != nil { - log.Fatal(err) - } - - // we don't need crypt4gh keys if copyheader disabled - var key *[32]byte - var publicKey *[32]byte - if config.CopyHeader() { - key, err = config.GetC4GHKey() - if err != nil { - log.Fatal(err) - } - - publicKey, err = config.GetC4GHPublicKey() - if err != nil { - log.Fatal(err) - } - } - - defer mq.Channel.Close() - defer mq.Connection.Close() - defer db.Close() - - go func() { - connError := mq.ConnectionWatcher() - log.Error(connError) - forever <- false - }() - - go func() { - connError := mq.ChannelWatcher() - log.Error(connError) - forever <- false - }() - - log.Info("Starting backup service") - var message schema.IngestionAccession - - go func() { - messages, err := mq.GetMessages(conf.Broker.Queue) - if err != nil { - log.Fatal(err) - } - for delivered := range messages { - log.Debugf("Received a message (corr-id: %s, message: %s)", delivered.CorrelationId, delivered.Body) - err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-accession.json", conf.Broker.SchemasPath), delivered.Body) - if err != nil { - log.Errorf("validation of incoming message (ingestion-accession) failed, reason: %v ", err) - - continue - } - - // we unmarshal the message in the validation step so this is safe to do - _ = json.Unmarshal(delivered.Body, &message) - // Extract the sha256 from the message and use it for the database - var checksumSha256 string - for _, checksum := range message.DecryptedChecksums { - if checksum.Type == "sha256" { - checksumSha256 = checksum.Value - } - } - - var filePath string - var fileSize int - if filePath, fileSize, err = db.GetArchived(message.User, message.FilePath, checksumSha256); err != nil { - log.Errorf("failed to get file archive information, reason: %v", err) - - // nack the message but requeue until we fixed the SQL retry. - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%v)", err) - } - - continue - } - - log.Debug("Backup initiated") - // Get size on disk, will also give some time for the file to - // appear if it has not already - diskFileSize, err := archive.GetFileSize(filePath) - if err != nil { - log.Errorf("failed to get size info for archived file, reason: %v", err) - if e := delivered.Nack(false, true); e != nil { - log.Errorf("failed to Nack message, reason: (%v)", err) - } - - continue - } - - if diskFileSize != int64(fileSize) { - log.Errorf("file size in archive does not match database for archive file") - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%v)", err) - } - - continue - } - - file, err := archive.NewFileReader(filePath) - if err != nil { - log.Errorf("failed to open archived file, reason: %v", err) - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%v)", err) - } - - continue - } - - // If the copy header is enabled, use the actual filepath to make backup - // This will be used in the BigPicture backup, enabling for ingestion of the file - if config.CopyHeader() { - filePath = message.FilePath - } - - dest, err := backupStorage.NewFileWriter(filePath) - if err != nil { - log.Errorf("failed to open backup file for writing, reason: %v", err) - //FIXME: should it retry? - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%v)", err) - } - - continue - } - - fileID, err := db.GetFileID(delivered.CorrelationId) - if err != nil { - log.Errorf("failed to get ID for file, reason: %v", err) - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%v)", err) - } - - continue - } - - // Check if the header is needed - // nolint:nestif - if config.CopyHeader() { - // Get the header from db - header, err := db.GetHeader(fileID) - if err != nil { - log.Errorf("failed to get header for archived file, reason: %v", err) - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%v)", err) - } - - continue - } - - // Reencrypt header - log.Debug("Reencrypt header") - pubkeyList := [][chacha20poly1305.KeySize]byte{} - pubkeyList = append(pubkeyList, *publicKey) - newHeader, err := headers.ReEncryptHeader(header, *key, pubkeyList) - if err != nil { - log.Errorf("failed to reencrypt the header, reason: %v)", err) - - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%v)", err) - } - } - - // write header to destination file - _, err = dest.Write(newHeader) - if err != nil { - log.Errorf("failed to write header to file, reason: %v)", err) - - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%v)", err) - } - } - } - - // Copy the file and check is sizes match - copiedSize, err := io.Copy(dest, file) - if err != nil || copiedSize != int64(fileSize) { - log.Errorf("failed to copy file, reason: %v)", err) - //FIXME: should it retry? - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%v)", err) - } - - continue - } - - file.Close() - dest.Close() - - // Mark file as "backed up" - if err := db.UpdateFileStatus(fileID, "backed up", delivered.CorrelationId, message.User, string(delivered.Body)); err != nil { - log.Errorf("MarkCompleted failed, reason: (%v)", err) - - continue - // this should really be hadled by the DB retry mechanism - } - - if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, conf.Broker.RoutingKey, delivered.Body); err != nil { - // TODO fix resend mechanism - log.Errorf("failed to publish message, reason: (%v)", err) - - continue - } - - if err := delivered.Ack(false); err != nil { - log.Errorf("failed to Ack message, reason: (%v)", err) - } - } - }() - - <-forever -} diff --git a/sda/cmd/backup/backup.md b/sda/cmd/backup/backup.md deleted file mode 100644 index bb55da27d..000000000 --- a/sda/cmd/backup/backup.md +++ /dev/null @@ -1,159 +0,0 @@ -# sda-pipeline: backup - -Moves data to backup storage and optionally merges it with the encryption header. - -## Configuration - -There are a number of options that can be set for the backup service. -These settings can be set by mounting a yaml-file at `/config.yaml` with settings. - -ex. - -```yaml -log: - level: "debug" - format: "json" -``` - -They may also be set using environment variables like: - -```bash -export LOG_LEVEL="debug" -export LOG_FORMAT="json" -``` - -### Backup specific settings - -- `BACKUP_COPYHEADER`: if `true`, the backup service will reencrypt and add headers to the backup files. - -#### Keyfile settings - -These settings control which crypt4gh keyfile is loaded. -These settings are only needed is `copyheader` is `true`. - -- `C4GH_FILEPATH`: path to the crypt4gh keyfile -- `C4GH_PASSPHRASE`: pass phrase to unlock the keyfile -- `C4GH_BACKUPPUBKEY`: path to the crypt4gh public key to use for reencrypting file headers. - -### RabbitMQ broker settings - -These settings control how backup connects to the RabbitMQ message broker. - -- `BROKER_HOST`: hostname of the rabbitmq server -- `BROKER_PORT`: rabbitmq broker port (commonly `5671` with TLS and `5672` without) -- `BROKER_QUEUE`: message queue to read messages from (commonly `backup`) -- `BROKER_ROUTINGKEY`: message queue to write success messages to (commonly `completed`) -- `BROKER_USER`: username to connect to rabbitmq -- `BROKER_PASSWORD`: password to connect to rabbitmq -- `BROKER_PREFETCHCOUNT`: Number of messages to pull from the message server at the time (default to 2) - -### PostgreSQL Database settings - -- `DB_HOST`: hostname for the postgresql database -- `DB_PORT`: database port (commonly 5432) -- `DB_USER`: username for the database -- `DB_PASSWORD`: password for the database -- `DB_DATABASE`: database name -- `DB_SSLMODE`: The TLS encryption policy to use for database connections. - Valid options are: - - `disable` - - `allow` - - `prefer` - - `require` - - `verify-ca` - - `verify-full` - - More information is available - [in the postgresql documentation](https://www.postgresql.org/docs/current/libpq-ssl.html#LIBPQ-SSL-PROTECTION) - - Note that if `DB_SSLMODE` is set to anything but `disable`, then `DB_CACERT` needs to be set, - and if set to `verify-full`, then `DB_CLIENTCERT`, and `DB_CLIENTKEY` must also be set - -- `DB_CLIENTKEY`: key-file for the database client certificate -- `DB_CLIENTCERT`: database client certificate file -- `DB_CACERT`: Certificate Authority (CA) certificate for the database to use - -### Storage settings - -Storage backend is defined by the `ARCHIVE_TYPE`, and `BACKUP_TYPE` variables. -Valid values for these options are `S3` or `POSIX` -(Defaults to `POSIX` on unknown values). - -The value of these variables define what other variables are read. -The same variables are available for all storage types, differing by prefix (`ARCHIVE_`, or `BACKUP_`) - -if `*_TYPE` is `S3` then the following variables are available: - -- `*_URL`: URL to the S3 system -- `*_ACCESSKEY`: The S3 access and secret key are used to authenticate to S3, -[more info at AWS](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) -- `*_SECRETKEY`: The S3 access and secret key are used to authenticate to S3, -[more info at AWS](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) -- `*_BUCKET`: The S3 bucket to use as the storage root -- `*_PORT`: S3 connection port (default: `443`) -- `*_REGION`: S3 region (default: `us-east-1`) -- `*_CHUNKSIZE`: S3 chunk size for multipart uploads. -- `*_CACERT`: Certificate Authority (CA) certificate for the storage system, tjhis is only needed if the S3 server has a certificate signed by a private entity - -and if `*_TYPE` is `POSIX`: - -- `*_LOCATION`: POSIX path to use as storage root - -### Logging settings - -- `LOG_FORMAT` can be set to “json” to get logs in json format. All other values result in text logging -- `LOG_LEVEL` can be set to one of the following, in increasing order of severity: - - `trace` - - `debug` - - `info` - - `warn` (or `warning`) - - `error` - - `fatal` - - `panic` - -## Service Description - -The backup service copies files from the archive storage to backup storage. If a public key is supplied and the copyHeader option is enabled the header will be re-encrypted and attached to the file before writing it to backup storage. - -When running, backup reads messages from the configured RabbitMQ queue (default "backup"). -For each message, these steps are taken (if not otherwise noted, errors halts progress, the message is Nack'ed, and the service moves on to the next message): - -1. The message is validated as valid JSON that matches either the "ingestion-completion" or "ingestion-accession" schema (based on configuration). -If the message can’t be validated it is discarded with an error message in the logs. - -1. The file path and file size is fetched from the database. - 1. In case the service is configured to copy headers, the path is replaced by the one of the incoming message and it is the original location where the file was uploaded in the inbox. - -1. The file size on disk is requested from the storage system. - -1. The database file size is compared against the disk file size. - -1. A file reader is created for the archive storage file, and a file writer is created for the backup storage file. - -1. If the service is configured to copy headers: - - 1. The header is read from the database. - On error, the error is written to the logs, but the message continues processing. - - 1. The header is decrypted. - If this causes an error, the error is written to the logs, the message is Nack'ed, but message processing continues. - - 1. The header is reencrypted. - If this causes an error, the error is written to the logs, the message is Nack'ed, but message processing continues. - - 1. The header is written to the backup file writer. - On error, the error is written to the logs, but the message continues processing. - -1. The file data is copied from the archive file reader to the backup file writer. - -1. A completed message is sent to RabbitMQ, if this fails a message is written to the logs, and the message is neither nack'ed nor ack'ed. - -1. The message is Ack'ed. - -## Communication - -- Backup reads messages from one rabbitmq queue (default `backup`) -- Backup writes messages to one rabbitmq queue (default `completed`) -- Backup optionally reads encryption headers from the database and can not be started without a database connection. -This is done using the `GetArchived`, and `GetHeaderForStableID` functions. -- Backup reads data from archive storage and writes data to backup storage. diff --git a/sda/cmd/backup/backup_test.go b/sda/cmd/backup/backup_test.go deleted file mode 100644 index 2fa0bf9ee..000000000 --- a/sda/cmd/backup/backup_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package main - -import ( - "testing" - - "github.com/spf13/viper" - "github.com/stretchr/testify/suite" -) - -type TestSuite struct { - suite.Suite -} - -func TestBackupTestSuite(t *testing.T) { - suite.Run(t, new(TestSuite)) -} - -func (suite *TestSuite) SetupTest() { - viper.Set("log.level", "debug") - viper.Set("archive.location", "../../dev_utils") - viper.Set("backup.location", "../../dev_utils") - - viper.Set("broker.host", "test") - viper.Set("broker.port", 123) - viper.Set("broker.user", "test") - viper.Set("broker.password", "test") - viper.Set("broker.queue", "test") - viper.Set("broker.routingkey", "test") - viper.Set("db.host", "test") - viper.Set("db.port", 123) - viper.Set("db.user", "test") - viper.Set("db.password", "test") - viper.Set("db.database", "test") -} diff --git a/sda/cmd/finalize/finalize.go b/sda/cmd/finalize/finalize.go index 7447ac438..4b47c6efa 100644 --- a/sda/cmd/finalize/finalize.go +++ b/sda/cmd/finalize/finalize.go @@ -5,18 +5,29 @@ package main import ( "encoding/json" "fmt" + "io" + "github.com/neicnordic/crypt4gh/model/headers" "github.com/neicnordic/sensitive-data-archive/internal/broker" "github.com/neicnordic/sensitive-data-archive/internal/config" "github.com/neicnordic/sensitive-data-archive/internal/database" "github.com/neicnordic/sensitive-data-archive/internal/schema" + "github.com/neicnordic/sensitive-data-archive/internal/storage" + "golang.org/x/crypto/chacha20poly1305" + amqp "github.com/rabbitmq/amqp091-go" log "github.com/sirupsen/logrus" ) +var db *database.SDAdb +var archive, backup storage.Backend +var conf *config.Config +var err error +var message schema.IngestionAccession + func main() { forever := make(chan bool) - conf, err := config.NewConfig("finalize") + conf, err = config.NewConfig("finalize") if err != nil { log.Fatal(err) } @@ -24,11 +35,23 @@ func main() { if err != nil { log.Fatal(err) } - db, err := database.NewSDAdb(conf.Database) + db, err = database.NewSDAdb(conf.Database) if err != nil { log.Fatal(err) } + if conf.Backup.Type != "" && conf.Archive.Type != "" { + log.Debugln("initiating storage backends") + backup, err = storage.NewBackend(conf.Backup) + if err != nil { + log.Fatal(err) + } + archive, err = storage.NewBackend(conf.Archive) + if err != nil { + log.Fatal(err) + } + } + defer mq.Channel.Close() defer mq.Connection.Close() defer db.Close() @@ -46,8 +69,6 @@ func main() { }() log.Info("Starting finalize service") - var message schema.IngestionAccession - go func() { messages, err := mq.GetMessages(conf.Broker.Queue) if err != nil { @@ -81,20 +102,6 @@ func main() { continue } - c := schema.IngestionCompletion{ - User: message.User, - FilePath: message.FilePath, - AccessionID: message.AccessionID, - DecryptedChecksums: message.DecryptedChecksums, - } - completeMsg, _ := json.Marshal(&c) - err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-completion.json", conf.Broker.SchemasPath), completeMsg) - if err != nil { - log.Errorf("Validation of outgoing message failed, reason: (%v)", err) - - continue - } - accessionIDExists, err := db.CheckAccessionIDExists(message.AccessionID) if err != nil { log.Errorf("CheckAccessionIdExists failed, reason: %v ", err) @@ -127,6 +134,17 @@ func main() { continue } + if conf.Backup.Type != "" && conf.Archive.Type != "" { + if err = backupFile(delivered); err != nil { + log.Errorf("Failed to backup file with corrID: %v, reason: %v", delivered.CorrelationId, err) + if err := delivered.Nack(false, true); err != nil { + log.Errorf("failed to Nack message, reason: (%v)", err) + } + + continue + } + } + fileID, err := db.GetFileID(delivered.CorrelationId) if err != nil { log.Errorf("failed to get ID for file, reason: %v", err) @@ -185,3 +203,105 @@ func main() { <-forever } + +func backupFile(delivered amqp.Delivery) error { + log.Debug("Backup initiated") + var checksumSha256 string + var key *[32]byte + var publicKey *[32]byte + + for _, checksum := range message.DecryptedChecksums { + if checksum.Type == "sha256" { + checksumSha256 = checksum.Value + } + } + + filePath, fileSize, err := db.GetArchived(message.User, message.FilePath, checksumSha256) + if err != nil { + return fmt.Errorf("failed to get file archive information, reason: %v", err) + } + + // If the copy header is enabled, use the actual filepath to make backup + // This will be used in the BigPicture backup, enabling for ingestion of the file + if config.CopyHeader() { + key, err = config.GetC4GHKey() + if err != nil { + log.Fatal(err) + } + + publicKey, err = config.GetC4GHPublicKey() + if err != nil { + log.Fatal(err) + } + + filePath = message.FilePath + } + + // Get size on disk, will also give some time for the file to + // appear if it has not already + diskFileSize, err := archive.GetFileSize(filePath) + if err != nil { + return fmt.Errorf("failed to get size info for archived file, reason: %v", err) + } + + if diskFileSize != int64(fileSize) { + return fmt.Errorf("file size in archive does not match database for archive file") + } + + file, err := archive.NewFileReader(filePath) + if err != nil { + return fmt.Errorf("failed to open archived file, reason: %v", err) + } + + dest, err := backup.NewFileWriter(filePath) + if err != nil { + return fmt.Errorf("failed to open backup file for writing, reason: %v", err) + } + + fileID, err := db.GetFileID(delivered.CorrelationId) + if err != nil { + return fmt.Errorf("failed to get ID for file, reason: %v", err) + } + + // Check if the header is needed + if config.CopyHeader() { + // Get the header from db + header, err := db.GetHeader(fileID) + if err != nil { + return fmt.Errorf("failed to get header for archived file, reason: %v", err) + } + + // Reencrypt header + log.Debug("Reencrypt header") + pubkeyList := [][chacha20poly1305.KeySize]byte{} + pubkeyList = append(pubkeyList, *publicKey) + newHeader, err := headers.ReEncryptHeader(header, *key, pubkeyList) + if err != nil { + return fmt.Errorf("failed to reencrypt the header, reason: %v)", err) + } + + // write header to destination file + _, err = dest.Write(newHeader) + if err != nil { + log.Errorf("failed to write header to file, reason: %v)", err) + } + } + + // Copy the file and check is sizes match + copiedSize, err := io.Copy(dest, file) + if err != nil || copiedSize != int64(fileSize) { + log.Errorf("failed to copy file, reason: %v)", err) + } + + file.Close() + dest.Close() + + // Mark file as "backed up" + if err := db.UpdateFileStatus(fileID, "backed up", delivered.CorrelationId, message.User, string(delivered.Body)); err != nil { + return fmt.Errorf("MarkCompleted failed, reason: (%v)", err) + } + + log.Debug("Backup completed") + + return nil +} diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index 2e7781b04..c585b24b0 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -131,7 +131,7 @@ func NewConfig(app string) (*Config, error) { "db.password", "db.database", } - case "backup": + case "ingest": requiredConfVars = []string{ "broker.host", "broker.port", @@ -155,17 +155,15 @@ func NewConfig(app string) (*Config, error) { return nil, fmt.Errorf("archive.type not set") } - switch viper.GetString("backup.type") { + switch viper.GetString("inbox.type") { case S3: - requiredConfVars = append(requiredConfVars, []string{"backup.url", "backup.accesskey", "backup.secretkey", "backup.bucket"}...) + requiredConfVars = append(requiredConfVars, []string{"inbox.url", "inbox.accesskey", "inbox.secretkey", "inbox.bucket"}...) case POSIX: - requiredConfVars = append(requiredConfVars, []string{"backup.location"}...) - case SFTP: - requiredConfVars = append(requiredConfVars, []string{"backup.sftp.host", "backup.sftp.port", "backup.sftp.userName", "backup.sftp.pemKeyPath", "backup.sftp.pemKeyPass"}...) + requiredConfVars = append(requiredConfVars, []string{"inbox.location"}...) default: - return nil, fmt.Errorf("backup.type not set") + return nil, fmt.Errorf("inbox.type not set") } - case "ingest": + case "finalize": requiredConfVars = []string{ "broker.host", "broker.port", @@ -185,34 +183,17 @@ func NewConfig(app string) (*Config, error) { requiredConfVars = append(requiredConfVars, []string{"archive.url", "archive.accesskey", "archive.secretkey", "archive.bucket"}...) case POSIX: requiredConfVars = append(requiredConfVars, []string{"archive.location"}...) - default: - return nil, fmt.Errorf("archive.type not set") } - switch viper.GetString("inbox.type") { + switch viper.GetString("backup.type") { case S3: - requiredConfVars = append(requiredConfVars, []string{"inbox.url", "inbox.accesskey", "inbox.secretkey", "inbox.bucket"}...) + requiredConfVars = append(requiredConfVars, []string{"backup.url", "backup.accesskey", "backup.secretkey", "backup.bucket"}...) case POSIX: - requiredConfVars = append(requiredConfVars, []string{"inbox.location"}...) - default: - return nil, fmt.Errorf("inbox.type not set") - } - case "finalize": - requiredConfVars = []string{ - "broker.host", - "broker.port", - "broker.user", - "broker.password", - "broker.queue", - "broker.routingkey", - "db.host", - "db.port", - "db.user", - "db.password", - "db.database", + requiredConfVars = append(requiredConfVars, []string{"backup.location"}...) + case SFTP: + requiredConfVars = append(requiredConfVars, []string{"backup.sftp.host", "backup.sftp.port", "backup.sftp.userName", "backup.sftp.pemKeyPath", "backup.sftp.pemKeyPass"}...) } case "intercept": - // Intercept does not require these extra settings requiredConfVars = []string{ "broker.host", "broker.port", @@ -354,22 +335,12 @@ func NewConfig(app string) (*Config, error) { if err != nil { return nil, err } - case "backup": - c.configArchive() - c.configBackup() - - err := c.configBroker() - if err != nil { - return nil, err - } - - err = c.configDatabase() - if err != nil { - return nil, err + case "finalize": + if viper.GetString("archive.type") != "" && viper.GetString("backup.type") != "" { + c.configArchive() + c.configBackup() } - c.configSchemas() - case "finalize": err := c.configBroker() if err != nil { return nil, err From 3a76bb4e4d4e9802caf0e1fb9d1167ac2791577c Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Fri, 18 Aug 2023 13:05:33 +0200 Subject: [PATCH 19/34] Update GO mod --- sda/go.mod | 2 +- sda/go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/sda/go.mod b/sda/go.mod index a3655481f..1bafa65bb 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -4,7 +4,7 @@ go 1.20 require ( github.com/DATA-DOG/go-sqlmock v1.5.0 - github.com/aws/aws-sdk-go v1.44.325 + github.com/aws/aws-sdk-go v1.44.326 github.com/golang-jwt/jwt/v4 v4.5.0 github.com/google/uuid v1.1.2 github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb diff --git a/sda/go.sum b/sda/go.sum index 84d6186cb..522bf46da 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -51,6 +51,8 @@ github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEV github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= github.com/aws/aws-sdk-go v1.44.325 h1:jF/L99fJSq/BfiLmUOflO/aM+LwcqBm0Fe/qTK5xxuI= github.com/aws/aws-sdk-go v1.44.325/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.326 h1:/6xD/9mKZ2RMTDfbhh9qCxw+CaTbJRvfHJ/NHPFbI38= +github.com/aws/aws-sdk-go v1.44.326/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= From 31f42eb66ef3daa02219f3b769368d20e34019b8 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 21 Aug 2023 12:36:45 +0200 Subject: [PATCH 20/34] Merge `mapper` from sda-pipeline --- .github/integration/sda-integration.yml | 26 +++ .../integration/tests/sda/40_mapper_test.sh | 156 +++++++++++++++ sda/cmd/mapper/mapper.go | 185 ++++++++++++++++++ sda/cmd/mapper/mapper.md | 108 ++++++++++ sda/cmd/mapper/mapper_test.go | 20 ++ sda/internal/database/db_functions.go | 116 +++++++++++ sda/internal/database/db_functions_test.go | 83 ++++++++ sda/internal/schema/schema.go | 14 ++ 8 files changed, 708 insertions(+) create mode 100644 .github/integration/tests/sda/40_mapper_test.sh create mode 100644 sda/cmd/mapper/mapper.go create mode 100644 sda/cmd/mapper/mapper.md create mode 100644 sda/cmd/mapper/mapper_test.go diff --git a/.github/integration/sda-integration.yml b/.github/integration/sda-integration.yml index ead597eb3..b8af281eb 100644 --- a/.github/integration/sda-integration.yml +++ b/.github/integration/sda-integration.yml @@ -184,6 +184,30 @@ services: - ./sda/config.yaml:/config.yaml - shared:/shared + mapper: + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + command: [ sda-mapper ] + container_name: mapper + depends_on: + credentials: + condition: service_completed_successfully + minio: + condition: service_healthy + postgres: + condition: service_healthy + rabbitmq: + condition: service_healthy + environment: + - BROKER_PASSWORD=mapper + - BROKER_USER=mapper + - BROKER_QUEUE=mappings + - DB_PASSWORD=mapper + - DB_USER=mapper + restart: always + volumes: + - ./sda/config.yaml:/config.yaml + - shared:/shared + oidc: container_name: oidc command: @@ -222,6 +246,8 @@ services: condition: service_started ingest: condition: service_started + mapper: + condition: service_started s3inbox: condition: service_started verify: diff --git a/.github/integration/tests/sda/40_mapper_test.sh b/.github/integration/tests/sda/40_mapper_test.sh new file mode 100644 index 000000000..7de50977f --- /dev/null +++ b/.github/integration/tests/sda/40_mapper_test.sh @@ -0,0 +1,156 @@ +#!/bin/bash +set -e + +cd shared || true + +## map files to dataset +properties=$( + jq -c -n \ + --argjson delivery_mode 2 \ + --arg content_encoding UTF-8 \ + --arg content_type application/json \ + '$ARGS.named' +) + +mappings=$( + jq -c -n \ + '$ARGS.positional' \ + --args "EGAF74900000001" \ + --args "EGAF74900000002" +) + +mapping_payload=$( + jq -r -c -n \ + --arg type mapping \ + --arg dataset_id EGAD74900000101 \ + --argjson accession_ids "$mappings" \ + '$ARGS.named|@base64' +) + +mapping_body=$( + jq -c -n \ + --arg vhost test \ + --arg name sda \ + --argjson properties "$properties" \ + --arg routing_key "mappings" \ + --arg payload_encoding base64 \ + --arg payload "$mapping_payload" \ + '$ARGS.named' +) + +curl -s -u guest:guest "http://rabbitmq:15672/api/exchanges/sda/sda/publish" \ + -H 'Content-Type: application/json;charset=UTF-8' \ + -d "$mapping_body" + +# check DB for dataset contents +RETRY_TIMES=0 +until [ "$(psql -U postgres -h postgres -d sda -At -c "select count(id) from sda.file_dataset where dataset_id = (select id from sda.datasets where stable_id = 'EGAD74900000101')")" -eq 2 ]; do + echo "waiting for mapper to complete" + RETRY_TIMES=$((RETRY_TIMES + 1)) + if [ "$RETRY_TIMES" -eq 30 ]; then + echo "::error::Time out while waiting for dataset to be mapped" + exit 1 + fi + sleep 2 +done + +## check that files has been removed form the inbox +for file in NA12878.bam.c4gh NA12878_20k_b37.bam.c4gh; do + result=$(s3cmd -c direct ls s3://inbox/test_dummy.org/"$file") + if [ "$result" != "" ]; then + echo "Failed to remove $file from inbox" + exit 1 + fi +done + +until [ "$(psql -U postgres -h postgres -d sda -At -c "select event from sda.file_event_log where file_id = (select id from sda.files where stable_id = 'EGAF74900000002') order by started_at DESC LIMIT 1")" = "ready" ]; do + echo "waiting for files be ready" + RETRY_TIMES=$((RETRY_TIMES + 1)) + if [ "$RETRY_TIMES" -eq 30 ]; then + echo "::error::Time out while waiting for files to be ready" + exit 1 + fi + sleep 2 +done + +until [ "$(psql -U postgres -h postgres -d sda -At -c "select event from sda.dataset_event_log where dataset_id = 'EGAD74900000101' order by event_date DESC LIMIT 1")" = "registered" ]; do + echo "waiting for dataset be registered" + RETRY_TIMES=$((RETRY_TIMES + 1)) + if [ "$RETRY_TIMES" -eq 30 ]; then + echo "::error::Time out while waiting for dataset to be registered" + exit 1 + fi + sleep 2 +done + +echo "dataset mapped successfully" + +## release dataset +release_payload=$( + jq -r -c -n \ + --arg type release \ + --arg dataset_id EGAD74900000101 \ + '$ARGS.named' +) + +release_body=$( + jq -c -n \ + --arg vhost test \ + --arg name sda \ + --argjson properties "$properties" \ + --arg routing_key "mappings" \ + --arg payload "$release_payload" \ + --arg payload_encoding string \ + '$ARGS.named' +) + +curl -s -u guest:guest "http://rabbitmq:15672/api/exchanges/sda/sda/publish" \ + -H 'Content-Type: application/json;charset=UTF-8' \ + -d "$release_body" + +until [ "$(psql -U postgres -h postgres -d sda -At -c "select event from sda.dataset_event_log where dataset_id = 'EGAD74900000101' order by event_date DESC LIMIT 1")" = "released" ]; do + echo "waiting for dataset be released" + RETRY_TIMES=$((RETRY_TIMES + 1)) + if [ "$RETRY_TIMES" -eq 30 ]; then + echo "::error::Time out while waiting for dataset to be released" + exit 1 + fi + sleep 2 +done + +echo "dataset released successfully" + +## deprecate dataset +deprecate_payload=$( + jq -r -c -n \ + --arg type deprecate \ + --arg dataset_id EGAD74900000101 \ + '$ARGS.named' +) + +deprecate_body=$( + jq -c -n \ + --arg vhost test \ + --arg name sda \ + --argjson properties "$properties" \ + --arg routing_key "mappings" \ + --arg payload "$deprecate_payload" \ + --arg payload_encoding string \ + '$ARGS.named' +) + +curl -s -u guest:guest "http://rabbitmq:15672/api/exchanges/sda/sda/publish" \ + -H 'Content-Type: application/json;charset=UTF-8' \ + -d "$deprecate_body" + +until [ "$(psql -U postgres -h postgres -d sda -At -c "select event from sda.dataset_event_log where dataset_id = 'EGAD74900000101' order by event_date DESC LIMIT 1")" = "deprecated" ]; do + echo "waiting for dataset be deprecated" + RETRY_TIMES=$((RETRY_TIMES + 1)) + if [ "$RETRY_TIMES" -eq 30 ]; then + echo "::error::Time out while waiting for dataset to be deprecated" + exit 1 + fi + sleep 2 +done + +echo "dataset deprecated successfully" \ No newline at end of file diff --git a/sda/cmd/mapper/mapper.go b/sda/cmd/mapper/mapper.go new file mode 100644 index 000000000..53c7f7042 --- /dev/null +++ b/sda/cmd/mapper/mapper.go @@ -0,0 +1,185 @@ +// The mapper service register mapping of accessionIDs +// (IDs for files) to datasetIDs. +package main + +import ( + "encoding/json" + "fmt" + + "github.com/neicnordic/sensitive-data-archive/internal/broker" + "github.com/neicnordic/sensitive-data-archive/internal/config" + "github.com/neicnordic/sensitive-data-archive/internal/database" + "github.com/neicnordic/sensitive-data-archive/internal/schema" + "github.com/neicnordic/sensitive-data-archive/internal/storage" + + log "github.com/sirupsen/logrus" +) + +func main() { + forever := make(chan bool) + conf, err := config.NewConfig("mapper") + if err != nil { + log.Fatal(err) + } + mq, err := broker.NewMQ(conf.Broker) + if err != nil { + log.Fatal(err) + } + db, err := database.NewSDAdb(conf.Database) + if err != nil { + log.Fatal(err) + } + inbox, err := storage.NewBackend(conf.Inbox) + if err != nil { + log.Fatal(err) + } + + defer mq.Channel.Close() + defer mq.Connection.Close() + defer db.Close() + + go func() { + connError := mq.ConnectionWatcher() + log.Error(connError) + forever <- false + }() + + go func() { + connError := mq.ChannelWatcher() + log.Error(connError) + forever <- false + }() + + log.Info("Starting mapper service") + var mappings schema.DatasetMapping + + go func() { + messages, err := mq.GetMessages(conf.Broker.Queue) + if err != nil { + log.Fatalf("Failed to get message from mq (error: %v)", err) + } + + for delivered := range messages { + log.Debugf("received a message: %s", delivered.Body) + schemaType, err := schemaFromDatasetOperation(delivered.Body) + if err != nil { + log.Errorf(err.Error()) + if err := delivered.Ack(false); err != nil { + log.Errorf("failed to ack message: %v", err) + } + if err := mq.SendMessage(delivered.CorrelationId, mq.Conf.Exchange, "error", delivered.Body); err != nil { + log.Errorf("failed to send error message: %v", err) + } + + continue + } + + err = schema.ValidateJSON(fmt.Sprintf("%s/%s.json", conf.Broker.SchemasPath, schemaType), delivered.Body) + if err != nil { + log.Errorf("validation of incoming message (%s) failed, reason: %v ", schemaType, err) + if err := delivered.Ack(false); err != nil { + log.Errorf("Failed acking canceled work, reason: %v", err) + } + + continue + } + + // we unmarshal the message in the validation step so this is safe to do + _ = json.Unmarshal(delivered.Body, &mappings) + + switch mappings.Type { + case "mapping": + log.Debug("Mapping type operation, mapping files to dataset") + if err := db.MapFilesToDataset(mappings.DatasetID, mappings.AccessionIDs); err != nil { + log.Errorf("failed to map files to dataset, reason: %v", err) + + // Nack message so the server gets notified that something is wrong and requeue the message + if err := delivered.Nack(false, true); err != nil { + log.Errorf("failed to Nack message, reason: (%v)", err) + } + + continue + } + + for _, aID := range mappings.AccessionIDs { + log.Debugf("Mapped file to dataset (corr-id: %s, datasetid: %s, accessionid: %s)", delivered.CorrelationId, mappings.DatasetID, aID) + filePath, err := db.GetInboxPath(aID) + if err != nil { + log.Errorf("failed to get inbox path for file with stable ID: %v", aID) + } + err = inbox.RemoveFile(filePath) + if err != nil { + log.Errorf("Remove file from inbox failed, reason: %v", err) + } + } + + if err := db.UpdateDatasetEvent(mappings.DatasetID, "registered", string(delivered.Body)); err != nil { + log.Errorf("failed to set dataset status for dataset: %v", mappings.DatasetID) + if err = delivered.Nack(false, false); err != nil { + log.Errorf("Failed to Nack message, reason: (%v)", err.Error()) + } + + continue + } + case "release": + log.Debug("Release type operation, marking dataset as released") + if err := db.UpdateDatasetEvent(mappings.DatasetID, "released", string(delivered.Body)); err != nil { + log.Errorf("failed to set dataset status for dataset: %v", mappings.DatasetID) + if err = delivered.Nack(false, false); err != nil { + log.Errorf("Failed to Nack message, reason: (%v)", err.Error()) + } + + continue + } + case "deprecate": + log.Debug("Deprecate type operation, marking dataset as deprecated") + if err := db.UpdateDatasetEvent(mappings.DatasetID, "deprecated", string(delivered.Body)); err != nil { + log.Errorf("failed to set dataset status for dataset: %v", mappings.DatasetID) + if err = delivered.Nack(false, false); err != nil { + log.Errorf("Failed to Nack message, reason: (%v)", err.Error()) + } + + continue + } + } + + if err := delivered.Ack(false); err != nil { + log.Errorf("failed to Ack message, reason: (%v)", err) + } + } + }() + + <-forever +} + +// schemaFromDatasetOperation returns the operation done with dataset +// supplied in body of the message +func schemaFromDatasetOperation(body []byte) (string, error) { + message := make(map[string]interface{}) + err := json.Unmarshal(body, &message) + if err != nil { + return "", err + } + + datasetMessageType, ok := message["type"] + if !ok { + return "", fmt.Errorf("malformed message, dataset message type is missing") + } + + datasetOpsType, ok := datasetMessageType.(string) + if !ok { + return "", fmt.Errorf("could not cast operation attribute to string") + } + + switch datasetOpsType { + case "mapping": + return "dataset-mapping", nil + case "release": + return "dataset-release", nil + case "deprecate": + return "dataset-deprecate", nil + default: + return "", fmt.Errorf("could not recognize mapping operation") + } + +} diff --git a/sda/cmd/mapper/mapper.md b/sda/cmd/mapper/mapper.md new file mode 100644 index 000000000..c34467b9b --- /dev/null +++ b/sda/cmd/mapper/mapper.md @@ -0,0 +1,108 @@ +# sda-pipeline: mapper + +The mapper service registers mapping of accessionIDs (stable ids for files) to datasetIDs. + +## Configuration + +There are a number of options that can be set for the mapper service. +These settings can be set by mounting a yaml-file at `/config.yaml` with settings. + +ex. +```yaml +log: + level: "debug" + format: "json" +``` +They may also be set using environment variables like: +```bash +export LOG_LEVEL="debug" +export LOG_FORMAT="json" +``` + +### RabbitMQ broker settings + +These settings control how mapper connects to the RabbitMQ message broker. + + - `BROKER_HOST`: hostname of the rabbitmq server + + - `BROKER_PORT`: rabbitmq broker port (commonly `5671` with TLS and `5672` without) + + - `BROKER_QUEUE`: message queue to read messages from (commonly `mapper`) + + - `BROKER_USER`: username to connect to rabbitmq + + - `BROKER_PASSWORD`: password to connect to rabbitmq + + - `BROKER_PREFETCHCOUNT`: Number of messages to pull from the message server at the time (default to 2) + +### PostgreSQL Database settings: + + - `DB_HOST`: hostname for the postgresql database + + - `DB_PORT`: database port (commonly 5432) + + - `DB_USER`: username for the database + + - `DB_PASSWORD`: password for the database + + - `DB_DATABASE`: database name + + - `DB_SSLMODE`: The TLS encryption policy to use for database connections. + Valid options are: + - `disable` + - `allow` + - `prefer` + - `require` + - `verify-ca` + - `verify-full` + + More information is available + [in the postgresql documentation](https://www.postgresql.org/docs/current/libpq-ssl.html#LIBPQ-SSL-PROTECTION) + + Note that if `DB_SSLMODE` is set to anything but `disable`, then `DB_CACERT` needs to be set, + and if set to `verify-full`, then `DB_CLIENTCERT`, and `DB_CLIENTKEY` must also be set + + - `DB_CLIENTKEY`: key-file for the database client certificate + + - `DB_CLIENTCERT`: database client certificate file + + - `DB_CACERT`: Certificate Authority (CA) certificate for the database to use + +### Logging settings: + + - `LOG_FORMAT` can be set to “json” to get logs in json format. + All other values result in text logging + + - `LOG_LEVEL` can be set to one of the following, in increasing order of severity: + - `trace` + - `debug` + - `info` + - `warn` (or `warning`) + - `error` + - `fatal` + - `panic` + +## Service Description + +The mapper service maps file accessionIDs to datasetIDs. + +When running, mapper reads messages from the configured RabbitMQ queue (default: "mappings"). +For each message, these steps are taken (if not otherwise noted, errors halt progress and the service moves on to the next message): + +1. The message is validated as valid JSON that matches the "dataset-mapping" schema (defined in sda-common). +If the message can’t be validated it is discarded with an error message in the logs. + +1. AccessionIDs from the message are mapped to a datasetID (also in the message) in the database. +On error the service sleeps for up to 5 minutes to allow for database recovery, after 5 minutes the message is Nacked, re-queued and an error message is written to the logs. + +1. The uploaded files for each AccessionID is removed from the inbox +If this fails an error will be written to the logs. + +2. The RabbitMQ message is Ack'ed. + + +## Communication + + - Mapper reads messages from one rabbitmq queue (default `mappings`). + + - Mapper maps files to datasets in the database using the `MapFilesToDataset` function. diff --git a/sda/cmd/mapper/mapper_test.go b/sda/cmd/mapper/mapper_test.go new file mode 100644 index 000000000..0f8a2819d --- /dev/null +++ b/sda/cmd/mapper/mapper_test.go @@ -0,0 +1,20 @@ +package main + +import ( + "testing" + + "github.com/spf13/viper" + "github.com/stretchr/testify/suite" +) + +type TestSuite struct { + suite.Suite +} + +func TestConfigTestSuite(t *testing.T) { + suite.Run(t, new(TestSuite)) +} + +func (suite *TestSuite) SetupTest() { + viper.Set("log.level", "debug") +} diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 3d43d96ec..287dc1bfa 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -8,6 +8,8 @@ import ( "fmt" "math" "time" + + log "github.com/sirupsen/logrus" ) // RegisterFile inserts a file in the database, along with a "registered" log @@ -352,3 +354,117 @@ func (dbs *SDAdb) setAccessionID(accessionID, fileID string) error { return nil } + +// MapFilesToDataset maps a set of files to a dataset in the database +func (dbs *SDAdb) MapFilesToDataset(datasetID string, accessionIDs []string) error { + var err error + // 2, 4, 8, 16, 32 seconds between each retry event. + for count := 1; count <= RetryTimes; count++ { + err = dbs.mapFilesToDataset(datasetID, accessionIDs) + if err == nil { + break + } + time.Sleep(time.Duration(math.Pow(2, float64(count))) * time.Second) + } + + return err +} +func (dbs *SDAdb) mapFilesToDataset(datasetID string, accessionIDs []string) error { + dbs.checkAndReconnectIfNeeded() + + const getID = "SELECT id FROM sda.files WHERE stable_id = $1;" + const dataset = "INSERT INTO sda.datasets (stable_id) VALUES ($1) ON CONFLICT DO NOTHING;" + const mapping = "INSERT INTO sda.file_dataset (file_id, dataset_id) SELECT $1, id FROM sda.datasets WHERE stable_id = $2 ON CONFLICT DO NOTHING;" + var fileID string + + db := dbs.DB + _, err := db.Exec(dataset, datasetID) + if err != nil { + return err + } + + transaction, _ := db.Begin() + for _, accessionID := range accessionIDs { + err := db.QueryRow(getID, accessionID).Scan(&fileID) + if err != nil { + log.Errorf("something went wrong with the DB query: %s", err) + if err := transaction.Rollback(); err != nil { + log.Errorf("failed to rollback the transaction: %s", err) + } + + return err + } + _, err = transaction.Exec(mapping, fileID, datasetID) + if err != nil { + log.Errorf("something went wrong with the DB transaction: %s", err) + if err := transaction.Rollback(); err != nil { + log.Errorf("failed to rollback the transaction: %s", err) + } + + return err + } + } + + return transaction.Commit() +} + +// GetInboxPath retrieves the submission_fie_path for a file with a given accessionID +func (dbs *SDAdb) GetInboxPath(stableID string) (string, error) { + var ( + err error + count int + inboxPath string + ) + + for count == 0 || (err != nil && count < RetryTimes) { + inboxPath, err = dbs.getInboxPath(stableID) + count++ + } + + return inboxPath, err +} +func (dbs *SDAdb) getInboxPath(stableID string) (string, error) { + dbs.checkAndReconnectIfNeeded() + db := dbs.DB + const getFileID = "SELECT submission_file_path from sda.files WHERE stable_id = $1;" + + var inboxPath string + err := db.QueryRow(getFileID, stableID).Scan(&inboxPath) + if err != nil { + return "", err + } + + return inboxPath, nil +} + +// UpdateDatasetEvent marks the files in a dataset as "ready" or "disabled" +func (dbs *SDAdb) UpdateDatasetEvent(datasetID, status, message string) error { + var err error + // 2, 4, 8, 16, 32 seconds between each retry event. + for count := 1; count <= RetryTimes; count++ { + err = dbs.updateDatasetEvent(datasetID, status, message) + if err == nil { + break + } + time.Sleep(time.Duration(math.Pow(2, float64(count))) * time.Second) + } + + return err +} +func (dbs *SDAdb) updateDatasetEvent(datasetID, status, message string) error { + dbs.checkAndReconnectIfNeeded() + db := dbs.DB + + const setStatus = "INSERT INTO sda.dataset_event_log(dataset_id, event, message) VALUES($1, $2, $3);" + result, err := db.Exec(setStatus, datasetID, status, message) + if err != nil { + return err + } + + if rowsAffected, _ := result.RowsAffected(); rowsAffected == 0 { + return errors.New("something went wrong with the query zero rows were changed") + } + + return nil + +} diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index 8586996b4..090eaccc8 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -249,3 +249,86 @@ func (suite *DatabaseTests) TestCheckAccessionIDExists() { assert.NoError(suite.T(), err, "got (%v) when getting file archive information", err) assert.True(suite.T(), ok, "CheckAccessionIDExists returned false when true was expected") } + +func (suite *DatabaseTests) TestMapFilesToDataset() { + db, err := NewSDAdb(suite.dbConf) + assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) + + accessions := []string{} + for i := 1; i < 12; i++ { + fileID, err := db.RegisterFile(fmt.Sprintf("/testuser/TestMapFilesToDataset-%d.c4gh", i), "testuser") + assert.NoError(suite.T(), err, "failed to register file in database") + + err = db.SetAccessionID(fmt.Sprintf("acession-%d", i), fileID) + assert.NoError(suite.T(), err, "got (%v) when getting file archive information", err) + + accessions = append(accessions, fmt.Sprintf("acession-%d", i)) + } + + diSet := map[string][]string{ + "dataset1": accessions[0:3], + "dataset2": accessions[3:5], + "dataset3": accessions[5:8], + "dataset4": accessions[8:9], + } + + for di, acs := range diSet { + err := db.MapFilesToDataset(di, acs) + assert.NoError(suite.T(), err, "failed to map file to dataset") + } +} + +func (suite *DatabaseTests) TestGetInboxPath() { + db, err := NewSDAdb(suite.dbConf) + assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) + + accessions := []string{} + for i := 0; i < 5; i++ { + fileID, err := db.RegisterFile(fmt.Sprintf("/testuser/TestGetInboxPath-00%d.c4gh", i), "testuser") + assert.NoError(suite.T(), err, "failed to register file in database") + + err = db.SetAccessionID(fmt.Sprintf("acession-00%d", i), fileID) + assert.NoError(suite.T(), err, "got (%v) when getting file archive information", err) + + accessions = append(accessions, fmt.Sprintf("acession-00%d", i)) + } + + for _, acc := range accessions { + path, err := db.getInboxPath(acc) + assert.NoError(suite.T(), err, "getInboxPath failed") + assert.Contains(suite.T(), path, "/testuser/TestGetInboxPath") + } +} + +func (suite *DatabaseTests) TestUpdateDatasetEvent() { + db, err := NewSDAdb(suite.dbConf) + assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) + + accessions := []string{} + for i := 0; i < 5; i++ { + fileID, err := db.RegisterFile(fmt.Sprintf("/testuser/TestGetInboxPath-00%d.c4gh", i), "testuser") + assert.NoError(suite.T(), err, "failed to register file in database") + + err = db.SetAccessionID(fmt.Sprintf("acession-00%d", i), fileID) + assert.NoError(suite.T(), err, "got (%v) when getting file archive information", err) + + accessions = append(accessions, fmt.Sprintf("acession-00%d", i)) + } + + diSet := map[string][]string{"DATASET:TEST-0001": accessions} + + for di, acs := range diSet { + err := db.MapFilesToDataset(di, acs) + assert.NoError(suite.T(), err, "failed to map file to dataset") + } + + dID := "DATASET:TEST-0001" + err = db.UpdateDatasetEvent(dID, "registered", "{\"type\": \"mapping\"}") + assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) + + err = db.UpdateDatasetEvent(dID, "released", "{\"type\": \"release\"}") + assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) + + err = db.UpdateDatasetEvent(dID, "deprecated", "{\"type\": \"deprecate\"}") + assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) +} diff --git a/sda/internal/schema/schema.go b/sda/internal/schema/schema.go index 5244c9776..9c2c68eec 100644 --- a/sda/internal/schema/schema.go +++ b/sda/internal/schema/schema.go @@ -36,8 +36,12 @@ func ValidateJSON(reference string, body []byte) error { func getStructName(path string) interface{} { switch strings.TrimSuffix(filepath.Base(path), filepath.Ext(path)) { + case "dataset-deprecate": + return new(DatasetDeprecate) case "dataset-mapping": return new(DatasetMapping) + case "dataset-release": + return new(DatasetRelease) case "inbox-remove": return new(InboxRemove) case "inbox-rename": @@ -68,12 +72,22 @@ type Checksums struct { Value string `json:"value"` } +type DatasetDeprecate struct { + Type string `json:"type"` + DatasetID string `json:"dataset_id"` +} + type DatasetMapping struct { Type string `json:"type"` DatasetID string `json:"dataset_id"` AccessionIDs []string `json:"accession_ids"` } +type DatasetRelease struct { + Type string `json:"type"` + DatasetID string `json:"dataset_id"` +} + type InfoError struct { Error string `json:"error"` Reason string `json:"reason"` From 4d935bf3092c366f60a6dd7a41c2f71b17f083b8 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 22 Aug 2023 09:35:13 +0200 Subject: [PATCH 21/34] Merge `intercept` from sda-pipeline --- .github/integration/rabbitmq-federation.yml | 2 +- sda/cmd/intercept/intercept.go | 117 ++++++++++++++++++ sda/cmd/intercept/intercept.md | 108 +++++++++++++++++ sda/cmd/intercept/intercept_test.go | 127 ++++++++++++++++++++ 4 files changed, 353 insertions(+), 1 deletion(-) create mode 100644 sda/cmd/intercept/intercept.go create mode 100644 sda/cmd/intercept/intercept.md create mode 100644 sda/cmd/intercept/intercept_test.go diff --git a/.github/integration/rabbitmq-federation.yml b/.github/integration/rabbitmq-federation.yml index 86d4c872e..59430e1d5 100644 --- a/.github/integration/rabbitmq-federation.yml +++ b/.github/integration/rabbitmq-federation.yml @@ -78,7 +78,7 @@ services: - BROKER_CLIENTKEY=/certs/client.key - BROKER_CACERT=/certs/ca.crt - LOG_LEVEL=debug - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-pipeline + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} restart: always volumes: - certs:/certs diff --git a/sda/cmd/intercept/intercept.go b/sda/cmd/intercept/intercept.go new file mode 100644 index 000000000..e916310c0 --- /dev/null +++ b/sda/cmd/intercept/intercept.go @@ -0,0 +1,117 @@ +// The intercept service relays message between the queue +// provided from the federated service and local queues. +package main + +import ( + "encoding/json" + "errors" + + "github.com/neicnordic/sensitive-data-archive/internal/broker" + "github.com/neicnordic/sensitive-data-archive/internal/config" + + log "github.com/sirupsen/logrus" +) + +const ( + msgAccession string = "accession" + msgCancel string = "cancel" + msgIngest string = "ingest" + msgMapping string = "mapping" + msgRelease string = "release" + msgDeprecate string = "deprecate" +) + +func main() { + forever := make(chan bool) + conf, err := config.NewConfig("intercept") + if err != nil { + log.Fatal(err) + } + mq, err := broker.NewMQ(conf.Broker) + if err != nil { + log.Fatal(err) + } + + defer mq.Channel.Close() + defer mq.Connection.Close() + + go func() { + connError := mq.ConnectionWatcher() + log.Error(connError) + forever <- false + }() + + go func() { + connError := mq.ChannelWatcher() + log.Error(connError) + forever <- false + }() + + log.Info("Starting intercept service") + + go func() { + messages, err := mq.GetMessages(conf.Broker.Queue) + if err != nil { + log.Fatal(err) + } + for delivered := range messages { + log.Debugf("Received a message: %s", delivered.Body) + + msgType, err := typeFromMessage(delivered.Body) + if err != nil { + log.Errorf("Failed to get type for message (%v), reason: %v", msgType, err.Error()) + if err := delivered.Ack(false); err != nil { + log.Errorf("Failed acking canceled work, reason: (%v)", err) + } + // Restart on new message + continue + } + + routing := map[string]string{ + msgAccession: "accession", + msgCancel: "ingest", + msgIngest: "ingest", + msgMapping: "mappings", + msgRelease: "mappings", + msgDeprecate: "mappings", + } + + routingKey := routing[msgType] + if routingKey == "" { + continue + } + + log.Infof("Routing message (corr-id: %s, routingkey: %s)", delivered.CorrelationId, routingKey) + if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, routingKey, delivered.Body); err != nil { + log.Errorf("failed so publish message, reason: (%v)", err) + } + if err := delivered.Ack(false); err != nil { + log.Errorf("failed to ack message for reason: %v", err) + } + } + }() + + <-forever +} + +// typeFromMessage returns the type value given a JSON structure for the message +// supplied in body +func typeFromMessage(body []byte) (string, error) { + message := make(map[string]interface{}) + err := json.Unmarshal(body, &message) + if err != nil { + return "", err + } + + msgTypeFetch, ok := message["type"] + if !ok { + return "", errors.New("malformed message, type is missing") + } + + msgType, ok := msgTypeFetch.(string) + if !ok { + return "", errors.New("could not cast type attribute to string") + } + + return msgType, nil +} diff --git a/sda/cmd/intercept/intercept.md b/sda/cmd/intercept/intercept.md new file mode 100644 index 000000000..4495267e7 --- /dev/null +++ b/sda/cmd/intercept/intercept.md @@ -0,0 +1,108 @@ +# sda-pipeline: intercept + +The intercept service relays messages between Central EGA and Federated EGA nodes. + +## Configuration + +There are a number of options that can be set for the intercept service. +These settings can be set by mounting a yaml-file at `/config.yaml` with settings. + +ex. +```yaml +log: + level: "debug" + format: "json" +``` +They may also be set using environment variables like: +```bash +export LOG_LEVEL="debug" +export LOG_FORMAT="json" +``` + +### RabbitMQ broker settings + +These settings control how intercept connects to the RabbitMQ message broker. + + - `BROKER_HOST`: hostname of the rabbitmq server + + - `BROKER_PORT`: rabbitmq broker port (commonly `5671` with TLS and `5672` without) + + - `BROKER_QUEUE`: message queue to read messages from (commonly `files`) + + - `BROKER_USER`: username to connect to rabbitmq + + - `BROKER_PASSWORD`: password to connect to rabbitmq + +### PostgreSQL Database settings: + + - `DB_HOST`: hostname for the postgresql database + + - `DB_PORT`: database port (commonly 5432) + + - `DB_USER`: username for the database + + - `DB_PASSWORD`: password for the database + + - `DB_DATABASE`: database name + + - `DB_SSLMODE`: The TLS encryption policy to use for database connections. + Valid options are: + - `disable` + - `allow` + - `prefer` + - `require` + - `verify-ca` + - `verify-full` + + More information is available + [in the postgresql documentation](https://www.postgresql.org/docs/current/libpq-ssl.html#LIBPQ-SSL-PROTECTION) + + Note that if `DB_SSLMODE` is set to anything but `disable`, then `DB_CACERT` needs to be set, + and if set to `verify-full`, then `DB_CLIENTCERT`, and `DB_CLIENTKEY` must also be set + + - `DB_CLIENTKEY`: key-file for the database client certificate + + - `DB_CLIENTCERT`: database client certificate file + + - `DB_CACERT`: Certificate Authority (CA) certificate for the database to use + +### Logging settings: + + - `LOG_FORMAT` can be set to “json” to get logs in json format. + All other values result in text logging + + - `LOG_LEVEL` can be set to one of the following, in increasing order of severity: + - `trace` + - `debug` + - `info` + - `warn` (or `warning`) + - `error` + - `fatal` + - `panic` + +## Service Description + +When running, intercept reads messages from the configured RabbitMQ queue (default: "files"). +For each message, these steps are taken (if not otherwise noted, errors halt progress, the message is Nack'ed, the error is written to the log, and to the rabbitMQ error queue. +Then the service moves on to the next message): + +1. The message type is read from the message "type" field. + +1. The message schema is read from the message "msgType" field. + +1. The message is validated as valid JSON following the schema read in the previous step. +If this fails an error is written to the logs, but not to the error queue and the message is not Ack'ed or Nack'ed. + +1. The correct queue for the message is decided based on message type. +This is not supposed to be able to fail. + +1. The message is re-sent to the correct queue. +This has no error handling as the resend-mechanism hasn't been finished. + +1. The message is Ack'ed. + +## Communication + + - Intercept reads messages from one rabbitmq queue (default `files`). + + - Intercept writes messages to three rabbitmq queues, `accessionIDs`, `ingest`, and `mappings`. diff --git a/sda/cmd/intercept/intercept_test.go b/sda/cmd/intercept/intercept_test.go new file mode 100644 index 000000000..a591cea4f --- /dev/null +++ b/sda/cmd/intercept/intercept_test.go @@ -0,0 +1,127 @@ +package main + +import ( + "encoding/json" + "testing" + + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type TestSuite struct { + suite.Suite +} + +func TestConfigTestSuite(t *testing.T) { + suite.Run(t, new(TestSuite)) +} + +func (suite *TestSuite) SetupTest() { + viper.Set("log.level", "debug") +} + +type accession struct { + Type string `json:"type"` + User string `json:"user"` + FilePath string `json:"filepath"` + AccessionID string `json:"accession_id"` + DecryptedChecksums []Checksums `json:"decrypted_checksums"` +} + +type Checksums struct { + Type string `json:"type"` + Value string `json:"value"` +} + +type ingest struct { + Type string `json:"type"` + User string `json:"user"` + FilePath string `json:"filepath"` +} + +type mapping struct { + Type string `json:"type"` + DatasetID string `json:"dataset_id"` + AcessionIDs []string `json:"accession_ids"` +} + +type missing struct { + User string `json:"user"` + FilePath string `json:"filepath"` +} + +func (suite *TestSuite) TestMessageSelection_Accession() { + msg := accession{ + Type: "accession", + User: "foo", + FilePath: "/tmp/foo", + AccessionID: "EGAF12345678901", + DecryptedChecksums: []Checksums{ + {"md5", "7Ac236b1a8dce2dac89e7cf45d2b48BD"}, + }, + } + message, _ := json.Marshal(&msg) + + msgType, err := typeFromMessage(message) + + assert.Nil(suite.T(), err, "Unexpected error from typeFromMessage") + assert.Equal(suite.T(), msgType, msgAccession, "message type from message does not match expected") +} + +func (suite *TestSuite) TestMessageSelection_Cancel() { + msg := ingest{ + Type: "cancel", + User: "foo", + FilePath: "/tmp/foo", + } + message, _ := json.Marshal(&msg) + + msgType, err := typeFromMessage(message) + + assert.Nil(suite.T(), err, "Unexpected error from typeFromMessage") + assert.Equal(suite.T(), msgType, msgCancel, "message type from message does not match expected") +} + +func (suite *TestSuite) TestMessageSelection_Ingest() { + msg := ingest{ + Type: "ingest", + User: "foo", + FilePath: "/tmp/foo", + } + message, _ := json.Marshal(&msg) + + msgType, err := typeFromMessage(message) + + assert.Nil(suite.T(), err, "Unexpected error from typeFromMessage") + assert.Equal(suite.T(), msgIngest, msgType, "message type from message does not match expected") +} + +func (suite *TestSuite) TestMessageSelection_Mapping() { + msg := mapping{ + Type: "mapping", + DatasetID: "EGAD12345678900", + AcessionIDs: []string{ + "EGAF12345678901", + }, + } + message, _ := json.Marshal(&msg) + + msgType, err := typeFromMessage(message) + + assert.Nil(suite.T(), err, "Unexpected error from typeFromMessage") + assert.Equal(suite.T(), msgMapping, msgType, "message type from message does not match expected") +} + +func (suite *TestSuite) TestMessageSelection_Notype() { + msg := missing{ + User: "foo", + FilePath: "/tmp/foo", + } + message, _ := json.Marshal(&msg) + + msgType, err := typeFromMessage(message) + + assert.Error(suite.T(), err, "Unexpected lack of error from typeFromMessage") + assert.Equal(suite.T(), "", msgType, "message type from message does not match expected") +} From 549e00f5e688888a8e25a96a5313c9f7dae8f0c2 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Wed, 23 Aug 2023 08:27:15 +0200 Subject: [PATCH 22/34] Remove old pipeline code tests --- .github/workflows/test.yml | 50 -------------------------------------- 1 file changed, 50 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e4bb85de5..6d718ee60 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -29,56 +29,6 @@ jobs: - name: Test run: cd sda-sftp-inbox && mvn test -B - test_pipeline: - name: Test ingestion pipeline - runs-on: ubuntu-latest - strategy: - matrix: - go-version: ['1.20'] - steps: - - - name: Set up Go ${{ matrix.go-version }} - uses: actions/setup-go@v4 - with: - go-version: ${{ matrix.go-version }} - id: go - - - name: Check out code into the Go module directory - uses: actions/checkout@v4 - - - name: Get dependencies - run: | - cd sda-pipeline - go get -v -t -d ./... - if [ -f Gopkg.toml ]; then - curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh - dep ensure - fi - - - name: Create certificates - run: | - cd sda-pipeline/dev_utils - bash ./make_certs.sh - cd .. - - - name: Start MQ and DB - run: | - cd sda-pipeline - docker-compose -f dev_utils/compose-no-tls.yml up -d db mq - - - name: Test - run: | - cd sda-pipeline - go test --tags=integration -v -coverprofile=coverage.txt -covermode=atomic ./... - - - name: Codecov - uses: codecov/codecov-action@v3.1.4 - with: - token: ${{ secrets.CODECOV_TOKEN }} - file: ./sda-pipeline/coverage.txt - flags: unittests - fail_ci_if_error: false - test_download: name: Test Download runs-on: ubuntu-latest From f9cf35476fcd1e0d5a4668965bbc7dceb52d21e5 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Wed, 23 Aug 2023 08:33:06 +0200 Subject: [PATCH 23/34] Remove linting action for pipeline --- .github/workflows/code-linter.yaml | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/.github/workflows/code-linter.yaml b/.github/workflows/code-linter.yaml index bed6ff117..e9912b646 100644 --- a/.github/workflows/code-linter.yaml +++ b/.github/workflows/code-linter.yaml @@ -53,29 +53,6 @@ jobs: args: -E bodyclose,gocritic,gofmt,gosec,govet,nestif,nlreturn,revive,rowserrcheck -e G401,G501,G107 --timeout 5m working-directory: sda-download - lint_pipeline: - name: Lint pipeline code - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - go-version: ['1.20'] - steps: - - name: Set up Go ${{ matrix.go-version }} - uses: actions/setup-go@v4 - with: - go-version: ${{ matrix.go-version }} - id: go - - - name: Check out code into the Go module directory - uses: actions/checkout@v4 - - - name: Run golangci-lint - uses: golangci/golangci-lint-action@v3.7.0 - with: - args: -E bodyclose,gocritic,gofmt,gosec,govet,nestif,nlreturn,rowserrcheck --timeout 5m - working-directory: sda-pipeline - lint_sda: name: Lint sda code runs-on: ubuntu-latest From 7d5312dd1dcc0481c746249b61dc33b3524c0164 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Wed, 23 Aug 2023 08:34:36 +0200 Subject: [PATCH 24/34] Don't build PR container for old pipeline --- .github/workflows/build_pr_container.yaml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/.github/workflows/build_pr_container.yaml b/.github/workflows/build_pr_container.yaml index 8dd5124f1..f9c9b5cc6 100644 --- a/.github/workflows/build_pr_container.yaml +++ b/.github/workflows/build_pr_container.yaml @@ -55,19 +55,6 @@ jobs: org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ') org.opencontainers.image.revision=${{ github.sha }} - - name: Build container for sda-pipeline - uses: docker/build-push-action@v4 - with: - context: ./sda-pipeline - push: true - tags: | - ghcr.io/${{ github.repository }}:sha-${{ github.sha }}-pipeline - ghcr.io/${{ github.repository }}:PR${{ github.event.number }}-pipeline - labels: | - org.opencontainers.image.source=${{ github.event.repository.clone_url }} - org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ') - org.opencontainers.image.revision=${{ github.sha }} - - name: Build container for sensitive-data-archive uses: docker/build-push-action@v4 with: From e16b3cbda9fc7ba260b230bc63c5f0013c40bd9e Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Wed, 23 Aug 2023 08:28:57 +0200 Subject: [PATCH 25/34] Remove old pipeline functionality tests --- .github/workflows/functionality.yml | 38 ----------------------------- 1 file changed, 38 deletions(-) diff --git a/.github/workflows/functionality.yml b/.github/workflows/functionality.yml index 920a4dd8f..918fc37de 100644 --- a/.github/workflows/functionality.yml +++ b/.github/workflows/functionality.yml @@ -3,9 +3,6 @@ name: Functionality tests on: pull_request: -env: - svc_list: 'finalize inbox ingest mapper verify' - jobs: sda-auth: runs-on: ubuntu-latest @@ -72,41 +69,6 @@ jobs: bash -x "$runscript"; done - sda-pipeline: - name: sda-pipeline-integration-${{ matrix.storagetype }} - runs-on: ubuntu-latest - env: - STORAGETYPE: ${{ matrix.storagetype }} - - strategy: - matrix: - storagetype: [s3, posix, s3notls, s3header, s3notlsheader, posixheader, sftp, sftpheader] - fail-fast: false - steps: - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - - name: Check out code into the Go module directory - uses: actions/checkout@v4 - - - name: Run setup scripts - run: | - cd sda-pipeline - ls -1 .github/integration/setup/{common,${{ matrix.storagetype }}}/*.sh 2>/dev/null | sort -t/ -k5 -n | while read -r runscript; do - echo "Executing setup script $runscript"; - bash -x "$runscript"; - done - - - name: Run tests - run: | - cd sda-pipeline - ls -1 .github/integration/tests/{common,${{ matrix.storagetype }}}/*.sh 2>/dev/null | sort -t/ -k5 -n | while read -r runscript; do - echo "Executing test script $runscript"; - bash -x "$runscript"; - done - sftp-inbox: runs-on: ubuntu-latest From 479b94fd92910b3728160d50601cb15bdc8ee3f2 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Wed, 23 Aug 2023 08:38:32 +0200 Subject: [PATCH 26/34] Update GO mod --- sda/go.mod | 2 +- sda/go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/sda/go.mod b/sda/go.mod index 1bafa65bb..b45131a7b 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -4,7 +4,7 @@ go 1.20 require ( github.com/DATA-DOG/go-sqlmock v1.5.0 - github.com/aws/aws-sdk-go v1.44.326 + github.com/aws/aws-sdk-go v1.44.329 github.com/golang-jwt/jwt/v4 v4.5.0 github.com/google/uuid v1.1.2 github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb diff --git a/sda/go.sum b/sda/go.sum index 522bf46da..e173b3358 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -53,6 +53,8 @@ github.com/aws/aws-sdk-go v1.44.325 h1:jF/L99fJSq/BfiLmUOflO/aM+LwcqBm0Fe/qTK5xx github.com/aws/aws-sdk-go v1.44.325/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go v1.44.326 h1:/6xD/9mKZ2RMTDfbhh9qCxw+CaTbJRvfHJ/NHPFbI38= github.com/aws/aws-sdk-go v1.44.326/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.329 h1:Rqy+wYI8h+iq+FphR59KKTsHR1Lz7YiwRqFzWa7xoYU= +github.com/aws/aws-sdk-go v1.44.329/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= From 0befc5b1490af88bede568e127911593308b2e16 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 31 Aug 2023 14:06:19 +0200 Subject: [PATCH 27/34] Update charts to match latest container code --- charts/sda-svc/templates/_helpers.yaml | 8 -- charts/sda-svc/templates/finalize-deploy.yaml | 84 ++++++++++++++++++- charts/sda-svc/templates/s3-inbox-deploy.yaml | 20 +++-- charts/sda-svc/templates/shared-secrets.yaml | 11 +++ 4 files changed, 106 insertions(+), 17 deletions(-) diff --git a/charts/sda-svc/templates/_helpers.yaml b/charts/sda-svc/templates/_helpers.yaml index 4e32654f6..cab89357c 100644 --- a/charts/sda-svc/templates/_helpers.yaml +++ b/charts/sda-svc/templates/_helpers.yaml @@ -297,14 +297,6 @@ Create chart name and version as used by the chart label. {{- end }} {{- end -}} -{{- define "S3ArchiveURL" -}} - {{- if .Values.global.inbox.s3Port }} - {{- printf "%s:%v" .Values.global.inbox.s3Url .Values.global.inbox.s3Port }} - {{- else }} - {{- printf "%s" .Values.global.inbox.s3Url }} - {{- end }} -{{- end -}} - {{- define "TLSissuer" -}} {{- if and .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} {{- fail "Only one of global.tls.issuer or global.tls.clusterIssuer should be set" }} diff --git a/charts/sda-svc/templates/finalize-deploy.yaml b/charts/sda-svc/templates/finalize-deploy.yaml index adc70bb89..d2c39aa09 100644 --- a/charts/sda-svc/templates/finalize-deploy.yaml +++ b/charts/sda-svc/templates/finalize-deploy.yaml @@ -66,6 +66,64 @@ spec: securityContext: allowPrivilegeEscalation: false env: +{{- if .Values.global.backupArchive.storageType }} + - name: ARCHIVE_TYPE + {{- if eq "s3" .Values.global.archive.storageType }} + value: "s3" + - name: ARCHIVE_URL + value: {{ required "S3 archive URL missing" .Values.global.archive.s3Url }} + {{- if .Values.global.archive.s3Port }} + - name: ARCHIVE_PORT + value: {{ .Values.global.archive.s3Port | quote }} + {{- end }} + - name: ARCHIVE_BUCKET + value: {{ required "S3 archive bucket missing" .Values.global.archive.s3Bucket }} + - name: ARCHIVE_REGION + value: {{ default "us-east-1" .Values.global.archive.s3Region }} + - name: ARCHIVE_CHUNKSIZE + value: {{ .Values.global.archive.s3ChunkSize | quote }} + {{- if and .Values.global.archive.s3CaFile .Values.global.tls.enabled }} + - name: ARCHIVE_CACERT + value: {{ template "tlsPath" . }}/ca.crt + {{- end }} + {{- else }} + value: "posix" + - name: ARCHIVE_LOCATION + value: "{{ .Values.global.archive.volumePath }}" + {{- end }} + - name: BACKUP_TYPE + {{- if eq "s3" .Values.global.backupArchive.storageType }} + value: "s3" + - name: BACKUP_URL + value: {{ required "S3 backup archive URL missing" .Values.global.backupArchive.s3Url }} + {{- if .Values.global.backupArchive.s3Port }} + - name: BACKUP_PORT + value: {{ .Values.global.backupArchive.s3Port | quote }} + {{- end }} + - name: BACKUP_BUCKET + value: {{ required "S3 backup archive bucket missing" .Values.global.backupArchive.s3Bucket }} + - name: BACKUP_REGION + value: {{ default "us-east-1" .Values.global.backupArchive.s3Region }} + - name: BACKUP_CHUNKSIZE + value: {{ .Values.global.backupArchive.s3ChunkSize | quote }} + {{- if and .Values.global.backupArchive.s3CaFile .Values.global.tls.enabled }} + - name: BACKUP_CACERT + value: {{ template "tlsPath" . }}/ca.crt + {{- end }} + {{- else }} + value: "posix" + - name: BACKUP_LOCATION + value: "{{ .Values.global.backupArchive.volumePath }}" + {{- end }} + - name: BACKUP_COPYHEADER + value: "{{ .Values.global.backupArchive.copyHeader }}" + {{- if .Values.global.backupArchive.copyHeader}} + - name: C4GH_FILEPATH + value: "{{ template "c4ghPath" . }}/{{ .Values.global.c4gh.keyFile }}" + - name: C4GH_BACKUPPUBKEY + value: "{{ template "c4ghPath" . }}/{{ .Values.global.c4gh.backupPubKey }}" + {{- end }} +{{- end }} - name: BROKER_DURABLE value: {{ .Values.global.broker.durable | quote }} - name: BROKER_EXCHANGE @@ -81,7 +139,7 @@ spec: - name: BROKER_ROUTINGERROR value: {{ .Values.global.broker.routingError }} - name: BROKER_ROUTINGKEY - value: {{ ternary .Values.global.broker.backupRoutingKey "completed" (.Values.backup.deploy) }} + value: "completed" - name: BROKER_VHOST value: {{ .Values.global.broker.vhost | quote }} - name: BROKER_SERVERNAME @@ -129,6 +187,30 @@ spec: - name: SCHEMA_TYPE value: {{ default "federated" .Values.global.schemaType }} {{- if not .Values.global.vaultSecrets }} + {{- if eq "s3" .Values.global.archive.storageType }} + - name: ARCHIVE_ACCESSKEY + valueFrom: + secretKeyRef: + name: {{ template "sda.fullname" . }}-s3archive-keys + key: s3ArchiveAccessKey + - name: ARCHIVE_SECRETKEY + valueFrom: + secretKeyRef: + name: {{ template "sda.fullname" . }}-s3archive-keys + key: s3ArchiveSecretKey + {{- end }} + {{- if eq "s3" .Values.global.backupArchive.storageType }} + - name: BACKUP_ACCESSKEY + valueFrom: + secretKeyRef: + name: {{ template "sda.fullname" . }}-s3backup-keys + key: s3BackupAccessKey + - name: BACKUP_SECRETKEY + valueFrom: + secretKeyRef: + name: {{ template "sda.fullname" . }}-s3backup-keys + key: s3BackupSecretKey + {{- end }} - name: BROKER_PASSWORD valueFrom: secretKeyRef: diff --git a/charts/sda-svc/templates/s3-inbox-deploy.yaml b/charts/sda-svc/templates/s3-inbox-deploy.yaml index 405e526d2..fa072dbe0 100644 --- a/charts/sda-svc/templates/s3-inbox-deploy.yaml +++ b/charts/sda-svc/templates/s3-inbox-deploy.yaml @@ -86,12 +86,12 @@ spec: {{- end }} env: {{- if not .Values.global.vaultSecrets }} - - name: AWS_ACCESSKEY + - name: INBOX_ACCESSKEY valueFrom: secretKeyRef: name: {{ template "sda.fullname" . }}-inbox key: s3InboxAccessKey - - name: AWS_SECRETKEY + - name: INBOX_SECRETKEY valueFrom: secretKeyRef: name: {{ template "sda.fullname" . }}-inbox @@ -117,20 +117,24 @@ spec: - name: SERVER_CONFFILE value: {{ include "confFile" .}} {{- end }} - - name: AWS_URL - value: {{ template "S3InboxURL" . }} + - name: INBOX_URL + value: {{ .Values.global.inbox.s3Url | quote }} + {{- if .Values.global.inbox.s3Port }} + - name: INBOX_PORT + value: {{ .Values.global.inbox.s3Port | quote }} + {{- end }} {{- if and .Values.global.inbox.s3CaFile .Values.global.tls.enabled }} - - name: AWS_CACERT + - name: INBOX_CACERT value: "{{ include "tlsPath" . }}/ca.crt" {{- end }} {{- if .Values.global.inbox.s3Region }} - - name: AWS_REGION + - name: INBOX_REGION value: {{ .Values.global.inbox.s3Region | quote }} {{- end }} - - name: AWS_BUCKET + - name: INBOX_BUCKET value: {{ .Values.global.inbox.s3Bucket | quote }} {{- if .Values.global.inbox.s3ReadyPath }} - - name: AWS_READYPATH + - name: INBOX_READYPATH value: {{ .Values.global.inbox.s3ReadyPath }} {{- end }} - name: BROKER_HOST diff --git a/charts/sda-svc/templates/shared-secrets.yaml b/charts/sda-svc/templates/shared-secrets.yaml index c83d876e9..06b3c69bd 100644 --- a/charts/sda-svc/templates/shared-secrets.yaml +++ b/charts/sda-svc/templates/shared-secrets.yaml @@ -10,6 +10,17 @@ data: s3ArchiveAccessKey: {{ .Values.global.archive.s3AccessKey | quote | trimall "\"" | b64enc }} s3ArchiveSecretKey: {{ .Values.global.archive.s3SecretKey | quote | trimall "\"" | b64enc }} {{- end }} +{{- if and .Values.global.backupArchive.s3AccessKey .Values.global.backupArchive.s3SecretKey }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "sda.fullname" . }}-s3backup-keys +type: Opaque +data: + s3BackupAccessKey: {{ .Values.global.backupArchive.s3AccessKey | quote | trimall "\"" | b64enc }} + s3BackupSecretKey: {{ .Values.global.backupArchive.s3SecretKey | quote | trimall "\"" | b64enc }} +{{- end }} {{- if and .Values.global.inbox.s3AccessKey .Values.global.inbox.s3SecretKey }} --- apiVersion: v1 From 2d775fe67a56f9f529a1fa1ec25ff7ca7afca9b9 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 31 Aug 2023 13:54:00 +0200 Subject: [PATCH 28/34] Remove `backup` from the helm chart --- .../integration/scripts/charts/values.yaml | 3 - charts/sda-svc/README.md | 11 - charts/sda-svc/templates/_helpers.yaml | 14 - .../sda-svc/templates/backup-certificate.yaml | 40 --- charts/sda-svc/templates/backup-deploy.yaml | 310 ------------------ charts/sda-svc/templates/backup-secrets.yaml | 29 -- charts/sda-svc/values.yaml | 24 -- 7 files changed, 431 deletions(-) delete mode 100644 charts/sda-svc/templates/backup-certificate.yaml delete mode 100644 charts/sda-svc/templates/backup-deploy.yaml delete mode 100644 charts/sda-svc/templates/backup-secrets.yaml diff --git a/.github/integration/scripts/charts/values.yaml b/.github/integration/scripts/charts/values.yaml index 3dd9e6bf8..8c7c73325 100644 --- a/.github/integration/scripts/charts/values.yaml +++ b/.github/integration/scripts/charts/values.yaml @@ -67,9 +67,6 @@ global: auth: replicaCount: 1 resources: null -backup: - deploy: true - resources: null doa: deploy: false download: diff --git a/charts/sda-svc/README.md b/charts/sda-svc/README.md index 73866f682..7e18fc4f7 100644 --- a/charts/sda-svc/README.md +++ b/charts/sda-svc/README.md @@ -89,7 +89,6 @@ Parameter | Description | Default `global.broker.vhost` | Virtual host to connect to. |`/` `global.broker.password` | Shared password to the message broker. |`/` `global.broker.username` | Shared user to the message broker. |`/` -`global.broker.backupRoutingKey` | routing key used to send messages to backup service |`""` `global.broker.prefetchCount` | Number of messages to retrieve from the broker at the time, setting this to `1` will create a round-robin behavior between consumers |`2` `global.cega.host` | Full URI to the EGA user authentication service. |`""` `global.cega.user` | Username for the EGA user authentication service. |`""` @@ -153,10 +152,6 @@ If no shared credentials for the message broker and database are used these shou Parameter | Description | Default --------- | ----------- | ------- -`credentials.backup.dbUser` | Databse user for backup | `""` -`credentials.backup.dbPassword` | Database password for backup | `""` -`credentials.backup.mqUser` | Broker user for backup | `""` -`credentials.backup.mqPassword` | Broker password for backup | `""` `credentials.doa.dbUser` | Databse user for doa | `""` `credentials.doa.dbPassword` | Database password for doa| `""` `credentials.download.dbUser` | Databse user for download | `""` @@ -192,12 +187,6 @@ Parameter | Description | Default `auth.resources.requests.cpu` | CPU request for container. |`100m` `auth.resources.limits.memory` | Memory limit for container. |`256Mi` `auth.resources.limits.cpu` | CPU limit for container. |`250m` -`backup.annotations` | Specific annotation for the backup pod | `{}` -`backup.resources.requests.memory` | Memory request for backup container. |`128Mi` -`backup.resources.requests.cpu` | CPU request for backup container. |`100m` -`backup.resources.limits.memory` | Memory limit for backup container. |`256Mi` -`backup.resources.limits.cpu` | CPU limit for backup container. |`250m` -`backup.deploy` | Set to true if the backup service should be active | `false` `doa.replicaCount` | desired number of replicas | `2` `doa.repository` | dataedge container image repository | `neicnordic/sda-doa` `doa.imageTag` | dataedge container image version | `"latest"` diff --git a/charts/sda-svc/templates/_helpers.yaml b/charts/sda-svc/templates/_helpers.yaml index cab89357c..52b1c676b 100644 --- a/charts/sda-svc/templates/_helpers.yaml +++ b/charts/sda-svc/templates/_helpers.yaml @@ -133,20 +133,6 @@ Create chart name and version as used by the chart label. {{ end }} {{- end -}} -{{/**/}} -{{- define "dbUserBackup" -}} -{{- ternary .Values.global.db.user .Values.credentials.backup.dbUser (empty .Values.credentials.backup.dbUser) -}} -{{- end -}} -{{- define "dbPassBackup" -}} -{{- ternary .Values.global.db.password .Values.credentials.backup.dbPassword (empty .Values.credentials.backup.dbPassword) -}} -{{- end -}} -{{- define "mqUserBackup" -}} -{{- ternary .Values.global.broker.username .Values.credentials.backup.mqUser (empty .Values.credentials.backup.mqUser) -}} -{{- end -}} -{{- define "mqPassBackup" -}} -{{- ternary .Values.global.broker.password .Values.credentials.backup.mqPassword (empty .Values.credentials.backup.mqPassword) -}} -{{- end -}} - {{/**/}} {{- define "dbUserDoa" -}} {{- ternary .Values.global.db.user .Values.credentials.doa.dbUser (empty .Values.credentials.doa.dbUser) -}} diff --git a/charts/sda-svc/templates/backup-certificate.yaml b/charts/sda-svc/templates/backup-certificate.yaml deleted file mode 100644 index cbde0c6d2..000000000 --- a/charts/sda-svc/templates/backup-certificate.yaml +++ /dev/null @@ -1,40 +0,0 @@ -{{- if .Values.global.tls.enabled }} -{{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} -{{- if .Values.backup.deploy}} -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: {{ template "sda.fullname" . }}-backup-certs -spec: - # Secret names are always required. - secretName: {{ template "sda.fullname" . }}-backup-certs - - duration: 2160h # 90d - - # The use of the common name field has been deprecated since 2000 and is - # discouraged from being used. - commonName: {{ template "sda.fullname" . }}-backup - isCA: false - privateKey: - algorithm: ECDSA - size: 256 - usages: - - client auth - # At least one of a DNS Name, URI, or IP address is required. - dnsNames: - - {{ template "sda.fullname" . }}-backup - - {{ template "sda.fullname" . }}-backup.{{ .Release.Namespace }}.svc - ipAddresses: - - 127.0.0.1 - # Issuer references are always required. - issuerRef: - name: {{ template "TLSissuer" . }} - # We can reference ClusterIssuers by changing the kind here. - # The default value is Issuer (i.e. a locally namespaced Issuer) - kind: {{ ternary "Issuer" "ClusterIssuer" (empty .Values.global.tls.clusterIssuer )}} - # This is optional since cert-manager will default to this value however - # if you are using an external issuer, change this to that issuer group. - group: cert-manager.io -{{- end -}} -{{- end -}} -{{- end -}} \ No newline at end of file diff --git a/charts/sda-svc/templates/backup-deploy.yaml b/charts/sda-svc/templates/backup-deploy.yaml deleted file mode 100644 index d41ad813b..000000000 --- a/charts/sda-svc/templates/backup-deploy.yaml +++ /dev/null @@ -1,310 +0,0 @@ -{{- if .Values.backup.deploy}} -{{- if or (or (eq "all" .Values.global.deploymentType) (eq "internal" .Values.global.deploymentType) ) (not .Values.global.deploymentType) }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ template "sda.fullname" . }}-backup - labels: - role: backup - app: {{ template "sda.name" . }} - chart: {{ .Chart.Name }}-{{ .Chart.Version }} - component: {{ .Release.Name }}-backup - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -spec: - replicas: 1 - revisionHistoryLimit: {{ default "3" .Values.global.revisionHistory }} - selector: - matchLabels: - app: {{ template "sda.name" . }}-backup - release: {{ .Release.Name }} - template: - metadata: - labels: - app: {{ template "sda.name" . }}-backup - role: backup - release: {{ .Release.Name }} - annotations: - {{- if not .Values.global.vaultSecrets }} - checksum/config: {{ include (print $.Template.BasePath "/backup-secrets.yaml") . | sha256sum }} - {{- end }} - {{- if .Values.global.podAnnotations }} -{{- toYaml .Values.global.podAnnotations | nindent 8 -}} - {{- end }} - {{- if .Values.backup.annotations }} -{{- toYaml .Values.backup.annotations | nindent 8 -}} - {{- end }} - spec: - {{- if .Values.global.rbacEnabled}} - serviceAccountName: {{ .Release.Name }} - {{- end }} - securityContext: - runAsUser: 65534 - runAsGroup: 65534 - fsGroup: 65534 - {{- if and .Values.global.pkiPermissions .Values.global.tls.enabled }} - initContainers: - - name: tls-init - image: busybox - command: ["/bin/sh", "-c"] - args: ["/bin/cp /tls-certs/* /tls/ && chown 65534:65534 /tls/* && chmod 0600 /tls/*"] - securityContext: - allowPrivilegeEscalation: false - {{- if .Values.global.extraSecurityContext }} -{{- toYaml .Values.global.extraSecurityContext | nindent 10 -}} - {{- end }} - volumeMounts: - - name: tls-certs - mountPath: /tls-certs - - name: tls - mountPath: /tls - {{- end }} - containers: - - name: backup - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}-pipeline" - imagePullPolicy: {{ .Values.image.pullPolicy | quote }} - command: ["sda-backup"] - securityContext: - allowPrivilegeEscalation: false - env: - - name: ARCHIVE_TYPE - {{- if eq "s3" .Values.global.archive.storageType }} - value: "s3" - - name: ARCHIVE_URL - value: {{ required "S3 archive URL missing" .Values.global.archive.s3Url }} - {{- if .Values.global.archive.s3Port }} - - name: ARCHIVE_PORT - value: {{ .Values.global.archive.s3Port | quote }} - {{- end }} - - name: ARCHIVE_BUCKET - value: {{ required "S3 archive bucket missing" .Values.global.archive.s3Bucket }} - - name: ARCHIVE_REGION - value: {{ default "us-east-1" .Values.global.archive.s3Region }} - - name: ARCHIVE_CHUNKSIZE - value: {{ .Values.global.archive.s3ChunkSize | quote }} - {{- if and .Values.global.archive.s3CaFile .Values.global.tls.enabled }} - - name: ARCHIVE_CACERT - value: {{ template "tlsPath" . }}/ca.crt - {{- end }} - {{- else }} - value: "posix" - - name: ARCHIVE_LOCATION - value: "{{ .Values.global.archive.volumePath }}" - {{- end }} - - name: BACKUP_TYPE - {{- if eq "s3" .Values.global.backupArchive.storageType }} - value: "s3" - - name: BACKUP_URL - value: {{ required "S3 backup archive URL missing" .Values.global.backupArchive.s3Url }} - {{- if .Values.global.backupArchive.s3Port }} - - name: BACKUP_PORT - value: {{ .Values.global.backupArchive.s3Port | quote }} - {{- end }} - - name: BACKUP_BUCKET - value: {{ required "S3 backup archive bucket missing" .Values.global.backupArchive.s3Bucket }} - - name: BACKUP_REGION - value: {{ default "us-east-1" .Values.global.backupArchive.s3Region }} - - name: BACKUP_CHUNKSIZE - value: {{ .Values.global.backupArchive.s3ChunkSize | quote }} - {{- if and .Values.global.backupArchive.s3CaFile .Values.global.tls.enabled }} - - name: BACKUP_CACERT - value: {{ template "tlsPath" . }}/ca.crt - {{- end }} - {{- else }} - value: "posix" - - name: BACKUP_LOCATION - value: "{{ .Values.global.backupArchive.volumePath }}" - {{- end }} - - name: BACKUP_COPYHEADER - value: "{{ .Values.global.backupArchive.copyHeader }}" - {{- if .Values.global.backupArchive.copyHeader}} - - name: C4GH_FILEPATH - value: "{{ template "c4ghPath" . }}/{{ .Values.global.c4gh.keyFile }}" - - name: C4GH_BACKUPPUBKEY - value: "{{ template "c4ghPath" . }}/{{ .Values.global.c4gh.backupPubKey }}" - {{- end }} - - name: BROKER_DURABLE - value: {{ .Values.global.broker.durable | quote }} - - name: BROKER_EXCHANGE - value: {{ default "sda" .Values.global.broker.exchange }} - - name: BROKER_QUEUE - value: {{ .Values.global.broker.backupRoutingKey }} - - name: BROKER_HOST - value: {{ required "A valid MQ host is required" .Values.global.broker.host | quote }} - - name: BROKER_PORT - value: {{ .Values.global.broker.port | quote }} - - name: BROKER_PREFETCHCOUNT - value: {{ .Values.global.broker.prefetchCount | quote }} - - name: BROKER_ROUTINGERROR - value: {{ .Values.global.broker.routingError }} - - name: BROKER_ROUTINGKEY - value: "completed" - - name: BROKER_VHOST - value: {{ .Values.global.broker.vhost | quote }} - - name: BROKER_SERVERNAME - value: {{ .Values.global.broker.host | quote }} - - name: BROKER_SSL - value: {{ .Values.global.tls.enabled | quote }} - {{- if .Values.global.tls.enabled }} - - name: BROKER_VERIFYPEER - value: {{ .Values.global.broker.verifyPeer | quote }} - - name: BROKER_CACERT - value: {{ include "tlsPath" . }}/ca.crt - {{- if .Values.global.broker.verifyPeer }} - - name: BROKER_CLIENTCERT - value: {{ include "tlsPath" . }}/tls.crt - - name: BROKER_CLIENTKEY - value: {{ include "tlsPath" . }}/tls.key - {{- end }} - {{- end }} - {{- if .Values.global.tls.enabled }} - - name: DB_CACERT - value: {{ include "tlsPath" . }}/ca.crt - {{- if ne "verify-none" .Values.global.db.sslMode }} - - name: DB_CLIENTCERT - value: {{ include "tlsPath" . }}/tls.crt - - name: DB_CLIENTKEY - value: {{ include "tlsPath" . }}/tls.key - {{- end }} - {{- end }} - - name: DB_DATABASE - value: {{ default "lega" .Values.global.db.name | quote }} - - name: DB_HOST - value: {{ required "A valid DB host is required" .Values.global.db.host | quote }} - - name: DB_PORT - value: {{ .Values.global.db.port | quote }} - - name: DB_SSLMODE - value: {{ template "dbSSLmode" . }} - {{- if .Values.global.log.format }} - - name: LOG_FORMAT - value: {{ .Values.global.log.format | quote }} - {{- end }} - {{- if .Values.global.log.level }} - - name: LOG_LEVEL - value: {{ .Values.global.log.level | quote }} - {{- end }} - - name: SCHEMA_TYPE - value: {{ default "federated" .Values.global.schemaType }} - {{- if not .Values.global.vaultSecrets }} - {{- if eq "s3" .Values.global.archive.storageType }} - - name: ARCHIVE_ACCESSKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3archive-keys - key: s3ArchiveAccessKey - - name: ARCHIVE_SECRETKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3archive-keys - key: s3ArchiveSecretKey - {{- end }} - {{- if eq "s3" .Values.global.backupArchive.storageType }} - - name: BACKUP_ACCESSKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3backup-keys - key: s3BackupAccessKey - - name: BACKUP_SECRETKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3backup-keys - key: s3BackupSecretKey - {{- end }} - - name: BROKER_PASSWORD - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-backup - key: mqPassword - - name: BROKER_USER - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-backup - key: mqUser - - name: DB_PASSWORD - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-backup - key: dbPassword - - name: DB_USER - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-backup - key: dbUser - {{ else }} - - name: CONFIGFILE - value: {{ include "confFile" . }} - {{- end }} - resources: -{{ toYaml .Values.backup.resources | trim | indent 10 }} - volumeMounts: - {{- if eq "posix" .Values.global.archive.storageType }} - - name: archive - mountPath: {{ .Values.global.archive.volumePath | quote }} - {{- end }} - {{- if .Values.global.backupArchive.copyHeader}} - - name: c4gh - mountPath: {{ template "c4ghPath" . }} - {{- end }} - {{- if eq "posix" .Values.global.backupArchive.storageType }} - - name: backup - mountPath: {{ .Values.global.backupArchive.volumePath | quote }} - {{- end }} - {{- if and (not .Values.global.pkiService) .Values.global.tls.enabled }} - - name: tls - mountPath: {{ template "tlsPath" . }} - {{- end }} - volumes: - {{- if and (not .Values.global.pkiService) .Values.global.tls.enabled }} - - name: {{ ternary "tls" "tls-certs" (empty .Values.global.pkiPermissions) }} - projected: - sources: - {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} - - secret: - name: {{ template "sda.fullname" . }}-backup-certs - {{- else }} - - secret: - name: {{ required "An certificate issuer or a TLS secret name is required for backup" .Values.backup.tls.secretName }} - {{- end }} - {{- if .Values.global.pkiPermissions }} - - name: tls - emptyDir: - medium: Memory - sizeLimit: 10Mi - {{- end }} - {{- if and (not .Values.global.vaultSecrets) .Values.global.backupArchive.copyHeader }} - - name: c4gh - secret: - defaultMode: 0440 - secretName: {{ required "A secret for the c4gh key is required" .Values.global.c4gh.secretName }} - items: - - key: {{ .Values.global.c4gh.keyFile }} - path: {{ .Values.global.c4gh.keyFile }} - - key: {{ .Values.global.c4gh.backupPubKey }} - path: {{ .Values.global.c4gh.backupPubKey }} - {{- end }} - {{- end }} - {{- if eq "posix" .Values.global.archive.storageType }} - - name: archive - {{- if .Values.global.archive.existingClaim }} - persistentVolumeClaim: - claimName: {{ .Values.global.archive.existingClaim }} - {{- else }} - nfs: - server: {{ required "An archive NFS server is required" .Values.global.archive.nfsServer | quote }} - path: {{ if .Values.global.archive.nfsPath }}{{ .Values.global.archive.nfsPath | quote }}{{ else }}{{ "/" }}{{ end }} - {{- end }} - {{- end }} - {{- if eq "posix" .Values.global.backupArchive.storageType }} - - name: backup - {{- if .Values.global.backupArchive.existingClaim }} - persistentVolumeClaim: - claimName: {{ .Values.global.backupArchive.existingClaim }} - {{- else }} - nfs: - server: {{ required "An backup NFS server is required" .Values.global.backupArchive.nfsServer | quote }} - path: {{ if .Values.global.backupArchive.nfsPath }}{{ .Values.global.backupArchive.nfsPath | quote }}{{ else }}{{ "/" }}{{ end }} - {{- end }} - {{- end }} - restartPolicy: Always -{{- end }} -{{- end }} diff --git a/charts/sda-svc/templates/backup-secrets.yaml b/charts/sda-svc/templates/backup-secrets.yaml deleted file mode 100644 index 746f3e05d..000000000 --- a/charts/sda-svc/templates/backup-secrets.yaml +++ /dev/null @@ -1,29 +0,0 @@ -{{- if .Values.backup.deploy}} -{{- if or (or (eq "all" .Values.global.deploymentType) (eq "internal" .Values.global.deploymentType) ) (not .Values.global.deploymentType)}} -{{- if not .Values.global.vaultSecrets }} ---- -apiVersion: v1 -kind: Secret -metadata: - name: {{ template "sda.fullname" . }}-backup -type: Opaque -data: - c4ghPassphrase: {{ .Values.global.c4gh.passphrase | b64enc }} - dbPassword: {{ include "dbPassBackup" . | b64enc }} - dbUser: {{ include "dbUserBackup" . | b64enc }} - mqPassword: {{ include "mqPassBackup" . | b64enc }} - mqUser: {{ include "mqUserBackup" . | b64enc }} -{{- if eq "s3" .Values.global.backupArchive.storageType }} ---- -apiVersion: v1 -kind: Secret -metadata: - name: {{ template "sda.fullname" . }}-s3backup-keys -type: Opaque -data: - s3BackupAccessKey: {{ required "S3 backup archive accesskey missing" .Values.global.backupArchive.s3AccessKey | quote | trimall "\"" | b64enc }} - s3BackupSecretKey: {{ required "S3 backup archive secretkey missing" .Values.global.backupArchive.s3SecretKey | quote | trimall "\"" | b64enc }} -{{- end }} -{{- end }} -{{- end }} -{{- end }} diff --git a/charts/sda-svc/values.yaml b/charts/sda-svc/values.yaml index 85993a1fa..6cde7979b 100644 --- a/charts/sda-svc/values.yaml +++ b/charts/sda-svc/values.yaml @@ -258,12 +258,6 @@ global: ################################## # service specific credentials credentials: - backup: - dbUser: "" - dbPassword: "" - mqUser: "" - mqPassword: "" - doa: dbUser: "" dbPassword: "" @@ -332,24 +326,6 @@ auth: tls: secretName: "" -backup: - name: backup - deploy: false - replicaCount: 1 - resources: - requests: - memory: "128Mi" - cpu: "100m" - limits: - memory: "512Mi" - cpu: "500m" -# Extra annotations to attach to the service pods -# This should be a multi-line string mapping directly to the a map of -# the annotations to apply to the service pods - annotations: {} - tls: - secretName: "" - doa: name: doa repository: ghcr.io/neicnordic/sda-doa From 6fdcfa7458a9c7be12e356b686f76b3f93880e2e Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 31 Aug 2023 13:57:05 +0200 Subject: [PATCH 29/34] Use `appVersion` from Chart.yaml as image tag --- .github/integration/scripts/charts/deploy_charts.sh | 4 ++-- charts/sda-db/Chart.yaml | 3 ++- charts/sda-db/templates/statefulset.yaml | 2 +- charts/sda-db/values.yaml | 2 +- charts/sda-mq/Chart.yaml | 3 ++- charts/sda-mq/templates/statefulset.yaml | 2 +- charts/sda-mq/values.yaml | 2 +- charts/sda-svc/Chart.yaml | 3 ++- charts/sda-svc/templates/auth-deploy.yaml | 2 +- charts/sda-svc/templates/download-deploy.yaml | 2 +- charts/sda-svc/templates/finalize-deploy.yaml | 2 +- charts/sda-svc/templates/ingest-deploy.yaml | 2 +- charts/sda-svc/templates/intercept-deploy.yaml | 2 +- charts/sda-svc/templates/mapper-deploy.yaml | 2 +- charts/sda-svc/templates/s3-inbox-deploy.yaml | 2 +- charts/sda-svc/templates/sftp-inbox-deploy.yaml | 2 +- charts/sda-svc/templates/verify-deploy.yaml | 2 +- charts/sda-svc/values.yaml | 2 +- 18 files changed, 22 insertions(+), 19 deletions(-) diff --git a/.github/integration/scripts/charts/deploy_charts.sh b/.github/integration/scripts/charts/deploy_charts.sh index 08ffd059a..fcf0e8946 100644 --- a/.github/integration/scripts/charts/deploy_charts.sh +++ b/.github/integration/scripts/charts/deploy_charts.sh @@ -14,7 +14,7 @@ fi if [ "$1" == "sda-db" ]; then ROOTPASS=$(yq e '.global.db.password' .github/integration/scripts/charts/values.yaml) helm install postgres charts/sda-db \ - --set image.tag="PR$2-postgres" \ + --set image.tag="PR$2" \ --set image.pullPolicy=IfNotPresent \ --set global.postgresAdminPassword="$ROOTPASS" \ --set global.tls.clusterIssuer=cert-issuer \ @@ -27,7 +27,7 @@ fi if [ "$1" == "sda-mq" ]; then ADMINPASS=$(yq e '.global.broker.password' .github/integration/scripts/charts/values.yaml) helm install broker charts/sda-mq \ - --set image.tag="PR$2-rabbitmq" \ + --set image.tag="PR$2" \ --set image.pullPolicy=IfNotPresent \ --set global.adminPassword="$ADMINPASS" \ --set global.adminUser=admin \ diff --git a/charts/sda-db/Chart.yaml b/charts/sda-db/Chart.yaml index a751eeb51..7d9f0627f 100644 --- a/charts/sda-db/Chart.yaml +++ b/charts/sda-db/Chart.yaml @@ -1,6 +1,7 @@ apiVersion: v2 name: sda-db -version: "0.6.0" +version: "0.7.0" +appVersion: "v0.0.75" description: Database component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io icon: https://neic.no/assets/images/logo.png diff --git a/charts/sda-db/templates/statefulset.yaml b/charts/sda-db/templates/statefulset.yaml index fd6da055a..44ae6c980 100644 --- a/charts/sda-db/templates/statefulset.yaml +++ b/charts/sda-db/templates/statefulset.yaml @@ -67,7 +67,7 @@ spec: {{- end }} containers: - name: postgresql - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + image: "{{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }}-postgres" imagePullPolicy: {{ .Values.image.pullPolicy | quote }} securityContext: allowPrivilegeEscalation: false diff --git a/charts/sda-db/values.yaml b/charts/sda-db/values.yaml index eb26e5722..922a1ba24 100644 --- a/charts/sda-db/values.yaml +++ b/charts/sda-db/values.yaml @@ -25,7 +25,7 @@ extraSecurityContext: {} image: repository: ghcr.io/neicnordic/sensitive-data-archive - tag: v0.0.65-postgres + tag: pullPolicy: IfNotPresent # utilize network isolation diff --git a/charts/sda-mq/Chart.yaml b/charts/sda-mq/Chart.yaml index 031bca048..14bce59e2 100644 --- a/charts/sda-mq/Chart.yaml +++ b/charts/sda-mq/Chart.yaml @@ -1,6 +1,7 @@ apiVersion: v2 name: sda-mq -version: "0.5.1" +version: "0.6.0" +appVersion: "v0.0.75" description: RabbitMQ component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io icon: https://neic.no/assets/images/logo.png diff --git a/charts/sda-mq/templates/statefulset.yaml b/charts/sda-mq/templates/statefulset.yaml index 59fe8d31a..17d92ddcc 100644 --- a/charts/sda-mq/templates/statefulset.yaml +++ b/charts/sda-mq/templates/statefulset.yaml @@ -55,7 +55,7 @@ spec: {{- end }} containers: - name: broker - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + image: "{{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }}-rabbitmq" imagePullPolicy: {{ .Values.image.pullPolicy | quote }} securityContext: allowPrivilegeEscalation: false diff --git a/charts/sda-mq/values.yaml b/charts/sda-mq/values.yaml index 8fab60cbc..e182dc9da 100644 --- a/charts/sda-mq/values.yaml +++ b/charts/sda-mq/values.yaml @@ -44,7 +44,7 @@ extraSecurityContext: {} image: repository: ghcr.io/neicnordic/sensitive-data-archive - tag: v0.0.65-rabbitmq + tag: pullPolicy: Always # utilize network isolation diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index 5402b48b6..2266544fe 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,6 +1,7 @@ apiVersion: v2 name: sda-svc -version: "0.20.1" +version: "0.21.0" +appVersion: "v0.0.75" kubeVersion: ">= 1.19.0-0" description: Components for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-svc/templates/auth-deploy.yaml b/charts/sda-svc/templates/auth-deploy.yaml index cef75c1dd..7c07336df 100644 --- a/charts/sda-svc/templates/auth-deploy.yaml +++ b/charts/sda-svc/templates/auth-deploy.yaml @@ -58,7 +58,7 @@ spec: fsGroup: 65534 containers: - name: auth - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}-auth" + image: "{{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }}-auth" imagePullPolicy: {{ .Values.image.pullPolicy | quote }} securityContext: allowPrivilegeEscalation: false diff --git a/charts/sda-svc/templates/download-deploy.yaml b/charts/sda-svc/templates/download-deploy.yaml index 9429b8319..1c89a8dba 100644 --- a/charts/sda-svc/templates/download-deploy.yaml +++ b/charts/sda-svc/templates/download-deploy.yaml @@ -75,7 +75,7 @@ spec: {{- end }} containers: - name: download - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}-download" + image: "{{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }}-download" imagePullPolicy: {{ .Values.image.pullPolicy | quote }} securityContext: allowPrivilegeEscalation: false diff --git a/charts/sda-svc/templates/finalize-deploy.yaml b/charts/sda-svc/templates/finalize-deploy.yaml index d2c39aa09..8148a1f85 100644 --- a/charts/sda-svc/templates/finalize-deploy.yaml +++ b/charts/sda-svc/templates/finalize-deploy.yaml @@ -60,7 +60,7 @@ spec: {{- end }} containers: - name: finalize - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}-pipeline" + image: "{{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy | quote }} command: ["sda-finalize"] securityContext: diff --git a/charts/sda-svc/templates/ingest-deploy.yaml b/charts/sda-svc/templates/ingest-deploy.yaml index 77e10fd69..754294fe7 100644 --- a/charts/sda-svc/templates/ingest-deploy.yaml +++ b/charts/sda-svc/templates/ingest-deploy.yaml @@ -61,7 +61,7 @@ spec: {{- end }} containers: - name: ingest - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}-pipeline" + image: "{{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy | quote }} command: ["sda-ingest"] securityContext: diff --git a/charts/sda-svc/templates/intercept-deploy.yaml b/charts/sda-svc/templates/intercept-deploy.yaml index 5a22791f6..3d2dfce83 100644 --- a/charts/sda-svc/templates/intercept-deploy.yaml +++ b/charts/sda-svc/templates/intercept-deploy.yaml @@ -44,7 +44,7 @@ spec: fsGroup: 65534 containers: - name: intercept - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}-pipeline" + image: "{{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy | quote }} command: ["sda-intercept"] securityContext: diff --git a/charts/sda-svc/templates/mapper-deploy.yaml b/charts/sda-svc/templates/mapper-deploy.yaml index 16f6ef8c4..ad282eaf3 100644 --- a/charts/sda-svc/templates/mapper-deploy.yaml +++ b/charts/sda-svc/templates/mapper-deploy.yaml @@ -60,7 +60,7 @@ spec: {{- end }} containers: - name: mapper - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}-pipeline" + image: "{{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy | quote }} command: ["sda-mapper"] securityContext: diff --git a/charts/sda-svc/templates/s3-inbox-deploy.yaml b/charts/sda-svc/templates/s3-inbox-deploy.yaml index fa072dbe0..0be8d06e0 100644 --- a/charts/sda-svc/templates/s3-inbox-deploy.yaml +++ b/charts/sda-svc/templates/s3-inbox-deploy.yaml @@ -76,7 +76,7 @@ spec: {{- end }} containers: - name: s3inbox - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + image: "{{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy | quote }} command: ["sda-s3inbox"] securityContext: diff --git a/charts/sda-svc/templates/sftp-inbox-deploy.yaml b/charts/sda-svc/templates/sftp-inbox-deploy.yaml index 75e42ef45..07262b6ba 100644 --- a/charts/sda-svc/templates/sftp-inbox-deploy.yaml +++ b/charts/sda-svc/templates/sftp-inbox-deploy.yaml @@ -94,7 +94,7 @@ spec: {{- end }} containers: - name: inbox - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}-sftp-inbox" + image: "{{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }}-sftp-inbox" imagePullPolicy: {{ .Values.image.pullPolicy | quote }} command: ["java", "-jar", "inbox-0.0.3-SNAPSHOT.jar"] securityContext: diff --git a/charts/sda-svc/templates/verify-deploy.yaml b/charts/sda-svc/templates/verify-deploy.yaml index f226828ac..61d038f3a 100644 --- a/charts/sda-svc/templates/verify-deploy.yaml +++ b/charts/sda-svc/templates/verify-deploy.yaml @@ -61,7 +61,7 @@ spec: {{- end }} containers: - name: verify - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}-pipeline" + image: "{{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy | quote }} command: ["sda-verify"] securityContext: diff --git a/charts/sda-svc/values.yaml b/charts/sda-svc/values.yaml index 6cde7979b..be4e4633d 100644 --- a/charts/sda-svc/values.yaml +++ b/charts/sda-svc/values.yaml @@ -2,7 +2,7 @@ # Declare variables to be passed into your templates. image: repository: "ghcr.io/neicnordic/sensitive-data-archive" - tag: "v0.0.65" + tag: pullPolicy: "Always" global: From 40c79e00eaab0357893e545428f6ae5a4f9dc914 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 31 Aug 2023 15:01:48 +0200 Subject: [PATCH 30/34] [actions][build pr container] add debug on failure --- .github/workflows/build_pr_container.yaml | 30 +++++++++++++++++------ 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build_pr_container.yaml b/.github/workflows/build_pr_container.yaml index f9c9b5cc6..d6c83d490 100644 --- a/.github/workflows/build_pr_container.yaml +++ b/.github/workflows/build_pr_container.yaml @@ -245,11 +245,11 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} - name: Initialise k3d + id: initK3D run: bash .github/integration/scripts/charts/k3d.sh ${{matrix.version}} shell: bash - - name: debug - if: failure() + if: steps.initK3D.outcome == 'failure' run: k3d version list k3s | grep ${{matrix.version}} shell: bash @@ -258,11 +258,25 @@ jobs: shell: bash - name: Deploy DB + id: deployDB run: bash .github/integration/scripts/charts/deploy_charts.sh sda-db ${{ github.event.number }} ${{matrix.tls}} + - name: debug + if: steps.deployDB.outcome == 'failure' + run: | + kubectl describe pod postgres-sda-db-0 + sleep 1 + kubectl logs postgres-sda-db-0 - name: Deploy MQ + id: deployMQ run: bash .github/integration/scripts/charts/deploy_charts.sh sda-mq ${{ github.event.number }} ${{matrix.tls}} shell: bash + - name: debug + if: steps.deployMQ.outcome == 'failure' + run: | + kubectl describe pod broker-sda-mq-0 + sleep 1 + kubectl logs broker-sda-mq-0 - name: Deploy pipeline run: bash .github/integration/scripts/charts/deploy_charts.sh sda-svc ${{ github.event.number }} ${{matrix.tls}} @@ -271,12 +285,12 @@ jobs: - name: test if: always() run: | - kubectl get secret broker-sda-mq -o json - kubectl get secret pipeline-sda-svc-mapper -o json kubectl get pods - echo "describe mapper" && kubectl describe pod -l role=mapper - sleep 1 - echo "logs mapper" && kubectl logs -l role=mapper sleep 1 - echo "describe broker" && kubectl logs -l role=broker + for svc in auth finalize inbox ingest mapper verify; do + echo "## describe $svc" && kubectl describe pod -l role="$svc" + sleep 1 + echo "## logs $svc" && kubectl logs -l role="$svc" + sleep 1 + done shell: bash \ No newline at end of file From 2df4be8474037522fd5483d28aa0813004cc622e Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 31 Aug 2023 18:24:37 +0200 Subject: [PATCH 31/34] [s3inbox] fix helthcheck url --- sda/cmd/s3inbox/healthchecks.go | 8 +++++--- sda/internal/config/config.go | 1 - 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/sda/cmd/s3inbox/healthchecks.go b/sda/cmd/s3inbox/healthchecks.go index ad57b7bb3..6f3e6c166 100644 --- a/sda/cmd/s3inbox/healthchecks.go +++ b/sda/cmd/s3inbox/healthchecks.go @@ -8,9 +8,8 @@ import ( "strconv" "time" - "github.com/neicnordic/sensitive-data-archive/internal/config" - "github.com/heptiolabs/healthcheck" + "github.com/neicnordic/sensitive-data-archive/internal/config" ) // HealthCheck registers and endpoint for healthchecking the service @@ -26,8 +25,11 @@ type HealthCheck struct { // the backend S3 storage and the Message Broker so it can report readiness. func NewHealthCheck(port int, db *sql.DB, conf *config.Config, tlsConfig *tls.Config) *HealthCheck { s3URL := conf.Inbox.S3.URL + if conf.Inbox.S3.Port != 0 { + s3URL = fmt.Sprintf("%s:%d", s3URL, conf.Inbox.S3.Port) + } if conf.Inbox.S3.Readypath != "" { - s3URL = conf.Inbox.S3.URL + conf.Inbox.S3.Readypath + s3URL += conf.Inbox.S3.Readypath } brokerURL := fmt.Sprintf("%s:%d", conf.Broker.Host, conf.Broker.Port) diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index c585b24b0..530a5831a 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -669,7 +669,6 @@ func configS3Storage(prefix string) storage.S3Conf { // Defaults (move to viper?) - s3.Port = 443 s3.Region = "us-east-1" s3.NonExistRetryTime = 2 * time.Minute From 97049e9c5e5cbbec2f91ce95d882b92435245f32 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 31 Aug 2023 18:46:44 +0200 Subject: [PATCH 32/34] [finalize] Update readme --- sda/cmd/finalize/finalize.md | 157 +++++++++++++++++++++-------------- 1 file changed, 95 insertions(+), 62 deletions(-) diff --git a/sda/cmd/finalize/finalize.md b/sda/cmd/finalize/finalize.md index d2af7d3a2..afdb59faa 100644 --- a/sda/cmd/finalize/finalize.md +++ b/sda/cmd/finalize/finalize.md @@ -2,19 +2,20 @@ Handles the so-called _Accession ID (stable ID)_ to filename mappings from Central EGA. - ## Configuration There are a number of options that can be set for the finalize service. These settings can be set by mounting a yaml-file at `/config.yaml` with settings. - ex. + ```yaml log: level: "debug" format: "json" ``` + They may also be set using environment variables like: + ```bash export LOG_LEVEL="debug" export LOG_FORMAT="json" @@ -24,91 +25,123 @@ export LOG_FORMAT="json" These settings control how finalize connects to the RabbitMQ message broker. - - `BROKER_HOST`: hostname of the rabbitmq server - - - `BROKER_PORT`: rabbitmq broker port (commonly `5671` with TLS and `5672` without) - - - `BROKER_QUEUE`: message queue to read messages from (commonly `accessionIDs`) +- `BROKER_HOST`: hostname of the rabbitmq server +- `BROKER_PORT`: rabbitmq broker port (commonly `5671` with TLS and `5672` without) +- `BROKER_QUEUE`: message queue to read messages from (commonly `accessionIDs`) +- `BROKER_ROUTINGKEY`: message queue to write success messages to (commonly `backup`) +- `BROKER_USER`: username to connect to rabbitmq +- `BROKER_PASSWORD`: password to connect to rabbitmq +- `BROKER_PREFETCHCOUNT`: Number of messages to pull from the message server at the time (default to 2) + +### PostgreSQL Database settings + +- `DB_HOST`: hostname for the postgresql database +- `DB_PORT`: database port (commonly 5432) +- `DB_USER`: username for the database +- `DB_PASSWORD`: password for the database +- `DB_DATABASE`: database name +- `DB_SSLMODE`: The TLS encryption policy to use for database connections. + Valid options are: + - `disable` + - `allow` + - `prefer` + - `require` + - `verify-ca` + - `verify-full` - - `BROKER_ROUTINGKEY`: message queue to write success messages to (commonly `backup`) - - - `BROKER_USER`: username to connect to rabbitmq - - - `BROKER_PASSWORD`: password to connect to rabbitmq - - - `BROKER_PREFETCHCOUNT`: Number of messages to pull from the message server at the time (default to 2) + More information is available + [in the postgresql documentation](https://www.postgresql.org/docs/current/libpq-ssl.html#LIBPQ-SSL-PROTECTION) -### PostgreSQL Database settings: + Note that if `DB_SSLMODE` is set to anything but `disable`, then `DB_CACERT` needs to be set, + and if set to `verify-full`, then `DB_CLIENTCERT`, and `DB_CLIENTKEY` must also be set - - `DB_HOST`: hostname for the postgresql database +- `DB_CLIENTKEY`: key-file for the database client certificate +- `DB_CLIENTCERT`: database client certificate file +- `DB_CACERT`: Certificate Authority (CA) certificate for the database to use - - `DB_PORT`: database port (commonly 5432) +### Logging settings - - `DB_USER`: username for the database +- `LOG_FORMAT` can be set to “json” to get logs in json format. + All other values result in text logging - - `DB_PASSWORD`: password for the database +- `LOG_LEVEL` can be set to one of the following, in increasing order of severity: + - `trace` + - `debug` + - `info` + - `warn` (or `warning`) + - `error` + - `fatal` + - `panic` - - `DB_DATABASE`: database name +#### Keyfile settings - - `DB_SSLMODE`: The TLS encryption policy to use for database connections. - Valid options are: - - `disable` - - `allow` - - `prefer` - - `require` - - `verify-ca` - - `verify-full` +These settings control which crypt4gh keyfile is loaded. +These settings are only needed is `copyheader` is `true`. - More information is available - [in the postgresql documentation](https://www.postgresql.org/docs/current/libpq-ssl.html#LIBPQ-SSL-PROTECTION) +- `C4GH_FILEPATH`: path to the crypt4gh keyfile +- `C4GH_PASSPHRASE`: pass phrase to unlock the keyfile +- `C4GH_BACKUPPUBKEY`: path to the crypt4gh public key to use for reencrypting file headers. - Note that if `DB_SSLMODE` is set to anything but `disable`, then `DB_CACERT` needs to be set, - and if set to `verify-full`, then `DB_CLIENTCERT`, and `DB_CLIENTKEY` must also be set +### Storage settings - - `DB_CLIENTKEY`: key-file for the database client certificate +Storage backend is defined by the `ARCHIVE_TYPE`, and `BACKUP_TYPE` variables. +Valid values for these options are `S3` or `POSIX` +(Defaults to `POSIX` on unknown values). - - `DB_CLIENTCERT`: database client certificate file +The value of these variables define what other variables are read. +The same variables are available for all storage types, differing by prefix (`ARCHIVE_`, or `BACKUP_`) - - `DB_CACERT`: Certificate Authority (CA) certificate for the database to use +if `*_TYPE` is `S3` then the following variables are available: -### Logging settings: +- `*_URL`: URL to the S3 system +- `*_ACCESSKEY`: The S3 access and secret key are used to authenticate to S3, +[more info at AWS](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) +- `*_SECRETKEY`: The S3 access and secret key are used to authenticate to S3, +[more info at AWS](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) +- `*_BUCKET`: The S3 bucket to use as the storage root +- `*_PORT`: S3 connection port (default: `443`) +- `*_REGION`: S3 region (default: `us-east-1`) +- `*_CHUNKSIZE`: S3 chunk size for multipart uploads. +- `*_CACERT`: Certificate Authority (CA) certificate for the storage system, tjhis is only needed if the S3 server has a certificate signed by a private entity - - `LOG_FORMAT` can be set to “json” to get logs in json format. - All other values result in text logging +and if `*_TYPE` is `POSIX`: - - `LOG_LEVEL` can be set to one of the following, in increasing order of severity: - - `trace` - - `debug` - - `info` - - `warn` (or `warning`) - - `error` - - `fatal` - - `panic` +- `*_LOCATION`: POSIX path to use as storage root ## Service Description + Finalize adds stable, shareable _Accession ID_'s to archive files. +If a backup location is configured it will perform backup of a file. When running, finalize reads messages from the configured RabbitMQ queue (default "accessionIDs"). For each message, these steps are taken (if not otherwise noted, errors halt progress and the service moves on to the next message): -1. The message is validated as valid JSON that matches the "ingestion-accession" schema (defined in sda-common). +1. The message is validated as valid JSON that matches the "ingestion-accession" schema. If the message can’t be validated it is discarded with an error message in the logs. - -1. if the type of the `DecryptedChecksums` field in the message is `sha256`, the value is stored. - -1. A new RabbitMQ "complete" message is created and validated against the "ingestion-completion" schema. +2. If the service is configured to perform backups the file path and file size is fetched from the database. + 1. In case the service is configured to copy headers, the path is replaced by the one of the incoming message and it is the original location where the file was uploaded in the inbox. + 2. The file size on disk is requested from the storage system. + 3. The database file size is compared against the disk file size. + 4. A file reader is created for the archive storage file, and a file writer is created for the backup storage file. + 5. If the service is configured to copy headers: + 1. The header is read from the database. + On error, the error is written to the logs, but the message continues processing. + 2. The header is decrypted. + If this causes an error, the error is written to the logs, the message is Nack'ed, but message processing continues. + 3. The header is reencrypted. + If this causes an error, the error is written to the logs, the message is Nack'ed, but message processing continues. + 4. The header is written to the backup file writer. + On error, the error is written to the logs, but the message continues processing. +3. The file data is copied from the archive file reader to the backup file writer. +4. if the type of the `DecryptedChecksums` field in the message is `sha256`, the value is stored. +5. A new RabbitMQ "complete" message is created and validated against the "ingestion-completion" schema. If the validation fails, an error message is written to the logs. - -1. The file accession ID in the message is marked as "ready" in the database. +6. The file accession ID in the message is marked as "ready" in the database. On error the service sleeps for up to 5 minutes to allow for database recovery, after 5 minutes the message is Nacked, re-queued and an error message is written to the logs. - -1. The complete message is sent to RabbitMQ. On error, a message is written to the logs. - -1. The original RabbitMQ message is Ack'ed. +7. The complete message is sent to RabbitMQ. On error, a message is written to the logs. +8. The original RabbitMQ message is Ack'ed. ## Communication - - Finalize reads messages from one rabbitmq queue (default `accessionIDs`). - - - Finalize writes messages to one rabbitmq queue (default `backup`). - - - Finalize assigns the accession ID to a file in the database using the `SetAccessionID` function. +- Finalize reads messages from one rabbitmq queue (default `accessionIDs`). +- Finalize writes messages to one rabbitmq queue (default `backup`). +- Finalize assigns the accession ID to a file in the database using the `SetAccessionID` function. From 2240f94a3cb9b26b8a564626494187d4f67e330e Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 11 Sep 2023 12:31:28 +0200 Subject: [PATCH 33/34] Graceful shutdown Use `syscall.sigint` instead of log fatal during startup --- sda/cmd/finalize/finalize.go | 42 ++++++++++++++++++++++++++++----- sda/cmd/ingest/ingest.go | 16 +++++++++---- sda/cmd/intercept/intercept.go | 30 +++++++++++++++++++++--- sda/cmd/mapper/mapper.go | 38 ++++++++++++++++++++++++++---- sda/cmd/s3inbox/s3inbox.go | 5 ++-- sda/cmd/verify/verify.go | 43 ++++++++++++++++++++++++++++------ 6 files changed, 147 insertions(+), 27 deletions(-) diff --git a/sda/cmd/finalize/finalize.go b/sda/cmd/finalize/finalize.go index 4b47c6efa..dc074931d 100644 --- a/sda/cmd/finalize/finalize.go +++ b/sda/cmd/finalize/finalize.go @@ -6,6 +6,9 @@ import ( "encoding/json" "fmt" "io" + "os" + "os/signal" + "syscall" "github.com/neicnordic/crypt4gh/model/headers" "github.com/neicnordic/sensitive-data-archive/internal/broker" @@ -27,28 +30,53 @@ var message schema.IngestionAccession func main() { forever := make(chan bool) + sigc := make(chan os.Signal, 1) + signal.Notify(sigc, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) + // Create a function to handle panic and exit gracefully + defer func() { + if err := recover(); err != nil { + log.Errorln("Could not recover, exiting") + forever <- false + } + }() + + go func() { + <-sigc + forever <- false + }() + conf, err = config.NewConfig("finalize") if err != nil { - log.Fatal(err) + log.Error(err) + sigc <- syscall.SIGINT + panic(err) } mq, err := broker.NewMQ(conf.Broker) if err != nil { - log.Fatal(err) + log.Error(err) + sigc <- syscall.SIGINT + panic(err) } db, err = database.NewSDAdb(conf.Database) if err != nil { - log.Fatal(err) + log.Error(err) + sigc <- syscall.SIGINT + panic(err) } if conf.Backup.Type != "" && conf.Archive.Type != "" { log.Debugln("initiating storage backends") backup, err = storage.NewBackend(conf.Backup) if err != nil { - log.Fatal(err) + log.Error(err) + sigc <- syscall.SIGINT + panic(err) } archive, err = storage.NewBackend(conf.Archive) if err != nil { - log.Fatal(err) + log.Error(err) + sigc <- syscall.SIGINT + panic(err) } } @@ -72,7 +100,9 @@ func main() { go func() { messages, err := mq.GetMessages(conf.Broker.Queue) if err != nil { - log.Fatal(err) + log.Error(err) + sigc <- syscall.SIGINT + panic(err) } for delivered := range messages { log.Debugf("Received a message (corr-id: %s, message: %s)", delivered.CorrelationId, delivered.Body) diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index 618a32e1b..263106930 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -25,16 +25,22 @@ import ( ) func main() { - sigc := make(chan os.Signal, 5) + forever := make(chan bool) + sigc := make(chan os.Signal, 1) signal.Notify(sigc, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) // Create a function to handle panic and exit gracefully defer func() { if err := recover(); err != nil { - log.Fatal("Could not recover, exiting") + log.Errorln("Could not recover, exiting") + forever <- false } }() - forever := make(chan bool) + go func() { + <-sigc + forever <- false + }() + conf, err := config.NewConfig("ingest") if err != nil { log.Error(err) @@ -99,7 +105,9 @@ func main() { go func() { messages, err := mq.GetMessages(conf.Broker.Queue) if err != nil { - log.Fatal(err) + log.Error(err) + sigc <- syscall.SIGINT + panic(err) } mainWorkLoop: for delivered := range messages { diff --git a/sda/cmd/intercept/intercept.go b/sda/cmd/intercept/intercept.go index e916310c0..8bb592088 100644 --- a/sda/cmd/intercept/intercept.go +++ b/sda/cmd/intercept/intercept.go @@ -5,6 +5,9 @@ package main import ( "encoding/json" "errors" + "os" + "os/signal" + "syscall" "github.com/neicnordic/sensitive-data-archive/internal/broker" "github.com/neicnordic/sensitive-data-archive/internal/config" @@ -23,13 +26,32 @@ const ( func main() { forever := make(chan bool) + sigc := make(chan os.Signal, 1) + signal.Notify(sigc, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) + // Create a function to handle panic and exit gracefully + defer func() { + if err := recover(); err != nil { + log.Errorln("Could not recover, exiting") + forever <- false + } + }() + + go func() { + <-sigc + forever <- false + }() + conf, err := config.NewConfig("intercept") if err != nil { - log.Fatal(err) + log.Error(err) + sigc <- syscall.SIGINT + panic(err) } mq, err := broker.NewMQ(conf.Broker) if err != nil { - log.Fatal(err) + log.Error(err) + sigc <- syscall.SIGINT + panic(err) } defer mq.Channel.Close() @@ -52,7 +74,9 @@ func main() { go func() { messages, err := mq.GetMessages(conf.Broker.Queue) if err != nil { - log.Fatal(err) + log.Error(err) + sigc <- syscall.SIGINT + panic(err) } for delivered := range messages { log.Debugf("Received a message: %s", delivered.Body) diff --git a/sda/cmd/mapper/mapper.go b/sda/cmd/mapper/mapper.go index 53c7f7042..882d10292 100644 --- a/sda/cmd/mapper/mapper.go +++ b/sda/cmd/mapper/mapper.go @@ -5,6 +5,9 @@ package main import ( "encoding/json" "fmt" + "os" + "os/signal" + "syscall" "github.com/neicnordic/sensitive-data-archive/internal/broker" "github.com/neicnordic/sensitive-data-archive/internal/config" @@ -17,21 +20,44 @@ import ( func main() { forever := make(chan bool) + sigc := make(chan os.Signal, 1) + signal.Notify(sigc, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) + // Create a function to handle panic and exit gracefully + defer func() { + if err := recover(); err != nil { + log.Errorln("Could not recover, exiting") + forever <- false + } + }() + + go func() { + <-sigc + forever <- false + }() + conf, err := config.NewConfig("mapper") if err != nil { - log.Fatal(err) + log.Error(err) + sigc <- syscall.SIGINT + panic(err) } mq, err := broker.NewMQ(conf.Broker) if err != nil { - log.Fatal(err) + log.Error(err) + sigc <- syscall.SIGINT + panic(err) } db, err := database.NewSDAdb(conf.Database) if err != nil { - log.Fatal(err) + log.Error(err) + sigc <- syscall.SIGINT + panic(err) } inbox, err := storage.NewBackend(conf.Inbox) if err != nil { - log.Fatal(err) + log.Error(err) + sigc <- syscall.SIGINT + panic(err) } defer mq.Channel.Close() @@ -56,7 +82,9 @@ func main() { go func() { messages, err := mq.GetMessages(conf.Broker.Queue) if err != nil { - log.Fatalf("Failed to get message from mq (error: %v)", err) + log.Error(err) + sigc <- syscall.SIGINT + panic(err) } for delivered := range messages { diff --git a/sda/cmd/s3inbox/s3inbox.go b/sda/cmd/s3inbox/s3inbox.go index c6e62f692..e7f4f20f4 100644 --- a/sda/cmd/s3inbox/s3inbox.go +++ b/sda/cmd/s3inbox/s3inbox.go @@ -19,13 +19,14 @@ import ( var Conf *config.Config func main() { - sigc := make(chan os.Signal, 5) + sigc := make(chan os.Signal, 1) signal.Notify(sigc, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) // Create a function to handle panic and exit gracefully defer func() { if err := recover(); err != nil { - log.Fatal("Could not recover, exiting") + log.Errorln("Could not recover, exiting") + os.Exit(1) } }() diff --git a/sda/cmd/verify/verify.go b/sda/cmd/verify/verify.go index 89b046472..96c46e41d 100644 --- a/sda/cmd/verify/verify.go +++ b/sda/cmd/verify/verify.go @@ -9,6 +9,9 @@ import ( "encoding/json" "fmt" "io" + "os" + "os/signal" + "syscall" "github.com/neicnordic/crypt4gh/streaming" "github.com/neicnordic/sensitive-data-archive/internal/broker" @@ -22,25 +25,50 @@ import ( func main() { forever := make(chan bool) + sigc := make(chan os.Signal, 1) + signal.Notify(sigc, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) + // Create a function to handle panic and exit gracefully + defer func() { + if err := recover(); err != nil { + log.Errorln("Could not recover, exiting") + forever <- false + } + }() + + go func() { + <-sigc + forever <- false + }() + conf, err := config.NewConfig("verify") if err != nil { - log.Fatal(err) + log.Error(err) + sigc <- syscall.SIGINT + panic(err) } mq, err := broker.NewMQ(conf.Broker) if err != nil { - log.Fatal(err) + log.Error(err) + sigc <- syscall.SIGINT + panic(err) } db, err := database.NewSDAdb(conf.Database) if err != nil { - log.Fatal(err) + log.Error(err) + sigc <- syscall.SIGINT + panic(err) } archive, err := storage.NewBackend(conf.Archive) if err != nil { - log.Fatal(err) + log.Error(err) + sigc <- syscall.SIGINT + panic(err) } key, err := config.GetC4GHKey() if err != nil { - log.Fatal(err) + log.Error(err) + sigc <- syscall.SIGINT + panic(err) } defer mq.Channel.Close() @@ -65,8 +93,9 @@ func main() { go func() { messages, err := mq.GetMessages(conf.Broker.Queue) if err != nil { - log.Fatalf("Failed to get messages (error: %v) ", - err) + log.Error(err) + sigc <- syscall.SIGINT + panic(err) } for delivered := range messages { log.Debugf("received a message (corr-id: %s, message: %s)", delivered.CorrelationId, delivered.Body) From 8d3212418a9726caef2fcef53c8b660bba839f21 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 12 Sep 2023 07:44:06 +0200 Subject: [PATCH 34/34] Update dependencies --- sda/go.mod | 12 ++++++------ sda/go.sum | 30 +++++++++++++----------------- 2 files changed, 19 insertions(+), 23 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index b45131a7b..dc2cbeedf 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -4,14 +4,14 @@ go 1.20 require ( github.com/DATA-DOG/go-sqlmock v1.5.0 - github.com/aws/aws-sdk-go v1.44.329 + github.com/aws/aws-sdk-go v1.45.7 github.com/golang-jwt/jwt/v4 v4.5.0 github.com/google/uuid v1.1.2 github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb github.com/lestrrat-go/jwx v1.2.26 github.com/lib/pq v1.10.9 github.com/minio/minio-go/v6 v6.0.57 - github.com/neicnordic/crypt4gh v1.7.6 + github.com/neicnordic/crypt4gh v1.8.2 github.com/ory/dockertest/v3 v3.10.0 github.com/pkg/errors v0.9.1 github.com/pkg/sftp v1.13.6 @@ -20,7 +20,7 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spf13/viper v1.16.0 github.com/stretchr/testify v1.8.4 - golang.org/x/crypto v0.12.0 + golang.org/x/crypto v0.13.0 ) require ( @@ -62,7 +62,7 @@ require ( github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.2 // indirect github.com/opencontainers/runc v1.1.7 // indirect - github.com/pelletier/go-toml/v2 v2.0.9 // indirect + github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.16.0 // indirect github.com/prometheus/client_model v0.4.0 // indirect @@ -77,8 +77,8 @@ require ( github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect golang.org/x/mod v0.9.0 // indirect - golang.org/x/sys v0.11.0 // indirect - golang.org/x/text v0.12.0 // indirect + golang.org/x/sys v0.12.0 // indirect + golang.org/x/text v0.13.0 // indirect golang.org/x/tools v0.7.0 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 // indirect diff --git a/sda/go.sum b/sda/go.sum index e173b3358..8b068597c 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -49,12 +49,8 @@ github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migc github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= -github.com/aws/aws-sdk-go v1.44.325 h1:jF/L99fJSq/BfiLmUOflO/aM+LwcqBm0Fe/qTK5xxuI= -github.com/aws/aws-sdk-go v1.44.325/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= -github.com/aws/aws-sdk-go v1.44.326 h1:/6xD/9mKZ2RMTDfbhh9qCxw+CaTbJRvfHJ/NHPFbI38= -github.com/aws/aws-sdk-go v1.44.326/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= -github.com/aws/aws-sdk-go v1.44.329 h1:Rqy+wYI8h+iq+FphR59KKTsHR1Lz7YiwRqFzWa7xoYU= -github.com/aws/aws-sdk-go v1.44.329/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.45.7 h1:k4QsvWZhm8409TYeRuTV1P6+j3lLKoe+giFA/j3VAps= +github.com/aws/aws-sdk-go v1.45.7/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= @@ -236,8 +232,8 @@ github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 h1:rzf0wL0CHVc8CEsgyygG0 github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/neicnordic/crypt4gh v1.7.6 h1:Vqcb8Yb950oaBBJFepDK1oLeu9rZzpywYWVHLmO0oI8= -github.com/neicnordic/crypt4gh v1.7.6/go.mod h1:rqmVXsprDFBRRLJkm1cK9kLETBPGEZmft9lHD/V40wk= +github.com/neicnordic/crypt4gh v1.8.2 h1:KNqYBBDU0qW296I6yLoA7l0GoNA/lfzhpy9RDkzNrRM= +github.com/neicnordic/crypt4gh v1.8.2/go.mod h1:VftsV+iUntv40/EB9TbnBnQ3/IDH40zEAqcMajrFVVg= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= @@ -246,8 +242,8 @@ github.com/opencontainers/runc v1.1.7 h1:y2EZDS8sNng4Ksf0GUYNhKbTShZJPJg1FiXJNH/ github.com/opencontainers/runc v1.1.7/go.mod h1:CbUumNnWCuTGFukNXahoo/RFBZvDAgRh/smNYNOhA50= github.com/ory/dockertest/v3 v3.10.0 h1:4K3z2VMe8Woe++invjaTB7VRyQXQy5UY+loujO4aNE4= github.com/ory/dockertest/v3 v3.10.0/go.mod h1:nr57ZbRWMqfsdGdFNLHz5jjNdDb7VVFnzAeW1n5N1Lg= -github.com/pelletier/go-toml/v2 v2.0.9 h1:uH2qQXheeefCCkuBBSLi7jCiSmj3VRh2+Goq2N7Xxu0= -github.com/pelletier/go-toml/v2 v2.0.9/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= +github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -332,8 +328,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= -golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= -golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= +golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -476,14 +472,14 @@ golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= +golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -495,8 +491,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= -golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=