From c28f0f8633bb26a4956e1ae687f7c34856527154 Mon Sep 17 00:00:00 2001 From: Steven Hartland Date: Mon, 2 Dec 2024 13:00:58 +0000 Subject: [PATCH] refactor(cockroachdb): to use request driven options (#2883) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: use recommended cluster settings for cockroachdb * fix: wrap errors * feat: add WithStatements option and expose recommended settings as a list of statements * Update modules/cockroachdb/options.go Co-authored-by: Steven Hartland * Update modules/cockroachdb/cockroachdb.go Co-authored-by: Steven Hartland * fix: return close error on defer * chore: improvement documentation and set default value for statements in defaultOptions * Update modules/cockroachdb/cockroachdb.go Co-authored-by: Manuel de la Peña * chore: remove unnecessary nolint directive and rename ClusterDefaults to DefaultStatements * fix: only use recommended settings in root user tests * feat!(cockroachdb): simplify connection handling Simplify the connection handling in cockroachdb so that ConnectionString can be used without the user doing extra work to handle TLS if enabled. Deprecate TLSConfig which is no longer needed separately. BREAKING_CHANGE: This now returns a registered connection string so is no longer compatible with pgx.ParseConfig, use ConnectionConfig for this use case instead. * refactor(cockroachdb): to use request driven options Refactor cockroachdb module to to use request driven options, simplifying the flow. * fix: various fixes and remove certs Various fixes and remove the ability to configure certs using the default generated ones instead. * fix: add missing data files Add the missing data file and remove unused constants. * chore: add reference to ErrTLSNotEnabled * fix: lint failure for missing test helper * feat: tls strategy and strategy walk Extract TLS certificate wait strategy into a dedicated wait type so it can be reused. Implement walk method which can be used to identify wait strategies in a request chain. Use embed to simplify wait test loading of certs. * chore(cockroachdb): remove customizer Remove the now unused customizer. * fix: check setOptions error return * fix: lint style Fix lint style issue for blank line. * chore: various clean ups * docs(cockroachdb): clarified run defaults Clarified the impact of run defaults. --------- Co-authored-by: Martin Asquino Co-authored-by: martskins Co-authored-by: Manuel de la Peña --- docs/features/tls.md | 4 +- docs/modules/cockroachdb.md | 33 +- modules/cockroachdb/certs.go | 67 ---- modules/cockroachdb/cockroachdb.go | 369 ++++++++++-------- modules/cockroachdb/cockroachdb_test.go | 247 +++--------- modules/cockroachdb/data/cluster_defaults.sql | 8 + modules/cockroachdb/examples_test.go | 91 ++++- modules/cockroachdb/go.mod | 1 - modules/cockroachdb/go.sum | 2 - modules/cockroachdb/options.go | 142 ++++--- modules/cockroachdb/testdata/__init.sql | 1 + modules/rabbitmq/examples_test.go | 4 + 12 files changed, 486 insertions(+), 483 deletions(-) delete mode 100644 modules/cockroachdb/certs.go create mode 100644 modules/cockroachdb/data/cluster_defaults.sql create mode 100644 modules/cockroachdb/testdata/__init.sql diff --git a/docs/features/tls.md b/docs/features/tls.md index fd8b95266d..130f789b5f 100644 --- a/docs/features/tls.md +++ b/docs/features/tls.md @@ -12,6 +12,6 @@ The example will also create a client that will connect to the server using the demonstrating how to use the generated certificate to communicate with a service. -[Create a self-signed certificate](../../modules/cockroachdb/certs.go) inside_block:exampleSelfSignedCert -[Sign a self-signed certificate](../../modules/cockroachdb/certs.go) inside_block:exampleSignSelfSignedCert +[Create a self-signed certificate](../../modules/rabbitmq/examples_test.go) inside_block:exampleSelfSignedCert +[Sign a self-signed certificate](../../modules/rabbitmq/examples_test.go) inside_block:exampleSignSelfSignedCert diff --git a/docs/modules/cockroachdb.md b/docs/modules/cockroachdb.md index 6bbdba0792..39956d5417 100644 --- a/docs/modules/cockroachdb.md +++ b/docs/modules/cockroachdb.md @@ -10,7 +10,7 @@ The Testcontainers module for CockroachDB. Please run the following command to add the CockroachDB module to your Go dependencies: -``` +```shell go get github.com/testcontainers/testcontainers-go/modules/cockroachdb ``` @@ -54,9 +54,11 @@ E.g. `Run(context.Background(), "cockroachdb/cockroach:latest-v23.1")`. Set the database that is created & dialled with `cockroachdb.WithDatabase`. -#### Password authentication +#### User and Password + +You can configured the container to create a user with a password by setting `cockroachdb.WithUser` and `cockroachdb.WithPassword`. -Disable insecure mode and connect with password authentication by setting `cockroachdb.WithUser` and `cockroachdb.WithPassword`. +`cockroachdb.WithPassword` is incompatible with `cockroachdb.WithInsecure`. #### Store size @@ -64,13 +66,21 @@ Control the maximum amount of memory used for storage, by default this is 100% b #### TLS authentication -`cockroachdb.WithTLS` lets you provide the CA certificate along with the certicate and key for the node & clients to connect with. -Internally CockroachDB requires a client certificate for the user to connect with. +`cockroachdb.WithInsecure` lets you disable the use of TLS on connections. + +`cockroachdb.WithInsecure` is incompatible with `cockroachdb.WithPassword`. + +#### Initialization Scripts + +`cockroachdb.WithInitScripts` adds the given scripts to those automatically run when the container starts. +These will be ignored if data exists in the `/cockroach/cockroach-data` directory within the container. -A helper `cockroachdb.NewTLSConfig` exists to generate all of this for you. +`cockroachdb.WithNoClusterDefaults` disables the default cluster settings script. -!!!warning - When TLS is enabled there's a very small, unlikely chance that the underlying driver can panic when registering the driver as part of waiting for CockroachDB to be ready to accept connections. If this is repeatedly happening please open an issue. +Without this option Cockroach containers run `data/cluster-defaults.sql` on startup +which configures the settings recommended by Cockroach Labs for +[local testing clusters](https://www.cockroachlabs.com/docs/stable/local-testing) +unless data exists in the `/cockroach/cockroach-data` directory within the container. ### Container Methods @@ -87,3 +97,10 @@ Same as `ConnectionString` but any error to generate the address will raise a pa #### TLSConfig Returns `*tls.Config` setup to allow you to dial your client over TLS, if enabled, else this will error with `cockroachdb.ErrTLSNotEnabled`. + +!!!info + The `TLSConfig()` function is deprecated and will be removed in the next major release of _Testcontainers for Go_. + +#### ConnectionConfig + +Returns `*pgx.ConnConfig` which can be passed to `pgx.ConnectConfig` to open a new connection. diff --git a/modules/cockroachdb/certs.go b/modules/cockroachdb/certs.go deleted file mode 100644 index afa12fcd1a..0000000000 --- a/modules/cockroachdb/certs.go +++ /dev/null @@ -1,67 +0,0 @@ -package cockroachdb - -import ( - "crypto/x509" - "errors" - "net" - "time" - - "github.com/mdelapenya/tlscert" -) - -type TLSConfig struct { - CACert *x509.Certificate - NodeCert []byte - NodeKey []byte - ClientCert []byte - ClientKey []byte -} - -// NewTLSConfig creates a new TLSConfig capable of running CockroachDB & connecting over TLS. -func NewTLSConfig() (*TLSConfig, error) { - // exampleSelfSignedCert { - caCert := tlscert.SelfSignedFromRequest(tlscert.Request{ - Name: "ca", - SubjectCommonName: "Cockroach Test CA", - Host: "localhost,127.0.0.1", - IsCA: true, - ValidFor: time.Hour, - }) - if caCert == nil { - return nil, errors.New("failed to generate CA certificate") - } - // } - - // exampleSignSelfSignedCert { - nodeCert := tlscert.SelfSignedFromRequest(tlscert.Request{ - Name: "node", - SubjectCommonName: "node", - Host: "localhost,127.0.0.1", - IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback}, - ValidFor: time.Hour, - Parent: caCert, // using the CA certificate as parent - }) - if nodeCert == nil { - return nil, errors.New("failed to generate node certificate") - } - // } - - clientCert := tlscert.SelfSignedFromRequest(tlscert.Request{ - Name: "client", - SubjectCommonName: defaultUser, - Host: "localhost,127.0.0.1", - ValidFor: time.Hour, - Parent: caCert, // using the CA certificate as parent - }) - if clientCert == nil { - return nil, errors.New("failed to generate client certificate") - } - - return &TLSConfig{ - CACert: caCert.Cert, - NodeCert: nodeCert.Bytes, - NodeKey: nodeCert.KeyBytes, - ClientCert: clientCert.Bytes, - ClientKey: clientCert.KeyBytes, - }, nil -} diff --git a/modules/cockroachdb/cockroachdb.go b/modules/cockroachdb/cockroachdb.go index 092efa4e2a..40da90fcd1 100644 --- a/modules/cockroachdb/cockroachdb.go +++ b/modules/cockroachdb/cockroachdb.go @@ -1,15 +1,14 @@ package cockroachdb import ( + "bytes" "context" "crypto/tls" - "crypto/x509" - "encoding/pem" + _ "embed" "errors" "fmt" "net" "net/url" - "path/filepath" "github.com/docker/go-connections/nat" "github.com/jackc/pgx/v5" @@ -19,11 +18,10 @@ import ( "github.com/testcontainers/testcontainers-go/wait" ) +// ErrTLSNotEnabled is returned when trying to get a TLS config from a container that does not have TLS enabled. var ErrTLSNotEnabled = errors.New("tls not enabled") const ( - certsDir = "/tmp" - defaultSQLPort = "26257/tcp" defaultAdminPort = "8080/tcp" @@ -31,15 +29,63 @@ const ( defaultPassword = "" defaultDatabase = "defaultdb" defaultStoreSize = "100%" + + // initDBPath is the path where the init scripts are placed in the container. + initDBPath = "/docker-entrypoint-initdb.d" + + // cockroachDir is the path where the CockroachDB files are placed in the container. + cockroachDir = "/cockroach" + + // clusterDefaultsContainerFile is the path to the default cluster settings script in the container. + clusterDefaultsContainerFile = initDBPath + "/__cluster_defaults.sql" + + // memStorageFlag is the flag to use in the start command to use an in-memory store. + memStorageFlag = "--store=type=mem,size=" + + // insecureFlag is the flag to use in the start command to disable TLS. + insecureFlag = "--insecure" + + // env vars. + envUser = "COCKROACH_USER" + envPassword = "COCKROACH_PASSWORD" + envDatabase = "COCKROACH_DATABASE" + + // cert files. + certsDir = cockroachDir + "/certs" + fileCACert = certsDir + "/ca.crt" ) +//go:embed data/cluster_defaults.sql +var clusterDefaults []byte + +// defaultsReader is a reader for the default settings scripts +// so that they can be identified and removed from the request. +type defaultsReader struct { + *bytes.Reader +} + +// newDefaultsReader creates a new reader for the default cluster settings script. +func newDefaultsReader(data []byte) *defaultsReader { + return &defaultsReader{Reader: bytes.NewReader(data)} +} + // CockroachDBContainer represents the CockroachDB container type used in the module type CockroachDBContainer struct { testcontainers.Container - opts options + options +} + +// options represents the options for the CockroachDBContainer type. +type options struct { + database string + user string + password string + tlsStrategy *wait.TLSStrategy } -// MustConnectionString panics if the address cannot be determined. +// MustConnectionString returns a connection string to open a new connection to CockroachDB +// as described by [CockroachDBContainer.ConnectionString]. +// It panics if an error occurs. func (c *CockroachDBContainer) MustConnectionString(ctx context.Context) string { addr, err := c.ConnectionString(ctx) if err != nil { @@ -48,35 +94,86 @@ func (c *CockroachDBContainer) MustConnectionString(ctx context.Context) string return addr } -// ConnectionString returns the dial address to open a new connection to CockroachDB. +// ConnectionString returns a connection string to open a new connection to CockroachDB. +// The returned string is suitable for use by [sql.Open] but is not be compatible with +// [pgx.ParseConfig], so if you want to call [pgx.ConnectConfig] use the +// [CockroachDBContainer.ConnectionConfig] method instead. func (c *CockroachDBContainer) ConnectionString(ctx context.Context) (string, error) { + cfg, err := c.ConnectionConfig(ctx) + if err != nil { + return "", fmt.Errorf("connection config: %w", err) + } + + return stdlib.RegisterConnConfig(cfg), nil +} + +// ConnectionConfig returns a [pgx.ConnConfig] for the CockroachDB container. +// This can be passed to [pgx.ConnectConfig] to open a new connection. +func (c *CockroachDBContainer) ConnectionConfig(ctx context.Context) (*pgx.ConnConfig, error) { port, err := c.MappedPort(ctx, defaultSQLPort) if err != nil { - return "", err + return nil, fmt.Errorf("mapped port: %w", err) } host, err := c.Host(ctx) if err != nil { - return "", err + return nil, fmt.Errorf("host: %w", err) } - return connString(c.opts, host, port), nil + return c.connConfig(host, port) } // TLSConfig returns config necessary to connect to CockroachDB over TLS. +// Returns [ErrTLSNotEnabled] if TLS is not enabled. +// +// Deprecated: use [CockroachDBContainer.ConnectionString] or +// [CockroachDBContainer.ConnectionConfig] instead. func (c *CockroachDBContainer) TLSConfig() (*tls.Config, error) { - return connTLS(c.opts) + if cfg := c.tlsStrategy.TLSConfig(); cfg != nil { + return cfg, nil + } + + return nil, ErrTLSNotEnabled } -// Deprecated: use Run instead +// Deprecated: use Run instead. // RunContainer creates an instance of the CockroachDB container type func RunContainer(ctx context.Context, opts ...testcontainers.ContainerCustomizer) (*CockroachDBContainer, error) { return Run(ctx, "cockroachdb/cockroach:latest-v23.1", opts...) } -// Run creates an instance of the CockroachDB container type +// Run start an instance of the CockroachDB container type using the given image and options. +// +// By default, the container will configured with: +// - Cluster: Single node +// - Storage: 100% in-memory +// - User: root +// - Password: "" +// - Database: defaultdb +// - Exposed ports: 26257/tcp (SQL), 8080/tcp (Admin UI) +// - Init Scripts: `data/cluster_defaults.sql` +// +// This supports CockroachDB images v22.2.0 and later, earlier versions will only work with +// customised options, such as disabling TLS and removing the wait for `init_success` using +// a [testcontainers.ContainerCustomizer]. +// +// The init script `data/cluster_defaults.sql` configures the settings recommended +// by Cockroach Labs for [local testing clusters] unless data exists in the +// `/cockroach/cockroach-data` directory within the container. Use [WithNoClusterDefaults] +// to disable this behaviour and provide your own settings using [WithInitScripts]. +// +// For more information see starting a [local cluster in docker]. +// +// [local cluster in docker]: https://www.cockroachlabs.com/docs/stable/start-a-local-cluster-in-docker-linux +// [local testing clusters]: https://www.cockroachlabs.com/docs/stable/local-testing func Run(ctx context.Context, img string, opts ...testcontainers.ContainerCustomizer) (*CockroachDBContainer, error) { - o := defaultOptions() + ctr := &CockroachDBContainer{ + options: options{ + database: defaultDatabase, + user: defaultUser, + password: defaultPassword, + }, + } req := testcontainers.GenericContainerRequest{ ContainerRequest: testcontainers.ContainerRequest{ Image: img, @@ -84,164 +181,80 @@ func Run(ctx context.Context, img string, opts ...testcontainers.ContainerCustom defaultSQLPort, defaultAdminPort, }, - LifecycleHooks: []testcontainers.ContainerLifecycleHooks{ - { - PreStarts: []testcontainers.ContainerHook{ - func(ctx context.Context, container testcontainers.Container) error { - return addTLS(ctx, container, o) - }, - }, - }, + Env: map[string]string{ + "COCKROACH_DATABASE": defaultDatabase, + "COCKROACH_USER": defaultUser, + "COCKROACH_PASSWORD": defaultPassword, + }, + Files: []testcontainers.ContainerFile{{ + Reader: newDefaultsReader(clusterDefaults), + ContainerFilePath: clusterDefaultsContainerFile, + FileMode: 0o644, + }}, + Cmd: []string{ + "start-single-node", + memStorageFlag + defaultStoreSize, }, + WaitingFor: wait.ForAll( + wait.ForFile(cockroachDir+"/init_success"), + wait.ForHTTP("/health").WithPort(defaultAdminPort), + wait.ForTLSCert( + certsDir+"/client."+defaultUser+".crt", + certsDir+"/client."+defaultUser+".key", + ).WithRootCAs(fileCACert).WithServerName("127.0.0.1"), + wait.ForSQL(defaultSQLPort, "pgx/v5", func(host string, port nat.Port) string { + connStr, err := ctr.connString(host, port) + if err != nil { + panic(err) + } + return connStr + }), + ), }, Started: true, } - // apply options for _, opt := range opts { - if apply, ok := opt.(Option); ok { - apply(&o) - } if err := opt.Customize(&req); err != nil { - return nil, err + return nil, fmt.Errorf("customize request: %w", err) } } - // modify request - for _, fn := range []modiferFunc{ - addEnvs, - addCmd, - addWaitingFor, - } { - if err := fn(&req, o); err != nil { - return nil, err - } - } - - container, err := testcontainers.GenericContainer(ctx, req) - var c *CockroachDBContainer - if container != nil { - c = &CockroachDBContainer{Container: container, opts: o} + if err := ctr.configure(&req); err != nil { + return nil, fmt.Errorf("set options: %w", err) } + var err error + ctr.Container, err = testcontainers.GenericContainer(ctx, req) if err != nil { - return c, fmt.Errorf("generic container: %w", err) + return ctr, fmt.Errorf("generic container: %w", err) } - return c, nil + return ctr, nil } -type modiferFunc func(*testcontainers.GenericContainerRequest, options) error - -func addCmd(req *testcontainers.GenericContainerRequest, opts options) error { - req.Cmd = []string{ - "start-single-node", - "--store=type=mem,size=" + opts.StoreSize, - } - - // authN - if opts.TLS != nil { - if opts.User != defaultUser { - return fmt.Errorf("unsupported user %s with TLS, use %s", opts.User, defaultUser) - } - if opts.Password != "" { - return errors.New("cannot use password authentication with TLS") - } - } - - switch { - case opts.TLS != nil: - req.Cmd = append(req.Cmd, "--certs-dir="+certsDir) - case opts.Password != "": - req.Cmd = append(req.Cmd, "--accept-sql-without-tls") - default: - req.Cmd = append(req.Cmd, "--insecure") - } - return nil -} - -func addEnvs(req *testcontainers.GenericContainerRequest, opts options) error { - if req.Env == nil { - req.Env = make(map[string]string) +// connString returns a connection string for the given host, port and options. +func (c *CockroachDBContainer) connString(host string, port nat.Port) (string, error) { + cfg, err := c.connConfig(host, port) + if err != nil { + return "", fmt.Errorf("connection config: %w", err) } - req.Env["COCKROACH_DATABASE"] = opts.Database - req.Env["COCKROACH_USER"] = opts.User - req.Env["COCKROACH_PASSWORD"] = opts.Password - return nil + return stdlib.RegisterConnConfig(cfg), nil } -func addWaitingFor(req *testcontainers.GenericContainerRequest, opts options) error { - var tlsConfig *tls.Config - if opts.TLS != nil { - cfg, err := connTLS(opts) - if err != nil { - return err - } - tlsConfig = cfg - } - - sqlWait := wait.ForSQL(defaultSQLPort, "pgx/v5", func(host string, port nat.Port) string { - connStr := connString(opts, host, port) - if tlsConfig == nil { - return connStr - } - - // register TLS config with pgx driver - connCfg, err := pgx.ParseConfig(connStr) - if err != nil { - panic(err) - } - connCfg.TLSConfig = tlsConfig - - return stdlib.RegisterConnConfig(connCfg) - }) - defaultStrategy := wait.ForAll( - wait.ForHTTP("/health").WithPort(defaultAdminPort), - sqlWait, - ) - - if req.WaitingFor == nil { - req.WaitingFor = defaultStrategy +// connConfig returns a [pgx.ConnConfig] for the given host, port and options. +func (c *CockroachDBContainer) connConfig(host string, port nat.Port) (*pgx.ConnConfig, error) { + var user *url.Userinfo + if c.password != "" { + user = url.UserPassword(c.user, c.password) } else { - req.WaitingFor = wait.ForAll(req.WaitingFor, defaultStrategy) - } - - return nil -} - -func addTLS(ctx context.Context, container testcontainers.Container, opts options) error { - if opts.TLS == nil { - return nil - } - - caBytes := pem.EncodeToMemory(&pem.Block{ - Type: "CERTIFICATE", - Bytes: opts.TLS.CACert.Raw, - }) - files := map[string][]byte{ - "ca.crt": caBytes, - "node.crt": opts.TLS.NodeCert, - "node.key": opts.TLS.NodeKey, - "client.root.crt": opts.TLS.ClientCert, - "client.root.key": opts.TLS.ClientKey, - } - for filename, contents := range files { - if err := container.CopyToContainer(ctx, contents, filepath.Join(certsDir, filename), 0o600); err != nil { - return err - } - } - return nil -} - -func connString(opts options, host string, port nat.Port) string { - user := url.User(opts.User) - if opts.Password != "" { - user = url.UserPassword(opts.User, opts.Password) + user = url.User(c.user) } sslMode := "disable" - if opts.TLS != nil { + tlsConfig := c.tlsStrategy.TLSConfig() + if tlsConfig != nil { sslMode = "verify-full" } params := url.Values{ @@ -252,29 +265,57 @@ func connString(opts options, host string, port nat.Port) string { Scheme: "postgres", User: user, Host: net.JoinHostPort(host, port.Port()), - Path: opts.Database, + Path: c.database, RawQuery: params.Encode(), } - return u.String() + cfg, err := pgx.ParseConfig(u.String()) + if err != nil { + return nil, fmt.Errorf("parse config: %w", err) + } + + cfg.TLSConfig = tlsConfig + + return cfg, nil } -func connTLS(opts options) (*tls.Config, error) { - if opts.TLS == nil { - return nil, ErrTLSNotEnabled +// configure sets the CockroachDBContainer options from the given request and updates the request +// wait strategies to match the options. +func (c *CockroachDBContainer) configure(req *testcontainers.GenericContainerRequest) error { + c.database = req.Env[envDatabase] + c.user = req.Env[envUser] + c.password = req.Env[envPassword] + + var insecure bool + for _, arg := range req.Cmd { + if arg == insecureFlag { + insecure = true + break + } } - keyPair, err := tls.X509KeyPair(opts.TLS.ClientCert, opts.TLS.ClientKey) - if err != nil { - return nil, err - } + // Walk the wait strategies to find the TLS strategy and either remove it or + // update the client certificate files to match the user and configure the + // container to use the TLS strategy. + if err := wait.Walk(&req.WaitingFor, func(strategy wait.Strategy) error { + if cert, ok := strategy.(*wait.TLSStrategy); ok { + if insecure { + // If insecure mode is enabled, the certificate strategy is removed. + return errors.Join(wait.VisitRemove, wait.VisitStop) + } - certPool := x509.NewCertPool() - certPool.AddCert(opts.TLS.CACert) + // Update the client certificate files to match the user which may have changed. + cert.WithCert(certsDir+"/client."+c.user+".crt", certsDir+"/client."+c.user+".key") - return &tls.Config{ - RootCAs: certPool, - Certificates: []tls.Certificate{keyPair}, - ServerName: "localhost", - }, nil + c.tlsStrategy = cert + + // Stop the walk as the certificate strategy has been found. + return wait.VisitStop + } + return nil + }); err != nil { + return fmt.Errorf("walk strategies: %w", err) + } + + return nil } diff --git a/modules/cockroachdb/cockroachdb_test.go b/modules/cockroachdb/cockroachdb_test.go index cc355e9168..e3a7bb1f12 100644 --- a/modules/cockroachdb/cockroachdb_test.go +++ b/modules/cockroachdb/cockroachdb_test.go @@ -2,221 +2,94 @@ package cockroachdb_test import ( "context" - "errors" - "net/url" - "strings" + "database/sql" "testing" - "time" "github.com/jackc/pgx/v5" "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/modules/cockroachdb" - "github.com/testcontainers/testcontainers-go/wait" ) -func TestCockroach_Insecure(t *testing.T) { - suite.Run(t, &AuthNSuite{ - url: "postgres://root@localhost:xxxxx/defaultdb?sslmode=disable", - }) -} +const testImage = "cockroachdb/cockroach:latest-v23.1" -func TestCockroach_NotRoot(t *testing.T) { - suite.Run(t, &AuthNSuite{ - url: "postgres://test@localhost:xxxxx/defaultdb?sslmode=disable", - opts: []testcontainers.ContainerCustomizer{ - cockroachdb.WithUser("test"), - }, - }) +func TestRun(t *testing.T) { + testContainer(t) } -func TestCockroach_Password(t *testing.T) { - suite.Run(t, &AuthNSuite{ - url: "postgres://foo:bar@localhost:xxxxx/defaultdb?sslmode=disable", - opts: []testcontainers.ContainerCustomizer{ - cockroachdb.WithUser("foo"), - cockroachdb.WithPassword("bar"), - }, - }) +func TestRun_WithAllOptions(t *testing.T) { + testContainer(t, + cockroachdb.WithDatabase("testDatabase"), + cockroachdb.WithStoreSize("50%"), + cockroachdb.WithUser("testUser"), + cockroachdb.WithPassword("testPassword"), + cockroachdb.WithNoClusterDefaults(), + cockroachdb.WithInitScripts("testdata/__init.sql"), + // WithInsecure is not present as it is incompatible with WithPassword. + ) } -func TestCockroach_TLS(t *testing.T) { - tlsCfg, err := cockroachdb.NewTLSConfig() - require.NoError(t, err) - - suite.Run(t, &AuthNSuite{ - url: "postgres://root@localhost:xxxxx/defaultdb?sslmode=verify-full", - opts: []testcontainers.ContainerCustomizer{ - cockroachdb.WithTLS(tlsCfg), - }, +func TestRun_WithInsecure(t *testing.T) { + t.Run("valid", func(t *testing.T) { + testContainer(t, cockroachdb.WithInsecure()) }) -} - -type AuthNSuite struct { - suite.Suite - url string - opts []testcontainers.ContainerCustomizer -} - -func (suite *AuthNSuite) TestConnectionString() { - ctx := context.Background() - - ctr, err := cockroachdb.Run(ctx, "cockroachdb/cockroach:latest-v23.1", suite.opts...) - testcontainers.CleanupContainer(suite.T(), ctr) - suite.Require().NoError(err) - - connStr, err := removePort(ctr.MustConnectionString(ctx)) - suite.Require().NoError(err) - - suite.Equal(suite.url, connStr) -} - -func (suite *AuthNSuite) TestPing() { - ctx := context.Background() - - inputs := []struct { - name string - opts []testcontainers.ContainerCustomizer - }{ - { - name: "defaults", - // opts: suite.opts - }, - { - name: "database", - opts: []testcontainers.ContainerCustomizer{ - cockroachdb.WithDatabase("test"), - }, - }, - } - - for _, input := range inputs { - suite.Run(input.name, func() { - opts := suite.opts - opts = append(opts, input.opts...) - - ctr, err := cockroachdb.Run(ctx, "cockroachdb/cockroach:latest-v23.1", opts...) - testcontainers.CleanupContainer(suite.T(), ctr) - suite.Require().NoError(err) - - conn, err := conn(ctx, ctr) - suite.Require().NoError(err) - defer conn.Close(ctx) - - err = conn.Ping(ctx) - suite.Require().NoError(err) - }) - } -} - -func (suite *AuthNSuite) TestQuery() { - ctx := context.Background() - - ctr, err := cockroachdb.Run(ctx, "cockroachdb/cockroach:latest-v23.1", suite.opts...) - testcontainers.CleanupContainer(suite.T(), ctr) - suite.Require().NoError(err) - - conn, err := conn(ctx, ctr) - suite.Require().NoError(err) - defer conn.Close(ctx) - _, err = conn.Exec(ctx, "CREATE TABLE test (id INT PRIMARY KEY)") - suite.Require().NoError(err) - - _, err = conn.Exec(ctx, "INSERT INTO test (id) VALUES (523123)") - suite.Require().NoError(err) - - var id int - err = conn.QueryRow(ctx, "SELECT id FROM test").Scan(&id) - suite.Require().NoError(err) - suite.Equal(523123, id) -} - -// TestWithWaitStrategyAndDeadline covers a previous regression, container creation needs to fail to cover that path. -func (suite *AuthNSuite) TestWithWaitStrategyAndDeadline() { - nodeStartUpCompleted := "node startup completed" - - suite.Run("Expected Failure To Run", func() { - ctx := context.Background() - - // This will never match a log statement - suite.opts = append(suite.opts, testcontainers.WithWaitStrategyAndDeadline(time.Millisecond*250, wait.ForLog("Won't Exist In Logs"))) - ctr, err := cockroachdb.Run(ctx, "cockroachdb/cockroach:latest-v23.1", suite.opts...) - testcontainers.CleanupContainer(suite.T(), ctr) - suite.Require().ErrorIs(err, context.DeadlineExceeded) + t.Run("invalid-password-insecure", func(t *testing.T) { + _, err := cockroachdb.Run(context.Background(), testImage, + cockroachdb.WithPassword("testPassword"), + cockroachdb.WithInsecure(), + ) + require.Error(t, err) }) - suite.Run("Expected Failure To Run But Would Succeed ", func() { - ctx := context.Background() - - // This will timeout as we didn't give enough time for intialization, but would have succeeded otherwise - suite.opts = append(suite.opts, testcontainers.WithWaitStrategyAndDeadline(time.Millisecond*20, wait.ForLog(nodeStartUpCompleted))) - ctr, err := cockroachdb.Run(ctx, "cockroachdb/cockroach:latest-v23.1", suite.opts...) - testcontainers.CleanupContainer(suite.T(), ctr) - suite.Require().ErrorIs(err, context.DeadlineExceeded) + t.Run("invalid-insecure-password", func(t *testing.T) { + _, err := cockroachdb.Run(context.Background(), testImage, + cockroachdb.WithInsecure(), + cockroachdb.WithPassword("testPassword"), + ) + require.Error(t, err) }) +} - suite.Run("Succeeds And Executes Commands", func() { - ctx := context.Background() +// testContainer runs a CockroachDB container and validates its functionality. +func testContainer(t *testing.T, opts ...testcontainers.ContainerCustomizer) { + t.Helper() - // This will succeed - suite.opts = append(suite.opts, testcontainers.WithWaitStrategyAndDeadline(time.Second*60, wait.ForLog(nodeStartUpCompleted))) - ctr, err := cockroachdb.Run(ctx, "cockroachdb/cockroach:latest-v23.1", suite.opts...) - testcontainers.CleanupContainer(suite.T(), ctr) - suite.Require().NoError(err) + ctx := context.Background() + ctr, err := cockroachdb.Run(ctx, testImage, opts...) + testcontainers.CleanupContainer(t, ctr) + require.NoError(t, err) + require.NotNil(t, ctr) - conn, err := conn(ctx, ctr) - suite.Require().NoError(err) - defer conn.Close(ctx) + // Check a raw connection with a ping. + cfg, err := ctr.ConnectionConfig(ctx) + require.NoError(t, err) - _, err = conn.Exec(ctx, "CREATE TABLE test (id INT PRIMARY KEY)") - suite.Require().NoError(err) + conn, err := pgx.ConnectConfig(ctx, cfg) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, conn.Close(ctx)) }) - suite.Run("Succeeds And Executes Commands Waiting on HTTP Endpoint", func() { - ctx := context.Background() + err = conn.Ping(ctx) + require.NoError(t, err) - // This will succeed - suite.opts = append(suite.opts, testcontainers.WithWaitStrategyAndDeadline(time.Second*60, wait.ForHTTP("/health").WithPort("8080/tcp"))) - ctr, err := cockroachdb.Run(ctx, "cockroachdb/cockroach:latest-v23.1", suite.opts...) - testcontainers.CleanupContainer(suite.T(), ctr) - suite.Require().NoError(err) + // Check an SQL connection with a queries. + addr, err := ctr.ConnectionString(ctx) + require.NoError(t, err) - conn, err := conn(ctx, ctr) - suite.Require().NoError(err) - defer conn.Close(ctx) + db, err := sql.Open("pgx/v5", addr) + require.NoError(t, err) - _, err = conn.Exec(ctx, "CREATE TABLE test (id INT PRIMARY KEY)") - suite.Require().NoError(err) - }) -} + _, err = db.ExecContext(ctx, "CREATE TABLE test (id INT PRIMARY KEY)") + require.NoError(t, err) -func conn(ctx context.Context, container *cockroachdb.CockroachDBContainer) (*pgx.Conn, error) { - cfg, err := pgx.ParseConfig(container.MustConnectionString(ctx)) - if err != nil { - return nil, err - } - - tlsCfg, err := container.TLSConfig() - switch { - case err != nil: - if !errors.Is(err, cockroachdb.ErrTLSNotEnabled) { - return nil, err - } - default: - // apply TLS config - cfg.TLSConfig = tlsCfg - } - - return pgx.ConnectConfig(ctx, cfg) -} + _, err = db.ExecContext(ctx, "INSERT INTO test (id) VALUES (523123)") + require.NoError(t, err) -func removePort(s string) (string, error) { - u, err := url.Parse(s) - if err != nil { - return "", err - } - return strings.Replace(s, ":"+u.Port(), ":xxxxx", 1), nil + var id int + err = db.QueryRowContext(ctx, "SELECT id FROM test").Scan(&id) + require.NoError(t, err) + require.Equal(t, 523123, id) } diff --git a/modules/cockroachdb/data/cluster_defaults.sql b/modules/cockroachdb/data/cluster_defaults.sql new file mode 100644 index 0000000000..78502d115e --- /dev/null +++ b/modules/cockroachdb/data/cluster_defaults.sql @@ -0,0 +1,8 @@ +SET CLUSTER SETTING kv.range_merge.queue_interval = '50ms'; +SET CLUSTER SETTING jobs.registry.interval.gc = '30s'; +SET CLUSTER SETTING jobs.registry.interval.cancel = '180s'; +SET CLUSTER SETTING jobs.retention_time = '15s'; +SET CLUSTER SETTING sql.stats.automatic_collection.enabled = false; +SET CLUSTER SETTING kv.range_split.by_load_merge_delay = '5s'; +ALTER RANGE default CONFIGURE ZONE USING "gc.ttlseconds" = 600; +ALTER DATABASE system CONFIGURE ZONE USING "gc.ttlseconds" = 600; diff --git a/modules/cockroachdb/examples_test.go b/modules/cockroachdb/examples_test.go index c06c97596b..a1259c218b 100644 --- a/modules/cockroachdb/examples_test.go +++ b/modules/cockroachdb/examples_test.go @@ -2,9 +2,11 @@ package cockroachdb_test import ( "context" + "database/sql" "fmt" "log" - "net/url" + + "github.com/jackc/pgx/v5" "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/modules/cockroachdb" @@ -33,20 +35,97 @@ func ExampleRun() { } fmt.Println(state.Running) + cfg, err := cockroachdbContainer.ConnectionConfig(ctx) + if err != nil { + log.Printf("failed to get connection string: %s", err) + return + } + + conn, err := pgx.ConnectConfig(ctx, cfg) + if err != nil { + log.Printf("failed to connect: %s", err) + return + } + + defer func() { + if err := conn.Close(ctx); err != nil { + log.Printf("failed to close connection: %s", err) + } + }() + + if err = conn.Ping(ctx); err != nil { + log.Printf("failed to ping: %s", err) + return + } + + // Output: + // true +} + +func ExampleRun_withInitOptions() { + ctx := context.Background() + + cockroachdbContainer, err := cockroachdb.Run(ctx, "cockroachdb/cockroach:latest-v23.1", + cockroachdb.WithNoClusterDefaults(), + cockroachdb.WithInitScripts("testdata/__init.sql"), + ) + defer func() { + if err := testcontainers.TerminateContainer(cockroachdbContainer); err != nil { + log.Printf("failed to terminate container: %s", err) + } + }() + if err != nil { + log.Printf("failed to start container: %s", err) + return + } + + state, err := cockroachdbContainer.State(ctx) + if err != nil { + log.Printf("failed to get container state: %s", err) + return + } + fmt.Println(state.Running) + addr, err := cockroachdbContainer.ConnectionString(ctx) if err != nil { log.Printf("failed to get connection string: %s", err) return } - u, err := url.Parse(addr) + + db, err := sql.Open("pgx/v5", addr) if err != nil { - log.Printf("failed to parse connection string: %s", err) + log.Printf("failed to open connection: %s", err) return } - u.Host = fmt.Sprintf("%s:%s", u.Hostname(), "xxx") - fmt.Println(u.String()) + defer func() { + if err := db.Close(); err != nil { + log.Printf("failed to close connection: %s", err) + } + }() + + var interval string + if err := db.QueryRow("SHOW CLUSTER SETTING kv.range_merge.queue_interval").Scan(&interval); err != nil { + log.Printf("failed to scan row: %s", err) + return + } + fmt.Println(interval) + + if err := db.QueryRow("SHOW CLUSTER SETTING jobs.registry.interval.gc").Scan(&interval); err != nil { + log.Printf("failed to scan row: %s", err) + return + } + fmt.Println(interval) + + var statsCollectionEnabled bool + if err := db.QueryRow("SHOW CLUSTER SETTING sql.stats.automatic_collection.enabled").Scan(&statsCollectionEnabled); err != nil { + log.Printf("failed to scan row: %s", err) + return + } + fmt.Println(statsCollectionEnabled) // Output: // true - // postgres://root@localhost:xxx/defaultdb?sslmode=disable + // 00:00:05 + // 00:00:50 + // true } diff --git a/modules/cockroachdb/go.mod b/modules/cockroachdb/go.mod index fbc0fd6f7a..cf31a35616 100644 --- a/modules/cockroachdb/go.mod +++ b/modules/cockroachdb/go.mod @@ -41,7 +41,6 @@ require ( github.com/kr/text v0.2.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/magiconair/properties v1.8.7 // indirect - github.com/mdelapenya/tlscert v0.1.0 github.com/moby/patternmatcher v0.6.0 // indirect github.com/moby/sys/sequential v0.5.0 // indirect github.com/moby/sys/user v0.1.0 // indirect diff --git a/modules/cockroachdb/go.sum b/modules/cockroachdb/go.sum index e8661eb69a..3877e20a9a 100644 --- a/modules/cockroachdb/go.sum +++ b/modules/cockroachdb/go.sum @@ -69,8 +69,6 @@ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= -github.com/mdelapenya/tlscert v0.1.0 h1:YTpF579PYUX475eOL+6zyEO3ngLTOUWck78NBuJVXaM= -github.com/mdelapenya/tlscert v0.1.0/go.mod h1:wrbyM/DwbFCeCeqdPX/8c6hNOqQgbf0rUDErE1uD+64= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= diff --git a/modules/cockroachdb/options.go b/modules/cockroachdb/options.go index a2211d77e7..9efac532c6 100644 --- a/modules/cockroachdb/options.go +++ b/modules/cockroachdb/options.go @@ -1,69 +1,119 @@ package cockroachdb -import "github.com/testcontainers/testcontainers-go" - -type options struct { - Database string - User string - Password string - StoreSize string - TLS *TLSConfig -} +import ( + "errors" + "path/filepath" + "strings" + + "github.com/testcontainers/testcontainers-go" +) + +// errInsecureWithPassword is returned when trying to use insecure mode with a password. +var errInsecureWithPassword = errors.New("insecure mode cannot be used with a password") -func defaultOptions() options { - return options{ - User: defaultUser, - Password: defaultPassword, - Database: defaultDatabase, - StoreSize: defaultStoreSize, +// WithDatabase sets the name of the database to create and use. +// This will be converted to lowercase as CockroachDB forces the database to be lowercase. +// The database creation will be skipped if data exists in the `/cockroach/cockroach-data` directory within the container. +func WithDatabase(database string) testcontainers.CustomizeRequestOption { + return func(req *testcontainers.GenericContainerRequest) error { + req.Env[envDatabase] = strings.ToLower(database) + return nil } } -// Compiler check to ensure that Option implements the testcontainers.ContainerCustomizer interface. -var _ testcontainers.ContainerCustomizer = (*Option)(nil) +// WithUser sets the name of the user to create and connect as. +// This will be converted to lowercase as CockroachDB forces the user to be lowercase. +// The user creation will be skipped if data exists in the `/cockroach/cockroach-data` directory within the container. +func WithUser(user string) testcontainers.CustomizeRequestOption { + return func(req *testcontainers.GenericContainerRequest) error { + req.Env[envUser] = strings.ToLower(user) + return nil + } +} -// Option is an option for the CockroachDB container. -type Option func(*options) +// WithPassword sets the password of the user to create and connect as. +// The user creation will be skipped if data exists in the `/cockroach/cockroach-data` directory within the container. +// This will error if insecure mode is enabled. +func WithPassword(password string) testcontainers.CustomizeRequestOption { + return func(req *testcontainers.GenericContainerRequest) error { + for _, arg := range req.Cmd { + if arg == insecureFlag { + return errInsecureWithPassword + } + } -// Customize is a NOOP. It's defined to satisfy the testcontainers.ContainerCustomizer interface. -func (o Option) Customize(*testcontainers.GenericContainerRequest) error { - // NOOP to satisfy interface. - return nil -} + req.Env[envPassword] = password -// WithDatabase sets the name of the database to use. -func WithDatabase(database string) Option { - return func(o *options) { - o.Database = database + return nil } } -// WithUser creates & sets the user to connect as. -func WithUser(user string) Option { - return func(o *options) { - o.User = user +// WithStoreSize sets the amount of available [in-memory storage]. +// +// [in-memory storage]: https://www.cockroachlabs.com/docs/stable/cockroach-start#store +func WithStoreSize(size string) testcontainers.CustomizeRequestOption { + return func(req *testcontainers.GenericContainerRequest) error { + for i, cmd := range req.Cmd { + if strings.HasPrefix(cmd, memStorageFlag) { + req.Cmd[i] = memStorageFlag + size + return nil + } + } + + // Wasn't found, add it. + req.Cmd = append(req.Cmd, memStorageFlag+size) + + return nil } } -// WithPassword sets the password when using password authentication. -func WithPassword(password string) Option { - return func(o *options) { - o.Password = password +// WithNoClusterDefaults disables the default cluster settings script. +// +// Without this option Cockroach containers run `data/cluster-defaults.sql` on startup +// which configures the settings recommended by Cockroach Labs for [local testing clusters] +// unless data exists in the `/cockroach/cockroach-data` directory within the container. +// +// [local testing clusters]: https://www.cockroachlabs.com/docs/stable/local-testing +func WithNoClusterDefaults() testcontainers.CustomizeRequestOption { + return func(req *testcontainers.GenericContainerRequest) error { + for i, file := range req.Files { + if _, ok := file.Reader.(*defaultsReader); ok && file.ContainerFilePath == clusterDefaultsContainerFile { + req.Files = append(req.Files[:i], req.Files[i+1:]...) + return nil + } + } + + return nil } } -// WithStoreSize sets the amount of available in-memory storage. -// See https://www.cockroachlabs.com/docs/stable/cockroach-start#store -func WithStoreSize(size string) Option { - return func(o *options) { - o.StoreSize = size +// WithInitScripts adds the given scripts to those automatically run when the container starts. +// These will be ignored if data exists in the `/cockroach/cockroach-data` directory within the container. +func WithInitScripts(scripts ...string) testcontainers.CustomizeRequestOption { + return func(req *testcontainers.GenericContainerRequest) error { + files := make([]testcontainers.ContainerFile, len(scripts)) + for i, script := range scripts { + files[i] = testcontainers.ContainerFile{ + HostFilePath: script, + ContainerFilePath: initDBPath + "/" + filepath.Base(script), + FileMode: 0o644, + } + } + req.Files = append(req.Files, files...) + + return nil } } -// WithTLS enables TLS on the CockroachDB container. -// Cert and key must be PEM-encoded. -func WithTLS(cfg *TLSConfig) Option { - return func(o *options) { - o.TLS = cfg +// WithInsecure enables insecure mode which disables TLS. +func WithInsecure() testcontainers.CustomizeRequestOption { + return func(req *testcontainers.GenericContainerRequest) error { + if req.Env[envPassword] != "" { + return errInsecureWithPassword + } + + req.Cmd = append(req.Cmd, insecureFlag) + + return nil } } diff --git a/modules/cockroachdb/testdata/__init.sql b/modules/cockroachdb/testdata/__init.sql new file mode 100644 index 0000000000..c2c82dd48a --- /dev/null +++ b/modules/cockroachdb/testdata/__init.sql @@ -0,0 +1 @@ +SET CLUSTER SETTING jobs.registry.interval.gc = '50s'; diff --git a/modules/rabbitmq/examples_test.go b/modules/rabbitmq/examples_test.go index bc6a849456..b9c4e9fdf2 100644 --- a/modules/rabbitmq/examples_test.go +++ b/modules/rabbitmq/examples_test.go @@ -102,6 +102,7 @@ func ExampleRun_withSSL() { defer os.RemoveAll(certDirs) // generates the CA certificate and the certificate + // exampleSelfSignedCert { caCert := tlscert.SelfSignedFromRequest(tlscert.Request{ Name: "ca", Host: "localhost,127.0.0.1", @@ -112,7 +113,9 @@ func ExampleRun_withSSL() { log.Print("failed to generate CA certificate") return } + // } + // exampleSignSelfSignedCert { cert := tlscert.SelfSignedFromRequest(tlscert.Request{ Name: "client", Host: "localhost,127.0.0.1", @@ -124,6 +127,7 @@ func ExampleRun_withSSL() { log.Print("failed to generate certificate") return } + // } sslSettings := rabbitmq.SSLSettings{ CACertFile: caCert.CertPath,