Rework to not assume schema, usernames

cmd/igor/main.go

@@ -7,14 +7,15 @@ import (
 )
 
 type optsDatabase struct {
-	DatabaseURL string `long:"database-url" env:"DATABASE_URL" description:"Database connection string" default:"postgres://$DATABASE_USER:$DATABASE_PASSWORD@localhost:5432/igor?sslmode=disable&search_path=igor"`
+	DatabaseURL string `long:"database-url" env:"DATABASE_URL" description:"Database connection string" default:"postgres://postgres:test@localhost:5432/igor?sslmode=disable"`
 }
 
 type optsQueue struct {
-	QueueURL       string `long:"queue-url" env:"QUEUE_URL" description:"Queue connection string" default:"redis://localhost:6379/0"`
 	QueueTLSCert   string `long:"queue-tls-cert" env:"QUEUE_TLS_CERT" description:"Queue TLS certificate"`
 	QueueTLSKey    string `long:"queue-tls-key" env:"QUEUE_TLS_KEY" description:"Queue TLS key"`
 	QueueTLSCaCert string `long:"queue-tls-ca-cert" env:"QUEUE_TLS_CA_CERT" description:"Queue TLS CA certificate"`
+
+	QueueURL string `long:"queue-url" env:"QUEUE_URL" default:"redis://localhost:6379/0" description:"Queue connection string. If your redis requires a username and password for Asynq you can set the following environment variables: ASYNQ_REDIS_USER, ASYNQ_REDIS_PASSWORD"`
 }
 
 type optsGeneral struct {

cmd/igor/migrate.go

@@ -3,7 +3,8 @@ package main
 import (
 	"errors"
 	"fmt"
-	"os"
+	"log"
+	"net/url"
 	"strings"
 	"time"
 
@@ -64,7 +65,14 @@ type cmdMigrateWait struct {
 	Timeout time.Duration `long:"timeout" env:"MIGRATIONS_WAIT_TIMEOUT" description:"Time to wait before erroring (optional)"`
 }
 
+type cmdMigrateSetup struct {
+	optsGeneral
+	optsDatabase
+}
+
 type optsMigrate struct {
+	Setup cmdMigrateSetup `command:"setup" hidden:"true" description:"Setup the database, should only be used in testing"`
+
 	Up cmdMigrateUp `command:"up" description:"Up version the database"`
 
 	Down cmdMigrateDown `command:"down" description:"Down version the database"`
@@ -74,6 +82,26 @@ type optsMigrate struct {
 	Wait cmdMigrateWait `command:"wait" description:"Wait (block) for the database to be at least the given version"`
 }
 
+func (c *cmdMigrateSetup) Execute(args []string) error {
+	// Used to setup the database for testing, not intended for use outside of testing
+	u, err := url.Parse(c.DatabaseURL)
+	if err != nil {
+		return err
+	}
+	database := strings.Trim(u.Path, "/")
+	u.Path = ""
+
+	// connect to postgres without the DB
+	db, err := sql.Open("postgres", u.String())
+	if err != nil {
+		return err
+	}
+	defer db.Close()
+	_, err = db.Exec(fmt.Sprintf("CREATE DATABASE %s;", database))
+	log.Println("Create database:", database, err)
+	return err
+}
+
 func (c *cmdMigrateForce) Execute(args []string) error {
 	m, db, err := buildMigrate(c.Source, &database.Options{URL: c.DatabaseURL})
 	if err != nil {
@@ -184,10 +212,6 @@ func getVersion(m *migrate.Migrate) (int, error) {
 }
 
 func buildMigrate(source string, opts *database.Options) (*migrate.Migrate, *sql.DB, error) {
-	opts.SetDefaults()
-	opts.URL = strings.Replace(opts.URL, "$"+opts.UsernameEnvVar, os.Getenv(opts.UsernameEnvVar), 1)
-	opts.URL = strings.Replace(opts.URL, "$"+opts.PasswordEnvVar, os.Getenv(opts.PasswordEnvVar), 1)
-
 	db, err := sql.Open("postgres", opts.URL)
 	if err != nil {
 		return nil, nil, err

cmd/igor/worker.go

@@ -3,6 +3,7 @@ package main
 import (
 	"os"
 	"os/signal"
+	"syscall"
 
 	"github.com/voidshard/igor/internal/utils"
 	"github.com/voidshard/igor/pkg/api"
@@ -43,7 +44,7 @@ func (c *optsWorker) Execute(args []string) error {
 	defer api.Close()
 
 	exit := make(chan os.Signal, 1)
-	signal.Notify(exit, os.Interrupt)
+	signal.Notify(exit, os.Interrupt, syscall.SIGTERM)
 	<-exit
 
 	return nil

migrations/README.md

@@ -1,6 +1,3 @@
 DB Migrations for Postgres.
 
-
-/dev contains migrations suitable for testing; it creates a database, user(s) with default weak passwords (again, for testing), schema etc.
-
 /prod creates our main tables & functions, implies database, schema & users exist already.

migrations/prod/100000_create_tables.up.sql

@@ -50,8 +50,6 @@ CREATE OR REPLACE FUNCTION create_partition_and_insert() RETURNS trigger AS $$
         ELSIF (TG_TABLE_NAME = LOWER('Task')) THEN
             EXECUTE 'CREATE TRIGGER ' || partition || '_notify_event AFTER INSERT OR UPDATE OR DELETE ON ' || partition || ' FOR EACH ROW EXECUTE PROCEDURE notify_event();';
         END IF;
-        EXECUTE 'GRANT SELECT ON TABLE ' || partition || ' TO igorreadonly;';
-        EXECUTE 'GRANT ALL ON TABLE ' || partition || ' TO igorreadwrite;';
       END IF;
       EXECUTE 'INSERT INTO ' || partition || ' SELECT(' || TG_TABLE_NAME || ' ' || quote_literal(NEW) || ').*;';
       RETURN NULL;

pkg/database/options.go

@@ -1,31 +1,7 @@
 package database
 
-const (
-	defaultPasswordEnvVar = "DATABASE_PASSWORD"
-	defaultUsernameEnvVar = "DATABASE_USER"
-)
-
 // Options are options for the database.
 type Options struct {
 	// URL encodes how we'll connect to the database.
 	URL string
-
-	// PasswordEnvVar is the environment variable that contains the password for the database.
-	// This value is substituted into the URL (ie. "postgres://username:$PASS@localhost:5432" will have "PASS" subbed in).
-	// Defaults to "DATABASE_PASSWORD".
-	PasswordEnvVar string
-
-	// UsernameEnvVar is the environment variable that contains the username for the database.
-	// This value is substituted into the URL (ie. "postgres://$USER:password@localhost:5432" will have "USER" subbed in).
-	// Defaults to "DATABASE_USER".
-	UsernameEnvVar string
-}
-
-func (o *Options) SetDefaults() {
-	if o.PasswordEnvVar == "" {
-		o.PasswordEnvVar = defaultPasswordEnvVar
-	}
-	if o.UsernameEnvVar == "" {
-		o.UsernameEnvVar = defaultUsernameEnvVar
-	}
 }

pkg/database/postgres.go

@@ -3,7 +3,6 @@ package database
 import (
 	"context"
 	"fmt"
-	"os"
 	"strings"
 	"time"
 
@@ -22,9 +21,6 @@ type Postgres struct {
 
 // NewPostgres returns a new Postgres database connection.
 func NewPostgres(opts *Options) (*Postgres, error) {
-	opts.SetDefaults()
-	opts.URL = strings.Replace(opts.URL, "$"+opts.UsernameEnvVar, os.Getenv(opts.UsernameEnvVar), 1)
-	opts.URL = strings.Replace(opts.URL, "$"+opts.PasswordEnvVar, os.Getenv(opts.PasswordEnvVar), 1)
 	pool, err := pgxpool.New(context.Background(), opts.URL)
 	return &Postgres{pool: pool, opts: opts}, err
 }

pkg/queue/asynq.go

@@ -21,6 +21,9 @@ const (
 	asyncAggMaxSize  = 1000
 	asyncAggMaxDelay = 2 * time.Second
 	asyncAggRune     = "¬"
+
+	asyncEnvVarUser     = "ASYNQ_REDIS_USER"
+	asyncEnvVarPassword = "ASYNQ_REDIS_PASSWORD"
 )
 
 // Asynq is a Queue implementation that uses asynq.
@@ -47,13 +50,10 @@ type Asynq struct {
 
 // NewAsynqQueue returns a new Asynq queue with the given settings
 func NewAsynqQueue(svc database.QueueDB, opts *Options) (*Asynq, error) {
-	opts.SetDefaults()
-	opts.URL = strings.Replace(opts.URL, "$"+opts.UsernameEnvVar, os.Getenv(opts.UsernameEnvVar), 1)
-	opts.URL = strings.Replace(opts.URL, "$"+opts.PasswordEnvVar, os.Getenv(opts.PasswordEnvVar), 1)
 	redisOpts := asynq.RedisClientOpt{
 		Addr:      opts.URL,
-		Username:  os.Getenv(opts.UsernameEnvVar),
-		Password:  os.Getenv(opts.PasswordEnvVar),
+		Username:  os.Getenv(asyncEnvVarUser),
+		Password:  os.Getenv(asyncEnvVarPassword),
 		TLSConfig: opts.TLSConfig,
 	}
 	ins := asynq.NewInspector(redisOpts)

pkg/queue/options.go

@@ -4,33 +4,11 @@ import (
 	"crypto/tls"
 )
 
-const (
-	defaultPasswordEnvVar = "QUEUE_PASSWORD"
-	defaultUsernameEnvVar = "QUEUE_USER"
-)
-
 // Options are options for the queue.
 type Options struct {
 	// URL encodes how we'll connect to the queue.
 	URL string
 
-	// PasswordEnvVar is the environment variable that contains the password for the database.
-	// Defaults to "QUEUE_PASSWORD".
-	PasswordEnvVar string
-
-	// UsernameEnvVar is the environment variable that contains the username for the database.
-	// Defaults to "QUEUE_USER".
-	UsernameEnvVar string
-
 	// TLSConfig needed to connect to the queue (optional).
 	TLSConfig *tls.Config
 }
-
-func (o *Options) SetDefaults() {
-	if o.PasswordEnvVar == "" {
-		o.PasswordEnvVar = defaultPasswordEnvVar
-	}
-	if o.UsernameEnvVar == "" {
-		o.UsernameEnvVar = defaultUsernameEnvVar
-	}
-}

tests/compose.yaml

@@ -5,7 +5,7 @@ services:
       context: ../
       dockerfile: tests/Dockerfile
     environment:
-      - DATABASE_URL=postgres://igorreadwrite:readwrite@postgres:5432/igor?sslmode=disable&search_path=igor
+      - DATABASE_URL=postgres://postgres:test@postgres:5432/igor?sslmode=disable
       - QUEUE_URL=redis:6379
     depends_on:
       - postgres
@@ -22,7 +22,7 @@ services:
     ports:
       - "8100:8100"
     environment:
-      - DATABASE_URL=postgres://igorreadwrite:readwrite@postgres:5432/igor?sslmode=disable&search_path=igor
+      - DATABASE_URL=postgres://postgres:test@postgres:5432/igor?sslmode=disable
       - QUEUE_URL=redis:6379
       - ADDR=0.0.0.0:8100
       - DEBUG=true
@@ -39,7 +39,7 @@ services:
       dockerfile: Dockerfile
     command: ["worker"]
     environment:
-      - DATABASE_URL=postgres://igorreadwrite:readwrite@postgres:5432/igor?sslmode=disable&search_path=igor
+      - DATABASE_URL=postgres://postgres:test@postgres:5432/igor?sslmode=disable
       - QUEUE_URL=redis:6379
       - DEBUG=true
     depends_on:

tests/run.sh

@@ -11,8 +11,8 @@ PGDATABASE=${PGDATABASE:-igor}
 # these test user/passwords are made in migrations/dev/
 OWNUSER=${OWNUSER:-postgres} # owner
 OWNPASS=${OWNPASS:-test}
-RWUSER=${RWUSER:-igorreadwrite} # readwrite
-RWPASS=${RWPASS:-readwrite}
+RWUSER=${RWUSER:-postgres} # readwrite
+RWPASS=${RWPASS:-test}
 
 REDISHOST=${REDISHOST:-localhost}
 REDISPORT=${REDISPORT:-6379}
@@ -42,17 +42,17 @@ done
 
 # apply db migrations
 # Create the DB
-# Nb. "migrate" refuses to acknowledge '\connect' to make & connect to the DB to make a schema .. so we have to do the first step manually
-PGHOST=$PGHOST PGPORT=$PGPORT PGUSER=$OWNUSER PGPASSWORD=$OWNPASS psql -f ${SCRIPT_DIR}/../migrations/dev/000000_setup_database.up.sql
+DATABASE_URL="postgres://${OWNUSER}:${OWNPASS}@${PGHOST}:${PGPORT}/igor?sslmode=disable"
+$IGOR migrate setup
 
 # Apply the migrations
-DATABASE_URL="postgres://${RWUSER}:${RWPASS}@${PGHOST}:${PGPORT}/${PGDATABASE}?sslmode=disable&search_path=igor" $IGOR migrate up --source file://${SCRIPT_DIR}/../migrations/prod
+$IGOR migrate up --source file://${SCRIPT_DIR}/../migrations/prod
 # Print the version
-DATABASE_URL="postgres://${RWUSER}:${RWPASS}@${PGHOST}:${PGPORT}/${PGDATABASE}?sslmode=disable&search_path=igor" $IGOR migrate version --source file://${SCRIPT_DIR}/../migrations/prod
+$IGOR migrate version --source file://${SCRIPT_DIR}/../migrations/prod
 
 # run the tests
 set +e 
-IGOR_TEST_API="http://localhost:${APIPORT}/api/v1" IGOR_TEST_DATA=${SCRIPT_DIR}/data IGOR_TEST_PG_URL="postgres://${RWUSER}:${RWPASS}@${PGHOST}:${PGPORT}/${PGDATABASE}?sslmode=disable&search_path=igor" IGOR_TEST_RD_URL="redis://${REDISHOST}:${REDISPORT}/${REDISDB}" go test -v ./...
+IGOR_TEST_API="http://localhost:${APIPORT}/api/v1" IGOR_TEST_DATA=${SCRIPT_DIR}/data IGOR_TEST_PG_URL=${DATABASE_URL} IGOR_TEST_RD_URL="redis://${REDISHOST}:${REDISPORT}/${REDISDB}" go test -v ./...
 
 # tear down & remove the test infra
 docker compose stop