testutil/compose: introduce new and auto commands (#600)

Refactors compose:
 - Decouple `new` from `define`: `new` only creates config.json file, `define` is now identical to `lock` and `run`; it only creates docker-compose files.
 - This adds support for using existing config files, and generating clusters from them
 - Add convenience function `compose auto` that runs `compose define && compose lock && compose run`, since this will be the most common use-case.
 
 Result:
 ```
 compose new --keygen==dkg && compose auto
 ``` 

category: refactor 
ticket: #568

testutil/compose/README.md

@@ -7,22 +7,23 @@ Compose is a tool that generates `docker-compose.yml` files such that different
 The aim is for developers to be able to debug features and check functionality of clusters on their local machines.
 
 The `compose` command should be executed in sequential steps:
- 1. `compose clean`: Cleans the compose directory of existing artifacts.
- 2. `compose define`: Defines the target cluster and how keys are to be created.
-    1. It outputs `config.json` which is the compose config
-    1. It also creates `docker-compose.yml` in order to create `cluster-definition.json` if `keygen==dkg`.
- 1. `compose lock`: Creates `docker-compose.yml` to create threshold key shares and the `cluster-lock.json` file.
- 1. `compose run`: Creates `docker-compose.yml` to run the cluster.
+ 1. `compose new`: Creates a new config.json that defines what will be composed.
+ 2. `compose define`: Creates a docker-compose.yml that executes `charon create dkg` if keygen==dkg.
+ 3. `compose lock`: Creates a docker-compose.yml that executes `charon create cluster` or `charon dkg`.
+ 4. `compose run`: Creates a docker-compose.yml that executes `charon run`.
+
+The `compose` command also includes some convenience functions.
+- `compose clean`: Cleans the compose directory of existing files.
+- `compose auto`: Runs `compose define && compose lock && compose run`.
 
 Note that compose automatically runs `docker-compose up` at the end of each command. This can be disabled via `--up=false`.
 
-The `compose define` step configures the target cluster and key generation process. It supports the following flags:
+The `compose new` step configures the target cluster and key generation process. It supports the following flags:
  - `--keygen`: Key generation process: `create` or `dkg`.
    - create` creates keys locally via `charon create cluster`
    - `dkg` creates keys via `charon create dkg` followed by `charon dkg`.
  - `--split-keys-dir`: Path to a folder containing keys to split. Only applicable to `--keygen=create`.
  - `--build-local`: Build a local charon binary from source. Note this requires the `CHARON_REPO` path env var. Devs are encouraged to put this in the bash profile.
- - `--seed`: Randomness seed, can be used to produce deterministic p2pkeys for dkg.
 
 ## Usage
 Install the `compose` binary:
@@ -45,7 +46,7 @@ cd charon-compose
 ```
 Create the default cluster:
 ```
-compose clean && compose define && compose lock && compose run
+compose clean && compose new && compose define && compose lock && compose run
 ```
 Monitor the cluster via `grafana` and `jaeger`:
 ```
@@ -54,8 +55,6 @@ open http://localhost:16686            # Open Jaeger dashboard
 ```
 Creating a DKG based cluster that uses locally built binary:
 ```
-compose clean
-compose define --keygen=dkg --build-local
-compose lock
-compose run
+compose new --keygen=dkg --build-local
+compose auto
 ```

testutil/compose/compose/main.go

@@ -16,23 +16,24 @@
 // Command compose provides a tool to run, test, debug local charon clusters
 // using docker-compose.
 //
-//  It consists of three steps:
-//   - compose define: Creates config.json (and p2pkeys) and a docker-compose.yml to create a cluster definition file.
-//   - compose lock: Creates docker-compose.yml to generates keys and cluster lock file.
-//   - compose run: Creates docker-compose.yml that runs the cluster.
+//  It consists of multiple steps:
+//   - compose new: Creates a new config.json that defines what will be composed.
+//   - compose define: Creates a docker-compose.yml that executes `charon create dkg` if keygen==dkg.
+//   - compose lock: Creates a docker-compose.yml that executes `charon create cluster` or `charon dkg`.
+//   - compose run: Creates a docker-compose.yml that executes `charon run`.
 package main
 
 import (
 	"context"
 	"os"
 	"os/exec"
-	"time"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 
 	"github.com/obolnetwork/charon/app/errors"
 	"github.com/obolnetwork/charon/app/log"
+	"github.com/obolnetwork/charon/app/z"
 	"github.com/obolnetwork/charon/testutil/compose"
 )
 
@@ -46,54 +47,87 @@ func newRootCmd() *cobra.Command {
 		Short: "Charon Compose - Run, test, and debug a developer-focussed insecure local charon cluster using docker-compose",
 	}
 
+	root.AddCommand(newNewCmd())
 	root.AddCommand(newCleanCmd())
-	root.AddCommand(newDefineCmd())
-	root.AddCommand(newLockCmd())
-	root.AddCommand(newRunCmd())
+	root.AddCommand(newAutoCmd())
+	root.AddCommand(newDockerCmd(
+		"define",
+		"Creates a docker-compose.yml that executes `charon create dkg` if keygen==dkg",
+		compose.Define,
+	))
+	root.AddCommand(newDockerCmd(
+		"lock",
+		"Creates a docker-compose.yml that executes `charon create cluster` or `charon dkg`",
+		compose.Lock,
+	))
+	root.AddCommand(newDockerCmd(
+		"run",
+		"Creates a docker-compose.yml that executes `charon run`",
+		compose.Run,
+	))
 
 	return root
 }
 
-func newRunCmd() *cobra.Command {
-	cmd := &cobra.Command{
-		Use:   "run",
-		Short: "Creates docker-compose.yml that runs the cluster.",
-	}
+// newDockerRunFunc returns a cobra run function that generates docker-compose.yml files and executes it.
+func newDockerRunFunc(topic string, dir *string, up *bool, runFunc func(context.Context, string) error) func(cmd *cobra.Command, _ []string) error {
+	return func(cmd *cobra.Command, _ []string) (err error) {
+		ctx := log.WithTopic(cmd.Context(), topic)
+		defer func() {
+			if err != nil {
+				log.Error(ctx, "Fatal error", err)
+			}
+		}()
 
-	up := addUpFlag(cmd.Flags())
-	dir := addDirFlag(cmd.Flags())
+		log.Info(ctx, "Running compose command", z.Str("command", topic))
 
-	cmd.RunE = func(cmd *cobra.Command, _ []string) error {
-		if err := compose.Run(cmd.Context(), *dir); err != nil {
+		if err := runFunc(ctx, *dir); err != nil {
 			return err
 		}
 
 		if *up {
-			return execUp(cmd.Context(), *dir)
+			return execUp(ctx, *dir)
 		}
 
 		return nil
 	}
+}
+
+// newDockerCmd returns a cobra command that generates docker-compose.yml files and executes it.
+func newDockerCmd(use string, short string, run func(context.Context, string) error) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   use,
+		Short: short,
+	}
+
+	up := addUpFlag(cmd.Flags())
+	dir := addDirFlag(cmd.Flags())
+	cmd.RunE = newDockerRunFunc(use, dir, up, run)
 
 	return cmd
 }
 
-func newLockCmd() *cobra.Command {
+func newAutoCmd() *cobra.Command {
 	cmd := &cobra.Command{
-		Use:   "lock",
-		Short: "Creates docker-compose.yml to generates keys and cluster lock file.",
+		Use:   "auto",
+		Short: "Convenience function that runs `compose define && compose lock && compose run`",
 	}
 
-	up := addUpFlag(cmd.Flags())
 	dir := addDirFlag(cmd.Flags())
+	up := true
 
-	cmd.RunE = func(cmd *cobra.Command, _ []string) error {
-		if err := compose.Lock(cmd.Context(), *dir); err != nil {
-			return err
-		}
+	runFuncs := []func(cmd *cobra.Command, _ []string) (err error){
+		newDockerRunFunc("define", dir, &up, compose.Define),
+		newDockerRunFunc("lock", dir, &up, compose.Lock),
+		newDockerRunFunc("run", dir, &up, compose.Run),
+	}
 
-		if *up {
-			return execUp(cmd.Context(), *dir)
+	cmd.RunE = func(cmd *cobra.Command, _ []string) (err error) {
+		for _, runFunc := range runFuncs {
+			err := runFunc(cmd, nil)
+			if err != nil {
+				return err
+			}
 		}
 
 		return nil
@@ -102,32 +136,28 @@ func newLockCmd() *cobra.Command {
 	return cmd
 }
 
-func newDefineCmd() *cobra.Command {
+func newNewCmd() *cobra.Command {
 	cmd := &cobra.Command{
-		Use:   "define",
-		Short: "Creates config.json (and p2pkeys) and a docker-compose.yml to create a cluster definition file",
+		Use:   "new",
+		Short: "Creates a new config.json that defines what will be composed",
 	}
 
 	conf := compose.NewDefaultConfig()
 
-	up := addUpFlag(cmd.Flags())
 	dir := addDirFlag(cmd.Flags())
-	seed := cmd.Flags().Int("seed", int(time.Now().UnixNano()), "Randomness seed")
 	keygen := cmd.Flags().String("keygen", string(conf.KeyGen), "Key generation process: create, split, dkg")
 	buildLocal := cmd.Flags().Bool("build-local", conf.BuildLocal, "Enables building a local charon binary from source. Note this requires the CHARON_REPO env var.")
 
 	cmd.RunE = func(cmd *cobra.Command, _ []string) error {
 		conf.KeyGen = compose.KeyGen(*keygen)
 		conf.BuildLocal = *buildLocal
 
-		if err := compose.Define(cmd.Context(), *dir, *seed, conf); err != nil {
+		ctx := log.WithTopic(cmd.Context(), "new")
+		if err := compose.New(ctx, *dir, conf); err != nil {
+			log.Error(ctx, "Fatal error", err)
 			return err
 		}
 
-		if *up {
-			return execUp(cmd.Context(), *dir)
-		}
-
 		return nil
 	}
 
@@ -137,7 +167,7 @@ func newDefineCmd() *cobra.Command {
 func newCleanCmd() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "clean",
-		Short: "Cleans compose files and artifacts",
+		Short: "Convenience function that cleans the compose directory",
 	}
 
 	dir := addDirFlag(cmd.Flags())
@@ -159,7 +189,6 @@ func addUpFlag(flags *pflag.FlagSet) *bool {
 
 // execUp executes `docker-compose up`.
 func execUp(ctx context.Context, dir string) error {
-	ctx = log.WithTopic(ctx, "cmd")
 	log.Info(ctx, "Executing docker-compose up")
 
 	cmd := exec.CommandContext(ctx, "docker-compose", "up", "--remove-orphans", "--build")

testutil/compose/config.go

@@ -56,6 +56,7 @@ const (
 type step string
 
 const (
+	stepNew     step = "new"
 	stepDefined step = "defined"
 	stepLocked  step = "locked"
 )

testutil/compose/define.go

@@ -20,12 +20,14 @@ import (
 	"crypto/ecdsa"
 	"encoding/json"
 	"fmt"
+	"io/fs"
 	"math/rand"
 	"os"
 	"os/exec"
 	"path"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/p2p/enode"
@@ -61,11 +63,14 @@ func Clean(ctx context.Context, dir string) error {
 }
 
 // Define defines a compose cluster; including both keygen and running definitions.
-func Define(ctx context.Context, dir string, seed int, conf Config) error {
-	ctx = log.WithTopic(ctx, "define")
-
-	if _, err := loadConfig(dir); err == nil {
-		return errors.New("compose config already defined; maybe try `compose clean` or `compose lock`")
+func Define(ctx context.Context, dir string) error {
+	conf, err := loadConfig(dir)
+	if errors.Is(err, fs.ErrNotExist) {
+		return errors.New("compose config not found; maybe try `compose new` first")
+	} else if err != nil {
+		return err
+	} else if conf.Step != stepNew {
+		return errors.New("compose config not new, so can't be defined", z.Any("step", conf.Step))
 	}
 
 	if conf.BuildLocal {
@@ -79,7 +84,7 @@ func Define(ctx context.Context, dir string, seed int, conf Config) error {
 		log.Info(ctx, "Creating node*/p2pkey for ENRs required for charon create dkg")
 
 		// charon create dkg requires operator ENRs, so we need to create p2pkeys now.
-		p2pkeys, err := newP2PKeys(conf.NumNodes, seed)
+		p2pkeys, err := newP2PKeys(conf.NumNodes)
 		if err != nil {
 			return err
 		}
@@ -233,9 +238,12 @@ func keyToENR(key *ecdsa.PrivateKey) (string, error) {
 	return p2p.EncodeENR(r)
 }
 
+// p2pSeed can be overridden in tests for deterministic p2pkeys.
+var p2pSeed = time.Now().UnixNano()
+
 // newP2PKeys returns a slice of newly generated ecdsa private keys.
-func newP2PKeys(n, seed int) ([]*ecdsa.PrivateKey, error) {
-	random := rand.New(rand.NewSource(int64(seed))) //nolint:gosec // Weak random is fine for testing.
+func newP2PKeys(n int) ([]*ecdsa.PrivateKey, error) {
+	random := rand.New(rand.NewSource(p2pSeed)) //nolint:gosec // Weak random is fine for testing.
 	var resp []*ecdsa.PrivateKey
 	for i := 0; i < n; i++ {
 		key, err := ecdsa.GenerateKey(crypto.S256(), random)

testutil/compose/define_test.go → testutil/compose/define_internal_test.go

@@ -13,7 +13,7 @@
 // You should have received a copy of the GNU General Public License along with
 // this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package compose_test
+package compose
 
 import (
 	"bytes"
@@ -25,30 +25,19 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/obolnetwork/charon/testutil"
-	"github.com/obolnetwork/charon/testutil/compose"
 )
 
-func TestDefineCompose(t *testing.T) {
-	dir, err := os.MkdirTemp("", "")
-	require.NoError(t, err)
-
-	err = compose.Define(context.Background(), dir, 1, compose.NewDefaultConfig())
-	require.NoError(t, err)
-
-	conf, err := os.ReadFile(path.Join(dir, "config.json"))
-	require.NoError(t, err)
-
-	testutil.RequireGoldenBytes(t, conf)
-}
-
 func TestDefineDKG(t *testing.T) {
 	dir, err := os.MkdirTemp("", "")
 	require.NoError(t, err)
 
-	conf := compose.NewDefaultConfig()
+	conf := NewDefaultConfig()
 	conf.KeyGen = "dkg"
+	conf.Step = stepNew
+	p2pSeed = 1
+	require.NoError(t, writeConfig(dir, conf))
 
-	err = compose.Define(context.Background(), dir, 1, conf)
+	err = Define(context.Background(), dir)
 	require.NoError(t, err)
 
 	dc, err := os.ReadFile(path.Join(dir, "docker-compose.yml"))
@@ -62,10 +51,12 @@ func TestDefineCreate(t *testing.T) {
 	dir, err := os.MkdirTemp("", "")
 	require.NoError(t, err)
 
-	conf := compose.NewDefaultConfig()
+	conf := NewDefaultConfig()
 	conf.KeyGen = "create"
+	conf.Step = stepNew
+	require.NoError(t, writeConfig(dir, conf))
 
-	err = compose.Define(context.Background(), dir, 1, conf)
+	err = Define(context.Background(), dir)
 	require.NoError(t, err)
 
 	dc, err := os.ReadFile(path.Join(dir, "docker-compose.yml"))

testutil/compose/lock.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/obolnetwork/charon/app/errors"
 	"github.com/obolnetwork/charon/app/log"
+	"github.com/obolnetwork/charon/app/z"
 )
 
 // Lock creates a docker-compose.yml from a charon-compose.yml for generating keys and a cluster lock file.
@@ -33,11 +34,11 @@ func Lock(ctx context.Context, dir string) error {
 
 	conf, err := loadConfig(dir)
 	if errors.Is(err, fs.ErrNotExist) {
-		return errors.New("compose config not found; maybe try `compose define` first")
+		return errors.New("compose config not found; maybe try `compose new` first")
 	} else if err != nil {
 		return err
-	} else if conf.Step == stepLocked {
-		return errors.New("compose config already locked; maybe try `compose clean` or `compose run`")
+	} else if conf.Step != stepDefined {
+		return errors.New("compose config not defined, so can't be locked", z.Any("step", conf.Step))
 	}
 
 	var data tmplData