Merge pull request #1833 from mesg-foundation/dev

Release v0.25.0

CHANGELOG.md

@@ -1,5 +1,21 @@
 # Changelog
 
+## [v0.25.0](https://github.com/mesg-foundation/engine/releases/tag/v0.25.0)
+
+#### Breaking Changes
+
+- ([#1830](https://github.com/mesg-foundation/engine/pull/1830)) Revert "Fix 2 issues with instance".
+
+#### Added
+
+- ([#1825](https://github.com/mesg-foundation/engine/pull/1825)) Add filter on execution list.
+
+#### Changed
+
+- ([#1827](https://github.com/mesg-foundation/engine/pull/1827)) Update dev script to use docker container instead of docker service.
+- ([#1828](https://github.com/mesg-foundation/engine/pull/1828)) Improve CLI commands.
+- ([#1831](https://github.com/mesg-foundation/engine/pull/1831)) Switch to json logger in orchestrator and daemon.
+
 ## [v0.24.0](https://github.com/mesg-foundation/engine/releases/tag/v0.24.0)
 
 #### Breaking Changes

Makefile

@@ -84,4 +84,6 @@ changelog:
 
 clean: 
 	- rm -rf bin
-	- docker image rm $(docker images | grep 'mesg')
+	- docker volume rm engine
+	- docker image rm $(shell docker images -q mesg/engine)
+	- docker image rm $(shell docker images -q mesg/tools)

app/app.go

@@ -241,9 +241,9 @@ func NewInitApp(
 
 	// Engine's module keepers
 	app.ownershipKeeper = ownership.NewKeeper(app.cdc, keys[ownership.StoreKey], app.bankKeeper)
-	app.serviceKeeper = service.NewKeeper(app.cdc, keys[service.StoreKey], app.ownershipKeeper)
-	app.instanceKeeper = instance.NewKeeper(app.cdc, keys[instance.StoreKey], app.serviceKeeper)
+	app.instanceKeeper = instance.NewKeeper(app.cdc, keys[instance.StoreKey])
 	app.processKeeper = process.NewKeeper(app.cdc, keys[process.StoreKey], app.instanceKeeper, app.ownershipKeeper, app.bankKeeper)
+	app.serviceKeeper = service.NewKeeper(app.cdc, keys[service.StoreKey], app.ownershipKeeper)
 	app.runnerKeeper = runner.NewKeeper(app.cdc, keys[runner.StoreKey], app.instanceKeeper, app.ownershipKeeper)
 	app.executionKeeper = execution.NewKeeper(
 		app.cdc,

cmd/mesg-cli/main.go

@@ -8,12 +8,10 @@ import (
 	"github.com/cosmos/cosmos-sdk/client"
 	"github.com/cosmos/cosmos-sdk/client/flags"
 	"github.com/cosmos/cosmos-sdk/client/keys"
-	"github.com/cosmos/cosmos-sdk/client/lcd"
 	"github.com/cosmos/cosmos-sdk/client/rpc"
 	"github.com/cosmos/cosmos-sdk/version"
 	"github.com/cosmos/cosmos-sdk/x/auth"
 	authcmd "github.com/cosmos/cosmos-sdk/x/auth/client/cli"
-	authrest "github.com/cosmos/cosmos-sdk/x/auth/client/rest"
 	"github.com/cosmos/cosmos-sdk/x/bank"
 	bankcmd "github.com/cosmos/cosmos-sdk/x/bank/client/cli"
 	"github.com/mesg-foundation/engine/app"
@@ -54,7 +52,7 @@ func main() {
 		flags.LineBreak,
 		orchestratorCmd(cdc),
 		flags.LineBreak,
-		lcd.ServeCommand(cdc, registerRoutes),
+		ServeCommand(cdc),
 		flags.LineBreak,
 		keys.Commands(),
 		signCommand(),
@@ -131,16 +129,6 @@ func txCmd(cdc *amino.Codec) *cobra.Command {
 	return txCmd
 }
 
-// registerRoutes registers the routes from the different modules for the LCD.
-// NOTE: details on the routes added for each module are in the module documentation
-// NOTE: If making updates here you also need to update the test helper in client/lcd/test_helper.go
-func registerRoutes(rs *lcd.RestServer) {
-	client.RegisterRoutes(rs.CliCtx, rs.Mux)
-	authrest.RegisterTxRoutes(rs.CliCtx, rs.Mux)
-	app.ModuleBasics.RegisterRESTRoutes(rs.CliCtx, rs.Mux)
-	cosmos.RegisterSimulateRoute(rs.CliCtx, rs.Mux)
-}
-
 func initConfig(cmd *cobra.Command) error {
 	home, err := cmd.PersistentFlags().GetString(cli.HomeFlag)
 	if err != nil {

cmd/mesg-cli/orchestrator.go

@@ -48,7 +48,7 @@ func startOrchestratorCmd(cdc *codec.Codec) *cobra.Command {
 				return fmt.Errorf("chain-id is required. use flag --chain-id or config file")
 			}
 
-			logger := log.NewTMLogger(log.NewSyncWriter(os.Stdout))
+			logger := log.NewTMJSONLogger(log.NewSyncWriter(os.Stdout))
 			client, err := cliCtx.GetNode()
 			if err != nil {
 				return err

cmd/mesg-cli/serve.go

@@ -0,0 +1,70 @@
+package main
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"time"
+
+	"github.com/cosmos/cosmos-sdk/client"
+	"github.com/cosmos/cosmos-sdk/client/context"
+	"github.com/cosmos/cosmos-sdk/client/flags"
+	"github.com/cosmos/cosmos-sdk/codec"
+	"github.com/cosmos/cosmos-sdk/server"
+	authrest "github.com/cosmos/cosmos-sdk/x/auth/client/rest"
+	"github.com/gorilla/mux"
+	"github.com/mesg-foundation/engine/app"
+	"github.com/mesg-foundation/engine/cosmos"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"github.com/tendermint/tendermint/libs/log"
+	rpcserver "github.com/tendermint/tendermint/rpc/lib/server"
+)
+
+// ServeCommand creates and starts the LCD server
+// adapted version of function from https://github.com/cosmos/cosmos-sdk/blob/v0.38.3/client/lcd/root.go#L74-L100
+func ServeCommand(cdc *codec.Codec) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "rest-server",
+		Short: "Start LCD (light-client daemon), a local REST server",
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			// new rest server
+			r := mux.NewRouter()
+			cliCtx := context.NewCLIContext().WithCodec(cdc)
+			logger := log.NewTMJSONLogger(log.NewSyncWriter(os.Stdout)).With("module", "rest-server")
+
+			// register routes
+			client.RegisterRoutes(cliCtx, r)
+			authrest.RegisterTxRoutes(cliCtx, r)
+			app.ModuleBasics.RegisterRESTRoutes(cliCtx, r)
+			cosmos.RegisterSimulateRoute(cliCtx, r)
+
+			// start
+			var listener net.Listener
+			server.TrapSignal(func() {
+				err := listener.Close()
+				logger.Error("error closing listener", "err", err)
+			})
+
+			cfg := rpcserver.DefaultConfig()
+			cfg.MaxOpenConnections = viper.GetInt(flags.FlagMaxOpenConnections)
+			cfg.ReadTimeout = time.Duration(uint(viper.GetInt(flags.FlagRPCReadTimeout))) * time.Second
+			cfg.WriteTimeout = time.Duration(uint(viper.GetInt(flags.FlagRPCWriteTimeout))) * time.Second
+
+			listener, err = rpcserver.Listen(viper.GetString(flags.FlagListenAddr), cfg)
+			if err != nil {
+				return
+			}
+			logger.Info(
+				fmt.Sprintf(
+					"Starting application REST service (chain-id: %q)...",
+					viper.GetString(flags.FlagChainID),
+				),
+			)
+
+			return rpcserver.StartHTTPServer(listener, r, logger, cfg)
+		},
+	}
+
+	return flags.RegisterRestServerFlags(cmd)
+}

cmd/mesg-daemon/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"encoding/json"
 	"io"
+	"os"
 
 	"github.com/cosmos/cosmos-sdk/baseapp"
 	"github.com/cosmos/cosmos-sdk/client/debug"
@@ -19,7 +20,9 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	abci "github.com/tendermint/tendermint/abci/types"
+	cfg "github.com/tendermint/tendermint/config"
 	"github.com/tendermint/tendermint/libs/cli"
+	tmflags "github.com/tendermint/tendermint/libs/cli/flags"
 	"github.com/tendermint/tendermint/libs/log"
 	tmtypes "github.com/tendermint/tendermint/types"
 	dbm "github.com/tendermint/tm-db"
@@ -35,12 +38,30 @@ func main() {
 	// init the config of cosmos
 	cosmos.InitConfig()
 
-	ctx := server.NewDefaultContext()
+	ctx := server.NewContext(
+		cfg.DefaultConfig(),
+		log.NewTMJSONLogger(log.NewSyncWriter(os.Stdout)),
+	)
 	cobra.EnableCommandSorting = false
 	rootCmd := &cobra.Command{
-		Use:               version.ServerName,
-		Short:             "Engine Daemon (server)",
-		PersistentPreRunE: server.PersistentPreRunEFn(ctx),
+		Use:   version.ServerName,
+		Short: "Engine Daemon (server)",
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			// adapted version of function from https://github.com/cosmos/cosmos-sdk/blob/v0.38.3/server/util.go#L49-L74
+			if err := server.PersistentPreRunEFn(ctx)(cmd, args); err != nil {
+				return err
+			}
+			logger, err := tmflags.ParseLogLevel(ctx.Config.LogLevel, log.NewTMJSONLogger(log.NewSyncWriter(os.Stdout)), cfg.DefaultLogLevel())
+			if err != nil {
+				return err
+			}
+			if viper.GetBool(cli.TraceFlag) {
+				logger = log.NewTracingLogger(logger)
+			}
+			logger = logger.With("module", "main")
+			ctx.Logger = logger
+			return nil
+		},
 	}
 
 	rootCmd.AddCommand(genutilcli.InitCmd(ctx, cdc, app.ModuleBasics, app.DefaultNodeHome))

e2e/api_test.go

@@ -145,13 +145,11 @@ func pollExecutionOfProcess(processHash hash.Hash, status execution.Status, node
 	timeout := time.After(pollingTimeout)
 	for {
 		var execs []*execution.Execution
-		if err := lcd.Get("execution/list", &execs); err != nil {
+		if err := lcd.Get(fmt.Sprintf("execution/list?processHash=%s&status=%s&nodeKey=%s", processHash, status, nodeKey), &execs); err != nil {
 			return nil, err
 		}
-		for _, exec := range execs {
-			if exec.ProcessHash.Equal(processHash) && exec.Status == status && exec.NodeKey == nodeKey {
-				return exec, nil
-			}
+		if len(execs) > 0 {
+			return execs[0], nil
 		}
 		select {
 		case <-time.After(pollingInterval):

protobuf/types/instance.proto

@@ -21,7 +21,7 @@ message Instance {
   ];
 
   bytes envHash = 3 [
-    (gogoproto.moretags) = 'hash:"name:3" validate:"omitempty,hash"',
+    (gogoproto.moretags) = 'hash:"name:3" validate:"required,hash"',
     (gogoproto.casttype) = "github.com/mesg-foundation/engine/hash.Hash"
   ];
 }

scripts/dev.sh

@@ -17,13 +17,10 @@ if [[ "$2" == "monitoring" ]]; then
 fi
 
 function onexit {
-  docker service rm $ENGINE_NAME
-  docker wait $(docker ps -f label=com.docker.swarm.service.name=$ENGINE_NAME -q) 2> /dev/null
+  docker stop $ENGINE_NAME
 
-  if $monitor; then
-    docker service rm engine-grafana engine-prometheus
-    docker wait $(docker ps -f label=com.docker.swarm.service.name=engine-grafana -q) 2> /dev/null
-    docker wait $(docker ps -f label=com.docker.swarm.service.name=engine-prometheus -q) 2> /dev/null
+  if $monitoring; then
+    docker stop engine-grafana engine-prometheus
   fi
 
   docker network remove $NETWORK_NAME
@@ -32,35 +29,41 @@ function onexit {
 trap onexit EXIT
 
 if [[ -z $(docker network list -f name="$NETWORK_NAME" -q) ]]; then
-  docker network create --driver overlay $NETWORK_NAME
+  docker network create $NETWORK_NAME
 fi
 
 if $monitoring; then
   echo "start monitoring"
-  docker service create \
+  docker run \
+    -d \
+    --rm \
+    --name=engine-grafana \
     -p 3001:3000 \
     --network $NETWORK_NAME \
-    --name=engine-grafana \
     --mount type=bind,source=$(pwd)/scripts/monitoring/datasource.yml,destination=/etc/grafana/provisioning/datasources/datasource.yml \
     --mount type=bind,source=$(pwd)/scripts/monitoring/dashboard.yml,destination=/etc/grafana/provisioning/dashboards/dashboard.yml \
     --mount type=bind,source=$(pwd)/scripts/monitoring/dashboards,destination=/var/lib/grafana/dashboards \
     grafana/grafana
 
-  docker service create \
+  docker run \
+    -d \
+    --rm \
+    --name=engine-prometheus \
     -p 9090:9090 \
     --network $NETWORK_NAME \
-    --name=engine-prometheus \
     --mount type=bind,source=$(pwd)/scripts/monitoring/prometheus.yml,destination=/etc/prometheus/prometheus.yml \
     prom/prometheus
 fi
 
-docker service create \
+docker run \
+  -d \
+  --rm \
   --name $ENGINE_NAME \
   -p 1317:1317 \
   -p 50052:50052 \
   -p 26657:26657 \
   --network $NETWORK_NAME \
-  --label com.docker.stack.namespace=$ENGINE_NAME \
+  --volume engine:/root/ \
   mesg/engine:$1-dev
 
-docker service logs --tail 1000 --follow --raw $ENGINE_NAME
+docker logs --tail 1000 --follow $ENGINE_NAME

x/execution/alias.go

@@ -25,8 +25,9 @@ var (
 
 	ModuleCdc = types.ModuleCdc
 
-	QueryGet  = types.QueryGet
-	QueryList = types.QueryList
+	QueryGet        = types.QueryGet
+	QueryList       = types.QueryList
+	QueryParameters = types.QueryParameters
 
 	M = keeper.M