fix(block-explorer): make the block-explorer creation more reliable (#…

…977)

cmd/block-explorer/run/run.go

@@ -4,11 +4,13 @@ import (
 	"fmt"
 	"path/filepath"
 
+	"github.com/pterm/pterm"
+	"github.com/spf13/cobra"
+
+	"github.com/dymensionxyz/roller/cmd/consts"
 	"github.com/dymensionxyz/roller/cmd/utils"
 	"github.com/dymensionxyz/roller/utils/blockexplorer"
 	"github.com/dymensionxyz/roller/utils/config/tomlconfig"
-	"github.com/pterm/pterm"
-	"github.com/spf13/cobra"
 )
 
 func Cmd() *cobra.Command {
@@ -19,17 +21,45 @@ func Cmd() *cobra.Command {
 		Args:  cobra.MaximumNArgs(1),
 		Run: func(cmd *cobra.Command, args []string) {
 			home := cmd.Flag(utils.FlagNames.Home).Value.String()
+			isFlagChanged := cmd.Flags().Changed("block-explorer-rpc-endpoint")
+			defaultBeRpcEndpoint, _ := cmd.Flags().GetString("block-explorer-rpc-endpoint")
+
+			var beRpcEndpoint string
+			if !isFlagChanged {
+				useDefaultEndpoint, _ := pterm.DefaultInteractiveConfirm.WithDefaultValue(false).
+					WithDefaultText(
+						fmt.Sprintf(
+							`'--block-explorer-rpc-endpoint' is not set,
+would you like to continue with the default endpoint (%s)?'
+press 'y' if you're running local node, press 'n' and provide the endpoint if you're running remote node`,
+							defaultBeRpcEndpoint,
+						),
+					).
+					Show()
+				if useDefaultEndpoint {
+					beRpcEndpoint = defaultBeRpcEndpoint
+				} else {
+					newBeRpcEndpoint, _ := pterm.DefaultInteractiveTextInput.Show()
+					beRpcEndpoint = newBeRpcEndpoint
+				}
+			}
 
 			rollerData, err := tomlconfig.LoadRollerConfig(home)
 			if err != nil {
 				pterm.Error.Println("failed to load roller config file", err)
 				return
 			}
 
-			fmt.Println("Run the block explorer")
-
-			beChainConfigPath := filepath.Join(home, "block-explorer", "config", "chains.yaml")
-			beChainConfig := blockexplorer.GenerateChainsYAML(rollerData.RollappID)
+			beChainConfigPath := filepath.Join(
+				home,
+				consts.ConfigDirName.BlockExplorer,
+				"config",
+				"chains.yaml",
+			)
+			beChainConfig := blockexplorer.GenerateChainsYAML(
+				rollerData.RollappID,
+				beRpcEndpoint,
+			)
 			err = blockexplorer.WriteChainsYAML(beChainConfigPath, beChainConfig)
 			if err != nil {
 				pterm.Error.Println("failed to generate block explorer config", err)
@@ -43,5 +73,8 @@ func Cmd() *cobra.Command {
 		},
 	}
 
+	cmd.Flags().
+		String("block-explorer-rpc-endpoint", "http://localhost:11100", "block explorer rpc endpoint")
+
 	return cmd
 }

cmd/block-explorer/run/utils.go

@@ -4,12 +4,10 @@ import (
 	"context"
 	"database/sql"
 	"fmt"
-	"log"
 	"os"
 	"path/filepath"
 	"runtime"
 	"strings"
-	"time"
 
 	"github.com/docker/docker/api/types/mount"
 	"github.com/docker/docker/api/types/network"
@@ -18,7 +16,11 @@ import (
 	"github.com/pterm/pterm"
 	"golang.org/x/exp/maps"
 
+	"github.com/dymensionxyz/roller/cmd/consts"
+	postgresqlutils "github.com/dymensionxyz/roller/utils/database/postgresql"
+	"github.com/dymensionxyz/roller/utils/dependencies/types"
 	dockerutils "github.com/dymensionxyz/roller/utils/docker"
+	"github.com/dymensionxyz/roller/utils/filesystem"
 )
 
 func createBlockExplorerContainers(home string) error {
@@ -36,18 +38,17 @@ func createBlockExplorerContainers(home string) error {
 	}
 
 	// Determine the host address to use
-	hostAddress := "host.docker.internal"
+	hostAddress := "18.197.167.214"
 	if runtime.GOOS == "linux" {
 		hostAddress = "172.17.0.1" // Default Docker bridge network gateway
 	}
 
 	beChainConfigPath := filepath.Join(
 		home,
-		"block-explorer",
+		consts.ConfigDirName.BlockExplorer,
 		"config",
 		"chains.yaml",
 	)
-	fmt.Println(beChainConfigPath)
 	containers := map[string]dockerutils.ContainerConfigOptions{
 		"db": {
 			Name:  "be-postgresql",
@@ -109,20 +110,15 @@ func createBlockExplorerContainers(home string) error {
 			return err
 		}
 
-		// Connect the container to the network
-		err = cc.NetworkConnect(
-			context.Background(),
-			networkName,
-			options.Name,
-			&network.EndpointSettings{},
-		)
+		// Connect the container to the network using the new function
+		err = connectContainerToNetwork(context.Background(), cc, networkName, options.Name)
 		if err != nil {
-			fmt.Printf("Failed to connect container %s to network: %v\n", options.Name, err)
+			fmt.Printf("Error with network connection for container %s: %v\n", options.Name, err)
 			return err
 		}
 	}
 
-	if err := runSQLMigration(); err != nil {
+	if err := runSQLMigration(home); err != nil {
 		fmt.Printf("Failed to apply migrations: %v\n", err)
 		return err
 	}
@@ -159,15 +155,48 @@ func ensureNetworkExists(cli *client.Client, networkName string) error {
 	return nil
 }
 
-func runSQLMigration() error {
-	// Database connection details
+func connectContainerToNetwork(
+	ctx context.Context,
+	cc *client.Client,
+	networkName, containerName string,
+) error {
+	networkResource, err := cc.NetworkInspect(ctx, networkName, network.InspectOptions{})
+	if err != nil {
+		return fmt.Errorf("failed to inspect network: %v", err)
+	}
+
+	for _, container := range networkResource.Containers {
+		if container.Name == containerName {
+			fmt.Printf(
+				"Container %s is already connected to network %s\n",
+				containerName,
+				networkName,
+			)
+			return nil
+		}
+	}
+
+	err = cc.NetworkConnect(
+		ctx,
+		networkName,
+		containerName,
+		&network.EndpointSettings{},
+	)
+	if err != nil {
+		return fmt.Errorf("failed to connect container %s to network: %v", containerName, err)
+	}
+
+	fmt.Printf("Connected container %s to network %s\n", containerName, networkName)
+	return nil
+}
+
+func runSQLMigration(home string) error {
 	dbHost := "localhost"
 	dbPort := "5432"
 	dbName := "blockexplorer"
 	dbUserAdmin := "be"
 	dbPassAdmin := "psw"
 
-	// Connect to the database as an admin user
 	dbConnStr := fmt.Sprintf(
 		"postgresql://%s:%s@%s:%s/%s?sslmode=disable",
 		dbUserAdmin,
@@ -176,44 +205,81 @@ func runSQLMigration() error {
 		dbPort,
 		dbName,
 	)
+
+	pterm.Info.Println("Retrieving SQL migration files")
+	dbMigrationsPath := filepath.Join(home, consts.ConfigDirName.BlockExplorer, "migrations")
+	dbMigrationsSchemaPath := filepath.Join(dbMigrationsPath, "schema.sql")
+	dbMigrationsEventsPath := filepath.Join(dbMigrationsPath, "events.sql")
+
+	err := os.MkdirAll(dbMigrationsPath, 0o755)
+	if err != nil {
+		return fmt.Errorf("failed to create migrations directory: %w", err)
+	}
+
+	migrationFiles := []types.PersistFile{
+		{
+			Source: "https://raw.githubusercontent.com/dymensionxyz/roller/main/migrations/block-explorer/schema.sql",
+			Target: dbMigrationsSchemaPath,
+		},
+		{
+			Source: "https://raw.githubusercontent.com/dymensionxyz/roller/main/migrations/block-explorer/events.sql",
+			Target: dbMigrationsEventsPath,
+		},
+	}
+
+	for _, file := range migrationFiles {
+		err := filesystem.DownloadFile(file.Source, file.Target)
+		if err != nil {
+			pterm.Error.Printf("Failed to retrieve SQL migration %s: %v\n", file.Target, err)
+			return err
+		}
+	}
+
 	dbAdmin, err := sql.Open("postgres", dbConnStr)
 	if err != nil {
 		return fmt.Errorf("failed to connect to database as admin: %w", err)
 	}
 	defer dbAdmin.Close()
 
-	// Wait for the database to be ready
-	time.Sleep(5 * time.Second)
-
-	// Connect to the new database as the local user
 	dbLocal, err := sql.Open("postgres", dbConnStr)
 	if err != nil {
 		return fmt.Errorf("failed to connect to database as local user: %w", err)
 	}
 	defer dbLocal.Close()
 
-	// Read and execute the SQL migration file
-	sqlFile, err := os.ReadFile("migrations/block-explorer/schema.sql")
+	// Create migration tracking table
+	const createMigrationTableSQL = `
+    CREATE TABLE IF NOT EXISTS applied_migrations (
+        id SERIAL PRIMARY KEY,
+        filename VARCHAR(255) NOT NULL UNIQUE,
+        applied_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+    );`
+
+	_, err = dbAdmin.Exec(createMigrationTableSQL)
 	if err != nil {
-		return fmt.Errorf("failed to read SQL file: %w", err)
+		return fmt.Errorf("failed to create migration table: %w", err)
 	}
 
-	_, err = dbLocal.Exec(string(sqlFile))
+	// Apply schema migration
+	schemaContent, err := os.ReadFile(dbMigrationsSchemaPath)
 	if err != nil {
-		return fmt.Errorf("failed to execute SQL migration: %w", err)
+		return fmt.Errorf("failed to read schema SQL file: %w", err)
 	}
-
-	// Execute additional SQL files if needed
-	superSchemaFile, err := os.ReadFile("migrations/block-explorer/events.sql")
+	err = postgresqlutils.ApplyMigration(dbLocal, "schema.sql", schemaContent)
 	if err != nil {
-		return fmt.Errorf("failed to read super-schema SQL file: %w", err)
+		return err
 	}
 
-	_, err = dbAdmin.Exec(string(superSchemaFile))
+	// Apply events migration
+	eventsContent, err := os.ReadFile(dbMigrationsEventsPath)
+	if err != nil {
+		return fmt.Errorf("failed to read events SQL file: %w", err)
+	}
+	err = postgresqlutils.ApplyMigration(dbAdmin, "events.sql", eventsContent)
 	if err != nil {
-		return fmt.Errorf("failed to execute super-schema SQL file: %w", err)
+		return err
 	}
 
-	log.Println("Migrations applied successfully")
+	pterm.Success.Println("Migrations checked and applied successfully")
 	return nil
 }

cmd/consts/consts.go

@@ -77,6 +77,7 @@ var ConfigDirName = struct {
 	RollappSequencerKeys string
 	LocalHub             string
 	Eibc                 string
+	BlockExplorer        string
 }{
 	Rollapp:              "rollapp",
 	Relayer:              "relayer",
@@ -85,6 +86,7 @@ var ConfigDirName = struct {
 	RollappSequencerKeys: "rollapp-sequencer-keys",
 	LocalHub:             "local-hub",
 	Eibc:                 ".eibc-client",
+	BlockExplorer:        "block-explorer",
 }
 
 var Denoms = struct {

utils/blockexplorer/blockexplorer.go

@@ -9,13 +9,13 @@ import (
 // GenerateChainsYAML generates the YAML content with the given chain_id
 // this configuration is used by the block-explorer to index the locally running
 // chain
-func GenerateChainsYAML(chainID string) string {
+func GenerateChainsYAML(chainID, beRpcEndpoint string) string {
 	template := `local:
   chain_id: %s
-  be_json_rpc_urls: [ "http://host.docker.internal:11100" ]
+  be_json_rpc_urls: [ "%s" ]
   # disable: true
 `
-	return fmt.Sprintf(template, chainID)
+	return fmt.Sprintf(template, chainID, beRpcEndpoint)
 }
 
 func WriteChainsYAML(filePath, content string) error {

utils/database/postgresql/utils.go

@@ -0,0 +1,40 @@
+package postgresql
+
+import (
+	"database/sql"
+	"fmt"
+)
+
+func isMigrationApplied(db *sql.DB, filename string) (bool, error) {
+	var count int
+	err := db.QueryRow("SELECT COUNT(*) FROM applied_migrations WHERE filename = $1", filename).
+		Scan(&count)
+	if err != nil {
+		return false, err
+	}
+	return count > 0, nil
+}
+
+func ApplyMigration(db *sql.DB, filename string, content []byte) error {
+	applied, err := isMigrationApplied(db, filename)
+	if err != nil {
+		return fmt.Errorf("failed to check if migration is applied: %w", err)
+	}
+	if applied {
+		fmt.Printf("Migration %s already applied, skipping\n", filename)
+		return nil
+	}
+
+	_, err = db.Exec(string(content))
+	if err != nil {
+		return fmt.Errorf("failed to execute migration %s: %w", filename, err)
+	}
+
+	_, err = db.Exec("INSERT INTO applied_migrations (filename) VALUES ($1)", filename)
+	if err != nil {
+		return fmt.Errorf("failed to record applied migration %s: %w", filename, err)
+	}
+
+	fmt.Printf("Migration %s applied successfully\n", filename)
+	return nil
+}