Add datalayer commands

cmd/datalayer/datalayer.go

@@ -0,0 +1,17 @@
+package datalayer
+
+import (
+	"github.com/spf13/cobra"
+
+	"github.com/chia-network/chia-tools/cmd"
+)
+
+// datalayerCmd represents the config command
+var datalayerCmd = &cobra.Command{
+	Use:   "data",
+	Short: "Utilities for working with chia data layer",
+}
+
+func init() {
+	cmd.RootCmd.AddCommand(datalayerCmd)
+}

cmd/datalayer/deletemirrors.go

@@ -0,0 +1,74 @@
+package datalayer
+
+import (
+	"github.com/chia-network/go-chia-libs/pkg/rpc"
+	"github.com/chia-network/go-chia-libs/pkg/types"
+	"github.com/chia-network/go-modules/pkg/slogs"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+// deleteMirrorsCmd Deletes all owned mirrors for all datalayer subscriptions
+var deleteMirrorsCmd = &cobra.Command{
+	Use:   "delete-mirrors",
+	Short: "Deletes all owned mirrors for all datalayer subscriptions",
+	Run: func(cmd *cobra.Command, args []string) {
+		client, err := rpc.NewClient(rpc.ConnectionModeHTTP, rpc.WithAutoConfig())
+		if err != nil {
+			slogs.Logr.Fatal("error creating chia RPC client", "error", err)
+		}
+
+		// Figure out what fee we are using
+		feeXCH := viper.GetFloat64("delete-mirror-fee")
+		feeMojos := uint64(feeXCH * 1000000000000)
+		slogs.Logr.Debug("fee for all transactions", "xch", feeXCH, "mojos", feeMojos)
+
+		subscriptions, _, err := client.DataLayerService.GetSubscriptions(&rpc.DatalayerGetSubscriptionsOptions{})
+		if err != nil {
+			slogs.Logr.Fatal("error getting list of datalayer subscriptions", "error", err)
+		}
+
+		for _, subscription := range subscriptions.StoreIDs {
+			slogs.Logr.Info("checking subscription", "store", subscription)
+
+			mirrors, _, err := client.DataLayerService.GetMirrors(&rpc.DatalayerGetMirrorsOptions{
+				ID: subscription,
+			})
+			if err != nil {
+				slogs.Logr.Fatal("error fetching mirrors for subscription", "store", subscription, "error", err)
+			}
+			var ownedMirrors []types.Bytes32
+
+			for _, mirror := range mirrors.Mirrors {
+				if mirror.Ours {
+					ownedMirrors = append(ownedMirrors, mirror.CoinID)
+				}
+			}
+
+			if len(ownedMirrors) == 0 {
+				slogs.Logr.Info("no owned mirrors for this datastore", "store", subscription)
+				continue
+			}
+
+			for _, coinID := range ownedMirrors {
+				resp, _, err := client.DataLayerService.DeleteMirror(&rpc.DatalayerDeleteMirrorOptions{
+					CoinID: coinID.String(),
+					Fee:    feeMojos,
+				})
+				if err != nil {
+					slogs.Logr.Fatal("error deleting mirror for store", "store", subscription, "mirror", coinID, "error", err)
+				}
+				if !resp.Success {
+					slogs.Logr.Fatal("unknown error when deleting mirror for store", "store", subscription, "mirror", coinID)
+				}
+			}
+		}
+	},
+}
+
+func init() {
+	deleteMirrorsCmd.PersistentFlags().Float64P("fee", "m", 0, "Fee to use when deleting the mirrors. The fee is used per mirror. Units are XCH")
+	cobra.CheckErr(viper.BindPFlag("delete-mirror-fee", deleteMirrorsCmd.PersistentFlags().Lookup("fee")))
+
+	datalayerCmd.AddCommand(deleteMirrorsCmd)
+}

cmd/datalayer/fixmirrors.go

@@ -0,0 +1,125 @@
+package datalayer
+
+import (
+	"strings"
+	"time"
+
+	"github.com/chia-network/go-chia-libs/pkg/rpc"
+	"github.com/chia-network/go-modules/pkg/slogs"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+// fixMirrorsCmd Replaces one mirror url with another for all mirrors with the url
+var fixMirrorsCmd = &cobra.Command{
+	Use:     "fix-mirrors",
+	Short:   "For all owned mirrors, replaces one url with a new url",
+	Example: "chia-tools data fix-mirrors -b 127.0.0.1 -n https://my-dl-domain.com -a 300 -m 0.00000001",
+	Run: func(cmd *cobra.Command, args []string) {
+		client, err := rpc.NewClient(rpc.ConnectionModeHTTP, rpc.WithAutoConfig())
+		if err != nil {
+			slogs.Logr.Fatal("error creating chia RPC client", "error", err)
+		}
+
+		// Figure out what fee we are using
+		feeXCH := viper.GetFloat64("fix-mirror-fee")
+		feeMojos := uint64(feeXCH * 1000000000000)
+		slogs.Logr.Debug("fee for all transactions", "xch", feeXCH, "mojos", feeMojos)
+
+		subscriptions, _, err := client.DataLayerService.GetSubscriptions(&rpc.DatalayerGetSubscriptionsOptions{})
+		if err != nil {
+			slogs.Logr.Fatal("error getting list of datalayer subscriptions", "error", err)
+		}
+
+		for _, sub := range subscriptions.StoreIDs {
+			foundAnyMirror := false
+
+			mirrors, _, err := client.DataLayerService.GetMirrors(&rpc.DatalayerGetMirrorsOptions{
+				ID: sub,
+			})
+			if err != nil {
+				slogs.Logr.Fatal("error getting mirrors for subscription", "store", sub, "error", err)
+			}
+			for _, mirror := range mirrors.Mirrors {
+				if !mirror.Ours {
+					continue
+				}
+				for _, url := range mirror.URLs {
+					if strings.EqualFold(url, viper.GetString("fix-mirror-bad-url")) {
+						foundAnyMirror = true
+						waitForAvailableBalance(client, feeMojos)
+						slogs.Logr.Info("deleting mirror", "store", sub, "mirror", mirror.CoinID.String())
+						_, _, err := client.DataLayerService.DeleteMirror(&rpc.DatalayerDeleteMirrorOptions{
+							CoinID: mirror.CoinID.String(),
+							Fee:    viper.GetUint64("fix-mirror-fee"),
+						})
+						if err != nil {
+							slogs.Logr.Fatal("error deleting mirror", "store", sub, "mirror", mirror.CoinID.String(), "error", err)
+						}
+						break
+					}
+				}
+			}
+
+			// Outside the mirror loop, in case there's a weird edge case where we have multiple mirrors on the same bad
+			// url, we consolidate down to just one
+			if foundAnyMirror {
+				mirrorAmount := viper.GetUint64("fix-mirror-amount")
+				waitForAvailableBalance(client, mirrorAmount+feeMojos)
+				slogs.Logr.Info("adding replacement mirror", "store", sub)
+				_, _, err = client.DataLayerService.AddMirror(&rpc.DatalayerAddMirrorOptions{
+					ID:     sub,
+					URLs:   []string{viper.GetString("fix-mirror-new-url")},
+					Amount: mirrorAmount,
+					Fee:    feeMojos,
+				})
+				if err != nil {
+					slogs.Logr.Fatal("error adding new mirror", "store", sub, "error", err)
+				}
+			}
+		}
+	},
+}
+
+// waitForAvailableBalance blocks execution until the wallet has at least one coin and the specified amount available to spend
+func waitForAvailableBalance(client *rpc.Client, amount uint64) {
+	for {
+		balance, _, err := client.WalletService.GetWalletBalance(&rpc.GetWalletBalanceOptions{WalletID: 1})
+		if err != nil {
+			slogs.Logr.Error("error checking wallet balance. Retrying in 5 seconds", "error", err)
+			time.Sleep(5 * time.Second)
+			continue
+		}
+
+		if balance.Balance.IsAbsent() {
+			slogs.Logr.Error("unknown error checking wallet balance. Retrying in 5 seconds")
+			time.Sleep(5 * time.Second)
+			continue
+		}
+
+		// Makes the assumption that wallet balance is not over uint64max mojos
+		// It is extremely unlikely to have that balance in one wallet
+		if !balance.Balance.MustGet().SpendableBalance.FitsInUint64() || balance.Balance.MustGet().SpendableBalance.Uint64() < amount {
+			slogs.Logr.Warn("wallet does not have enough funds to continue. Waiting...", "need", amount, "spendable", balance.Balance.MustGet().SpendableBalance.Uint64())
+			time.Sleep(5 * time.Second)
+			continue
+		}
+
+		// Have enough, so return
+		return
+	}
+}
+
+func init() {
+	fixMirrorsCmd.PersistentFlags().Float64P("fee", "m", 0, "Fee to use when deleting and launching the mirrors. The fee is used per mirror. Units are XCH")
+	fixMirrorsCmd.PersistentFlags().StringP("new-url", "n", "", "New mirror URL (required)")
+	fixMirrorsCmd.PersistentFlags().StringP("bad-url", "b", "", "Old mirror URL to replace (required)")
+	fixMirrorsCmd.PersistentFlags().Uint64P("amount", "a", 100, "Mirror coin amount in mojos")
+
+	cobra.CheckErr(viper.BindPFlag("fix-mirror-fee", fixMirrorsCmd.PersistentFlags().Lookup("fee")))
+	cobra.CheckErr(viper.BindPFlag("fix-mirror-new-url", fixMirrorsCmd.PersistentFlags().Lookup("new-url")))
+	cobra.CheckErr(viper.BindPFlag("fix-mirror-bad-url", fixMirrorsCmd.PersistentFlags().Lookup("bad-url")))
+	cobra.CheckErr(viper.BindPFlag("fix-mirror-amount", fixMirrorsCmd.PersistentFlags().Lookup("amount")))
+
+	datalayerCmd.AddCommand(fixMirrorsCmd)
+}

cmd/datalayer/unsuball.go

@@ -0,0 +1,53 @@
+package datalayer
+
+import (
+	"slices"
+
+	"github.com/chia-network/go-chia-libs/pkg/rpc"
+	"github.com/chia-network/go-modules/pkg/slogs"
+	"github.com/spf13/cobra"
+)
+
+// unsubAllCmd Unsubscribes from all non-owned datalayer stores
+var unsubAllCmd = &cobra.Command{
+	Use:   "unsub-all",
+	Short: "Unsubscribes from all datalayer stores except for owned stores",
+	Run: func(cmd *cobra.Command, args []string) {
+		client, err := rpc.NewClient(rpc.ConnectionModeHTTP, rpc.WithAutoConfig())
+		if err != nil {
+			slogs.Logr.Fatal("error creating chia RPC client", "error", err)
+		}
+
+		ownedStores, _, err := client.DataLayerService.GetOwnedStores(&rpc.DatalayerGetOwnedStoresOptions{})
+		if err != nil {
+			slogs.Logr.Fatal("error getting list of owned data stores", "error", err)
+		}
+
+		subscriptions, _, err := client.DataLayerService.GetSubscriptions(&rpc.DatalayerGetSubscriptionsOptions{})
+		if err != nil {
+			slogs.Logr.Fatal("error getting list of datalayer subscriptions", "error", err)
+		}
+
+		for _, subscription := range subscriptions.StoreIDs {
+			if slices.Contains(ownedStores.StoreIDs, subscription) {
+				slogs.Logr.Info("Owned store found, skipping", "store", subscription)
+				continue
+			}
+			slogs.Logr.Info("Unsubscribing from subscription", "store", subscription)
+			resp, _, err := client.DataLayerService.Unsubscribe(&rpc.DatalayerUnsubscribeOptions{
+				ID:         subscription,
+				RetainData: true,
+			})
+			if err != nil {
+				slogs.Logr.Fatal("error unsubscribing from store", "store", subscription, "error", err)
+			}
+			if !resp.Success {
+				slogs.Logr.Fatal("unknown error when unsubscribing from store", "store", subscription)
+			}
+		}
+	},
+}
+
+func init() {
+	datalayerCmd.AddCommand(unsubAllCmd)
+}

go.mod

@@ -3,7 +3,7 @@ module github.com/chia-network/chia-tools
 go 1.22.4
 
 require (
-	github.com/chia-network/go-chia-libs v0.8.5
+	github.com/chia-network/go-chia-libs v0.8.6-0.20240802191521-ac5fb4a152f1
 	github.com/chia-network/go-modules v0.0.5
 	github.com/spf13/cobra v1.8.1
 	github.com/spf13/viper v1.19.0
@@ -12,10 +12,14 @@ require (
 
 require (
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/google/go-querystring v1.1.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/gorilla/websocket v1.5.3 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
 	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect

go.sum

@@ -1,5 +1,5 @@
-github.com/chia-network/go-chia-libs v0.8.5 h1:IZKpQQH6VcbafDA/DlDlFkjerr3KzDn5vWD4C5nmzHA=
-github.com/chia-network/go-chia-libs v0.8.5/go.mod h1:npTqaFSjTdMxE7hc0LOmWJmWGqcs+IERarK5fDxXk/I=
+github.com/chia-network/go-chia-libs v0.8.6-0.20240802191521-ac5fb4a152f1 h1:zmeRuQcnMEjBKEbQcOgEr3JyAGxZubOrXy8gKOek60w=
+github.com/chia-network/go-chia-libs v0.8.6-0.20240802191521-ac5fb4a152f1/go.mod h1:npTqaFSjTdMxE7hc0LOmWJmWGqcs+IERarK5fDxXk/I=
 github.com/chia-network/go-modules v0.0.5 h1:5luTVlP6RgBXodnFcWFBk2sLdJn+6vQ4wObim683C7c=
 github.com/chia-network/go-modules v0.0.5/go.mod h1:5AiYBxQSvf2aFSOizTqFXXSeb9AucZWrWmRCVwUMO3A=
 github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
@@ -11,8 +11,15 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk
 github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
 github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
 github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
+github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
+github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
@@ -25,6 +32,8 @@ github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0V
 github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
+github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
 github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -73,6 +82,7 @@ golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
 golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

main.go

@@ -4,6 +4,7 @@ import (
 	"github.com/chia-network/chia-tools/cmd"
 	_ "github.com/chia-network/chia-tools/cmd/certs"
 	_ "github.com/chia-network/chia-tools/cmd/config"
+	_ "github.com/chia-network/chia-tools/cmd/datalayer"
 )
 
 func main() {