Fix migrations

backend/cmd/mkdb/main.go

@@ -0,0 +1,64 @@
+// Program mkdb is supposed to be executed with `go run` from the root of the repository.
+// It will create a (more or less) deterministic snapshot of the current schema of our database,
+// which we can then use in tests to verify our migration scripts actually end up where we want them to.
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"mintter/backend/config"
+	"mintter/backend/core/coretest"
+	"mintter/backend/daemon"
+	"os"
+
+	"github.com/burdiyan/go/mainutil"
+)
+
+func main() {
+	mainutil.Run(run)
+}
+
+func run() error {
+	ctx, cancel := context.WithCancel(mainutil.TrapSignals())
+	defer cancel()
+
+	alice := coretest.NewTester("alice")
+
+	cfg := config.Default()
+	cfg.P2P.NoRelay = true
+	cfg.P2P.BootstrapPeers = nil
+	cfg.RepoPath = "/tmp/mintter-test-db-snapshot"
+
+	if err := os.RemoveAll(cfg.RepoPath); err != nil {
+		return err
+	}
+
+	if err := os.MkdirAll(cfg.RepoPath, 0750); err != nil {
+		return err
+	}
+
+	dir, err := daemon.InitRepo(cfg, alice.Device.Wrapped())
+	if err != nil {
+		return err
+	}
+
+	app, err := daemon.LoadWithStorage(ctx, cfg, dir)
+	if err != nil {
+		return err
+	}
+
+	if err := app.RPC.Daemon.RegisterAccount(ctx, alice.Account); err != nil {
+		return err
+	}
+
+	cancel()
+
+	err = app.Wait()
+	fmt.Println("Database has been saved in:", cfg.RepoPath)
+	if errors.Is(err, context.Canceled) {
+		return nil
+	}
+
+	return err
+}

backend/daemon/api/daemon/v1alpha/daemon.go

@@ -92,11 +92,7 @@ func (srv *Server) Register(ctx context.Context, req *daemon.RegisterRequest) (*
 
 // RegisterAccount performs registration given an existing account key pair.
 func (srv *Server) RegisterAccount(ctx context.Context, acc core.KeyPair) error {
-	if err := srv.repo.CommitAccount(acc.PublicKey); err != nil {
-		return err
-	}
-
-	_, err := Register(ctx, srv.blobs, acc, srv.repo.Device().PublicKey, time.Now().UTC())
+	_, err := RegisterWithRepo(ctx, srv.repo, srv.blobs, acc, srv.repo.Device().PublicKey, time.Now().UTC())
 	if err != nil {
 		return err
 	}
@@ -108,6 +104,15 @@ func (srv *Server) RegisterAccount(ctx context.Context, acc core.KeyPair) error
 	return nil
 }
 
+// RegisterWithRepo creates key delegation from account to device and stores the keys in files.
+func RegisterWithRepo(ctx context.Context, repo Repo, bs *hyper.Storage, acc core.KeyPair, device core.PublicKey, at time.Time) (cid.Cid, error) {
+	if err := repo.CommitAccount(acc.PublicKey); err != nil {
+		return cid.Undef, err
+	}
+
+	return Register(ctx, bs, acc, device, at)
+}
+
 // Register creates key delegation from account to device.
 func Register(ctx context.Context, bs *hyper.Storage, account core.KeyPair, device core.PublicKey, at time.Time) (cid.Cid, error) {
 	kd, err := hyper.NewKeyDelegation(account, device, time.Now().UTC())

backend/daemon/daemon.go

@@ -104,15 +104,17 @@ func Load(ctx context.Context, cfg config.Config, grpcOpt ...grpc.ServerOption)
 			return nil, err
 		}
 	}
-	r, err := initRepo(cfg, deviceKey)
+	r, err := InitRepo(cfg, deviceKey)
 	if err != nil {
 		return nil, err
 	}
 
-	return loadApp(ctx, cfg, r, grpcOpt...)
+	return LoadWithStorage(ctx, cfg, r, grpcOpt...)
 }
 
-func loadApp(ctx context.Context, cfg config.Config, r *storage.Dir, grpcOpt ...grpc.ServerOption) (a *App, err error) {
+// LoadWithStorage is the same as Load, but allows to pass a custom storage.
+// The storage must be created using the [InitRepo] function.
+func LoadWithStorage(ctx context.Context, cfg config.Config, r *storage.Dir, grpcOpt ...grpc.ServerOption) (a *App, err error) {
 	a = &App{
 		log:     logging.New("mintter/daemon", "debug"),
 		Storage: r,
@@ -257,7 +259,9 @@ func (a *App) Wait() error {
 	return a.g.Wait()
 }
 
-func initRepo(cfg config.Config, device crypto.PrivKey) (r *storage.Dir, err error) {
+// InitRepo initializes the storage directory.
+// Device can be nil in which case a random new device key will be generated.
+func InitRepo(cfg config.Config, device crypto.PrivKey) (r *storage.Dir, err error) {
 	log := logging.New("mintter/repo", "debug")
 	if device == nil {
 		r, err = storage.New(cfg.RepoPath, log)

backend/daemon/daemon_e2e_test.go

@@ -537,10 +537,10 @@ func makeTestApp(t *testing.T, name string, cfg config.Config, register bool) *A
 
 	u := coretest.NewTester(name)
 
-	repo, err := initRepo(cfg, u.Device.Wrapped())
+	repo, err := InitRepo(cfg, u.Device.Wrapped())
 	require.NoError(t, err)
 
-	app, err := loadApp(ctx, cfg, repo)
+	app, err := LoadWithStorage(ctx, cfg, repo)
 	require.NoError(t, err)
 	t.Cleanup(func() {
 		cancel()

backend/daemon/storage/migrations.go

@@ -51,6 +51,14 @@ var migrations = []migration{
 	{Version: "2023-08-30.01", Run: func(d *Dir, conn *sqlite.Conn) error {
 		return nil
 	}},
+	{Version: "2023-09-12.01", Run: func(d *Dir, conn *sqlite.Conn) error {
+		return sqlitex.ExecScript(conn, `
+			DROP TABLE sites;
+			DROP TABLE invite_tokens;
+			DROP TABLE site_members;
+			DROP TABLE web_publications;
+		`)
+	}},
 }
 
 const (

backend/daemon/storage/migrations_test.go

@@ -17,27 +17,14 @@ import (
 )
 
 func TestMigrateMatchesFreshSchema(t *testing.T) {
-	// We made a new breaking change so we have no migrations now.
-	// We can skip this test until we add at least one new migration,
-	// in which case we need to generate the initial data dir snapshot,
-	// and store it in testdata. Then we should apply migrations
-	// on top of this snapshot and verify that it has the same structure
-	// as new freshly created data dir.
-
-	if len(migrations) == 1 {
-		t.SkipNow()
-	}
-
-	t.Fatalf("We now have some migrations. Fix this text!")
-
 	// We have manually snapshot the data dir from before the migration framework was implemented.
 	// It's stored in ./testdata/initial-data-dir.
 	// We want to test that the data dir with all applied migrations matches the data dir created from scratch.
 	// So we copy the initial snapshot to a temporary directory, apply all migrations to it,
 	// and then compare it with a fresh directory.
 
 	tmpDir := t.TempDir()
-	err := copyDir("./testdata/initial-data-dir", tmpDir)
+	err := copyDir("./testdata/mintter-test-db-snapshot", tmpDir)
 	require.NoError(t, err)
 
 	oldDir, err := New(tmpDir, zap.NewNop())

backend/daemon/storage/schema.sql

@@ -225,58 +225,6 @@ CREATE TABLE wallets (
     balance INTEGER DEFAULT 0
 );
 
--- Stores sites that user has manually added
-CREATE TABLE sites (
-    -- Site unique identification. The hostname of the site with protocol https://example.com
-    hostname TEXT PRIMARY KEY CHECK(hostname <> ''),
-    -- The role we play in the site ROLE_UNSPECIFIED = 0 | OWNER = 1 | EDITOR = 2
-    role INTEGER NOT NULL DEFAULT 0,
-    -- P2P addresses to connect to that site in the format of multiaddresses. Space separated.
-    addresses TEXT NOT NULL CHECK(addresses <> ''),
-    -- The account ID of the site. We need a previous connection to the site so the 
-    -- actual account is inserted in the accounts table when handshake.
-    account_id INTEGER REFERENCES lookup (id) ON DELETE CASCADE NOT NULL
-) WITHOUT ROWID;
-
-CREATE INDEX sites_by_account ON sites (account_id);
-
--- Table that stores all the tokens not yet redeemed inside a site. Although this table is relevant only
--- for sites at the beginning, keep in mind that any regular node can be upgraded to a site.
-CREATE TABLE invite_tokens (
-    -- Unique token identification. Random string.
-    token TEXT PRIMARY KEY CHECK(token <> ''),
-    -- The member role for the user that will redeem the token.
-    -- OWNER = 1 | EDITOR = 2.
-    role INTEGER NOT NULL CHECK (role != 0),
-    -- Timestamp since the token will no longer be eligible to be redeemed. Seconds since  Jan 1, 1970
-    expire_time INTEGER NOT NULL CHECK (expire_time > 0)
-) WITHOUT ROWID;
-
--- Table that stores the role each account has inside a site. Although this table is relevant only
--- for sites at the beginning, keep in mind that any regular node can be upgraded to a site.
-CREATE TABLE site_members (
-    -- The account id that has been linked to a role on this site
-    account_id INTEGER REFERENCES lookup (id) ON DELETE CASCADE NOT NULL,
-    -- The role of the site member.
-    -- OWNER = 1 | EDITOR = 2.
-    role INTEGER NOT NULL CHECK (role != 0),
-    PRIMARY KEY (account_id)
-) WITHOUT ROWID;
-
--- We currently only allow one owner per site.
-CREATE UNIQUE INDEX idx_site_owner ON site_members (role) WHERE role = 1;
-
--- Stores all the records published on this site. Although this table is relevant only
--- for sites at the beginning, keep in mind that any regular node can be upgraded to a site.
-CREATE TABLE web_publications (
-    -- Entity ID of the published document.
-    eid TEXT PRIMARY KEY CHECK (eid <> ''),
-    -- doc version of the base document published. Not its references.
-    version TEXT NOT NULL,
-    -- Path this publication is published to. If NULL is not listed.
-    path TEXT UNIQUE
-);
-
 -- Stores all the sites served locally. Sites are a a group + domain.
 -- for sites at the beginning, keep in mind that any regular node can be upgraded to a site.
 CREATE TABLE served_sites (

backend/daemon/storage/testdata/mintter-test-db-snapshot/VERSION

@@ -0,0 +1 @@
+2023-08-30.01

backend/daemon/storage/testdata/mintter-test-db-snapshot/keys/libp2p_id_ed25519

@@ -0,0 +1 @@
+
@�����Z�(\u�-w@^�����8HHchT�1RA>�;^�àB��g8��&�����i���+ڑE2�

backend/daemon/storage/testdata/mintter-test-db-snapshot/keys/mintter_id_ed25519.pub

@@ -0,0 +1 @@
+
 ��Z����&�R6﹛�3A��0��"�`