refactor: reference schema field (#1077)

* ref field migration * fix lint * add missing translations * add some comments * add new error * fix migration * fix(server): ja translations (#1078) * fix ja translations * fix ja translation * migration test * fix: graphql * fix gql resolver * fix: reference item searching * fix migration --------- Co-authored-by: caichi <[email protected]> Co-authored-by: Kazuma Tsuchiya <[email protected]>
reearth · Mar 6, 2024 · 7f6d5b3 · 7f6d5b3
1 parent 5b5d662
commit 7f6d5b3
Show file tree

Hide file tree

Showing 41 changed files with 839 additions and 569 deletions.
diff --git a/server/cmd/db-migrations/common_test.go b/server/cmd/db-migrations/common_test.go
@@ -0,0 +1,7 @@
+package main
+
+import "github.com/reearth/reearthx/mongox/mongotest"
+
+func init() {
+	mongotest.Env = "REEARTH_CMS_DB"
+}
diff --git a/server/cmd/db-migrations/main.go b/server/cmd/db-migrations/main.go
@@ -0,0 +1,57 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"os"
+
+	"github.com/joho/godotenv"
+)
+
+type command = func(ctx context.Context, dbURL, dbName string, wetRun bool) error
+
+var commands = map[string]command{
+	"ref-field-schema": RefFieldSchema,
+}
+
+func main() {
+	wet := flag.Bool("wet-run", false, "wet run (default: dry-run)")
+	cmd := flag.String("cmd", "", "migration to be executed name")
+	flag.Parse()
+
+	if *cmd == "" {
+		fmt.Print("command is not set")
+		return
+	}
+
+	command := commands[*cmd]
+	if command == nil {
+		fmt.Printf("command '%s' not found", *cmd)
+		return
+	}
+
+	// load .env
+	if err := godotenv.Load(".env"); err != nil && !os.IsNotExist(err) {
+		fmt.Printf("load .env failed: %s\n", err)
+		return
+	} else if err == nil {
+		fmt.Printf("config: .env loaded\n")
+	}
+
+	// get db url
+	dbURL := os.Getenv("REEARTH_CMS_DB")
+	if dbURL == "" {
+		fmt.Print("REEARTH_CMS_DB is not set")
+		return
+	}
+
+	// exec command
+	fmt.Printf("command: '%s' ", *cmd)
+	ctx := context.Background()
+	if err := command(ctx, dbURL, "reearth_cms", *wet); err != nil {
+		fmt.Printf("faild: %s.\n", err)
+		return
+	}
+	fmt.Printf("succeeded.\n")
+}
diff --git a/server/cmd/db-migrations/ref_field_schema.go b/server/cmd/db-migrations/ref_field_schema.go
@@ -0,0 +1,178 @@
+package main
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/samber/lo"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+)
+
+type Schema struct {
+	ID     string   `bson:"id"`
+	Fields []*Field `bson:"fields"`
+}
+
+type Field struct {
+	ID           string        `bson:"id"`
+	TypeProperty *TypeProperty `bson:"typeproperty"`
+}
+
+type TypeProperty struct {
+	Type      string     `bson:"type"`
+	Reference *Reference `bson:"reference"`
+}
+
+type Reference struct {
+	Model               string  `bson:"model"`
+	Schema              string  `bson:"schema"`
+	CorrespondingSchema *string `bson:"correspondingschema"`
+}
+
+func (r *Reference) SetSchema(s string) {
+	r.Schema = s
+}
+
+type Model struct {
+	ID     string `bson:"id"`
+	Schema string `bson:"schema"`
+}
+
+func RefFieldSchema(ctx context.Context, dbURL, dbName string, wetRun bool) error {
+	testID := ""
+
+	client, err := mongo.Connect(ctx, options.Client().ApplyURI(dbURL))
+	if err != nil {
+		return fmt.Errorf("db: failed to init client err: %w", err)
+	}
+	sCol := client.Database(dbName).Collection("schema")
+	mCol := client.Database(dbName).Collection("model")
+
+	schemas, err := loadSchemas(ctx, sCol)
+	if err != nil {
+		return err
+	}
+
+	if len(schemas) == 0 {
+		return fmt.Errorf("no docs found")
+	}
+
+	mIds := lo.FlatMap(schemas, func(s Schema, _ int) []string {
+		return lo.FilterMap(s.Fields, func(f *Field, _ int) (string, bool) {
+			if f.TypeProperty.Type != "reference" {
+				return "", false
+			}
+			return f.TypeProperty.Reference.Model, true
+		})
+	})
+	models, err := loadModels(ctx, mIds, mCol)
+	if err != nil {
+		return err
+	}
+
+	fmt.Printf("%d docs found, first id: %s\n", len(schemas), schemas[0].ID)
+
+	if testID != "" {
+		schemas = lo.Filter(schemas, func(s Schema, _ int) bool {
+			return s.ID == testID
+		})
+		fmt.Printf("test mode: filter on '%s', now %d docs selcted\n", testID, len(schemas))
+	}
+
+	lo.ForEach(schemas, func(s Schema, _ int) {
+		lo.ForEach(s.Fields, func(f *Field, _ int) {
+			if f.TypeProperty.Type != "reference" {
+				return
+			}
+			m, ok := models[f.TypeProperty.Reference.Model]
+			if ok {
+				f.TypeProperty.Reference.SetSchema(m.Schema)
+				return
+			}
+			if f.TypeProperty.Reference.CorrespondingSchema != nil {
+				f.TypeProperty.Reference.SetSchema(*f.TypeProperty.Reference.CorrespondingSchema)
+				return
+			}
+			fmt.Printf("no model found for schema '%s' model id '%s'\n", s.ID, f.TypeProperty.Reference.Model)
+		})
+	})
+
+	// update all documents in col
+	writes := lo.FlatMap(schemas, func(s Schema, _ int) []mongo.WriteModel {
+		return lo.FilterMap(s.Fields, func(f *Field, _ int) (mongo.WriteModel, bool) {
+			if f.TypeProperty.Type != "reference" {
+				return nil, false
+			}
+			fmt.Printf("updating schema '%s' field '%s' referenced schema '%s'\n", s.ID, f.ID, f.TypeProperty.Reference.Schema)
+			return mongo.NewUpdateOneModel().
+				SetFilter(bson.M{
+					"id":        s.ID,
+					"fields.id": f.ID,
+				}).
+				SetUpdate(bson.M{
+					"$set": bson.M{
+						"fields.$[f].typeproperty.reference.schema": f.TypeProperty.Reference.Schema,
+					},
+				}).
+				SetArrayFilters(options.ArrayFilters{
+					Filters: []interface{}{bson.M{"f.id": f.ID}},
+				}), true
+		})
+	})
+
+	if !wetRun {
+		fmt.Printf("dry run\n")
+		fmt.Printf("%d docs will be updated\n", len(writes))
+		return nil
+	}
+
+	fmt.Printf("writing docs...")
+	res, err := sCol.BulkWrite(ctx, writes)
+	if err != nil {
+		return fmt.Errorf("failed to bulk write: %w", err)
+	}
+
+	fmt.Printf("%d docs updated\n", res.ModifiedCount)
+	return nil
+}
+
+func loadSchemas(ctx context.Context, col *mongo.Collection) ([]Schema, error) {
+	cur, err := col.Find(
+		ctx,
+		bson.M{"fields.typeproperty.type": "reference"},
+		options.Find().SetProjection(bson.M{"id": 1, "fields": 1}),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to find schemas docs: %w", err)
+	}
+
+	var schemas []Schema
+	err = cur.All(ctx, &schemas)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode schemas docs: %w", err)
+	}
+	return schemas, nil
+}
+
+func loadModels(ctx context.Context, sIDs []string, col *mongo.Collection) (map[string]Model, error) {
+	cur, err := col.Find(
+		ctx,
+		bson.M{"id": bson.M{"$in": sIDs}},
+		options.Find().SetProjection(bson.M{"id": 1, "schema": 1}),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to find models docs: %w", err)
+	}
+
+	var models []Model
+	err = cur.All(ctx, &models)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode models docs: %w", err)
+	}
+	return lo.SliceToMap(models, func(m Model) (string, Model) {
+		return m.ID, m
+
+	}), nil
+}
diff --git a/server/cmd/db-migrations/ref_field_schema_test.go b/server/cmd/db-migrations/ref_field_schema_test.go
@@ -0,0 +1,153 @@
+package main
+
+import (
+	"context"
+	"os"
+	"testing"
+
+	"github.com/reearth/reearth-cms/server/internal/infrastructure/mongo/mongodoc"
+	"github.com/reearth/reearth-cms/server/pkg/id"
+	"github.com/reearth/reearthx/log"
+	"github.com/reearth/reearthx/mongox/mongotest"
+	"github.com/stretchr/testify/assert"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+func TestRefFieldSchema(t *testing.T) {
+	s1ID, s2ID, s3ID := id.NewSchemaID().String(), id.NewSchemaID().String(), id.NewSchemaID().String()
+	m1ID, m2ID, m3ID := id.NewModelID().String(), id.NewModelID().String(), id.NewModelID().String()
+	s1 := map[string]any{
+		"id": s1ID,
+		"fields": []map[string]any{
+			{
+				"id": "1",
+				"typeproperty": map[string]any{
+					"type": "reference",
+					"reference": map[string]any{
+						"model":               m2ID,
+						"correspondingschema": s2ID,
+					},
+				},
+			},
+			{
+				"id": "2",
+				"typeproperty": map[string]any{
+					"type": "reference",
+					"reference": map[string]any{
+						"model":               m3ID,
+						"correspondingschema": s3ID,
+						"correspondingfield":  "3",
+					},
+				},
+			},
+		},
+	}
+	m1 := map[string]any{
+		"id":     m1ID,
+		"schema": s1ID,
+	}
+	s2 := map[string]any{
+		"id":     s2ID,
+		"fields": nil,
+	}
+	m2 := map[string]any{
+		"id":     m2ID,
+		"schema": s2ID,
+	}
+	s3 := map[string]any{
+		"id": s3ID,
+		"fields": []map[string]any{
+			{
+				"id": "3",
+				"typeproperty": map[string]any{
+					"type": "reference",
+					"reference": map[string]any{
+						"model":               m1ID,
+						"correspondingschema": s1ID,
+						"correspondingfield":  "2",
+					},
+				},
+			},
+		},
+	}
+	m3 := map[string]any{
+		"id":     m3ID,
+		"schema": s3ID,
+	}
+
+	db := mongotest.Connect(t)(t)
+	log.Infof("test: new db created with name: %v", db.Name())
+
+	ctx := context.Background()
+	sCol := db.Collection("schema")
+	mCol := db.Collection("model")
+
+	_, err := sCol.InsertMany(ctx, []any{s1, s2, s3})
+	assert.NoError(t, err)
+
+	_, err = mCol.InsertMany(ctx, []any{m1, m2, m3})
+	assert.NoError(t, err)
+
+	err = RefFieldSchema(ctx, os.Getenv("REEARTH_CMS_DB"), db.Name(), true)
+	assert.NoError(t, err)
+
+	s1Updated := map[string]any{}
+	err = sCol.FindOne(ctx, bson.M{"id": s1ID}).Decode(&s1Updated)
+	assert.NoError(t, err)
+	assert.Equal(t, map[string]any{
+		"_id": s1Updated["_id"],
+		"id":  s1ID,
+		"fields": primitive.A{
+			map[string]any{
+				"id": "1",
+				"typeproperty": map[string]any{
+					"type": "reference",
+					"reference": map[string]any{
+						"model":               m2ID,
+						"schema":              s2ID,
+						"correspondingschema": s2ID,
+					},
+				},
+			},
+			map[string]any{
+				"id": "2",
+				"typeproperty": map[string]any{
+					"type": "reference",
+					"reference": map[string]any{
+						"model":               m3ID,
+						"schema":              s3ID,
+						"correspondingschema": s3ID,
+						"correspondingfield":  "3",
+					},
+				},
+			},
+		},
+	}, s1Updated)
+
+	s2Updated := mongodoc.SchemaDocument{}
+	err = sCol.FindOne(ctx, bson.M{"id": s2ID}).Decode(&s2Updated)
+	assert.NoError(t, err)
+
+	s3Updated := map[string]any{}
+	err = sCol.FindOne(ctx, bson.M{"id": s3ID}).Decode(&s3Updated)
+	assert.NoError(t, err)
+	assert.Equal(t, map[string]any{
+		"_id": s3Updated["_id"],
+		"id":  s3ID,
+		"fields": primitive.A{
+			map[string]any{
+				"id": "3",
+				"typeproperty": map[string]any{
+					"type": "reference",
+					"reference": map[string]any{
+						"model":               m1ID,
+						"schema":              s1ID,
+						"correspondingschema": s1ID,
+						"correspondingfield":  "2",
+					},
+				},
+			},
+		},
+	}, s3Updated)
+}