Merge branch 'main' into feat/support-multiple-tenants

.github/workflows/ci_server.yml

@@ -35,7 +35,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v5
         with:
-          go-version: 1.21
+          go-version: 1.22
       - name: checkout
         uses: actions/checkout@v4
       - name: cache

.github/workflows/ci_worker.yml

@@ -34,7 +34,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v5
         with:
-          go-version: 1.21
+          go-version: 1.22
       - name: checkout
         uses: actions/checkout@v4
       - name: cache

.github/workflows/cron_ion_token_test.yml

@@ -0,0 +1,43 @@
+name: Update Cesium Ion access token (test)
+on:
+  schedule:
+    - cron: "0 0 2 * *"
+  workflow_dispatch:
+env:
+  GCS_DOMAIN: gs://cms.test.reearth.dev
+  REEARTH_CONFIG_FILENAME: reearth_config.json
+jobs:
+  update_ion_token:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: google-github-actions/auth@v2
+        with:
+          credentials_json: ${{ secrets.GCP_SA_KEY }}
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@v2
+      - name: Download reearth config
+        run: gsutil cp "${{ env.GCS_DOMAIN }}/${{ env.REEARTH_CONFIG_FILENAME }}" .
+      - name: Get Cesium Ion token
+        id: ion_token
+        run: |
+          ION_TOKEN=$( \
+          curl -s https://raw.githubusercontent.com/CesiumGS/cesium/main/packages/engine/Source/Core/Ion.js | \
+          node -e " \
+            process.stdin.resume(); \
+            process.stdin.setEncoding('utf8'); \
+            let data = ''; \
+            process.stdin.on('data', chunk => { data += chunk; }); \
+            process.stdin.on('end', () => { \
+              const defaultAccessTokenRegex = /const defaultAccessToken =(\n| ).*\"(.*)\";/; \
+              const match = data.match(defaultAccessTokenRegex); \
+              console.log(match ? match[2] : 'Token not found'); \
+            }); \
+          " \
+          )
+          echo "token=${ION_TOKEN}" >> $GITHUB_OUTPUT
+      - name: Update Ion token in reearth config
+        run: |
+          echo $(cat ${{ env.REEARTH_CONFIG_FILENAME }} | jq -r '.cesiumIonAccessToken |= "${{ steps.ion_token.outputs.token }}"') > ${{ env.REEARTH_CONFIG_FILENAME }}
+          echo $(cat ${{ env.REEARTH_CONFIG_FILENAME }})
+      - name: Upload reearth config
+        run: gsutil -h "Cache-Control:no-store" cp reearth_config.json "${{ env.GCS_DOMAIN }}/${{ env.REEARTH_CONFIG_FILENAME }}"

go.work

@@ -1,4 +1,4 @@
-go 1.21
+go 1.22.1
 
 use (
 	./server

server/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.21-alpine AS build
+FROM golang:1.22.1-alpine AS build
 ARG TAG=release
 ARG REV
 ARG VERSION

server/cmd/db-migrations/common_test.go

@@ -0,0 +1,7 @@
+package main
+
+import "github.com/reearth/reearthx/mongox/mongotest"
+
+func init() {
+	mongotest.Env = "REEARTH_CMS_DB"
+}

server/cmd/db-migrations/main.go

@@ -0,0 +1,57 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"os"
+
+	"github.com/joho/godotenv"
+)
+
+type command = func(ctx context.Context, dbURL, dbName string, wetRun bool) error
+
+var commands = map[string]command{
+	"ref-field-schema": RefFieldSchema,
+}
+
+func main() {
+	wet := flag.Bool("wet-run", false, "wet run (default: dry-run)")
+	cmd := flag.String("cmd", "", "migration to be executed name")
+	flag.Parse()
+
+	if *cmd == "" {
+		fmt.Print("command is not set")
+		return
+	}
+
+	command := commands[*cmd]
+	if command == nil {
+		fmt.Printf("command '%s' not found", *cmd)
+		return
+	}
+
+	// load .env
+	if err := godotenv.Load(".env"); err != nil && !os.IsNotExist(err) {
+		fmt.Printf("load .env failed: %s\n", err)
+		return
+	} else if err == nil {
+		fmt.Printf("config: .env loaded\n")
+	}
+
+	// get db url
+	dbURL := os.Getenv("REEARTH_CMS_DB")
+	if dbURL == "" {
+		fmt.Print("REEARTH_CMS_DB is not set")
+		return
+	}
+
+	// exec command
+	fmt.Printf("command: '%s' ", *cmd)
+	ctx := context.Background()
+	if err := command(ctx, dbURL, "reearth_cms", *wet); err != nil {
+		fmt.Printf("faild: %s.\n", err)
+		return
+	}
+	fmt.Printf("succeeded.\n")
+}

server/cmd/db-migrations/ref_field_schema.go

@@ -0,0 +1,178 @@
+package main
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/samber/lo"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+)
+
+type Schema struct {
+	ID     string   `bson:"id"`
+	Fields []*Field `bson:"fields"`
+}
+
+type Field struct {
+	ID           string        `bson:"id"`
+	TypeProperty *TypeProperty `bson:"typeproperty"`
+}
+
+type TypeProperty struct {
+	Type      string     `bson:"type"`
+	Reference *Reference `bson:"reference"`
+}
+
+type Reference struct {
+	Model               string  `bson:"model"`
+	Schema              string  `bson:"schema"`
+	CorrespondingSchema *string `bson:"correspondingschema"`
+}
+
+func (r *Reference) SetSchema(s string) {
+	r.Schema = s
+}
+
+type Model struct {
+	ID     string `bson:"id"`
+	Schema string `bson:"schema"`
+}
+
+func RefFieldSchema(ctx context.Context, dbURL, dbName string, wetRun bool) error {
+	testID := ""
+
+	client, err := mongo.Connect(ctx, options.Client().ApplyURI(dbURL))
+	if err != nil {
+		return fmt.Errorf("db: failed to init client err: %w", err)
+	}
+	sCol := client.Database(dbName).Collection("schema")
+	mCol := client.Database(dbName).Collection("model")
+
+	schemas, err := loadSchemas(ctx, sCol)
+	if err != nil {
+		return err
+	}
+
+	if len(schemas) == 0 {
+		return fmt.Errorf("no docs found")
+	}
+
+	mIds := lo.FlatMap(schemas, func(s Schema, _ int) []string {
+		return lo.FilterMap(s.Fields, func(f *Field, _ int) (string, bool) {
+			if f.TypeProperty.Type != "reference" {
+				return "", false
+			}
+			return f.TypeProperty.Reference.Model, true
+		})
+	})
+	models, err := loadModels(ctx, mIds, mCol)
+	if err != nil {
+		return err
+	}
+
+	fmt.Printf("%d docs found, first id: %s\n", len(schemas), schemas[0].ID)
+
+	if testID != "" {
+		schemas = lo.Filter(schemas, func(s Schema, _ int) bool {
+			return s.ID == testID
+		})
+		fmt.Printf("test mode: filter on '%s', now %d docs selcted\n", testID, len(schemas))
+	}
+
+	lo.ForEach(schemas, func(s Schema, _ int) {
+		lo.ForEach(s.Fields, func(f *Field, _ int) {
+			if f.TypeProperty.Type != "reference" {
+				return
+			}
+			m, ok := models[f.TypeProperty.Reference.Model]
+			if ok {
+				f.TypeProperty.Reference.SetSchema(m.Schema)
+				return
+			}
+			if f.TypeProperty.Reference.CorrespondingSchema != nil {
+				f.TypeProperty.Reference.SetSchema(*f.TypeProperty.Reference.CorrespondingSchema)
+				return
+			}
+			fmt.Printf("no model found for schema '%s' model id '%s'\n", s.ID, f.TypeProperty.Reference.Model)
+		})
+	})
+
+	// update all documents in col
+	writes := lo.FlatMap(schemas, func(s Schema, _ int) []mongo.WriteModel {
+		return lo.FilterMap(s.Fields, func(f *Field, _ int) (mongo.WriteModel, bool) {
+			if f.TypeProperty.Type != "reference" {
+				return nil, false
+			}
+			fmt.Printf("updating schema '%s' field '%s' referenced schema '%s'\n", s.ID, f.ID, f.TypeProperty.Reference.Schema)
+			return mongo.NewUpdateOneModel().
+				SetFilter(bson.M{
+					"id":        s.ID,
+					"fields.id": f.ID,
+				}).
+				SetUpdate(bson.M{
+					"$set": bson.M{
+						"fields.$[f].typeproperty.reference.schema": f.TypeProperty.Reference.Schema,
+					},
+				}).
+				SetArrayFilters(options.ArrayFilters{
+					Filters: []interface{}{bson.M{"f.id": f.ID}},
+				}), true
+		})
+	})
+
+	if !wetRun {
+		fmt.Printf("dry run\n")
+		fmt.Printf("%d docs will be updated\n", len(writes))
+		return nil
+	}
+
+	fmt.Printf("writing docs...")
+	res, err := sCol.BulkWrite(ctx, writes)
+	if err != nil {
+		return fmt.Errorf("failed to bulk write: %w", err)
+	}
+
+	fmt.Printf("%d docs updated\n", res.ModifiedCount)
+	return nil
+}
+
+func loadSchemas(ctx context.Context, col *mongo.Collection) ([]Schema, error) {
+	cur, err := col.Find(
+		ctx,
+		bson.M{"fields.typeproperty.type": "reference"},
+		options.Find().SetProjection(bson.M{"id": 1, "fields": 1}),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to find schemas docs: %w", err)
+	}
+
+	var schemas []Schema
+	err = cur.All(ctx, &schemas)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode schemas docs: %w", err)
+	}
+	return schemas, nil
+}
+
+func loadModels(ctx context.Context, sIDs []string, col *mongo.Collection) (map[string]Model, error) {
+	cur, err := col.Find(
+		ctx,
+		bson.M{"id": bson.M{"$in": sIDs}},
+		options.Find().SetProjection(bson.M{"id": 1, "schema": 1}),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to find models docs: %w", err)
+	}
+
+	var models []Model
+	err = cur.All(ctx, &models)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode models docs: %w", err)
+	}
+	return lo.SliceToMap(models, func(m Model) (string, Model) {
+		return m.ID, m
+
+	}), nil
+}