fix(server): add schema field ID to dataset REST API (#547)

reearth · Jul 2, 2023 · 2410112 · 2410112
1 parent 765c29e
commit 2410112
Show file tree

Hide file tree

Showing 24 changed files with 110 additions and 61 deletions.
diff --git a/server/e2e/dataset_export_test.go b/server/e2e/dataset_export_test.go
@@ -1,13 +1,11 @@
 package e2e
 
 import (
-	"encoding/json"
 	"net/http"
 	"testing"
 
 	"github.com/reearth/reearth/server/internal/app/config"
 	"github.com/reearth/reearth/server/pkg/dataset"
-	"github.com/samber/lo"
 )
 
 func TestDatasetExport(t *testing.T) {
@@ -53,26 +51,33 @@ func TestDatasetExport(t *testing.T) {
 		Status(http.StatusOK).
 		ContentType("application/json")
 	res.Header("Content-Disposition").Equal("attachment;filename=test.csv.json")
-	res.Body().Equal(string(lo.Must(json.Marshal([]map[string]any{
-		{
+
+	res.JSON().Equal(map[string]any{
+		"schema": map[string]any{
 			"$schema": "http://json-schema.org/draft-07/schema#",
+			"$id":     "#/schemas/" + dssID.String(),
 			"title":   "test.csv",
 			"type":    "object",
 			"properties": map[string]any{
 				"": map[string]any{
 					"title": "ID",
+					"$id":   "#/properties/id",
 					"type":  "string",
 				},
 				"f1": map[string]any{
+					"$id":  "#/properties/" + dsfID1.String(),
 					"type": "string",
 				},
 				"f2": map[string]any{
+					"$id":  "#/properties/" + dsfID2.String(),
 					"type": "number",
 				},
 				"f3": map[string]any{
+					"$id":  "#/properties/" + dsfID3.String(),
 					"type": "boolean",
 				},
 				"location": map[string]any{
+					"$id":   "#/properties/" + dsfID4.String(),
 					"type":  "object",
 					"title": "LatLng",
 					"required": []string{
@@ -90,15 +95,17 @@ func TestDatasetExport(t *testing.T) {
 				},
 			},
 		},
-		{
-			"":   dsID.String(),
-			"f1": "test",
-			"f2": 123,
-			"f3": true,
-			"location": map[string]any{
-				"lat": 11.0,
-				"lng": 12.0,
+		"datasets": []map[string]any{
+			{
+				"":   dsID.String(),
+				"f1": "test",
+				"f2": 123,
+				"f3": true,
+				"location": map[string]any{
+					"lat": 11.0,
+					"lng": 12.0,
+				},
 			},
 		},
-	}))) + "\n")
+	})
 }
diff --git a/server/e2e/seeder.go b/server/e2e/seeder.go
@@ -23,9 +23,13 @@ var (
 	pID    = id.NewProjectID()
 	pAlias = "PROJECT_ALIAS"
 
-	sID   = id.NewSceneID()
-	dssID = id.NewDatasetSchemaID()
-	dsID  = id.NewDatasetID()
+	sID    = id.NewSceneID()
+	dssID  = id.NewDatasetSchemaID()
+	dsID   = id.NewDatasetID()
+	dsfID1 = dataset.NewFieldID()
+	dsfID2 = dataset.NewFieldID()
+	dsfID3 = dataset.NewFieldID()
+	dsfID4 = dataset.NewFieldID()
 
 	now = time.Date(2022, time.January, 1, 0, 0, 0, 0, time.UTC)
 )
@@ -63,15 +67,14 @@ func baseSeeder(ctx context.Context, r *repo.Container) error {
 		return err
 	}
 
-	fId1, fId2, fId3, fId4 := dataset.NewFieldID(), dataset.NewFieldID(), dataset.NewFieldID(), dataset.NewFieldID()
 	dss := dataset.NewSchema().ID(dssID).
 		Name("test.csv").
 		Scene(sID).
 		Fields([]*dataset.SchemaField{
-			dataset.NewSchemaField().ID(fId1).Name("f1").Type(dataset.ValueTypeString).MustBuild(),
-			dataset.NewSchemaField().ID(fId2).Name("f2").Type(dataset.ValueTypeNumber).MustBuild(),
-			dataset.NewSchemaField().ID(fId3).Name("f3").Type(dataset.ValueTypeBool).MustBuild(),
-			dataset.NewSchemaField().ID(fId4).Name("location").Type(dataset.ValueTypeLatLng).MustBuild(),
+			dataset.NewSchemaField().ID(dsfID1).Name("f1").Type(dataset.ValueTypeString).MustBuild(),
+			dataset.NewSchemaField().ID(dsfID2).Name("f2").Type(dataset.ValueTypeNumber).MustBuild(),
+			dataset.NewSchemaField().ID(dsfID3).Name("f3").Type(dataset.ValueTypeBool).MustBuild(),
+			dataset.NewSchemaField().ID(dsfID4).Name("location").Type(dataset.ValueTypeLatLng).MustBuild(),
 		}).
 		Source("file:///dss.csv").
 		MustBuild()
@@ -81,10 +84,10 @@ func baseSeeder(ctx context.Context, r *repo.Container) error {
 
 	ds := dataset.New().ID(dsID).Schema(dss.ID()).Scene(sID).
 		Fields([]*dataset.Field{
-			dataset.NewField(fId1, dataset.ValueTypeString.ValueFrom("test"), ""),
-			dataset.NewField(fId2, dataset.ValueTypeNumber.ValueFrom(123), ""),
-			dataset.NewField(fId3, dataset.ValueTypeBool.ValueFrom(true), ""),
-			dataset.NewField(fId4, dataset.ValueTypeLatLng.ValueFrom(dataset.LatLng{Lat: 11, Lng: 12}), ""),
+			dataset.NewField(dsfID1, dataset.ValueTypeString.ValueFrom("test"), ""),
+			dataset.NewField(dsfID2, dataset.ValueTypeNumber.ValueFrom(123), ""),
+			dataset.NewField(dsfID3, dataset.ValueTypeBool.ValueFrom(true), ""),
+			dataset.NewField(dsfID4, dataset.ValueTypeLatLng.ValueFrom(dataset.LatLng{Lat: 11, Lng: 12}), ""),
 		}).
 		MustBuild()
 	if err := r.Dataset.Save(ctx, ds); err != nil {

diff --git a/server/pkg/dataset/dataset.go b/server/pkg/dataset/dataset.go
@@ -109,6 +109,12 @@ func (d *Dataset) Interface(s *Schema, idkey string) map[string]interface{} {
 	m[idkey] = d.ID().String()
 	for _, f := range d.fields {
 		key := s.Field(f.Field()).Name()
+		if key == "" {
+			key = f.Field().String()
+		}
+		if key == "" {
+			continue
+		}
 		m[key] = f.Value().Interface()
 	}
 	return m

diff --git a/server/pkg/dataset/dataset_test.go b/server/pkg/dataset/dataset_test.go
@@ -9,6 +9,7 @@ import (
 func TestDataset_Interface(t *testing.T) {
 	f1 := NewFieldID()
 	f2 := NewFieldID()
+	f3 := NewFieldID()
 	sid := NewSchemaID()
 	did := NewID()
 
@@ -24,16 +25,19 @@ func TestDataset_Interface(t *testing.T) {
 			schema: NewSchema().ID(sid).Scene(NewSceneID()).Fields([]*SchemaField{
 				NewSchemaField().ID(f1).Name("foo").Type(ValueTypeNumber).MustBuild(),
 				NewSchemaField().ID(f2).Name("bar").Type(ValueTypeLatLng).MustBuild(),
+				NewSchemaField().ID(f3).Name("").Type(ValueTypeString).MustBuild(),
 			}).MustBuild(),
 			dataset: New().ID(did).Scene(NewSceneID()).Schema(sid).Fields([]*Field{
 				NewField(f1, ValueTypeNumber.ValueFrom(1), ""),
 				NewField(f2, ValueTypeLatLng.ValueFrom(LatLng{Lat: 1, Lng: 2}), ""),
+				NewField(f3, ValueTypeString.ValueFrom("aaa"), ""),
 			}).MustBuild(),
 			idkey: "",
 			want: map[string]interface{}{
-				"":    did.String(),
-				"foo": float64(1),
-				"bar": LatLng{Lat: 1, Lng: 2},
+				"":          did.String(),
+				"foo":       float64(1),
+				"bar":       LatLng{Lat: 1, Lng: 2},
+				f3.String(): "aaa",
 			},
 		},
 		{

diff --git a/server/pkg/dataset/export.go b/server/pkg/dataset/export.go
@@ -82,7 +82,7 @@ func ExportCSV(w io.Writer, ds *Schema, printSchema bool, loader func(func(*Data
 }
 
 func ExportJSON(w io.Writer, ds *Schema, printSchema bool, loader func(func(*Dataset) error) error) error {
-	if _, err := w.Write([]byte("[")); err != nil {
+	if _, err := w.Write([]byte(`{"schema":`)); err != nil {
 		return err
 	}
 
@@ -95,7 +95,7 @@ func ExportJSON(w io.Writer, ds *Schema, printSchema bool, loader func(func(*Dat
 		if _, err := w.Write(b); err != nil {
 			return err
 		}
-		if _, err := w.Write([]byte(",")); err != nil {
+		if _, err := w.Write([]byte(`,"datasets":[`)); err != nil {
 			return err
 		}
 	}
@@ -129,7 +129,7 @@ func ExportJSON(w io.Writer, ds *Schema, printSchema bool, loader func(func(*Dat
 		}
 	}
 
-	if _, err := w.Write([]byte("]\n")); err != nil {
+	if _, err := w.Write([]byte("]}\n")); err != nil {
 		return err
 	}
 	return nil

diff --git a/server/pkg/dataset/export_test.go b/server/pkg/dataset/export_test.go
@@ -52,26 +52,34 @@ func TestExportJSON(t *testing.T) {
 	})
 	assert.NoError(t, err)
 
-	assert.Equal(t, string(lo.Must(json.Marshal([]map[string]any{
-		{
+	var res map[string]any
+	lo.Must0(json.Unmarshal(buf.Bytes(), &res))
+
+	assert.Equal(t, map[string]any{
+		"schema": map[string]any{
 			"$schema": "http://json-schema.org/draft-07/schema#",
+			"$id":     "#/schemas/" + ds.ID().String(),
 			"title":   "aaa",
 			"type":    "object",
 			"properties": map[string]any{
 				"": map[string]any{
+					"$id":   "#/properties/id",
 					"title": "ID",
 					"type":  "string",
 				},
 				"a": map[string]any{
+					"$id":  "#/properties/" + ds.Fields()[0].ID().String(),
 					"type": "number",
 				},
 				"b": map[string]any{
+					"$id":  "#/properties/" + ds.Fields()[1].ID().String(),
 					"type": "string",
 				},
 				"c": map[string]any{
+					"$id":   "#/properties/" + ds.Fields()[2].ID().String(),
 					"type":  "object",
 					"title": "LatLng",
-					"required": []string{
+					"required": []any{
 						"lat",
 						"lng",
 					},
@@ -86,14 +94,16 @@ func TestExportJSON(t *testing.T) {
 				},
 			},
 		},
-		{
-			"":  d.ID().String(),
-			"a": 1,
-			"b": "2",
-			"c": map[string]any{
-				"lat": 1.0,
-				"lng": 2.0,
+		"datasets": []any{
+			map[string]any{
+				"":  d.ID().String(),
+				"a": float64(1),
+				"b": "2",
+				"c": map[string]any{
+					"lat": 1.0,
+					"lng": 2.0,
+				},
 			},
 		},
-	})))+"\n", buf.String())
+	}, res)
 }
diff --git a/server/pkg/dataset/schema.go b/server/pkg/dataset/schema.go
@@ -1,5 +1,7 @@
 package dataset
 
+import "fmt"
+
 type Schema struct {
 	id                  SchemaID
 	source              string
@@ -122,14 +124,23 @@ func (d *Schema) JSONSchema() map[string]any {
 		"": map[string]any{
 			"title": "ID",
 			"type":  "string",
+			"$id":   "#/properties/id",
 		},
 	}
 	for _, f := range d.fields {
-		properties[f.Name()] = f.JSONSchema()
+		name := f.Name()
+		if name == "" {
+			name = f.ID().String()
+		}
+		if name == "" {
+			continue
+		}
+		properties[name] = f.JSONSchema()
 	}
 
 	m := map[string]any{
 		"$schema":    "http://json-schema.org/draft-07/schema#",
+		"$id":        fmt.Sprintf("#/schemas/%s", d.ID()),
 		"title":      d.name,
 		"type":       "object",
 		"properties": properties,

diff --git a/server/pkg/dataset/schema_field.go b/server/pkg/dataset/schema_field.go
@@ -1,5 +1,7 @@
 package dataset
 
+import "fmt"
+
 type SchemaField struct {
 	id       FieldID
 	name     string
@@ -64,10 +66,12 @@ func (d *SchemaField) Clone() *SchemaField {
 }
 
 // JSONSchema prints a JSON schema for the schema field.
-func (d *SchemaField) JSONSchema() any {
+func (d *SchemaField) JSONSchema() map[string]any {
 	if d == nil {
 		return nil
 	}
 
-	return d.dataType.JSONSchema()
+	s := d.dataType.JSONSchema()
+	s["$id"] = fmt.Sprintf("#/properties/%s", d.ID())
+	return s
 }
diff --git a/server/pkg/dataset/schema_test.go b/server/pkg/dataset/schema_test.go
@@ -14,17 +14,21 @@ func TestSchema_JSONSchema(t *testing.T) {
 
 	want := map[string]any{
 		"$schema": "http://json-schema.org/draft-07/schema#",
+		"$id":     "#/schemas/" + ds.ID().String(),
 		"title":   ds.Name(),
 		"type":    "object",
 		"properties": map[string]any{
 			"": map[string]any{
+				"$id":   "#/properties/id",
 				"title": "ID",
 				"type":  "string",
 			},
 			"foo": map[string]any{
+				"$id":  "#/properties/" + ds.Fields()[0].ID().String(),
 				"type": "number",
 			},
 			"bar": map[string]any{
+				"$id":   "#/properties/" + ds.Fields()[1].ID().String(),
 				"type":  "object",
 				"title": "LatLng",
 				"required": []string{

diff --git a/server/pkg/dataset/value.go b/server/pkg/dataset/value.go
@@ -55,7 +55,7 @@ func (vt ValueType) None() *OptionalValue {
 	return NewOptionalValue(vt, nil)
 }
 
-func (vt ValueType) JSONSchema() any {
+func (vt ValueType) JSONSchema() map[string]any {
 	return value.Type(vt).JSONSchema(nil)
 }
 

diff --git a/server/pkg/property/value_camera.go b/server/pkg/property/value_camera.go
@@ -70,7 +70,7 @@ func (p *typePropertyCamera) String(i interface{}) string {
 	// return i.(Camera).String()
 }
 
-func (v *typePropertyCamera) JSONSchema() any {
+func (v *typePropertyCamera) JSONSchema() map[string]any {
 	return map[string]any{
 		"type":  "object",
 		"title": "Camera",

diff --git a/server/pkg/property/value_typography.go b/server/pkg/property/value_typography.go
@@ -133,7 +133,7 @@ func (p *typePropertyTypography) String(i interface{}) string {
 	// return i.(Typography).String()
 }
 
-func (v *typePropertyTypography) JSONSchema() any {
+func (v *typePropertyTypography) JSONSchema() map[string]any {
 	return map[string]any{
 		"type":  "object",
 		"title": "Typography",

diff --git a/server/pkg/value/bool.go b/server/pkg/value/bool.go
@@ -42,7 +42,7 @@ func (p *propertyBool) String(i any) string {
 	return strconv.FormatBool(i.(bool))
 }
 
-func (v *propertyBool) JSONSchema() any {
+func (v *propertyBool) JSONSchema() map[string]any {
 	return map[string]any{
 		"type": "boolean",
 	}

diff --git a/server/pkg/value/coordinates.go b/server/pkg/value/coordinates.go
@@ -87,7 +87,7 @@ func (p *propertyCoordinates) String(i any) string {
 	return i.(Coordinates).String()
 }
 
-func (v *propertyCoordinates) JSONSchema() any {
+func (v *propertyCoordinates) JSONSchema() map[string]any {
 	return map[string]any{
 		"type":  "array",
 		"title": "Coordinates",

diff --git a/server/pkg/value/latlng.go b/server/pkg/value/latlng.go
@@ -68,7 +68,7 @@ func (p *propertyLatLng) String(i any) string {
 	return i.(LatLng).String()
 }
 
-func (v *propertyLatLng) JSONSchema() any {
+func (v *propertyLatLng) JSONSchema() map[string]any {
 	return map[string]any{
 		"type":  "object",
 		"title": "LatLng",