diff --git a/server/e2e/dataset_export_test.go b/server/e2e/dataset_export_test.go index 8fcb3b6968..4a8fe14519 100644 --- a/server/e2e/dataset_export_test.go +++ b/server/e2e/dataset_export_test.go @@ -1,13 +1,11 @@ package e2e import ( - "encoding/json" "net/http" "testing" "github.com/reearth/reearth/server/internal/app/config" "github.com/reearth/reearth/server/pkg/dataset" - "github.com/samber/lo" ) func TestDatasetExport(t *testing.T) { @@ -53,26 +51,33 @@ func TestDatasetExport(t *testing.T) { Status(http.StatusOK). ContentType("application/json") res.Header("Content-Disposition").Equal("attachment;filename=test.csv.json") - res.Body().Equal(string(lo.Must(json.Marshal([]map[string]any{ - { + + res.JSON().Equal(map[string]any{ + "schema": map[string]any{ "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "#/schemas/" + dssID.String(), "title": "test.csv", "type": "object", "properties": map[string]any{ "": map[string]any{ "title": "ID", + "$id": "#/properties/id", "type": "string", }, "f1": map[string]any{ + "$id": "#/properties/" + dsfID1.String(), "type": "string", }, "f2": map[string]any{ + "$id": "#/properties/" + dsfID2.String(), "type": "number", }, "f3": map[string]any{ + "$id": "#/properties/" + dsfID3.String(), "type": "boolean", }, "location": map[string]any{ + "$id": "#/properties/" + dsfID4.String(), "type": "object", "title": "LatLng", "required": []string{ @@ -90,15 +95,17 @@ func TestDatasetExport(t *testing.T) { }, }, }, - { - "": dsID.String(), - "f1": "test", - "f2": 123, - "f3": true, - "location": map[string]any{ - "lat": 11.0, - "lng": 12.0, + "datasets": []map[string]any{ + { + "": dsID.String(), + "f1": "test", + "f2": 123, + "f3": true, + "location": map[string]any{ + "lat": 11.0, + "lng": 12.0, + }, }, }, - }))) + "\n") + }) } diff --git a/server/e2e/seeder.go b/server/e2e/seeder.go index 138b32a474..3051fb933e 100644 --- a/server/e2e/seeder.go +++ b/server/e2e/seeder.go @@ -23,9 +23,13 @@ var ( pID = id.NewProjectID() pAlias = "PROJECT_ALIAS" - sID = id.NewSceneID() - dssID = id.NewDatasetSchemaID() - dsID = id.NewDatasetID() + sID = id.NewSceneID() + dssID = id.NewDatasetSchemaID() + dsID = id.NewDatasetID() + dsfID1 = dataset.NewFieldID() + dsfID2 = dataset.NewFieldID() + dsfID3 = dataset.NewFieldID() + dsfID4 = dataset.NewFieldID() now = time.Date(2022, time.January, 1, 0, 0, 0, 0, time.UTC) ) @@ -63,15 +67,14 @@ func baseSeeder(ctx context.Context, r *repo.Container) error { return err } - fId1, fId2, fId3, fId4 := dataset.NewFieldID(), dataset.NewFieldID(), dataset.NewFieldID(), dataset.NewFieldID() dss := dataset.NewSchema().ID(dssID). Name("test.csv"). Scene(sID). Fields([]*dataset.SchemaField{ - dataset.NewSchemaField().ID(fId1).Name("f1").Type(dataset.ValueTypeString).MustBuild(), - dataset.NewSchemaField().ID(fId2).Name("f2").Type(dataset.ValueTypeNumber).MustBuild(), - dataset.NewSchemaField().ID(fId3).Name("f3").Type(dataset.ValueTypeBool).MustBuild(), - dataset.NewSchemaField().ID(fId4).Name("location").Type(dataset.ValueTypeLatLng).MustBuild(), + dataset.NewSchemaField().ID(dsfID1).Name("f1").Type(dataset.ValueTypeString).MustBuild(), + dataset.NewSchemaField().ID(dsfID2).Name("f2").Type(dataset.ValueTypeNumber).MustBuild(), + dataset.NewSchemaField().ID(dsfID3).Name("f3").Type(dataset.ValueTypeBool).MustBuild(), + dataset.NewSchemaField().ID(dsfID4).Name("location").Type(dataset.ValueTypeLatLng).MustBuild(), }). Source("file:///dss.csv"). MustBuild() @@ -81,10 +84,10 @@ func baseSeeder(ctx context.Context, r *repo.Container) error { ds := dataset.New().ID(dsID).Schema(dss.ID()).Scene(sID). Fields([]*dataset.Field{ - dataset.NewField(fId1, dataset.ValueTypeString.ValueFrom("test"), ""), - dataset.NewField(fId2, dataset.ValueTypeNumber.ValueFrom(123), ""), - dataset.NewField(fId3, dataset.ValueTypeBool.ValueFrom(true), ""), - dataset.NewField(fId4, dataset.ValueTypeLatLng.ValueFrom(dataset.LatLng{Lat: 11, Lng: 12}), ""), + dataset.NewField(dsfID1, dataset.ValueTypeString.ValueFrom("test"), ""), + dataset.NewField(dsfID2, dataset.ValueTypeNumber.ValueFrom(123), ""), + dataset.NewField(dsfID3, dataset.ValueTypeBool.ValueFrom(true), ""), + dataset.NewField(dsfID4, dataset.ValueTypeLatLng.ValueFrom(dataset.LatLng{Lat: 11, Lng: 12}), ""), }). MustBuild() if err := r.Dataset.Save(ctx, ds); err != nil { diff --git a/server/pkg/dataset/dataset.go b/server/pkg/dataset/dataset.go index 3963980acb..5202c74073 100644 --- a/server/pkg/dataset/dataset.go +++ b/server/pkg/dataset/dataset.go @@ -109,6 +109,12 @@ func (d *Dataset) Interface(s *Schema, idkey string) map[string]interface{} { m[idkey] = d.ID().String() for _, f := range d.fields { key := s.Field(f.Field()).Name() + if key == "" { + key = f.Field().String() + } + if key == "" { + continue + } m[key] = f.Value().Interface() } return m diff --git a/server/pkg/dataset/dataset_test.go b/server/pkg/dataset/dataset_test.go index f2c434228c..e5c026d5c6 100644 --- a/server/pkg/dataset/dataset_test.go +++ b/server/pkg/dataset/dataset_test.go @@ -9,6 +9,7 @@ import ( func TestDataset_Interface(t *testing.T) { f1 := NewFieldID() f2 := NewFieldID() + f3 := NewFieldID() sid := NewSchemaID() did := NewID() @@ -24,16 +25,19 @@ func TestDataset_Interface(t *testing.T) { schema: NewSchema().ID(sid).Scene(NewSceneID()).Fields([]*SchemaField{ NewSchemaField().ID(f1).Name("foo").Type(ValueTypeNumber).MustBuild(), NewSchemaField().ID(f2).Name("bar").Type(ValueTypeLatLng).MustBuild(), + NewSchemaField().ID(f3).Name("").Type(ValueTypeString).MustBuild(), }).MustBuild(), dataset: New().ID(did).Scene(NewSceneID()).Schema(sid).Fields([]*Field{ NewField(f1, ValueTypeNumber.ValueFrom(1), ""), NewField(f2, ValueTypeLatLng.ValueFrom(LatLng{Lat: 1, Lng: 2}), ""), + NewField(f3, ValueTypeString.ValueFrom("aaa"), ""), }).MustBuild(), idkey: "", want: map[string]interface{}{ - "": did.String(), - "foo": float64(1), - "bar": LatLng{Lat: 1, Lng: 2}, + "": did.String(), + "foo": float64(1), + "bar": LatLng{Lat: 1, Lng: 2}, + f3.String(): "aaa", }, }, { diff --git a/server/pkg/dataset/export.go b/server/pkg/dataset/export.go index 19c254b71e..c296b79ab8 100644 --- a/server/pkg/dataset/export.go +++ b/server/pkg/dataset/export.go @@ -82,7 +82,7 @@ func ExportCSV(w io.Writer, ds *Schema, printSchema bool, loader func(func(*Data } func ExportJSON(w io.Writer, ds *Schema, printSchema bool, loader func(func(*Dataset) error) error) error { - if _, err := w.Write([]byte("[")); err != nil { + if _, err := w.Write([]byte(`{"schema":`)); err != nil { return err } @@ -95,7 +95,7 @@ func ExportJSON(w io.Writer, ds *Schema, printSchema bool, loader func(func(*Dat if _, err := w.Write(b); err != nil { return err } - if _, err := w.Write([]byte(",")); err != nil { + if _, err := w.Write([]byte(`,"datasets":[`)); err != nil { return err } } @@ -129,7 +129,7 @@ func ExportJSON(w io.Writer, ds *Schema, printSchema bool, loader func(func(*Dat } } - if _, err := w.Write([]byte("]\n")); err != nil { + if _, err := w.Write([]byte("]}\n")); err != nil { return err } return nil diff --git a/server/pkg/dataset/export_test.go b/server/pkg/dataset/export_test.go index 81d686ea05..554345464c 100644 --- a/server/pkg/dataset/export_test.go +++ b/server/pkg/dataset/export_test.go @@ -52,26 +52,34 @@ func TestExportJSON(t *testing.T) { }) assert.NoError(t, err) - assert.Equal(t, string(lo.Must(json.Marshal([]map[string]any{ - { + var res map[string]any + lo.Must0(json.Unmarshal(buf.Bytes(), &res)) + + assert.Equal(t, map[string]any{ + "schema": map[string]any{ "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "#/schemas/" + ds.ID().String(), "title": "aaa", "type": "object", "properties": map[string]any{ "": map[string]any{ + "$id": "#/properties/id", "title": "ID", "type": "string", }, "a": map[string]any{ + "$id": "#/properties/" + ds.Fields()[0].ID().String(), "type": "number", }, "b": map[string]any{ + "$id": "#/properties/" + ds.Fields()[1].ID().String(), "type": "string", }, "c": map[string]any{ + "$id": "#/properties/" + ds.Fields()[2].ID().String(), "type": "object", "title": "LatLng", - "required": []string{ + "required": []any{ "lat", "lng", }, @@ -86,14 +94,16 @@ func TestExportJSON(t *testing.T) { }, }, }, - { - "": d.ID().String(), - "a": 1, - "b": "2", - "c": map[string]any{ - "lat": 1.0, - "lng": 2.0, + "datasets": []any{ + map[string]any{ + "": d.ID().String(), + "a": float64(1), + "b": "2", + "c": map[string]any{ + "lat": 1.0, + "lng": 2.0, + }, }, }, - })))+"\n", buf.String()) + }, res) } diff --git a/server/pkg/dataset/schema.go b/server/pkg/dataset/schema.go index 00c2f240e1..a0b7398905 100644 --- a/server/pkg/dataset/schema.go +++ b/server/pkg/dataset/schema.go @@ -1,5 +1,7 @@ package dataset +import "fmt" + type Schema struct { id SchemaID source string @@ -122,14 +124,23 @@ func (d *Schema) JSONSchema() map[string]any { "": map[string]any{ "title": "ID", "type": "string", + "$id": "#/properties/id", }, } for _, f := range d.fields { - properties[f.Name()] = f.JSONSchema() + name := f.Name() + if name == "" { + name = f.ID().String() + } + if name == "" { + continue + } + properties[name] = f.JSONSchema() } m := map[string]any{ "$schema": "http://json-schema.org/draft-07/schema#", + "$id": fmt.Sprintf("#/schemas/%s", d.ID()), "title": d.name, "type": "object", "properties": properties, diff --git a/server/pkg/dataset/schema_field.go b/server/pkg/dataset/schema_field.go index 51a2df4c90..9449bc4d4b 100644 --- a/server/pkg/dataset/schema_field.go +++ b/server/pkg/dataset/schema_field.go @@ -1,5 +1,7 @@ package dataset +import "fmt" + type SchemaField struct { id FieldID name string @@ -64,10 +66,12 @@ func (d *SchemaField) Clone() *SchemaField { } // JSONSchema prints a JSON schema for the schema field. -func (d *SchemaField) JSONSchema() any { +func (d *SchemaField) JSONSchema() map[string]any { if d == nil { return nil } - return d.dataType.JSONSchema() + s := d.dataType.JSONSchema() + s["$id"] = fmt.Sprintf("#/properties/%s", d.ID()) + return s } diff --git a/server/pkg/dataset/schema_test.go b/server/pkg/dataset/schema_test.go index afaef60a90..dcfc5790cf 100644 --- a/server/pkg/dataset/schema_test.go +++ b/server/pkg/dataset/schema_test.go @@ -14,17 +14,21 @@ func TestSchema_JSONSchema(t *testing.T) { want := map[string]any{ "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "#/schemas/" + ds.ID().String(), "title": ds.Name(), "type": "object", "properties": map[string]any{ "": map[string]any{ + "$id": "#/properties/id", "title": "ID", "type": "string", }, "foo": map[string]any{ + "$id": "#/properties/" + ds.Fields()[0].ID().String(), "type": "number", }, "bar": map[string]any{ + "$id": "#/properties/" + ds.Fields()[1].ID().String(), "type": "object", "title": "LatLng", "required": []string{ diff --git a/server/pkg/dataset/value.go b/server/pkg/dataset/value.go index b6c77a4a3f..1795b613fd 100644 --- a/server/pkg/dataset/value.go +++ b/server/pkg/dataset/value.go @@ -55,7 +55,7 @@ func (vt ValueType) None() *OptionalValue { return NewOptionalValue(vt, nil) } -func (vt ValueType) JSONSchema() any { +func (vt ValueType) JSONSchema() map[string]any { return value.Type(vt).JSONSchema(nil) } diff --git a/server/pkg/property/value_camera.go b/server/pkg/property/value_camera.go index 47bcd4545b..cd3f32eee3 100644 --- a/server/pkg/property/value_camera.go +++ b/server/pkg/property/value_camera.go @@ -70,7 +70,7 @@ func (p *typePropertyCamera) String(i interface{}) string { // return i.(Camera).String() } -func (v *typePropertyCamera) JSONSchema() any { +func (v *typePropertyCamera) JSONSchema() map[string]any { return map[string]any{ "type": "object", "title": "Camera", diff --git a/server/pkg/property/value_typography.go b/server/pkg/property/value_typography.go index 00a7f9f1d5..423688148d 100644 --- a/server/pkg/property/value_typography.go +++ b/server/pkg/property/value_typography.go @@ -133,7 +133,7 @@ func (p *typePropertyTypography) String(i interface{}) string { // return i.(Typography).String() } -func (v *typePropertyTypography) JSONSchema() any { +func (v *typePropertyTypography) JSONSchema() map[string]any { return map[string]any{ "type": "object", "title": "Typography", diff --git a/server/pkg/value/bool.go b/server/pkg/value/bool.go index 33e094041b..381accfc1d 100644 --- a/server/pkg/value/bool.go +++ b/server/pkg/value/bool.go @@ -42,7 +42,7 @@ func (p *propertyBool) String(i any) string { return strconv.FormatBool(i.(bool)) } -func (v *propertyBool) JSONSchema() any { +func (v *propertyBool) JSONSchema() map[string]any { return map[string]any{ "type": "boolean", } diff --git a/server/pkg/value/coordinates.go b/server/pkg/value/coordinates.go index 8cf404e04b..a5a853e92c 100644 --- a/server/pkg/value/coordinates.go +++ b/server/pkg/value/coordinates.go @@ -87,7 +87,7 @@ func (p *propertyCoordinates) String(i any) string { return i.(Coordinates).String() } -func (v *propertyCoordinates) JSONSchema() any { +func (v *propertyCoordinates) JSONSchema() map[string]any { return map[string]any{ "type": "array", "title": "Coordinates", diff --git a/server/pkg/value/latlng.go b/server/pkg/value/latlng.go index 37c83a3d6a..ee2d00f9a3 100644 --- a/server/pkg/value/latlng.go +++ b/server/pkg/value/latlng.go @@ -68,7 +68,7 @@ func (p *propertyLatLng) String(i any) string { return i.(LatLng).String() } -func (v *propertyLatLng) JSONSchema() any { +func (v *propertyLatLng) JSONSchema() map[string]any { return map[string]any{ "type": "object", "title": "LatLng", diff --git a/server/pkg/value/latlngheight.go b/server/pkg/value/latlngheight.go index dbabd8a4a0..5ee85b707a 100644 --- a/server/pkg/value/latlngheight.go +++ b/server/pkg/value/latlngheight.go @@ -70,7 +70,7 @@ func (p *propertyLatLngHeight) String(i any) string { return i.(LatLngHeight).String() } -func (v *propertyLatLngHeight) JSONSchema() any { +func (v *propertyLatLngHeight) JSONSchema() map[string]any { return map[string]any{ "type": "object", "title": "LatLngHeight", diff --git a/server/pkg/value/number.go b/server/pkg/value/number.go index 3fc6620aa4..261ca909fa 100644 --- a/server/pkg/value/number.go +++ b/server/pkg/value/number.go @@ -136,7 +136,7 @@ func (p *propertyNumber) String(i any) string { return fmt.Sprintf("%g", i.(float64)) } -func (v *propertyNumber) JSONSchema() any { +func (v *propertyNumber) JSONSchema() map[string]any { return map[string]any{ "type": "number", } diff --git a/server/pkg/value/polygon.go b/server/pkg/value/polygon.go index 774197eafd..c3860fe890 100644 --- a/server/pkg/value/polygon.go +++ b/server/pkg/value/polygon.go @@ -61,7 +61,7 @@ func (p *propertyPolygon) String(i any) string { return i.(Polygon).String() } -func (v *propertyPolygon) JSONSchema() any { +func (v *propertyPolygon) JSONSchema() map[string]any { return map[string]any{ "type": "array", "title": "Polygon", diff --git a/server/pkg/value/rect.go b/server/pkg/value/rect.go index de329105ec..e5efceb8ae 100644 --- a/server/pkg/value/rect.go +++ b/server/pkg/value/rect.go @@ -55,7 +55,7 @@ func (p *propertyRect) String(i any) string { return i.(Rect).String() } -func (v *propertyRect) JSONSchema() any { +func (v *propertyRect) JSONSchema() map[string]any { return map[string]any{ "type": "object", "title": "Rect", diff --git a/server/pkg/value/ref.go b/server/pkg/value/ref.go index f5915417b0..7ce764e6d8 100644 --- a/server/pkg/value/ref.go +++ b/server/pkg/value/ref.go @@ -38,7 +38,7 @@ func (p *propertyRef) String(i any) string { return i.(string) } -func (v *propertyRef) JSONSchema() any { +func (v *propertyRef) JSONSchema() map[string]any { return map[string]any{ "type": "string", "title": "Ref", diff --git a/server/pkg/value/string.go b/server/pkg/value/string.go index 8b8006e1f5..941fc1e887 100644 --- a/server/pkg/value/string.go +++ b/server/pkg/value/string.go @@ -44,7 +44,7 @@ func (p *propertyString) String(i any) string { return i.(string) } -func (v *propertyString) JSONSchema() any { +func (v *propertyString) JSONSchema() map[string]any { return map[string]any{ "type": "string", } diff --git a/server/pkg/value/type.go b/server/pkg/value/type.go index 13697382fc..3b819e5ad1 100644 --- a/server/pkg/value/type.go +++ b/server/pkg/value/type.go @@ -7,7 +7,7 @@ type TypeProperty interface { V2I(interface{}) (interface{}, bool) Validate(interface{}) bool String(any) string - JSONSchema() any + JSONSchema() map[string]any } type TypePropertyMap = map[Type]TypeProperty @@ -58,7 +58,7 @@ func (t Type) ValueFrom(i interface{}, p TypePropertyMap) *Value { return nil } -func (vt Type) JSONSchema(p TypePropertyMap) any { +func (vt Type) JSONSchema(p TypePropertyMap) map[string]any { if vt == TypeUnknown { return nil } diff --git a/server/pkg/value/type_test.go b/server/pkg/value/type_test.go index 5646752c41..f4596d52b9 100644 --- a/server/pkg/value/type_test.go +++ b/server/pkg/value/type_test.go @@ -153,7 +153,7 @@ func TestType_JSONSchema(t *testing.T) { tests := []struct { name string tr Type - want any + want map[string]any }{ { name: "default", diff --git a/server/pkg/value/url.go b/server/pkg/value/url.go index 606bf218ac..4f86822337 100644 --- a/server/pkg/value/url.go +++ b/server/pkg/value/url.go @@ -51,7 +51,7 @@ func (p *propertyURL) String(i any) string { return i.(*url.URL).String() } -func (v *propertyURL) JSONSchema() any { +func (v *propertyURL) JSONSchema() map[string]any { return map[string]any{ "type": "string", "title": "URL",