Skip to content

Commit

Permalink
feat: add properties to JSON output (#278)
Browse files Browse the repository at this point in the history
  • Loading branch information
@jacobmarble
jacobmarble authored Aug 1, 2023
1 parent c7992e2 commit 3a17528
Show file tree
Hide file tree
Showing 2 changed files with 177 additions and 91 deletions.
260 changes: 169 additions & 91 deletions schema.go
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
package avro

import (
"bytes"
"crypto/md5"
"crypto/sha256"
"errors"
"fmt"
"hash"
"sort"
"strconv"
"strings"
"sync"
Expand Down Expand Up @@ -314,6 +316,23 @@ func (p properties) Prop(name string) any {
return p.props[name]
}

func (p properties) marshalPropertiesToJSON(buf *bytes.Buffer) error {
sortedPropertyKeys := make([]string, 0, len(p.props))
for k := range p.props {
sortedPropertyKeys = append(sortedPropertyKeys, k)
}
sort.Strings(sortedPropertyKeys)
for _, k := range sortedPropertyKeys {
vv, err := jsoniter.Marshal(p.props[k])
if err != nil {
return err
}
buf.WriteString(`,"` + k + `":`)
buf.Write(vv)
}
return nil
}

type schemaConfig struct {
aliases []string
doc string
Expand Down Expand Up @@ -404,7 +423,26 @@ func (s *PrimitiveSchema) String() string {

// MarshalJSON marshals the schema to json.
func (s *PrimitiveSchema) MarshalJSON() ([]byte, error) {
return []byte(s.String()), nil
if s.logical == nil && len(s.props) == 0 {
return jsoniter.Marshal(s.typ)
}

buf := new(bytes.Buffer)
buf.WriteString(`{"type":"` + string(s.typ) + `"`)
if s.logical != nil {
buf.WriteString(`,"logicalType":"` + string(s.logical.Type()) + `"`)
if d, ok := s.logical.(*DecimalLogicalSchema); ok {
buf.WriteString(`,"precision":` + strconv.Itoa(d.prec))
if d.scale > 0 {
buf.WriteString(`,"scale":` + strconv.Itoa(d.scale))
}
}
}
if err := s.marshalPropertiesToJSON(buf); err != nil {
return nil, err
}
buf.WriteString("}")
return buf.Bytes(), nil
}

// Fingerprint returns the SHA256 fingerprint of the schema.
Expand Down Expand Up @@ -500,26 +538,35 @@ func (s *RecordSchema) String() string {

// MarshalJSON marshals the schema to json.
func (s *RecordSchema) MarshalJSON() ([]byte, error) {
typ := "record"
buf := new(bytes.Buffer)
buf.WriteString(`{"name":"` + s.full + `"`)
if len(s.aliases) > 0 {
aliasesJSON, err := jsoniter.Marshal(s.aliases)
if err != nil {
return nil, err
}
buf.WriteString(`,"aliases":`)
buf.Write(aliasesJSON)
}
if s.doc != "" {
buf.WriteString(`,"doc":"` + s.doc + `"`)
}
if s.isError {
typ = "error"
buf.WriteString(`,"type":"error"`)
} else {
buf.WriteString(`,"type":"record"`)
}

ss := struct {
Name string `json:"name"`
Aliases []string `json:"aliases,omitempty"`
Doc string `json:"doc,omitempty"`
Type string `json:"type"`
Fields []*Field `json:"fields"`
}{
Name: s.full,
Aliases: s.aliases,
Doc: s.doc,
Type: typ,
Fields: s.fields,
fieldsJSON, err := jsoniter.Marshal(s.fields)
if err != nil {
return nil, err
}

return jsoniter.Marshal(ss)
buf.WriteString(`,"fields":`)
buf.Write(fieldsJSON)
if err := s.marshalPropertiesToJSON(buf); err != nil {
return nil, err
}
buf.WriteString("}")
return buf.Bytes(), nil
}

// Fingerprint returns the SHA256 fingerprint of the schema.
Expand Down Expand Up @@ -643,36 +690,41 @@ func (f *Field) String() string {

// MarshalJSON marshals the schema to json.
func (f *Field) MarshalJSON() ([]byte, error) {
type base struct {
Name string `json:"name"`
Aliases []string `json:"aliases,omitempty"`
Doc string `json:"doc,omitempty"`
Type Schema `json:"type"`
Order string `json:"order,omitempty"`
}
type ext struct {
base
Default any `json:"default"`
buf := new(bytes.Buffer)
buf.WriteString(`{"name":"` + f.name + `"`)
if len(f.aliases) > 0 {
aliasesJSON, err := jsoniter.Marshal(f.aliases)
if err != nil {
return nil, err
}
buf.WriteString(`,"aliases":`)
buf.Write(aliasesJSON)
}

b := base{
Name: f.name,
Aliases: f.aliases,
Doc: f.doc,
Type: f.typ,
if f.doc != "" {
buf.WriteString(`,"doc":"` + f.doc + `"`)
}
if f.order != Asc {
b.Order = string(f.order)
typeJSON, err := jsoniter.Marshal(f.typ)
if err != nil {
return nil, err
}

var s any = b
buf.WriteString(`,"type":`)
buf.Write(typeJSON)
if f.hasDef {
s = ext{
base: s.(base),
Default: f.Default(),
defaultValueJSON, err := jsoniter.Marshal(f.Default())
if err != nil {
return nil, err
}
buf.WriteString(`,"default":`)
buf.Write(defaultValueJSON)
}
if f.order != "" && f.order != Asc {
buf.WriteString(`,"order":"` + string(f.order) + `"`)
}
return jsoniter.Marshal(s)
if err := f.marshalPropertiesToJSON(buf); err != nil {
return nil, err
}
buf.WriteString("}")
return buf.Bytes(), nil
}

// EnumSchema is an Avro enum type schema.
Expand Down Expand Up @@ -769,22 +821,34 @@ func (s *EnumSchema) String() string {

// MarshalJSON marshals the schema to json.
func (s *EnumSchema) MarshalJSON() ([]byte, error) {
ss := struct {
Name string `json:"name"`
Aliases []string `json:"aliases,omitempty"`
Doc string `json:"doc,omitempty"`
Type string `json:"type"`
Symbols []string `json:"symbols"`
Default string `json:"default,omitempty"`
}{
Name: s.full,
Aliases: s.aliases,
Doc: s.doc,
Type: "enum",
Symbols: s.symbols,
Default: s.def,
}
return jsoniter.Marshal(ss)
buf := new(bytes.Buffer)
buf.WriteString(`{"name":"` + s.full + `"`)
if len(s.aliases) > 0 {
aliasesJSON, err := jsoniter.Marshal(s.aliases)
if err != nil {
return nil, err
}
buf.WriteString(`,"aliases":`)
buf.Write(aliasesJSON)
}
if s.doc != "" {
buf.WriteString(`,"doc":"` + s.doc + `"`)
}
buf.WriteString(`,"type":"enum"`)
symbolsJSON, err := jsoniter.Marshal(s.symbols)
if err != nil {
return nil, err
}
buf.WriteString(`,"symbols":`)
buf.Write(symbolsJSON)
if s.def != "" {
buf.WriteString(`,"default":"` + s.def + `"`)
}
if err := s.marshalPropertiesToJSON(buf); err != nil {
return nil, err
}
buf.WriteString("}")
return buf.Bytes(), nil
}

// Fingerprint returns the SHA256 fingerprint of the schema.
Expand Down Expand Up @@ -835,14 +899,19 @@ func (s *ArraySchema) String() string {

// MarshalJSON marshals the schema to json.
func (s *ArraySchema) MarshalJSON() ([]byte, error) {
ss := struct {
Type string `json:"type"`
Items Schema `json:"items"`
}{
Type: "array",
Items: s.items,
buf := new(bytes.Buffer)
buf.WriteString(`{"type":"array"`)
itemsJSON, err := jsoniter.Marshal(s.items)
if err != nil {
return nil, err
}
buf.WriteString(`,"items":`)
buf.Write(itemsJSON)
if err = s.marshalPropertiesToJSON(buf); err != nil {
return nil, err
}
return jsoniter.Marshal(ss)
buf.WriteString("}")
return buf.Bytes(), nil
}

// Fingerprint returns the SHA256 fingerprint of the schema.
Expand Down Expand Up @@ -893,14 +962,19 @@ func (s *MapSchema) String() string {

// MarshalJSON marshals the schema to json.
func (s *MapSchema) MarshalJSON() ([]byte, error) {
ss := struct {
Type string `json:"type"`
Values Schema `json:"values"`
}{
Type: "map",
Values: s.values,
buf := new(bytes.Buffer)
buf.WriteString(`{"type":"map"`)
valuesJSON, err := jsoniter.Marshal(s.values)
if err != nil {
return nil, err
}
buf.WriteString(`,"values":`)
buf.Write(valuesJSON)
if err := s.marshalPropertiesToJSON(buf); err != nil {
return nil, err
}
return jsoniter.Marshal(ss)
buf.WriteString("}")
return buf.Bytes(), nil
}

// Fingerprint returns the SHA256 fingerprint of the schema.
Expand Down Expand Up @@ -1065,28 +1139,32 @@ func (s *FixedSchema) String() string {

// MarshalJSON marshals the schema to json.
func (s *FixedSchema) MarshalJSON() ([]byte, error) {
ss := struct {
Name string `json:"name"`
Aliases []string `json:"aliases,omitempty"`
Type string `json:"type"`
Size int `json:"size"`
}{
Name: s.full,
Aliases: s.aliases,
Type: "fixed",
Size: s.size,
}
json, err := jsoniter.MarshalToString(ss)
if err != nil {
return nil, err
buf := new(bytes.Buffer)
buf.WriteString(`{"name":"` + s.full + `"`)
if len(s.aliases) > 0 {
aliasesJSON, err := jsoniter.Marshal(s.aliases)
if err != nil {
return nil, err
}
buf.WriteString(`,"aliases":`)
buf.Write(aliasesJSON)
}

var logical string
buf.WriteString(`,"type":"fixed"`)
buf.WriteString(`,"size":` + strconv.Itoa(s.size))
if s.logical != nil {
logical = "," + s.logical.String()
buf.WriteString(`,"logicalType":"` + string(s.logical.Type()) + `"`)
if d, ok := s.logical.(*DecimalLogicalSchema); ok {
buf.WriteString(`,"precision":` + strconv.Itoa(d.prec))
if d.scale > 0 {
buf.WriteString(`,"scale":` + strconv.Itoa(d.scale))
}
}
}

return []byte(json[:len(json)-1] + logical + "}"), nil
if err := s.marshalPropertiesToJSON(buf); err != nil {
return nil, err
}
buf.WriteString("}")
return buf.Bytes(), nil
}

// Fingerprint returns the SHA256 fingerprint of the schema.
Expand Down
8 changes: 8 additions & 0 deletions schema_json_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,10 @@ func TestSchema_JSON(t *testing.T) {
input: `{"type":"long"}`,
json: `"long"`,
},
{
input: `{"property-b": "value-bar", "type":"long", "property-a": "value-foo"}`,
json: `{"type":"long","property-a":"value-foo","property-b":"value-bar"}`,
},
{
input: `{"type":"long","logicalType":"time-micros"}`,
json: `{"type":"long","logicalType":"time-micros"}`,
Expand Down Expand Up @@ -155,6 +159,10 @@ func TestSchema_JSON(t *testing.T) {
input: `{"fields":[], "type":"record", "name":"foo", "doc":"foo", "aliases":["foo","bar"]}`,
json: `{"name":"foo","aliases":["foo","bar"],"doc":"foo","type":"record","fields":[]}`,
},
{
input: `{"fields":[], "property-foo": "value-bar", "type":"record", "name":"foo", "doc":"foo", "aliases":["foo","bar"]}`,
json: `{"name":"foo","aliases":["foo","bar"],"doc":"foo","type":"record","fields":[],"property-foo":"value-bar"}`,
},
{
input: `{"fields":[{"type":{"type":"boolean"}, "name":"f1"}], "type":"record", "name":"foo"}`,
json: `{"name":"foo","type":"record","fields":[{"name":"f1","type":"boolean"}]}`,
Expand Down

0 comments on commit 3a17528

Please sign in to comment.