-
Notifications
You must be signed in to change notification settings - Fork 21
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* feat: init jq * fix: jq test * test: jq * docs: jq * refactor: ctx
- Loading branch information
Showing
6 changed files
with
250 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,89 @@ | ||
| package process | ||
|
|
||
| import ( | ||
| "context" | ||
| gojson "encoding/json" | ||
| "fmt" | ||
|
|
||
| "github.com/brexhq/substation/config" | ||
| "github.com/brexhq/substation/internal/errors" | ||
| "github.com/itchyny/gojq" | ||
| ) | ||
|
|
||
| // errJqNoOutputGenerated is returned when the jq query generates no output. | ||
| const errJqNoOutputGenerated = errors.Error("no output generated") | ||
|
|
||
| // jq processes data by applying jq queries. | ||
| // | ||
| // This processor supports the data handling pattern. | ||
| type procJQ struct { | ||
| process | ||
| Options procJQOptions `json:"options"` | ||
| } | ||
|
|
||
| type procJQOptions struct { | ||
| // Query is the jq query applied to data. | ||
| Query string `json:"query"` | ||
| } | ||
|
|
||
| // String returns the processor settings as an object. | ||
| func (p procJQ) String() string { | ||
| return toString(p) | ||
| } | ||
|
|
||
| // Closes resources opened by the processor. | ||
| func (p procJQ) Close(context.Context) error { | ||
| return nil | ||
| } | ||
|
|
||
| // Batch processes one or more capsules with the processor. Conditions are | ||
| // optionally applied to the data to enable processing. | ||
| func (p procJQ) Batch(ctx context.Context, capsules ...config.Capsule) ([]config.Capsule, error) { | ||
| return batchApply(ctx, capsules, p, p.Condition) | ||
| } | ||
|
|
||
| // Apply processes encapsulated data with the processor. | ||
| func (p procJQ) Apply(ctx context.Context, capsule config.Capsule) (config.Capsule, error) { | ||
| query, err := gojq.Parse(p.Options.Query) | ||
| if err != nil { | ||
| return capsule, err | ||
| } | ||
|
|
||
| var i interface{} | ||
| if err := gojson.Unmarshal(capsule.Data(), &i); err != nil { | ||
| return capsule, fmt.Errorf("process: jq: %v", err) | ||
| } | ||
|
|
||
| var arr []interface{} | ||
| iter := query.RunWithContext(ctx, i) | ||
|
|
||
| for { | ||
| v, ok := iter.Next() | ||
| if !ok { | ||
| break | ||
| } | ||
| if err, ok := v.(error); ok { | ||
| return capsule, fmt.Errorf("process: jq: %v", err) | ||
| } | ||
|
|
||
| arr = append(arr, v) | ||
| } | ||
|
|
||
| var b []byte | ||
| switch len(arr) { | ||
| case 0: | ||
| err = errJqNoOutputGenerated | ||
| case 1: | ||
| b, err = gojson.Marshal(arr[0]) | ||
| capsule.SetData(b) | ||
| default: | ||
| b, err = gojson.Marshal(arr) | ||
| capsule.SetData(b) | ||
| } | ||
|
|
||
| if err != nil { | ||
| return capsule, fmt.Errorf("process: jq: %v", err) | ||
| } | ||
|
|
||
| return capsule, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,134 @@ | ||
| package process | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "context" | ||
| "testing" | ||
|
|
||
| "github.com/brexhq/substation/config" | ||
| ) | ||
|
|
||
| var jsonTests = []struct { | ||
| name string | ||
| proc procJQ | ||
| test []byte | ||
| expected []byte | ||
| err error | ||
| }{ | ||
| { | ||
| "access", | ||
| procJQ{ | ||
| process: process{}, | ||
| Options: procJQOptions{ | ||
| Query: `.a`, | ||
| }, | ||
| }, | ||
| []byte(`{"a":"b"}`), | ||
| []byte(`"b"`), | ||
| nil, | ||
| }, | ||
| { | ||
| "access", | ||
| procJQ{ | ||
| process: process{}, | ||
| Options: procJQOptions{ | ||
| Query: `.a, .c`, | ||
| }, | ||
| }, | ||
| []byte(`{"a":"b","c":"d"}`), | ||
| []byte(`["b","d"]`), | ||
| nil, | ||
| }, | ||
| { | ||
| "access", | ||
| procJQ{ | ||
| process: process{}, | ||
| Options: procJQOptions{ | ||
| Query: `.a`, | ||
| }, | ||
| }, | ||
| []byte(`{"a":{"b":"c"}}`), | ||
| []byte(`{"b":"c"}`), | ||
| nil, | ||
| }, | ||
| { | ||
| "array", | ||
| procJQ{ | ||
| process: process{}, | ||
| Options: procJQOptions{ | ||
| Query: `.a`, | ||
| }, | ||
| }, | ||
| []byte(`{"a":["b","c","d"]}`), | ||
| []byte(`["b","c","d"]`), | ||
| nil, | ||
| }, | ||
| { | ||
| "slice", | ||
| procJQ{ | ||
| process: process{}, | ||
| Options: procJQOptions{ | ||
| Query: `.a[-1:]`, | ||
| }, | ||
| }, | ||
| []byte(`{"a":["b","c","d","e","f","g"]}`), | ||
| []byte(`["g"]`), | ||
| nil, | ||
| }, | ||
| { | ||
| "recursion", | ||
| procJQ{ | ||
| process: process{}, | ||
| Options: procJQOptions{ | ||
| Query: `walk( if type == "object" then | ||
| with_entries( select( | ||
| (.value != "") and | ||
| (.value != {}) and | ||
| (.value != null) | ||
| ) ) | ||
| else | ||
| . end)`, | ||
| }, | ||
| }, | ||
| []byte(`{"a":{"b":{"c":""}},"d":null,"e":"f"}`), | ||
| []byte(`{"e":"f"}`), | ||
| nil, | ||
| }, | ||
| } | ||
|
|
||
| func TestJq(t *testing.T) { | ||
| ctx := context.TODO() | ||
| capsule := config.NewCapsule() | ||
|
|
||
| for _, test := range jsonTests { | ||
| capsule.SetData(test.test) | ||
|
|
||
| result, err := test.proc.Apply(ctx, capsule) | ||
| if err != nil { | ||
| t.Error(err) | ||
| } | ||
|
|
||
| if !bytes.Equal(result.Data(), test.expected) { | ||
| t.Errorf("expected %s, got %s", test.expected, result.Data()) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| func benchmarkJq(b *testing.B, applier procJQ, test config.Capsule) { | ||
| ctx := context.TODO() | ||
| for i := 0; i < b.N; i++ { | ||
| _, _ = applier.Apply(ctx, test) | ||
| } | ||
| } | ||
|
|
||
| func BenchmarkJq(b *testing.B) { | ||
| capsule := config.NewCapsule() | ||
| for _, test := range jsonTests { | ||
| b.Run(test.name, | ||
| func(b *testing.B) { | ||
| capsule.SetData(test.test) | ||
| benchmarkJq(b, test.proc, capsule) | ||
| }, | ||
| ) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters