diff --git a/cmd/data.go b/cmd/data.go index fd8ea73b..6641cdf5 100644 --- a/cmd/data.go +++ b/cmd/data.go @@ -1,13 +1,93 @@ package cmd import ( + "context" + "encoding/json" "fmt" + "os" + "regexp" + "github.com/TylerBrock/colorjson" "github.com/spf13/cobra" + "golang.org/x/exp/slices" + + "github.com/blackstork-io/fabric/parser" + "github.com/blackstork-io/fabric/parser/definitions" + "github.com/blackstork-io/fabric/pkg/diagnostics" + "github.com/blackstork-io/fabric/plugin" ) -func data(target string) { - fmt.Println("data called on", target) +var dataTgtRe = regexp.MustCompile(`(?:document\.([^.]+)\.data\.([^.]+)\.([^.\n]+))|(?:data\.([^.]+)\.([^.]+))`) + +func Data(ctx context.Context, blocks *parser.DefinedBlocks, caller *parser.Caller, target string) (result plugin.MapData, diags diagnostics.Diag) { + // docName, pluginName, blockName + // target: document..data.. + tgt := dataTgtRe.FindStringSubmatch(target) + if tgt == nil { + diags.Add( + "Incorrect target", + "Target should have the format 'document..data..' or 'data..'", + ) + return + } + + var data *definitions.ParsedData + + if tgt[1] != "" { + // document..data.. + doc, found := blocks.Documents[tgt[1]] + if !found { + diags.Add( + "Document not found", + fmt.Sprintf( + "Definition for document named '%s' not found", + tgt[1], + ), + ) + return + } + + pd, diag := blocks.ParseDocument(doc) + if diags.Extend(diag) { + return + } + + idx := slices.IndexFunc(pd.Data, func(data *definitions.ParsedData) bool { + return data.PluginName == tgt[2] && data.BlockName == tgt[3] + }) + if idx == -1 { + diags.Add( + "Data block not found", + fmt.Sprintf("Data block '%s.%s' not found in document '%s'", tgt[2], tgt[3], tgt[1]), + ) + return + } + data = pd.Data[idx] + } else { + // data.. + defPlugin, found := blocks.Plugins[definitions.Key{ + PluginKind: definitions.BlockKindData, + PluginName: tgt[4], + BlockName: tgt[5], + }] + if !found { + diags.Add( + "Data block not found", + fmt.Sprintf("Data block '%s.%s' not found in global scope", tgt[4], tgt[5]), + ) + return + } + res, diag := blocks.ParsePlugin(defPlugin) + if diags.Extend(diag) { + return + } + data = (*definitions.ParsedData)(res) + } + res, diag := caller.CallData(ctx, data.PluginName, data.Config, data.Invocation) + if diags.Extend(diag) { + return + } + return res, diags } // dataCmd represents the data command @@ -15,8 +95,41 @@ var dataCmd = &cobra.Command{ Use: "data TARGET", Short: "Execute a single data block", Long: `Execute the data block and print out prettified JSON to stdout`, - Run: func(_ *cobra.Command, args []string) { - data(args[0]) + RunE: func(cmd *cobra.Command, args []string) (err error) { + var diags diagnostics.Diag + eval := NewEvaluator(cliArgs.pluginsDir) + defer func() { + err = eval.Cleanup(diags) + }() + diags = eval.ParseFabricFiles(os.DirFS(cliArgs.sourceDir)) + if diags.HasErrors() { + return + } + if diags.Extend(eval.LoadRunner()) { + return + } + + res, diag := Data(cmd.Context(), eval.Blocks, eval.PluginCaller(), args[0]) + if diags.Extend(diag) { + return + } + + val := res.Any() + var ser []byte + if cliArgs.colorize { + fmt := colorjson.NewFormatter() + fmt.Indent = 4 + ser, err = fmt.Marshal(val) + } else { + ser, err = json.MarshalIndent(val, "", " ") + } + if diags.AppendErr(err, "Failed to serialize data output to json") { + return + } + _, err = os.Stdout.Write(ser) + + diags.AppendErr(err, "Failed to output json data") + return }, Args: cobra.ExactArgs(1), } diff --git a/cmd/evaluator.go b/cmd/evaluator.go new file mode 100644 index 00000000..f63dd0b7 --- /dev/null +++ b/cmd/evaluator.go @@ -0,0 +1,73 @@ +package cmd + +import ( + "io/fs" + "os" + + "github.com/hashicorp/hcl/v2" + + "github.com/blackstork-io/fabric/internal/builtin" + "github.com/blackstork-io/fabric/parser" + "github.com/blackstork-io/fabric/pkg/diagnostics" + "github.com/blackstork-io/fabric/plugin/runner" +) + +type Evaluator struct { + PluginsDir string + Blocks *parser.DefinedBlocks + Runner *runner.Runner + FileMap map[string]*hcl.File +} + +func NewEvaluator(pluginsDir string) *Evaluator { + return &Evaluator{ + PluginsDir: pluginsDir, + } +} + +func (e *Evaluator) Cleanup(diags diagnostics.Diag) error { + if e.Runner != nil { + diags.ExtendHcl(e.Runner.Close()) + } + diagnostics.PrintDiags(os.Stderr, diags, e.FileMap, cliArgs.colorize) + // Errors have been already displayed + if diags.HasErrors() { + rootCmd.SilenceErrors = true + rootCmd.SilenceUsage = true + return diags + } + return nil +} + +func (e *Evaluator) ParseFabricFiles(sourceDir fs.FS) (diags diagnostics.Diag) { + e.Blocks, e.FileMap, diags = parser.ParseDir(sourceDir) + if diags.HasErrors() { + return + } + if e.PluginsDir == "" && e.Blocks.GlobalConfig != nil && e.Blocks.GlobalConfig.PluginRegistry != nil { + // use pluginsDir from config, unless overridden by cli arg + e.PluginsDir = e.Blocks.GlobalConfig.PluginRegistry.MirrorDir + } + return +} + +func (e *Evaluator) LoadRunner() diagnostics.Diag { + var pluginVersions runner.VersionMap + if e.Blocks.GlobalConfig != nil { + pluginVersions = e.Blocks.GlobalConfig.PluginVersions + } + var stdDiag hcl.Diagnostics + + e.Runner, stdDiag = runner.Load( + runner.WithBuiltIn( + builtin.Plugin(version), + ), + runner.WithPluginDir(e.PluginsDir), + runner.WithPluginVersions(pluginVersions), + ) + return diagnostics.Diag(stdDiag) +} + +func (e *Evaluator) PluginCaller() *parser.Caller { + return parser.NewPluginCaller(e.Runner) +} diff --git a/cmd/render.go b/cmd/render.go index fafea3c7..53cd4f02 100644 --- a/cmd/render.go +++ b/cmd/render.go @@ -2,34 +2,20 @@ package cmd import ( "bufio" + "context" "fmt" "io" - "io/fs" "os" "strings" - "github.com/hashicorp/hcl/v2" "github.com/spf13/cobra" - "github.com/blackstork-io/fabric/internal/builtin" "github.com/blackstork-io/fabric/parser" "github.com/blackstork-io/fabric/parser/definitions" "github.com/blackstork-io/fabric/pkg/diagnostics" - "github.com/blackstork-io/fabric/plugin/runner" ) -func Render(pluginsDir string, sourceDir fs.FS, docName string) (results []string, fileMap map[string]*hcl.File, diags diagnostics.Diag) { - blocks, fileMap, diags := parser.ParseDir(sourceDir) - if diags.HasErrors() { - return - } - if len(fileMap) == 0 { - diags.Add( - "No correct fabric files found", - fmt.Sprintf("There are no *.fabric files at '%s' or all of them have failed to parse", cliArgs.sourceDir), - ) - return - } +func Render(ctx context.Context, blocks *parser.DefinedBlocks, pluginCaller *parser.Caller, docName string) (results []string, diags diagnostics.Diag) { doc, found := blocks.Documents[docName] if !found { diags.Add( @@ -42,33 +28,12 @@ func Render(pluginsDir string, sourceDir fs.FS, docName string) (results []strin return } - if pluginsDir == "" && blocks.GlobalConfig != nil && blocks.GlobalConfig.PluginRegistry != nil { - // use pluginsDir from config, unless overridden by cli arg - pluginsDir = blocks.GlobalConfig.PluginRegistry.MirrorDir - } - - var pluginVersions runner.VersionMap - if blocks.GlobalConfig != nil { - pluginVersions = blocks.GlobalConfig.PluginVersions - } - - runner, stdDiag := runner.Load( - runner.WithBuiltIn( - builtin.Plugin(version), - ), - runner.WithPluginDir(pluginsDir), - runner.WithPluginVersions(pluginVersions), - ) - if diags.ExtendHcl(stdDiag) { + pd, diag := blocks.ParseDocument(doc) + if diags.Extend(diag) { return } - defer func() { diags.ExtendHcl(runner.Close()) }() - eval := parser.NewEvaluator( - parser.NewPluginCaller(runner), - blocks, - ) - results, diag := eval.EvaluateDocument(doc) + results, diag = pd.Render(ctx, pluginCaller) if diags.Extend(diag) { return } @@ -101,7 +66,7 @@ var renderCmd = &cobra.Command{ Short: "Render the document", Long: `Render the specified document into Markdown and output it either to stdout or to a file`, Args: cobra.ExactArgs(1), - RunE: func(_ *cobra.Command, args []string) (err error) { + RunE: func(cmd *cobra.Command, args []string) (err error) { target := strings.TrimSpace(args[0]) const docPrefix = definitions.BlockKindDocument + "." switch { @@ -111,28 +76,36 @@ var renderCmd = &cobra.Command{ return fmt.Errorf("target should have the format '%s'", docPrefix) } - var dest *os.File - if outFile == "" { - dest = os.Stdout - } else { + dest := os.Stdout + if outFile != "" { dest, err = os.Create(outFile) if err != nil { return fmt.Errorf("can't create the out-file: %w", err) } defer dest.Close() } - res, fileMap, diags := Render(cliArgs.pluginsDir, os.DirFS(cliArgs.sourceDir), target) - if !diags.HasErrors() { - diags.Extend(writeResults(dest, res)) - } - diagnostics.PrintDiags(os.Stderr, diags, fileMap, cliArgs.colorize) + + var diags diagnostics.Diag + eval := NewEvaluator(cliArgs.pluginsDir) + defer func() { + err = eval.Cleanup(diags) + }() + diags = eval.ParseFabricFiles(os.DirFS(cliArgs.sourceDir)) if diags.HasErrors() { - // Errors have been already displayed - rootCmd.SilenceErrors = true - rootCmd.SilenceUsage = true - return diags + return } - return nil + diag := eval.LoadRunner() + if diags.Extend(diag) { + return + } + res, diag := Render(cmd.Context(), eval.Blocks, eval.PluginCaller(), target) + if diags.Extend(diag) { + return + } + diags.Extend( + writeResults(dest, res), + ) + return }, } diff --git a/cmd/root.go b/cmd/root.go index c8842a76..8a960720 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -15,6 +15,7 @@ import ( "github.com/spf13/cobra" "golang.org/x/term" + "github.com/blackstork-io/fabric/pkg/clicontext" "github.com/blackstork-io/fabric/pkg/utils" ) @@ -149,7 +150,7 @@ var rootCmd = &cobra.Command{ // Execute adds all child commands to the root command and sets flags appropriately. // This is called by main.main(). It only needs to happen once to the rootCmd. func Execute() { - err := rootCmd.Execute() + err := rootCmd.ExecuteContext(clicontext.New()) if err != nil { os.Exit(1) } diff --git a/go.mod b/go.mod index a5468c3e..21a4528a 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ toolchain go1.21.1 require ( github.com/Masterminds/semver/v3 v3.2.1 + github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2 github.com/elastic/go-elasticsearch/v8 v8.11.1 github.com/golang-cz/devslog v0.0.8 github.com/google/go-github/v58 v58.0.0 @@ -54,7 +55,7 @@ require ( github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/elastic/elastic-transport-go/v8 v8.3.0 // indirect - github.com/fatih/color v1.7.0 // indirect + github.com/fatih/color v1.16.0 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.3 // indirect @@ -62,6 +63,7 @@ require ( github.com/google/go-querystring v1.1.0 // indirect github.com/google/uuid v1.4.0 // indirect github.com/hashicorp/yamux v0.1.1 // indirect + github.com/hokaccha/go-prettyjson v0.0.0-20211117102719-0474bc63780f // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/itchyny/timefmt-go v0.1.5 // indirect github.com/klauspost/compress v1.17.0 // indirect diff --git a/go.sum b/go.sum index caf53b18..32272415 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,8 @@ github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migc github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7fz8= github.com/Microsoft/hcsshim v0.11.4/go.mod h1:smjE4dvqPX9Zldna+t5FG3rnoHhaB7QYxPRqGcpAD9w= +github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2 h1:ZBbLwSJqkHBuFDA6DUhhse0IGJ7T5bemHyNILUjvOq4= +github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2/go.mod h1:VSw57q4QFiWDbRnjdX8Cb3Ow0SFncRw+bA/ofY6Q83w= github.com/agext/levenshtein v1.2.1 h1:QmvMAjj2aEICytGiWzmxoE0x2KZvE0fvmqMOfy2tjT8= github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= github.com/apparentlymart/go-dump v0.0.0-20180507223929-23540a00eaa3 h1:ZSTrOEhiM5J5RFxEaFvMZVEAM1KvT1YzbEOwB2EAGjA= @@ -51,8 +53,9 @@ github.com/elastic/elastic-transport-go/v8 v8.3.0 h1:DJGxovyQLXGr62e9nDMPSxRyWIO github.com/elastic/elastic-transport-go/v8 v8.3.0/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI= github.com/elastic/go-elasticsearch/v8 v8.11.1 h1:1VgTgUTbpqQZ4uE+cPjkOvy/8aw1ZvKcU0ZUE5Cn1mc= github.com/elastic/go-elasticsearch/v8 v8.11.1/go.mod h1:GU1BJHO7WeamP7UhuElYwzzHtvf9SDmeVpSSy9+o6Qg= -github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68= @@ -84,6 +87,8 @@ github.com/hashicorp/hcl/v2 v2.19.1 h1://i05Jqznmb2EXqa39Nsvyan2o5XyMowW5fnCKW5R github.com/hashicorp/hcl/v2 v2.19.1/go.mod h1:ThLC89FV4p9MPW804KVbe/cEXoQ8NZEh+JtMeeGErHE= github.com/hashicorp/yamux v0.1.1 h1:yrQxtgseBDrq9Y652vSRDvsKCJKOUD+GzTS4Y0Y8pvE= github.com/hashicorp/yamux v0.1.1/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ= +github.com/hokaccha/go-prettyjson v0.0.0-20211117102719-0474bc63780f h1:7LYC+Yfkj3CTRcShK0KOL/w6iTiKyqqBA9a41Wnggw8= +github.com/hokaccha/go-prettyjson v0.0.0-20211117102719-0474bc63780f/go.mod h1:pFlLw2CfqZiIBOx6BuCeRLCrfxBJipTY0nIOF/VbGcI= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/itchyny/gojq v0.12.14 h1:6k8vVtsrhQSYgSGg827AD+PVVaB1NLXEdX+dda2oZCc= diff --git a/parser/caller.go b/parser/caller.go index cdc8f300..3647a185 100644 --- a/parser/caller.go +++ b/parser/caller.go @@ -21,10 +21,6 @@ type pluginData struct { ConfigSpec hcldec.Spec InvocationSpec hcldec.Spec } -type PluginCaller interface { - CallContent(name string, config evaluation.Configuration, invocation evaluation.Invocation, context map[string]any) (result string, diag diagnostics.Diag) - CallData(name string, config evaluation.Configuration, invocation evaluation.Invocation) (result map[string]any, diag diagnostics.Diag) -} type Caller struct { plugins *runner.Runner @@ -36,7 +32,7 @@ func NewPluginCaller(r *runner.Runner) *Caller { } } -var _ PluginCaller = (*Caller)(nil) +var _ evaluation.PluginCaller = (*Caller)(nil) func (c *Caller) pluginData(kind, name string) (pluginData, diagnostics.Diag) { switch kind { @@ -69,35 +65,27 @@ func (c *Caller) pluginData(kind, name string) (pluginData, diagnostics.Diag) { } } -func (c *Caller) callPlugin(kind, name string, config evaluation.Configuration, invocation evaluation.Invocation, dataCtx map[string]any) (res any, diags diagnostics.Diag) { +func (c *Caller) callPlugin(ctx context.Context, kind, name string, config evaluation.Configuration, invocation evaluation.Invocation, dataCtx plugin.MapData) (res any, diags diagnostics.Diag) { data, diags := c.pluginData(kind, name) if diags.HasErrors() { return } - dataCtxAny, err := plugin.ParseDataMapAny(dataCtx) - if err != nil { - diags.Add("Error while parsing context", err.Error()) - return - } - acceptsConfig := !utils.IsNil(data.ConfigSpec) hasConfig := config.Exists() var configVal cty.Value if acceptsConfig { - var stdDiag diagnostics.Diag - configVal, stdDiag = config.ParseConfig(data.ConfigSpec) - if !diags.Extend(stdDiag) { + var diag diagnostics.Diag + configVal, diag = config.ParseConfig(data.ConfigSpec) + if !diags.Extend(diag) { typ := hcldec.ImpliedType(data.ConfigSpec) errs := configVal.Type().TestConformance(typ) if errs != nil { // Attempt a conversion var err error configVal, err = convert.Convert(configVal, typ) - if err != nil { - diags.AppendErr(err, "Error while serializing config") - } + diags.AppendErr(err, "Error while serializing config") } } } else if hasConfig { @@ -114,9 +102,6 @@ func (c *Caller) callPlugin(kind, name string, config evaluation.Configuration, pluginArgs, diag := invocation.ParseInvocation(data.InvocationSpec) diags.Extend(diag) - if diag.HasErrors() { - return - } if data.InvocationSpec != nil { typ := hcldec.ImpliedType(data.InvocationSpec) errs := pluginArgs.Type().TestConformance(typ) @@ -124,84 +109,68 @@ func (c *Caller) callPlugin(kind, name string, config evaluation.Configuration, // Attempt a conversion var err error pluginArgs, err = convert.Convert(pluginArgs, typ) - if err != nil { - diag.AppendErr(err, "Error while serializing args") - - return nil, diag - } + diags.AppendErr(err, "Error while serializing args") } } - - var result struct { - Result any - Diags hcl.Diagnostics + if diags.HasErrors() { + return } + switch kind { case "data": - source, diags := c.plugins.DataSource(name) - if diags.HasErrors() { - return nil, diagnostics.Diag(diags) + source, diag := c.plugins.DataSource(name) + if diags.ExtendHcl(diag) { + return } - data, diags := source.Execute(context.Background(), &plugin.RetrieveDataParams{ + data, diag := source.Execute(ctx, &plugin.RetrieveDataParams{ Config: configVal, Args: pluginArgs, }) - if data != nil { - result.Result = data.Any() - } - result.Diags = diags + res = data + diags.ExtendHcl(diag) case "content": - provider, diags := c.plugins.ContentProvider(name) - if diags.HasErrors() { - return nil, diagnostics.Diag(diags) + provider, diag := c.plugins.ContentProvider(name) + if diags.ExtendHcl(diag) { + return } - content, diags := provider.Execute(context.Background(), &plugin.ProvideContentParams{ + content, diag := provider.Execute(ctx, &plugin.ProvideContentParams{ Config: configVal, Args: pluginArgs, - DataContext: dataCtxAny, + DataContext: dataCtx, }) - result.Result = "" + res = "" if content != nil { - result.Result = content.Markdown + res = content.Markdown } - result.Diags = diags - } - - for _, d := range result.Diags { - diags = append(diags, d) + diags.ExtendHcl(diag) } - res = result.Result return } -func (c *Caller) CallContent(name string, config evaluation.Configuration, invocation evaluation.Invocation, context map[string]any) (result string, diag diagnostics.Diag) { +func (c *Caller) CallContent(ctx context.Context, name string, config evaluation.Configuration, invocation evaluation.Invocation, dataCtx plugin.MapData) (result string, diag diagnostics.Diag) { var ok bool var res any - res, diag = c.callPlugin(definitions.BlockKindContent, name, config, invocation, context) + res, diag = c.callPlugin(ctx, definitions.BlockKindContent, name, config, invocation, dataCtx) + if diag.HasErrors() { + return + } result, ok = res.(string) - if !diag.HasErrors() && !ok { - diag.Append(&hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Incorrect result type", - Detail: "Plugin returned incorrect data type. Please contact fabric developers about this issue", - Subject: invocation.DefRange().Ptr(), - }) + if !ok { + panic("Incorrect plugin result type") } return } -func (c *Caller) CallData(name string, config evaluation.Configuration, invocation evaluation.Invocation) (result map[string]any, diag diagnostics.Diag) { +func (c *Caller) CallData(ctx context.Context, name string, config evaluation.Configuration, invocation evaluation.Invocation) (result plugin.MapData, diag diagnostics.Diag) { var ok bool var res any - res, diag = c.callPlugin(definitions.BlockKindData, name, config, invocation, nil) - result, ok = res.(map[string]any) - if !diag.HasErrors() && !ok { - diag.Append(&hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Incorrect result type", - Detail: "Plugin returned incorrect data type. Please contact fabric developers about this issue", - Subject: invocation.DefRange().Ptr(), - }) + res, diag = c.callPlugin(ctx, definitions.BlockKindData, name, config, invocation, nil) + if diag.HasErrors() { + return + } + result, ok = res.(plugin.MapData) + if !ok { + panic("Incorrect plugin result type") } return } diff --git a/parser/definedBlocks.go b/parser/definedBlocks.go index 9d9e3688..4ddf8dc8 100644 --- a/parser/definedBlocks.go +++ b/parser/definedBlocks.go @@ -98,9 +98,13 @@ func (db *DefinedBlocks) AsValueMap() map[string]cty.Value { } func (db *DefinedBlocks) DefaultConfigFor(plugin *definitions.Plugin) (config *definitions.Config) { + return db.DefaultConfig(plugin.Kind(), plugin.Name()) +} + +func (db *DefinedBlocks) DefaultConfig(pluginKind, pluginName string) (config *definitions.Config) { return db.Config[definitions.Key{ - PluginKind: plugin.Kind(), - PluginName: plugin.Name(), + PluginKind: pluginKind, + PluginName: pluginName, BlockName: "", }] } diff --git a/parser/definitions/config.go b/parser/definitions/config.go index 21d8d1a1..39456b27 100644 --- a/parser/definitions/config.go +++ b/parser/definitions/config.go @@ -111,3 +111,5 @@ func DefineConfig(block *hclsyntax.Block) (config *Config, diags diagnostics.Dia } return } + +type ConfigResolver func(pluginKind string, pluginName string) (config *Config) diff --git a/parser/definitions/meta.go b/parser/definitions/meta.go index 02666db3..aa3247e9 100644 --- a/parser/definitions/meta.go +++ b/parser/definitions/meta.go @@ -1,5 +1,7 @@ package definitions +import "github.com/blackstork-io/fabric/plugin" + type MetaBlock struct { // XXX: is empty sting enougth or use a proper ptr-nil-if-missing? Author string `hcl:"author,optional"` @@ -7,3 +9,14 @@ type MetaBlock struct { // TODO: ?store def range defRange hcl.Range } + +func (m *MetaBlock) AsJQData() plugin.Data { + tags := make(plugin.ListData, len(m.Tags)) + for i, tag := range m.Tags { + tags[i] = plugin.StringData(tag) + } + return plugin.ConvMapData{ + "author": plugin.StringData(m.Author), + "tags": tags, + } +} diff --git a/parser/definitions/parsed_document.go b/parser/definitions/parsed_document.go new file mode 100644 index 00000000..813ea400 --- /dev/null +++ b/parser/definitions/parsed_document.go @@ -0,0 +1,83 @@ +package definitions + +import ( + "context" + + "github.com/hashicorp/hcl/v2" + + "github.com/blackstork-io/fabric/parser/evaluation" + "github.com/blackstork-io/fabric/pkg/diagnostics" + "github.com/blackstork-io/fabric/plugin" +) + +type ParsedDocument struct { + Meta *MetaBlock + Content []Renderable + Data []*ParsedData +} + +// result has a shape map[plugin_name]map[block_name]plugin_result. +func (d *ParsedDocument) evalData(ctx context.Context, caller evaluation.DataCaller) (result plugin.MapData, diags diagnostics.Diag) { + // TODO: can be parallel: + + result = plugin.MapData{} + for _, node := range d.Data { + res, diag := caller.CallData( + ctx, + node.PluginName, + node.Config, + node.Invocation, + ) + if diags.Extend(diag) { + continue + } + + var pluginNameRes plugin.MapData + if m, found := result[node.PluginName]; found { + pluginNameRes = m.(plugin.MapData) + } else { + pluginNameRes = plugin.MapData{} + result[node.PluginName] = pluginNameRes + } + + if _, found := pluginNameRes[node.BlockName]; found { + diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagWarning, + Summary: "Data conflict", + Detail: ("Result of this block overwrites results from the previous invocation. " + + "Creating multiple anonymous 'data ref' with the same 'base' is ill-advised, " + + "we recommend naming all 'data ref' blocks uniquely"), + Subject: node.Invocation.DefRange().Ptr(), + }) + } + pluginNameRes[node.BlockName] = res + } + return +} + +func (d *ParsedDocument) Render(ctx context.Context, caller evaluation.PluginCaller) (result []string, diags diagnostics.Diag) { + dataResult, diags := d.evalData(ctx, caller) + if diags.HasErrors() { + return + } + var res evaluation.Result + document := plugin.ConvMapData{ + BlockKindContent: &res, + } + if d.Meta != nil { + document[BlockKindMeta] = d.Meta.AsJQData() + } + + dataCtx := evaluation.NewDataContext(plugin.ConvMapData{ + BlockKindData: dataResult, + BlockKindDocument: document, + }) + + for _, content := range d.Content { + diags.Extend( + content.Render(ctx, caller, dataCtx.Share(), &res), + ) + } + result = res.AsGoType() + return +} diff --git a/parser/definitions/parsed_plugin.go b/parser/definitions/parsed_plugin.go new file mode 100644 index 00000000..515d253d --- /dev/null +++ b/parser/definitions/parsed_plugin.go @@ -0,0 +1,124 @@ +package definitions + +import ( + "context" + "fmt" + + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hcldec" + "github.com/itchyny/gojq" + "github.com/zclconf/go-cty/cty" + + "github.com/blackstork-io/fabric/parser/evaluation" + "github.com/blackstork-io/fabric/pkg/diagnostics" + "github.com/blackstork-io/fabric/pkg/utils" + "github.com/blackstork-io/fabric/plugin" +) + +type ParsedPlugin struct { + PluginName string + BlockName string + Meta *MetaBlock + Config evaluation.Configuration + Invocation evaluation.Invocation +} + +func (pe *ParsedPlugin) GetBlockInvocation() *evaluation.BlockInvocation { + res, ok := pe.Invocation.(*evaluation.BlockInvocation) + if !ok { + panic("This Plugin does not store a BlockInvocation!") + } + return res +} + +type ( + ParsedContent ParsedPlugin + ParsedData ParsedPlugin +) + +// Render implements Renderable. +func (c *ParsedContent) Render(ctx context.Context, caller evaluation.ContentCaller, dataCtx evaluation.DataContext, result *evaluation.Result) (diags diagnostics.Diag) { + if c.Meta != nil { + dataCtx.Set(BlockKindContent, plugin.ConvMapData{ + BlockKindMeta: c.Meta.AsJQData(), + }) + } else { + dataCtx.Delete(BlockKindContent) + } + + diags.Extend(c.EvalQuery(&dataCtx)) + // TODO: #28 #29 + if diags.HasErrors() { + return + } + + resultStr, diag := caller.CallContent(ctx, c.PluginName, c.Config, c.Invocation, dataCtx.AsJQData().(plugin.MapData)) + if diags.Extend(diag) { + // XXX: What to do if we have errors while executing content blocks? + // just skipping the value for now... + return + } + result.Append(resultStr) + return +} + +func (c *ParsedContent) EvalQuery(dataCtx *evaluation.DataContext) (diags diagnostics.Diag) { + body := c.Invocation.GetBody() + attr, found := body.Attributes["query"] + if !found { + return + } + val, newBody, dgs := hcldec.PartialDecode(body, &hcldec.ObjectSpec{ + "query": &hcldec.AttrSpec{ + Name: "query", + Type: cty.String, + Required: true, + }, + }, nil) + c.Invocation.SetBody(utils.ToHclsyntaxBody(newBody)) + if diags.ExtendHcl(dgs) { + return + } + query := val.GetAttr("query").AsString() + + dataCtx.Set("query", plugin.StringData(query)) + queryResult, err := runQuery(query, dataCtx) + if err != nil { + diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Failed to execute the query", + Detail: err.Error(), + Subject: &attr.SrcRange, + }) + return + } + dataCtx.Set("query_result", queryResult) + return +} + +func runQuery(query string, dataCtx *evaluation.DataContext) (result plugin.Data, err error) { + jqQuery, err := gojq.Parse(query) + if err != nil { + err = fmt.Errorf("failed to parse the query: %w", err) + return + } + + code, err := gojq.Compile(jqQuery) + if err != nil { + err = fmt.Errorf("failed to compile the query: %w", err) + return + } + res, hasResult := code.Run(dataCtx.Any()).Next() + if !hasResult { + return + } + result, err = plugin.ParseDataAny(res) + if err != nil { + err = fmt.Errorf("incorrect query result type: %w", err) + } + return +} + +type Renderable interface { + Render(ctx context.Context, caller evaluation.ContentCaller, dataCtx evaluation.DataContext, result *evaluation.Result) diagnostics.Diag +} diff --git a/parser/definitions/plugin.go b/parser/definitions/plugin.go index c5fa82a1..273ca42c 100644 --- a/parser/definitions/plugin.go +++ b/parser/definitions/plugin.go @@ -7,39 +7,17 @@ import ( "github.com/hashicorp/hcl/v2/hclsyntax" "github.com/zclconf/go-cty/cty" - "github.com/blackstork-io/fabric/parser/evaluation" "github.com/blackstork-io/fabric/pkg/diagnostics" ) type Plugin struct { Block *hclsyntax.Block - // Current plugin name. For unevaluated refs is "ref", - // after evaluation will change to the referenced plugin name. Once sync.Once Parsed bool ParseResult *ParsedPlugin } -// inversion of control: pass caller to plugin (as an interface) to execute it -// allows us to add more fields to different plugins (content plugin needs query parsed) - -type ParsedPlugin struct { - PluginName string - BlockName string - Meta *MetaBlock - Config evaluation.Configuration - Invocation evaluation.Invocation -} - -func (pe *ParsedPlugin) GetBlockInvocation() *evaluation.BlockInvocation { - res, ok := pe.Invocation.(*evaluation.BlockInvocation) - if !ok { - panic("This Plugin does not store a BlockInvocation!") - } - return res -} - func (p *Plugin) DefRange() hcl.Range { return p.Block.DefRange() } diff --git a/parser/definitions/section.go b/parser/definitions/section.go index d90113fa..dbb52f1b 100644 --- a/parser/definitions/section.go +++ b/parser/definitions/section.go @@ -1,13 +1,16 @@ package definitions import ( + "context" "sync" "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/hclsyntax" "github.com/zclconf/go-cty/cty" + "github.com/blackstork-io/fabric/parser/evaluation" "github.com/blackstork-io/fabric/pkg/diagnostics" + "github.com/blackstork-io/fabric/plugin" ) type Section struct { @@ -17,20 +20,30 @@ type Section struct { ParseResult *ParsedSection } -type ContentPluginOrSection interface { - contentPluginOrSection() +type ParsedSection struct { + Meta *MetaBlock + Title *ParsedContent + Content []Renderable } -func (*ParsedPlugin) contentPluginOrSection() {} - -func (*ParsedSection) contentPluginOrSection() {} - -var _ ContentPluginOrSection = (*ParsedPlugin)(nil) +func (s ParsedSection) Render(ctx context.Context, caller evaluation.ContentCaller, dataCtx evaluation.DataContext, result *evaluation.Result) (diags diagnostics.Diag) { + if s.Meta != nil { + dataCtx.Set(BlockKindSection, plugin.ConvMapData{ + BlockKindMeta: s.Meta.AsJQData(), + }) + } else { + dataCtx.Delete(BlockKindSection) + } + if title := s.Title; title != nil { + diags.Extend(title.Render(ctx, caller, dataCtx.Share(), result)) + } -type ParsedSection struct { - Meta *MetaBlock - Title *hclsyntax.Attribute - Content []ContentPluginOrSection + for _, content := range s.Content { + diags.Extend( + content.Render(ctx, caller, dataCtx.Share(), result), + ) + } + return } func (s *Section) IsRef() bool { diff --git a/parser/definitions/title.go b/parser/definitions/title.go index 550081cb..d247800e 100644 --- a/parser/definitions/title.go +++ b/parser/definitions/title.go @@ -11,12 +11,12 @@ import ( ) // Desugars `title = "foo"` into appropriate `context` invocation. -type TitleInvocation struct { +type titleInvocation struct { hcl.Expression } // GetBody implements evaluation.Invocation. -func (t *TitleInvocation) GetBody() *hclsyntax.Body { +func (t *titleInvocation) GetBody() *hclsyntax.Body { rng := t.Expression.Range() return &hclsyntax.Body{ SrcRange: rng, @@ -29,19 +29,19 @@ func (t *TitleInvocation) GetBody() *hclsyntax.Body { } // SetBody implements evaluation.Invocation. -func (*TitleInvocation) SetBody(*hclsyntax.Body) {} +func (*titleInvocation) SetBody(*hclsyntax.Body) {} -var _ evaluation.Invocation = (*TitleInvocation)(nil) +var _ evaluation.Invocation = (*titleInvocation)(nil) -func (t *TitleInvocation) DefRange() hcl.Range { +func (t *titleInvocation) DefRange() hcl.Range { return t.Expression.Range() } -func (t *TitleInvocation) MissingItemRange() hcl.Range { +func (t *titleInvocation) MissingItemRange() hcl.Range { return t.Expression.Range() } -func (t *TitleInvocation) ParseInvocation(spec hcldec.Spec) (val cty.Value, diags diagnostics.Diag) { +func (t *titleInvocation) ParseInvocation(spec hcldec.Spec) (val cty.Value, diags diagnostics.Diag) { // Titles can only be rendered once, so there's no reason to put `sync.Once` like in proper blocks expr, ok := t.Expression.(hclsyntax.Expression) if !ok { @@ -73,8 +73,13 @@ func (t *TitleInvocation) ParseInvocation(spec hcldec.Spec) (val cty.Value, diag return } -func NewTitle(title hcl.Expression) *TitleInvocation { - return &TitleInvocation{ - Expression: title, +func NewTitle(title *hclsyntax.Attribute, resolver ConfigResolver) *ParsedContent { + pluginName := "text" + return &ParsedContent{ + PluginName: pluginName, + Config: resolver(BlockKindContent, pluginName), + Invocation: &titleInvocation{ + Expression: title.Expr, + }, } } diff --git a/parser/evaluation/dataContext.go b/parser/evaluation/dataContext.go new file mode 100644 index 00000000..d587e06e --- /dev/null +++ b/parser/evaluation/dataContext.go @@ -0,0 +1,47 @@ +package evaluation + +import ( + "maps" + + "github.com/blackstork-io/fabric/plugin" +) + +type DataContext struct { + plugin.ConvMapData + mapOwned bool +} + +// Passed-in map must not be modified afterwards. +func NewDataContext(m plugin.ConvMapData) DataContext { + return DataContext{ + ConvMapData: m, + mapOwned: true, + } +} + +func (dc *DataContext) Share() DataContext { + dc.mapOwned = false + return *dc +} + +func (dc *DataContext) makeOwned() { + dc.ConvMapData = maps.Clone(dc.ConvMapData) + dc.mapOwned = true +} + +func (dc *DataContext) Delete(key string) { + if _, found := dc.ConvMapData[key]; !found { + return + } + if !dc.mapOwned { + dc.makeOwned() + } + delete(dc.ConvMapData, key) +} + +func (dc *DataContext) Set(key string, val plugin.ConvertableData) { + if !dc.mapOwned { + dc.makeOwned() + } + dc.ConvMapData[key] = val +} diff --git a/parser/evaluation/plugincaller.go b/parser/evaluation/plugincaller.go new file mode 100644 index 00000000..2de5c053 --- /dev/null +++ b/parser/evaluation/plugincaller.go @@ -0,0 +1,21 @@ +package evaluation + +import ( + "context" + + "github.com/blackstork-io/fabric/pkg/diagnostics" + "github.com/blackstork-io/fabric/plugin" +) + +type DataCaller interface { + CallData(ctx context.Context, name string, config Configuration, invocation Invocation) (result plugin.MapData, diag diagnostics.Diag) +} + +type ContentCaller interface { + CallContent(ctx context.Context, name string, config Configuration, invocation Invocation, context plugin.MapData) (result string, diag diagnostics.Diag) +} + +type PluginCaller interface { + DataCaller + ContentCaller +} diff --git a/parser/evaluation/result.go b/parser/evaluation/result.go new file mode 100644 index 00000000..6e459aeb --- /dev/null +++ b/parser/evaluation/result.go @@ -0,0 +1,22 @@ +package evaluation + +import "github.com/blackstork-io/fabric/plugin" + +type Result plugin.ListData + +func (d *Result) AsJQData() plugin.Data { + return plugin.ListData(*d) +} + +func (d Result) AsGoType() (result []string) { + result = make([]string, len(d)) + for i, s := range d { + // The only way to modify resultsList is through append, so this is always correct + result[i] = string(s.(plugin.StringData)) + } + return +} + +func (d *Result) Append(s string) { + *d = append(*d, plugin.StringData(s)) +} diff --git a/parser/evaluator.go b/parser/evaluator.go deleted file mode 100644 index 6f061240..00000000 --- a/parser/evaluator.go +++ /dev/null @@ -1,252 +0,0 @@ -package parser - -import ( - "fmt" - "maps" - - "github.com/hashicorp/hcl/v2" - "github.com/hashicorp/hcl/v2/gohcl" - "github.com/hashicorp/hcl/v2/hcldec" - "github.com/itchyny/gojq" - "github.com/zclconf/go-cty/cty" - - "github.com/blackstork-io/fabric/parser/definitions" - "github.com/blackstork-io/fabric/pkg/diagnostics" - "github.com/blackstork-io/fabric/pkg/jsontools" - "github.com/blackstork-io/fabric/pkg/utils" -) - -// Evaluates a chosen document - -type Evaluator struct { - caller PluginCaller - contentCalls []*definitions.ParsedPlugin - topLevelBlocks *DefinedBlocks - context map[string]any -} - -func NewEvaluator(caller PluginCaller, blocks *DefinedBlocks) *Evaluator { - return &Evaluator{ - caller: caller, - topLevelBlocks: blocks, - context: map[string]any{}, - } -} - -func (e *Evaluator) evaluateQuery(call *definitions.ParsedPlugin) (context map[string]any, diags diagnostics.Diag) { - context = e.context - body := call.Invocation.GetBody() - queryAttr, found := body.Attributes["query"] - if !found { - return - } - val, newBody, dgs := hcldec.PartialDecode(body, &hcldec.ObjectSpec{ - "query": &hcldec.AttrSpec{ - Name: "query", - Type: cty.String, - Required: true, - }, - }, nil) - call.Invocation.SetBody(utils.ToHclsyntaxBody(newBody)) - if diags.ExtendHcl(dgs) { - return - } - query := val.GetAttr("query").AsString() - q, err := gojq.Parse(query) - if err != nil { - diags.Append(&hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Failed to parse the query", - Detail: err.Error(), - Subject: &queryAttr.SrcRange, - }) - return - } - - code, err := gojq.Compile(q) - if err != nil { - diags.Append(&hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Failed to compile the query", - Detail: err.Error(), - Subject: &queryAttr.SrcRange, - }) - return - } - queryResultIter := code.Run(context) - queryResult, ok := queryResultIter.Next() - if ok { - context = maps.Clone(context) - context["query_result"] = queryResult - } - return -} - -func (e *Evaluator) EvaluateDocument(d *definitions.Document) (results []string, diags diagnostics.Diag) { - // TODO: Combining parsing and evaluation of the document is flawed - // perhaps a simpler approach is using more steps? - // Currently: - // 1) Define - // 2) ParseAndEvaluate (data) - // 3) Evaluate (content) - // This introduces compelexity with local content context (such as meta blocks) - // Switch to: - // 1) Define - // 2) ParseDocument - // 3) Evaluate (data) - // 4) Evaluate (content) - // May be a bit slower, but allows us to have access to full evaluation context at each step - - diags = e.parseAndEvaluateDocument(d) - if diags.HasErrors() { - return - } - - results = make([]string, 0, len(e.contentCalls)) - for _, call := range e.contentCalls { - context, _ := e.evaluateQuery(call) - // if diags.Extend(diag) { - // query failed, but context is always valid - // TODO: #28 #29 - // } - result, diag := e.caller.CallContent(call.PluginName, call.Config, call.Invocation, context) - if diags.Extend(diag) { - // XXX: What to do if we have errors while executing content blocks? - // just skipping the value for now... - continue - } - results = append(results, result) - // TODO: Here's the place to implement local context #17 - // However I think we need to rework it a bit before done - } - return -} - -func (e *Evaluator) parseAndEvaluateDocument(d *definitions.Document) (diags diagnostics.Diag) { - if title := d.Block.Body.Attributes["title"]; title != nil { - pluginName := "text" - e.contentCalls = append(e.contentCalls, &definitions.ParsedPlugin{ - PluginName: pluginName, - Config: e.topLevelBlocks.Config[definitions.Key{ - PluginKind: definitions.BlockKindContent, - PluginName: pluginName, - BlockName: "", - }], // use default config - Invocation: definitions.NewTitle(title.Expr), - }) - } - - var origMeta *hcl.Range - - for _, block := range d.Block.Body.Blocks { - switch block.Type { - case definitions.BlockKindContent, definitions.BlockKindData: - plugin, diag := definitions.DefinePlugin(block, false) - if diags.Extend(diag) { - continue - } - call, diag := e.topLevelBlocks.ParsePlugin(plugin) - if diags.Extend(diag) { - continue - } - switch block.Type { - case definitions.BlockKindContent: - // delaying content calls until all data calls are completed - // TODO: contentCalls must also store a ref to context (here - to the meta of the document) - // also requires to parse meta first, not in declaration order - e.contentCalls = append(e.contentCalls, call) - case definitions.BlockKindData: - res, diag := e.caller.CallData( - call.PluginName, - call.Config, - call.Invocation, - ) - if diags.Extend(diag) { - continue - } - var err error - e.context, err = jsontools.MapSet(e.context, []string{ - definitions.BlockKindData, - call.PluginName, - call.BlockName, - }, res) - diags.AppendErr(err, "Failed to save data plugin result") - default: - panic("must be exhaustive") - } - - case definitions.BlockKindMeta: - if origMeta != nil { - diags.Append(&hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Meta block redefinition", - Detail: fmt.Sprintf( - "%s block allows at most one meta block, original meta block was defined at %s:%d", - d.Block.Type, origMeta.Filename, origMeta.Start.Line, - ), - Subject: block.DefRange().Ptr(), - Context: d.Block.Body.Range().Ptr(), - }) - continue - } - var meta definitions.MetaBlock - if diags.ExtendHcl(gohcl.DecodeBody(block.Body, nil, &meta)) { - continue - } - d.Meta = &meta - origMeta = block.DefRange().Ptr() - case definitions.BlockKindSection: - section, diag := definitions.DefineSection(block, false) - if diags.Extend(diag) { - continue - } - parsedSection, diag := e.topLevelBlocks.ParseSection(section) - if diags.Extend(diag) { - continue - } - e.evaluateSection(parsedSection) - default: - diags.Append(definitions.NewNestingDiag( - d.Block.Type, - block, - d.Block.Body, - []string{ - definitions.BlockKindContent, - definitions.BlockKindData, - definitions.BlockKindMeta, - definitions.BlockKindSection, - }, - )) - continue - } - } - - return -} - -func (e *Evaluator) evaluateSection(s *definitions.ParsedSection) { - if title := s.Title; title != nil { - pluginName := "text" - e.contentCalls = append(e.contentCalls, &definitions.ParsedPlugin{ - PluginName: pluginName, - Config: e.topLevelBlocks.Config[definitions.Key{ - PluginKind: definitions.BlockKindContent, - PluginName: pluginName, - BlockName: "", - }], // use default config - Invocation: definitions.NewTitle(title.Expr), - }) - } - - for _, content := range s.Content { - switch contentT := content.(type) { - case *definitions.ParsedPlugin: - // TODO: contentCalls must also store a ref to context (here - to the meta of the section) - e.contentCalls = append(e.contentCalls, contentT) - case *definitions.ParsedSection: - e.evaluateSection(contentT) - default: - panic("must be exhaustive") - } - } -} diff --git a/parser/mockPluginCaller.go b/parser/mockPluginCaller.go deleted file mode 100644 index c3dc6a28..00000000 --- a/parser/mockPluginCaller.go +++ /dev/null @@ -1,80 +0,0 @@ -package parser - -import ( - "strings" - - "github.com/sanity-io/litter" - "golang.org/x/exp/maps" - - "github.com/blackstork-io/fabric/parser/definitions" - "github.com/blackstork-io/fabric/parser/evaluation" - "github.com/blackstork-io/fabric/pkg/diagnostics" - "github.com/blackstork-io/fabric/pkg/utils" -) - -type MockCaller struct{} - -func (c *MockCaller) dumpContext(context map[string]any) string { - return litter.Sdump("Context:", context) -} - -func (c *MockCaller) dumpConfig(config evaluation.Configuration) string { - if utils.IsNil(config) { - return "NoConfig" - } - switch c := config.(type) { - case *definitions.ConfigPtr: - attrs, _ := c.Cfg.Body.JustAttributes() - return litter.Sdump("ConfigPtr", maps.Keys(attrs)) - case *definitions.Config: - attrs, _ := c.Block.Body.JustAttributes() - return litter.Sdump("Config", maps.Keys(attrs)) - default: - return "UnknownConfig " + litter.Sdump(c) - } -} - -func (c *MockCaller) dumpInvocation(invoke evaluation.Invocation) string { - if utils.IsNil(invoke) { - return "NoConfig" - } - switch inv := invoke.(type) { - case *evaluation.BlockInvocation: - attrStringed := map[string]string{} - attrs, _ := inv.Body.JustAttributes() - for k, v := range attrs { - val, _ := v.Expr.Value(nil) - attrStringed[k] = val.GoString() - } - - return litter.Sdump("BlockInvocation", attrStringed) - case *definitions.TitleInvocation: - val, _ := inv.Expression.Value(nil) - return litter.Sdump("TitleInvocation", val.GoString()) - default: - return "UnknownInvocation " + litter.Sdump(inv) - } -} - -// CallContent implements PluginCaller. -func (c *MockCaller) CallContent(name string, config evaluation.Configuration, invocation evaluation.Invocation, context map[string]any) (result string, diag diagnostics.Diag) { - dump := []string{ - "Call to content:", - } - dump = append(dump, c.dumpConfig(config)) - dump = append(dump, c.dumpInvocation(invocation)) - dump = append(dump, c.dumpContext(context)) - return strings.Join(dump, "\n") + "\n\n", nil -} - -// CallData implements PluginCaller. -func (c *MockCaller) CallData(name string, config evaluation.Configuration, invocation evaluation.Invocation) (result map[string]any, diag diagnostics.Diag) { - dump := []string{ - "Call to data:", - } - dump = append(dump, c.dumpConfig(config)) - dump = append(dump, c.dumpInvocation(invocation)) - return map[string]any{"dumpResult": strings.Join(dump, "\n")}, nil -} - -var _ PluginCaller = (*MockCaller)(nil) diff --git a/parser/parseDocument.go b/parser/parseDocument.go new file mode 100644 index 00000000..4bc433a4 --- /dev/null +++ b/parser/parseDocument.go @@ -0,0 +1,87 @@ +package parser + +import ( + "fmt" + + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/gohcl" + + "github.com/blackstork-io/fabric/parser/definitions" + "github.com/blackstork-io/fabric/pkg/diagnostics" +) + +func (db *DefinedBlocks) ParseDocument(d *definitions.Document) (doc *definitions.ParsedDocument, diags diagnostics.Diag) { + doc = &definitions.ParsedDocument{} + if title := d.Block.Body.Attributes["title"]; title != nil { + doc.Content = append(doc.Content, definitions.NewTitle(title, db.DefaultConfig)) + } + + var origMeta *hcl.Range + + for _, block := range d.Block.Body.Blocks { + switch block.Type { + case definitions.BlockKindContent, definitions.BlockKindData: + plugin, diag := definitions.DefinePlugin(block, false) + if diags.Extend(diag) { + continue + } + call, diag := db.ParsePlugin(plugin) + if diags.Extend(diag) { + continue + } + switch block.Type { + case definitions.BlockKindContent: + doc.Content = append(doc.Content, (*definitions.ParsedContent)(call)) + case definitions.BlockKindData: + doc.Data = append(doc.Data, (*definitions.ParsedData)(call)) + default: + panic("must be exhaustive") + } + + case definitions.BlockKindMeta: + if origMeta != nil { + diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Meta block redefinition", + Detail: fmt.Sprintf( + "%s block allows at most one meta block, original meta block was defined at %s:%d", + d.Block.Type, origMeta.Filename, origMeta.Start.Line, + ), + Subject: block.DefRange().Ptr(), + Context: d.Block.Body.Range().Ptr(), + }) + continue + } + var meta definitions.MetaBlock + if diags.ExtendHcl(gohcl.DecodeBody(block.Body, nil, &meta)) { + continue + } + doc.Meta = &meta + origMeta = block.DefRange().Ptr() + case definitions.BlockKindSection: + section, diag := definitions.DefineSection(block, false) + if diags.Extend(diag) { + continue + } + parsedSection, diag := db.ParseSection(section) + if diags.Extend(diag) { + continue + } + doc.Content = append(doc.Content, parsedSection) + default: + diags.Append(definitions.NewNestingDiag( + d.Block.Type, + block, + d.Block.Body, + []string{ + definitions.BlockKindContent, + definitions.BlockKindData, + definitions.BlockKindMeta, + definitions.BlockKindSection, + }, + )) + continue + } + } + return +} diff --git a/parser/parsePluginBlock.go b/parser/parsePlugin.go similarity index 93% rename from parser/parsePluginBlock.go rename to parser/parsePlugin.go index 663f70b3..3a96580c 100644 --- a/parser/parsePluginBlock.go +++ b/parser/parsePlugin.go @@ -75,7 +75,7 @@ func (db *DefinedBlocks) parsePlugin(plugin *definitions.Plugin) (parsed *defini Subject: blk.DefRange().Ptr(), Context: plugin.Block.Range().Ptr(), }) - return true + break } configBlock = blk @@ -88,12 +88,12 @@ func (db *DefinedBlocks) parsePlugin(plugin *definitions.Plugin) (parsed *defini Subject: blk.DefRange().Ptr(), Context: plugin.Block.Range().Ptr(), }) - return true + break } var meta definitions.MetaBlock if diags.ExtendHcl(gohcl.DecodeBody(blk.Body, nil, &meta)) { - return true + break } res.Meta = &meta @@ -128,20 +128,6 @@ func (db *DefinedBlocks) parsePlugin(plugin *definitions.Plugin) (parsed *defini refBaseConfig = baseEval.Config if res.BlockName == "" { res.BlockName = baseEval.BlockName - if plugin.Kind() == definitions.BlockKindData { - diags.Append(&hcl.Diagnostic{ - Severity: hcl.DiagWarning, - Summary: "Potential data conflict", - Detail: fmt.Sprintf( - "This 'data ref' will inherit its name from its base (%q). "+ - "Creating another anonymous 'data ref' with the same 'base' would "+ - "override the current block's execution results. "+ - "We recommend namingall 'data ref' blocks uniquely", - res.BlockName, - ), - Subject: plugin.DefRange().Ptr(), - }) - } } updateRefBody(invocation.Body, baseEval.GetBlockInvocation().Body) diff --git a/parser/parseSectionBlock.go b/parser/parseSection.go similarity index 95% rename from parser/parseSectionBlock.go rename to parser/parseSection.go index d6bdbc71..e5eb8464 100644 --- a/parser/parseSectionBlock.go +++ b/parser/parseSection.go @@ -47,8 +47,9 @@ func (db *DefinedBlocks) ParseSection(section *definitions.Section) (res *defini } func (db *DefinedBlocks) parseSection(section *definitions.Section) (parsed *definitions.ParsedSection, diags diagnostics.Diag) { - res := definitions.ParsedSection{ - Title: section.Block.Body.Attributes["title"], + res := definitions.ParsedSection{} + if title := section.Block.Body.Attributes["title"]; title != nil { + res.Title = definitions.NewTitle(title, db.DefaultConfig) } var origMeta *hcl.Range @@ -99,7 +100,7 @@ func (db *DefinedBlocks) parseSection(section *definitions.Section) (parsed *def if diags.Extend(diag) { continue } - res.Content = append(res.Content, call) + res.Content = append(res.Content, (*definitions.ParsedContent)(call)) case definitions.BlockKindMeta: if origMeta != nil { diags.Append(&hcl.Diagnostic{ diff --git a/pkg/clicontext/context.go b/pkg/clicontext/context.go new file mode 100644 index 00000000..16d631e7 --- /dev/null +++ b/pkg/clicontext/context.go @@ -0,0 +1,52 @@ +package clicontext + +import ( + "context" + "fmt" + "log/slog" + "os" + "os/signal" +) + +type cleanupCtxKeyT struct{} + +var cleanupCtxKey = cleanupCtxKeyT{} + +// Returns the cleanup context given the main context. +// Use cleanup context in, for example, deferred statements +func GetCleanupCtx(ctx context.Context) context.Context { + cleanupCtx := ctx.Value(cleanupCtxKey) + if cleanupCtx == nil { + return context.Background() + } + return cleanupCtx.(context.Context) +} + +// Returns a cli-appropriate context (cancelable by ctrl+c). +func New() context.Context { + cleanupCtx, cleanupCancel := context.WithCancelCause(context.Background()) + valCtx := context.WithValue(cleanupCtx, cleanupCtxKey, cleanupCtx) + mainCtx, mainCancel := context.WithCancelCause(valCtx) + + c := make(chan os.Signal, 2) + signal.Notify(c, os.Interrupt) + + go func() { + caught := 0 + for range c { + switch caught { + case 0: + slog.Warn("Received os.Interrupt") + mainCancel(fmt.Errorf("got termination request (gentle)")) + case 1: + slog.Error("Received second os.Interrupt") + cleanupCancel(fmt.Errorf("got termination request (forceful)")) + default: + slog.Error("Rough exit (3 interrupts received, probably deadlocked)") + os.Exit(1) + } + caught++ + } + }() + return mainCtx +} diff --git a/plugin/data.go b/plugin/data.go index 1166cbd2..55bf0d69 100644 --- a/plugin/data.go +++ b/plugin/data.go @@ -8,6 +8,7 @@ import ( type Data interface { Any() any data() + ConvertableData } func (NumberData) data() {} @@ -16,6 +17,12 @@ func (BoolData) data() {} func (MapData) data() {} func (ListData) data() {} +func (d NumberData) AsJQData() Data { return d } +func (d StringData) AsJQData() Data { return d } +func (d BoolData) AsJQData() Data { return d } +func (d MapData) AsJQData() Data { return d } +func (d ListData) AsJQData() Data { return d } + type NumberData float64 func (d NumberData) Any() any { @@ -105,6 +112,9 @@ func ParseDataAny(v any) (Data, error) { case string: return StringData(v), nil case []any: + // TODO: potential bug: + // this case would trigger only for []any, not, for example, []string + // this can be worked around using reflection dst := make(ListData, len(v)) for i, e := range v { d, err := ParseDataAny(e) @@ -135,3 +145,49 @@ func ParseDataMapAny(v map[string]any) (MapData, error) { } return dst, nil } + +type ConvertableData interface { + AsJQData() Data +} + +type ConvMapData map[string]ConvertableData + +func (d ConvMapData) AsJQData() Data { + dst := make(MapData, len(d)) + for k, v := range d { + if v == nil { + dst[k] = nil + } else { + dst[k] = v.AsJQData() + } + } + return dst +} + +func (d ConvMapData) Any() any { + dst := make(map[string]any, len(d)) + for k, v := range d { + dst[k] = v.AsJQData().Any() + } + return dst +} +func (d ConvMapData) data() {} + +type ConvListData []ConvertableData + +func (d ConvListData) AsJQData() Data { + dst := make(ListData, len(d)) + for k, v := range d { + dst[k] = v.AsJQData() + } + return dst +} + +func (d ConvListData) Any() any { + dst := make([]any, len(d)) + for k, v := range d { + dst[k] = v.AsJQData().Any() + } + return dst +} +func (d ConvListData) data() {} diff --git a/test/e2e/data_test.go b/test/e2e/data_test.go new file mode 100644 index 00000000..a4abdc33 --- /dev/null +++ b/test/e2e/data_test.go @@ -0,0 +1,102 @@ +package e2e_test + +import ( + "context" + "fmt" + "log/slog" + "os" + "testing" + "testing/fstest" + + "github.com/stretchr/testify/assert" + + "github.com/blackstork-io/fabric/cmd" + "github.com/blackstork-io/fabric/pkg/diagnostics" + "github.com/blackstork-io/fabric/plugin" + "github.com/blackstork-io/fabric/test/e2e/diag_test" +) + +func dataTest(t *testing.T, testName string, files []string, target string, expectedResult plugin.MapData, diagAsserts [][]diag_test.Assert) { + t.Helper() + t.Run(testName, func(t *testing.T) { + t.Parallel() + t.Helper() + + sourceDir := fstest.MapFS{} + for i, content := range files { + sourceDir[fmt.Sprintf("file_%d.fabric", i)] = &fstest.MapFile{ + Data: []byte(content), + Mode: 0o777, + } + } + eval := cmd.NewEvaluator("") + defer func() { + eval.Cleanup(nil) + }() + + var res plugin.MapData + diags := eval.ParseFabricFiles(sourceDir) + if !diags.HasErrors() { + if !diags.Extend(eval.LoadRunner()) { + var diag diagnostics.Diag + res, diag = cmd.Data(context.Background(), eval.Blocks, eval.PluginCaller(), target) + diags.Extend(diag) + } + } + + assert.Equal(t, expectedResult, res) + if !diag_test.MatchBiject(diags, diagAsserts) { + assert.Fail(t, "Diagnostics do not match", diags) + } + }) +} + +func TestE2EData(t *testing.T) { + slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{ + AddSource: true, + Level: slog.LevelWarn, + }))) + + t.Parallel() + dataTest( + t, "Basic", + []string{ + ` + document "hello" { + data inline "test" { + hello = "world" + } + + content text { + text = "hello" + } + } + `, + }, + "document.hello.data.inline.test", + plugin.MapData{ + "hello": plugin.StringData("world"), + }, + [][]diag_test.Assert{}, + ) + dataTest( + t, "Basic", + []string{ + ` + data inline "test" { + hello = "world" + } + document "hello" { + content text { + text = "hello" + } + } + `, + }, + "data.inline.test", + plugin.MapData{ + "hello": plugin.StringData("world"), + }, + [][]diag_test.Assert{}, + ) +} diff --git a/test/e2e/render_test.go b/test/e2e/render_test.go index 4c2a171e..c90736cf 100644 --- a/test/e2e/render_test.go +++ b/test/e2e/render_test.go @@ -1,6 +1,7 @@ package e2e_test import ( + "context" "fmt" "log/slog" "os" @@ -10,6 +11,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/blackstork-io/fabric/cmd" + "github.com/blackstork-io/fabric/pkg/diagnostics" "github.com/blackstork-io/fabric/test/e2e/diag_test" ) @@ -26,8 +28,21 @@ func renderTest(t *testing.T, testName string, files []string, docName string, e Mode: 0o777, } } + eval := cmd.NewEvaluator("") + defer func() { + eval.Cleanup(nil) + }() + + var res []string + diags := eval.ParseFabricFiles(sourceDir) + if !diags.HasErrors() { + if !diags.Extend(eval.LoadRunner()) { + var diag diagnostics.Diag + res, diag = cmd.Render(context.Background(), eval.Blocks, eval.PluginCaller(), docName) + diags.Extend(diag) + } + } - res, _, diags := cmd.Render("", sourceDir, docName) if len(expectedResult) == 0 { // so nil == []string{} assert.Empty(t, res) @@ -309,6 +324,26 @@ func TestE2ERender(t *testing.T) { }, [][]diag_test.Assert{}, ) + renderTest( + t, "Data ref name warning missing", + []string{ + ` + data inline "name" { + inline { + a = "1" + } + } + document "test-doc" { + data ref { + base = data.inline.name + } + } + `, + }, + "test-doc", + []string{}, + [][]diag_test.Assert{}, + ) renderTest( t, "Data ref name warning", []string{ @@ -322,13 +357,16 @@ func TestE2ERender(t *testing.T) { data ref { base = data.inline.name } + data ref { + base = data.inline.name + } } `, }, "test-doc", []string{}, [][]diag_test.Assert{ - {diag_test.IsWarning, diag_test.SummaryContains("Potential data conflict")}, + {diag_test.IsWarning, diag_test.SummaryContains("Data conflict")}, }, ) renderTest( @@ -397,4 +435,128 @@ func TestE2ERender(t *testing.T) { []string{"There are 3 items"}, [][]diag_test.Assert{}, ) + renderTest( + t, "Document meta", + []string{ + ` + document "test" { + meta { + author = "foo" + } + content text { + query = ".document.meta.author" + text = "author = {{ .query_result }}" + } + } + `, + }, + "test", + []string{"author = foo"}, + [][]diag_test.Assert{}, + ) + renderTest( + t, "Document and content meta", + []string{ + ` + document "test" { + meta { + author = "foo" + } + section { + meta { + author = "bar" + } + content text { + meta { + author = "baz" + } + query = "(.document.meta.author + .section.meta.author + .content.meta.author)" // + text = "author = {{ .query_result }}" + } + } + } + `, + }, + "test", + []string{"author = foobarbaz"}, + [][]diag_test.Assert{}, + ) + renderTest( + t, "Meta scoping and nesting", + []string{ + ` + content text get_section_author { + query = ".section.meta.author // \"unknown\"" + text = "author = {{ .query_result }}" + } + document "test" { + content ref { + base = content.text.get_section_author + } + section { + content ref { + base = content.text.get_section_author + } + section { + meta { + author = "foo" + } + content ref { + base = content.text.get_section_author + } + section { + content ref { + base = content.text.get_section_author + } + section { + meta { + author = "bar" + } + content ref { + base = content.text.get_section_author + } + } + } + } + } + } + `, + }, + "test", + []string{ + "author = unknown", + "author = unknown", + "author = foo", + "author = unknown", + "author = bar", + }, + [][]diag_test.Assert{}, + ) + renderTest( + t, "Reference rendered blocks", + []string{ + ` + document "test" { + content text { + text = "first result" + } + content text { + query = ".document.content[0]" + text = "content[0] = {{ .query_result }}" + } + content text { + query = ".document.content[1]" + text = "content[1] = {{ .query_result }}" + } + } + `, + }, + "test", + []string{ + "first result", + "content[0] = first result", + "content[1] = content[0] = first result", + }, + [][]diag_test.Assert{}, + ) }