From 4ab2b4e4277783d30c1c7414f81e96e1fede681e Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Thu, 27 May 2021 17:16:17 -0700 Subject: [PATCH 01/16] [wip]: Visualize graphs on command line Signed-off-by: Ketan Umare Signed-off-by: Prafulla Mahindrakar --- cmd/get/workflow.go | 11 ++- cmd/root.go | 2 +- go.mod | 1 + go.sum | 14 ++++ pkg/printer/outputformat_enumer.go | 13 ++-- pkg/printer/printer.go | 3 + pkg/visualize/graphviz.go | 121 +++++++++++++++++++++++++++++ 7 files changed, 157 insertions(+), 8 deletions(-) create mode 100644 pkg/visualize/graphviz.go diff --git a/cmd/get/workflow.go b/cmd/get/workflow.go index cdefe894..b6606696 100644 --- a/cmd/get/workflow.go +++ b/cmd/get/workflow.go @@ -2,6 +2,7 @@ package get import ( "context" + "github.com/flyteorg/flytectl/pkg/visualize" workflowconfig "github.com/flyteorg/flytectl/cmd/config/subcommand/workflow" "github.com/flyteorg/flytectl/pkg/ext" @@ -96,7 +97,15 @@ func getWorkflowFunc(ctx context.Context, args []string, cmdCtx cmdCore.CommandC return err } logger.Debugf(ctx, "Retrieved %v workflow", len(workflows)) - return adminPrinter.Print(config.GetConfig().MustOutputFormat(), workflowColumns, WorkflowToProtoMessages(workflows)...) + if config.GetConfig().MustOutputFormat() == printer.OutputFormatSVG { + return visualize.RenderWorkflow(workflows[0].Closure.CompiledWorkflow, "/tmp/test.svg") + } else { + err = adminPrinter.Print(config.GetConfig().MustOutputFormat(), workflowColumns, WorkflowToProtoMessages(workflows)...) + if err != nil { + return err + } + } + return nil } workflows, err = cmdCtx.AdminFetcherExt().FetchAllVerOfWorkflow(ctx, "", config.GetConfig().Project, config.GetConfig().Domain, workflowconfig.DefaultConfig.Filter) diff --git a/cmd/root.go b/cmd/root.go index a3e942b4..6d43702a 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -53,7 +53,7 @@ func newRootCmd() *cobra.Command { // --root.project, this adds a convenience on top to allow --project to be used rootCmd.PersistentFlags().StringVarP(&(config.GetConfig().Project), "project", "p", "", "Specifies the Flyte project.") rootCmd.PersistentFlags().StringVarP(&(config.GetConfig().Domain), "domain", "d", "", "Specifies the Flyte project's domain.") - rootCmd.PersistentFlags().StringVarP(&(config.GetConfig().Output), "output", "o", printer.OutputFormatTABLE.String(), fmt.Sprintf("Specifies the output type - supported formats %s", printer.OutputFormats())) + rootCmd.PersistentFlags().StringVarP(&(config.GetConfig().Output), "output", "o", printer.OutputFormatTABLE.String(), fmt.Sprintf("Specifies the output type - supported formats %s. NOTE: SVG image is only supported for Workflow", printer.OutputFormats())) rootCmd.AddCommand(viper.GetConfigCommand()) rootCmd.AddCommand(get.CreateGetCommand()) rootCmd.AddCommand(create.RemoteCreateCommand()) diff --git a/go.mod b/go.mod index 9c2e7a55..9523d88d 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/flyteorg/flyteidl v0.19.3 github.com/flyteorg/flytestdlib v0.3.24 github.com/ghodss/yaml v1.0.0 + github.com/goccy/go-graphviz v0.0.9 github.com/golang/protobuf v1.4.3 github.com/google/go-github v17.0.0+incompatible github.com/google/go-querystring v1.1.0 // indirect diff --git a/go.sum b/go.sum index 3b2f0835..07f80f64 100644 --- a/go.sum +++ b/go.sum @@ -269,6 +269,8 @@ github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+ github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/corona10/goimagehash v1.0.2 h1:pUfB0LnsJASMPGEZLj7tGY251vF+qLGqOgEP4rUs6kA= +github.com/corona10/goimagehash v1.0.2/go.mod h1:/l9umBhvcHQXVtQO1V6Gp1yD20STawkhRnnX0D1bvVI= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -340,6 +342,8 @@ github.com/flyteorg/flyteidl v0.19.3/go.mod h1:576W2ViEyjTpT+kEVHAGbrTP3HARNUZ/e github.com/flyteorg/flytestdlib v0.3.13/go.mod h1:Tz8JCECAbX6VWGwFT6cmEQ+RJpZ/6L9pswu3fzWs220= github.com/flyteorg/flytestdlib v0.3.24 h1:Eu5TMKch9ihOavPKufgTBI677eVYjJpOAPPg9hfZIzU= github.com/flyteorg/flytestdlib v0.3.24/go.mod h1:1XG0DwYTUm34Yrffm1Qy9Tdr/pWQypEqTq5dUxw3/cM= +github.com/fogleman/gg v1.3.0 h1:/7zJX8F6AaYQc57WQCyN9cAIz+4bCJGO9B+dyW29am8= +github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= @@ -378,6 +382,8 @@ github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M= github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= +github.com/goccy/go-graphviz v0.0.9 h1:s/FMMJ1Joj6La3S5ApO3Jk2cwM4LpXECC2muFx3IPQQ= +github.com/goccy/go-graphviz v0.0.9/go.mod h1:wXVsXxmyMQU6TN3zGRttjNn3h+iCAS7xQFC6TlNvLhk= github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e h1:BWhy2j3IXJhjCbC68FptL43tDKIq8FladmaTs3Xs7Z8= @@ -395,6 +401,8 @@ github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -534,6 +542,7 @@ github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NH github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -650,6 +659,8 @@ github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ github.com/ncw/swift v1.0.49/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= github.com/ncw/swift v1.0.53 h1:luHjjTNtekIEvHg5KdAFIBaH7bWfNkefwFnpDffSIks= github.com/ncw/swift v1.0.53/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= +github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 h1:BvoENQQU+fZ9uukda/RzCAL/191HHwJA5b13R6diVlY= +github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= @@ -919,6 +930,7 @@ golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -937,6 +949,8 @@ golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EH golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200119044424-58c23975cae1 h1:5h3ngYt7+vXCDZCup/HkCQgW5XwmSvR/nA2JmJ0RErg= +golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= diff --git a/pkg/printer/outputformat_enumer.go b/pkg/printer/outputformat_enumer.go index b10b4514..d1a3a27b 100644 --- a/pkg/printer/outputformat_enumer.go +++ b/pkg/printer/outputformat_enumer.go @@ -8,9 +8,9 @@ import ( "fmt" ) -const _OutputFormatName = "TABLEJSONYAML" +const _OutputFormatName = "TABLEJSONYAMLSVG" -var _OutputFormatIndex = [...]uint8{0, 5, 9, 13} +var _OutputFormatIndex = [...]uint8{0, 5, 9, 13, 16} func (i OutputFormat) String() string { if i >= OutputFormat(len(_OutputFormatIndex)-1) { @@ -19,12 +19,13 @@ func (i OutputFormat) String() string { return _OutputFormatName[_OutputFormatIndex[i]:_OutputFormatIndex[i+1]] } -var _OutputFormatValues = []OutputFormat{0, 1, 2} +var _OutputFormatValues = []OutputFormat{0, 1, 2, 3} var _OutputFormatNameToValueMap = map[string]OutputFormat{ - _OutputFormatName[0:5]: 0, - _OutputFormatName[5:9]: 1, - _OutputFormatName[9:13]: 2, + _OutputFormatName[0:5]: 0, + _OutputFormatName[5:9]: 1, + _OutputFormatName[9:13]: 2, + _OutputFormatName[13:16]: 3, } // OutputFormatString retrieves an enum value from the enum constants string name. diff --git a/pkg/printer/printer.go b/pkg/printer/printer.go index fa86127b..e7e47788 100644 --- a/pkg/printer/printer.go +++ b/pkg/printer/printer.go @@ -22,6 +22,7 @@ const ( OutputFormatTABLE OutputFormat = iota OutputFormatJSON OutputFormatYAML + OutputFormatSVG ) func OutputFormats() []string { @@ -115,6 +116,8 @@ func (p Printer) Print(format OutputFormat, columns []Column, messages ...proto. // Factory Method for all printer switch format { + case OutputFormatSVG: + return fmt.Errorf("OutputFormat SVG is only support for Workflow type of object currently") case OutputFormatJSON, OutputFormatYAML: // Print protobuf to json buf := new(bytes.Buffer) encoder := json.NewEncoder(buf) diff --git a/pkg/visualize/graphviz.go b/pkg/visualize/graphviz.go new file mode 100644 index 00000000..35a0f1aa --- /dev/null +++ b/pkg/visualize/graphviz.go @@ -0,0 +1,121 @@ +package visualize + +import ( + "bytes" + "context" + "fmt" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" + "github.com/flyteorg/flytestdlib/errors" + "github.com/flyteorg/flytestdlib/logger" + "github.com/goccy/go-graphviz" + "github.com/goccy/go-graphviz/cgraph" +) + +func constructGraph(prefix string, graph *cgraph.Graph, w *core.CompiledWorkflow, subWf map[string]*cgraph.Graph) error { + grapNodes := make(map[string]*cgraph.Node) + for _, n := range w.Template.Nodes { + name := n.Id + if prefix != "" { + name = prefix + "-" + n.Id + } + + if n.Id == "start-node" || n.Id == "end-node"{ + gn, err := graph.CreateNode(name) + if err != nil { + return err + } + grapNodes[name] = gn + } else { + switch n.Target.(type) { + case *core.Node_TaskNode: + gn, err := graph.CreateNode(name) + if err != nil { + return err + } + grapNodes[name] = gn + case *core.Node_BranchNode: + gn, err := graph.CreateNode(name) + if err != nil { + return err + } + grapNodes[name] = gn + case *core.Node_WorkflowNode: + gn, err := graph.CreateNode(name) + if err != nil { + return err + } + grapNodes[name] = gn + } + } + } + + for name, node := range grapNodes { + upstreamNodes, _ := w.Connections.Upstream[name] + downstreamNodes, _ := w.Connections.Downstream[name] + if downstreamNodes != nil { + for _, n := range downstreamNodes.Ids { + dNode, ok := grapNodes[n] + if !ok { + return fmt.Errorf("node[%s], downstream from[%s] referenced before creation", n, name) + } + _, err := graph.CreateEdge(fmt.Sprintf("%s-%s", name, n), node, dNode) + if err != nil { + return err + } + } + } + if upstreamNodes != nil { + for _, n := range upstreamNodes.Ids { + uNode, ok := grapNodes[n] + if !ok { + return fmt.Errorf("node[%s], upstream from[%s] referenced before creation", n, name) + } + _, err := graph.CreateEdge(fmt.Sprintf("%s-%s", n, name), node, uNode) + if err != nil { + return err + } + } + } + } + return nil +} + +func CompiledWorkflowClosureToGraph(g *graphviz.Graphviz, w *core.CompiledWorkflowClosure) (*cgraph.Graph, error) { + graph, err := g.Graph(graphviz.Directed) + if err != nil { + return nil, errors.Wrapf("GraphInitFailure", err, "failed to initialize graphviz") + } + + defer func() { + if err := graph.Close(); err != nil { + logger.Fatalf(context.TODO(), "Failed to close the graphviz Graph. err: %s", err) + } + }() + + return graph, constructGraph("", graph, w.Primary, nil) +} + +// RenderWorkflow Renders the workflow graph to the given file +func RenderWorkflow(w *core.CompiledWorkflowClosure, file string) error { + g := graphviz.New() + defer func() { + if err := g.Close(); err != nil { + logger.Fatalf(context.TODO(), "failed to close graphviz. err: %s", err) + } + }() + graph, err := CompiledWorkflowClosureToGraph(g, w) + if err != nil { + return err + } + + logger.Infof(context.TODO(), "outputing file: %s", file) + var buf bytes.Buffer + if err := g.Render(graph, graphviz.XDOT, &buf); err != nil { + return err + } + logger.Infof(context.TODO(), buf.String()) + if err := g.RenderFilename(graph, graphviz.SVG, file); err != nil { + return err + } + return nil +} From 6b5715709223d7a04f8b5c898403da0f7326f072 Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Sun, 30 May 2021 19:51:35 -0700 Subject: [PATCH 02/16] [wip] working version Signed-off-by: Ketan Umare Signed-off-by: Prafulla Mahindrakar --- cmd/get/workflow.go | 7 +- pkg/visualize/graphviz.go | 200 +++++-- pkg/visualize/graphviz_test.go | 26 + .../compiled_closure_branch_nested.json | 553 ++++++++++++++++++ 4 files changed, 728 insertions(+), 58 deletions(-) create mode 100644 pkg/visualize/graphviz_test.go create mode 100644 pkg/visualize/testdata/compiled_closure_branch_nested.json diff --git a/cmd/get/workflow.go b/cmd/get/workflow.go index b6606696..ceaae9d8 100644 --- a/cmd/get/workflow.go +++ b/cmd/get/workflow.go @@ -2,6 +2,7 @@ package get import ( "context" + "fmt" "github.com/flyteorg/flytectl/pkg/visualize" workflowconfig "github.com/flyteorg/flytectl/cmd/config/subcommand/workflow" @@ -98,7 +99,11 @@ func getWorkflowFunc(ctx context.Context, args []string, cmdCtx cmdCore.CommandC } logger.Debugf(ctx, "Retrieved %v workflow", len(workflows)) if config.GetConfig().MustOutputFormat() == printer.OutputFormatSVG { - return visualize.RenderWorkflow(workflows[0].Closure.CompiledWorkflow, "/tmp/test.svg") + b, err := visualize.RenderWorkflow(workflows[0].Closure.CompiledWorkflow) + if err != nil { + return err + } + fmt.Println(string(b)) } else { err = adminPrinter.Print(config.GetConfig().MustOutputFormat(), workflowColumns, WorkflowToProtoMessages(workflows)...) if err != nil { diff --git a/pkg/visualize/graphviz.go b/pkg/visualize/graphviz.go index 35a0f1aa..bb1379a8 100644 --- a/pkg/visualize/graphviz.go +++ b/pkg/visualize/graphviz.go @@ -11,68 +11,151 @@ import ( "github.com/goccy/go-graphviz/cgraph" ) -func constructGraph(prefix string, graph *cgraph.Graph, w *core.CompiledWorkflow, subWf map[string]*cgraph.Graph) error { - grapNodes := make(map[string]*cgraph.Node) - for _, n := range w.Template.Nodes { - name := n.Id - if prefix != "" { - name = prefix + "-" + n.Id +func getName(prefix, id string) string { + if prefix != "" { + return prefix + "-" + id + } + return id +} + +type graphBuilder struct { + graphNodes map[string]*cgraph.Node + graphEdges map[string]*cgraph.Edge + subWf map[string]*cgraph.Graph +} + +func (gb *graphBuilder) addSubNodeEdge(graph *cgraph.Graph, parentNode, n *cgraph.Node) error { + edgeName := fmt.Sprintf("%s-%s", parentNode.Name(), n.Name()) + if _, ok := gb.graphEdges[edgeName]; !ok { + edge, err := graph.CreateEdge(edgeName, parentNode, n) + if err != nil { + return err } + gb.graphEdges[edgeName] = edge + } + return nil +} + +func (gb *graphBuilder) constructBranchNode(prefix string, graph *cgraph.Graph, n *core.Node) (*cgraph.Node, error) { + parentBranchNodeName := getName(prefix, n.Id) + parentBranchNode, err := graph.CreateNode(parentBranchNodeName) + if err != nil { + return nil, err + } + parentBranchNode.SetLabel(n.Metadata.Name) + gb.graphNodes[parentBranchNodeName] = parentBranchNode + + if n.GetBranchNode().GetIfElse() == nil { + return parentBranchNode, nil + } - if n.Id == "start-node" || n.Id == "end-node"{ - gn, err := graph.CreateNode(name) + subNode, err := gb.constructNode(prefix, graph, n.GetBranchNode().GetIfElse().Case.ThenNode) + if err != nil { + return nil, err + } + if err := gb.addSubNodeEdge(graph, parentBranchNode, subNode); err != nil { + return nil, err + } + + if n.GetBranchNode().GetIfElse().GetError() != nil { + name := fmt.Sprintf("%s-error", parentBranchNode.Name()) + subNode, err := graph.CreateNode(name) + subNode.SetLabel(n.GetBranchNode().GetIfElse().GetError().Message) + if err != nil { + return nil, err + } + gb.graphNodes[name] = subNode + if err := gb.addSubNodeEdge(graph, parentBranchNode, subNode); err != nil { + return nil, err + } + } else { + subNode, err := gb.constructNode(prefix, graph, n.GetBranchNode().GetIfElse().GetElseNode()) + if err != nil { + return nil, err + } + if err := gb.addSubNodeEdge(graph, parentBranchNode, subNode); err != nil { + return nil, err + } + } + + if n.GetBranchNode().GetIfElse().GetOther() != nil { + for _, c := range n.GetBranchNode().GetIfElse().GetOther() { + subNode, err := gb.constructNode(prefix, graph, c.ThenNode) if err != nil { - return err + return nil, err } - grapNodes[name] = gn - } else { - switch n.Target.(type) { - case *core.Node_TaskNode: - gn, err := graph.CreateNode(name) - if err != nil { - return err - } - grapNodes[name] = gn - case *core.Node_BranchNode: - gn, err := graph.CreateNode(name) - if err != nil { - return err - } - grapNodes[name] = gn - case *core.Node_WorkflowNode: - gn, err := graph.CreateNode(name) - if err != nil { - return err - } - grapNodes[name] = gn + if err := gb.addSubNodeEdge(graph, parentBranchNode, subNode); err != nil { + return nil, err } } } + return parentBranchNode, nil +} + +func (gb *graphBuilder) constructNode(prefix string, graph *cgraph.Graph, n *core.Node) (*cgraph.Node, error) { + name := getName(prefix, n.Id) + var err error + var gn *cgraph.Node - for name, node := range grapNodes { + if n.Id == "start-node" || n.Id == "end-node" { + gn, err = graph.CreateNode(name) + } else { + switch n.Target.(type) { + case *core.Node_TaskNode: + gn, err = graph.CreateNode(name) + case *core.Node_BranchNode: + branch := graph.SubGraph(fmt.Sprintf("cluster_"+n.Metadata.Name), 2) + gn, err = gb.constructBranchNode(prefix, branch, n) + case *core.Node_WorkflowNode: + gn, err = graph.CreateNode(name) + } + } + if err != nil { + return nil, err + } + gb.graphNodes[name] = gn + return gn, nil +} + +func (gb *graphBuilder) constructGraph(prefix string, graph *cgraph.Graph, w *core.CompiledWorkflow) error { + for _, n := range w.Template.Nodes { + if _, err := gb.constructNode(prefix, graph, n); err != nil { + return err + } + } + + for name, node := range gb.graphNodes { upstreamNodes, _ := w.Connections.Upstream[name] downstreamNodes, _ := w.Connections.Downstream[name] if downstreamNodes != nil { for _, n := range downstreamNodes.Ids { - dNode, ok := grapNodes[n] + dNode, ok := gb.graphNodes[n] if !ok { return fmt.Errorf("node[%s], downstream from[%s] referenced before creation", n, name) } - _, err := graph.CreateEdge(fmt.Sprintf("%s-%s", name, n), node, dNode) - if err != nil { - return err + edgeName := fmt.Sprintf("%s-%s", name, n) + if _, ok := gb.graphEdges[edgeName]; !ok { + edge, err := graph.CreateEdge(edgeName, node, dNode) + if err != nil { + return err + } + gb.graphEdges[edgeName] = edge } } } if upstreamNodes != nil { for _, n := range upstreamNodes.Ids { - uNode, ok := grapNodes[n] + uNode, ok := gb.graphNodes[n] if !ok { return fmt.Errorf("node[%s], upstream from[%s] referenced before creation", n, name) } - _, err := graph.CreateEdge(fmt.Sprintf("%s-%s", n, name), node, uNode) - if err != nil { - return err + edgeName := fmt.Sprintf("%s-%s", n, name) + if _, ok := gb.graphEdges[edgeName]; !ok { + edge, err := graph.CreateEdge(edgeName, uNode, node) + if err != nil { + return err + } + gb.graphEdges[edgeName] = edge } } } @@ -80,42 +163,45 @@ func constructGraph(prefix string, graph *cgraph.Graph, w *core.CompiledWorkflow return nil } -func CompiledWorkflowClosureToGraph(g *graphviz.Graphviz, w *core.CompiledWorkflowClosure) (*cgraph.Graph, error) { +func (gb *graphBuilder) CompiledWorkflowClosureToGraph(g *graphviz.Graphviz, w *core.CompiledWorkflowClosure) (*cgraph.Graph, error) { graph, err := g.Graph(graphviz.Directed) if err != nil { return nil, errors.Wrapf("GraphInitFailure", err, "failed to initialize graphviz") } - defer func() { - if err := graph.Close(); err != nil { - logger.Fatalf(context.TODO(), "Failed to close the graphviz Graph. err: %s", err) - } - }() + return graph, gb.constructGraph("", graph, w.Primary) +} - return graph, constructGraph("", graph, w.Primary, nil) +func NewGraphBuilder() *graphBuilder { + return &graphBuilder{ + graphNodes: make(map[string]*cgraph.Node), + graphEdges: make(map[string]*cgraph.Edge), + subWf: make(map[string]*cgraph.Graph), + } } // RenderWorkflow Renders the workflow graph to the given file -func RenderWorkflow(w *core.CompiledWorkflowClosure, file string) error { +func RenderWorkflow(w *core.CompiledWorkflowClosure) ([]byte, error) { g := graphviz.New() defer func() { if err := g.Close(); err != nil { logger.Fatalf(context.TODO(), "failed to close graphviz. err: %s", err) } }() - graph, err := CompiledWorkflowClosureToGraph(g, w) + gb := NewGraphBuilder() + graph, err := gb.CompiledWorkflowClosureToGraph(g, w) if err != nil { - return err + return nil, err } + defer func() { + if err := graph.Close(); err != nil { + logger.Fatalf(context.TODO(), "Failed to close the graphviz Graph. err: %s", err) + } + }() - logger.Infof(context.TODO(), "outputing file: %s", file) var buf bytes.Buffer - if err := g.Render(graph, graphviz.XDOT, &buf); err != nil { - return err - } - logger.Infof(context.TODO(), buf.String()) - if err := g.RenderFilename(graph, graphviz.SVG, file); err != nil { - return err + if err := g.Render(graph, graphviz.SVG, &buf); err != nil { + return nil, err } - return nil + return buf.Bytes(), nil } diff --git a/pkg/visualize/graphviz_test.go b/pkg/visualize/graphviz_test.go new file mode 100644 index 00000000..abbb1c55 --- /dev/null +++ b/pkg/visualize/graphviz_test.go @@ -0,0 +1,26 @@ +package visualize + +import ( + "bytes" + "fmt" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" + "github.com/golang/protobuf/jsonpb" + "github.com/stretchr/testify/assert" + "io/ioutil" + "testing" +) + +func TestRenderWorkflow(t *testing.T) { + r, err := ioutil.ReadFile("testdata/compiled_closure_branch_nested.json") + assert.NoError(t, err) + + i := bytes.NewReader(r) + + c := &core.CompiledWorkflowClosure{} + err = jsonpb.Unmarshal(i, c) + assert.NoError(t, err) + b, err := RenderWorkflow(c) + assert.NoError(t, err) + assert.NotNil(t, b) + fmt.Println(string(b)) +} \ No newline at end of file diff --git a/pkg/visualize/testdata/compiled_closure_branch_nested.json b/pkg/visualize/testdata/compiled_closure_branch_nested.json new file mode 100644 index 00000000..baae3d99 --- /dev/null +++ b/pkg/visualize/testdata/compiled_closure_branch_nested.json @@ -0,0 +1,553 @@ +{ + "primary": { + "template": { + "id": { + "resourceType": "WORKFLOW", + "project": "flytesnacks", + "domain": "development", + "name": "core.control_flow.run_conditions.so_nested", + "version": "v1" + }, + "metadata": {}, + "interface": { + "inputs": { + "variables": { + "my_input": { + "type": { + "simple": "FLOAT" + }, + "description": "my_input" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "nodes": [ + { + "id": "start-node" + }, + { + "id": "end-node", + "inputs": [ + { + "var": "o0", + "binding": { + "promise": { + "nodeId": "n0", + "var": "o0" + } + } + } + ] + }, + { + "id": "n0", + "metadata": { + "name": "fractions", + "retries": {}, + "interruptible": false + }, + "inputs": [ + { + "var": ".my_input", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "my_input" + } + } + } + ], + "branchNode": { + "ifElse": { + "case": { + "condition": { + "conjunction": { + "leftExpression": { + "comparison": { + "operator": "GT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 0.1 + } + } + } + }, + "rightExpression": { + "comparison": { + "operator": "LT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 1 + } + } + } + } + } + }, + "thenNode": { + "id": "n0-n0", + "metadata": { + "name": "inner_fractions", + "retries": {}, + "interruptible": false + }, + "inputs": [ + { + "var": ".my_input", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "my_input" + } + } + } + ], + "branchNode": { + "ifElse": { + "case": { + "condition": { + "comparison": { + "operator": "LT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 0.5 + } + } + } + }, + "thenNode": { + "id": "n0-n0-n0-n0", + "metadata": { + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.double", + "retries": {}, + "interruptible": false + }, + "inputs": [ + { + "var": "n", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "my_input" + } + } + } + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "flytesnacks", + "domain": "development", + "name": "core.control_flow.run_conditions.double", + "version": "v1" + } + } + } + }, + "other": [ + { + "condition": { + "conjunction": { + "leftExpression": { + "comparison": { + "operator": "GT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 0.5 + } + } + } + }, + "rightExpression": { + "comparison": { + "operator": "LT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 0.7 + } + } + } + } + } + }, + "thenNode": { + "id": "n0-n0-n0-n1", + "metadata": { + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.square", + "retries": {}, + "interruptible": false + }, + "inputs": [ + { + "var": "n", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "my_input" + } + } + } + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "flytesnacks", + "domain": "development", + "name": "core.control_flow.run_conditions.square", + "version": "v1" + } + } + } + } + ], + "error": { + "failedNodeId": "inner_fractions", + "message": "Only \u003c0.7 allowed" + } + } + } + } + }, + "other": [ + { + "condition": { + "conjunction": { + "leftExpression": { + "comparison": { + "operator": "GT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 1 + } + } + } + }, + "rightExpression": { + "comparison": { + "operator": "LT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 10 + } + } + } + } + } + }, + "thenNode": { + "id": "n0-n1", + "metadata": { + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.square", + "retries": {}, + "interruptible": false + }, + "inputs": [ + { + "var": "n", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "my_input" + } + } + } + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "flytesnacks", + "domain": "development", + "name": "core.control_flow.run_conditions.square", + "version": "v1" + } + } + } + } + ], + "elseNode": { + "id": "n0-n2", + "metadata": { + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.double", + "retries": {}, + "interruptible": false + }, + "inputs": [ + { + "var": "n", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "my_input" + } + } + } + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "flytesnacks", + "domain": "development", + "name": "core.control_flow.run_conditions.double", + "version": "v1" + } + } + } + } + } + } + ], + "outputs": [ + { + "var": "o0", + "binding": { + "promise": { + "nodeId": "n0", + "var": "o0" + } + } + } + ], + "metadataDefaults": {} + }, + "connections": { + "downstream": { + "n0": { + "ids": [ + "end-node" + ] + }, + "start-node": { + "ids": [ + "n0" + ] + } + }, + "upstream": { + "end-node": { + "ids": [ + "n0" + ] + }, + "n0": { + "ids": [ + "start-node" + ] + }, + "n0-n0": { + "ids": [ + "start-node" + ] + }, + "n0-n1": { + "ids": [ + "start-node" + ] + }, + "n0-n2": { + "ids": [ + "start-node" + ] + }, + "n0-n0-n0-n0": { + "ids": [ + "start-node" + ] + }, + "n0-n0-n0-n1": { + "ids": [ + "start-node" + ] + }, + "n1": { + "ids": [ + "start-node" + ] + }, + "n2": { + "ids": [ + "start-node" + ] + } + } + } + }, + "tasks": [ + { + "template": { + "id": { + "resourceType": "TASK", + "project": "flytesnacks", + "domain": "development", + "name": "core.control_flow.run_conditions.double", + "version": "v1" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.0.0+develop", + "flavor": "python" + }, + "retries": {}, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "n": { + "type": { + "simple": "FLOAT" + }, + "description": "n" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "flytecookbook:core-d5fa3ecfaca02f9b83957c68fd5fe3c9082ccc59", + "args": [ + "pyflyte-execute", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}", + "--resolver", + "flytekit.core.python_auto_container.default_task_resolver", + "--", + "task-module", + "core.control_flow.run_conditions", + "task-name", + "double" + ], + "resources": {}, + "env": [ + { + "key": "FLYTE_INTERNAL_CONFIGURATION_PATH", + "value": "/root/sandbox.config" + }, + { + "key": "FLYTE_INTERNAL_IMAGE", + "value": "flytecookbook:core-d5fa3ecfaca02f9b83957c68fd5fe3c9082ccc59" + } + ] + } + } + }, + { + "template": { + "id": { + "resourceType": "TASK", + "project": "flytesnacks", + "domain": "development", + "name": "core.control_flow.run_conditions.square", + "version": "v1" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.0.0+develop", + "flavor": "python" + }, + "retries": {}, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "n": { + "type": { + "simple": "FLOAT" + }, + "description": "n" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "flytecookbook:core-d5fa3ecfaca02f9b83957c68fd5fe3c9082ccc59", + "args": [ + "pyflyte-execute", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}", + "--resolver", + "flytekit.core.python_auto_container.default_task_resolver", + "--", + "task-module", + "core.control_flow.run_conditions", + "task-name", + "square" + ], + "resources": {}, + "env": [ + { + "key": "FLYTE_INTERNAL_CONFIGURATION_PATH", + "value": "/root/sandbox.config" + }, + { + "key": "FLYTE_INTERNAL_IMAGE", + "value": "flytecookbook:core-d5fa3ecfaca02f9b83957c68fd5fe3c9082ccc59" + } + ] + } + } + } + ] +} \ No newline at end of file From cc9c35aef757bcf5dae739da80858679d853e4df Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Sun, 30 May 2021 19:54:34 -0700 Subject: [PATCH 03/16] updated Signed-off-by: Ketan Umare Signed-off-by: Prafulla Mahindrakar --- cmd/get/workflow.go | 1 + pkg/visualize/graphviz.go | 15 ++++++++------- pkg/visualize/graphviz_test.go | 7 ++++--- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/cmd/get/workflow.go b/cmd/get/workflow.go index ceaae9d8..9cfc35a6 100644 --- a/cmd/get/workflow.go +++ b/cmd/get/workflow.go @@ -3,6 +3,7 @@ package get import ( "context" "fmt" + "github.com/flyteorg/flytectl/pkg/visualize" workflowconfig "github.com/flyteorg/flytectl/cmd/config/subcommand/workflow" diff --git a/pkg/visualize/graphviz.go b/pkg/visualize/graphviz.go index bb1379a8..88b44698 100644 --- a/pkg/visualize/graphviz.go +++ b/pkg/visualize/graphviz.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" "github.com/flyteorg/flytestdlib/errors" "github.com/flyteorg/flytestdlib/logger" @@ -19,9 +20,9 @@ func getName(prefix, id string) string { } type graphBuilder struct { - graphNodes map[string]*cgraph.Node - graphEdges map[string]*cgraph.Edge - subWf map[string]*cgraph.Graph + graphNodes map[string]*cgraph.Node + graphEdges map[string]*cgraph.Edge + subWf map[string]*cgraph.Graph } func (gb *graphBuilder) addSubNodeEdge(graph *cgraph.Graph, parentNode, n *cgraph.Node) error { @@ -125,8 +126,8 @@ func (gb *graphBuilder) constructGraph(prefix string, graph *cgraph.Graph, w *co } for name, node := range gb.graphNodes { - upstreamNodes, _ := w.Connections.Upstream[name] - downstreamNodes, _ := w.Connections.Downstream[name] + upstreamNodes := w.Connections.Upstream[name] + downstreamNodes := w.Connections.Downstream[name] if downstreamNodes != nil { for _, n := range downstreamNodes.Ids { dNode, ok := gb.graphNodes[n] @@ -172,7 +173,7 @@ func (gb *graphBuilder) CompiledWorkflowClosureToGraph(g *graphviz.Graphviz, w * return graph, gb.constructGraph("", graph, w.Primary) } -func NewGraphBuilder() *graphBuilder { +func newGraphBuilder() *graphBuilder { return &graphBuilder{ graphNodes: make(map[string]*cgraph.Node), graphEdges: make(map[string]*cgraph.Edge), @@ -188,7 +189,7 @@ func RenderWorkflow(w *core.CompiledWorkflowClosure) ([]byte, error) { logger.Fatalf(context.TODO(), "failed to close graphviz. err: %s", err) } }() - gb := NewGraphBuilder() + gb := newGraphBuilder() graph, err := gb.CompiledWorkflowClosureToGraph(g, w) if err != nil { return nil, err diff --git a/pkg/visualize/graphviz_test.go b/pkg/visualize/graphviz_test.go index abbb1c55..5647bfca 100644 --- a/pkg/visualize/graphviz_test.go +++ b/pkg/visualize/graphviz_test.go @@ -3,11 +3,12 @@ package visualize import ( "bytes" "fmt" + "io/ioutil" + "testing" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" "github.com/golang/protobuf/jsonpb" "github.com/stretchr/testify/assert" - "io/ioutil" - "testing" ) func TestRenderWorkflow(t *testing.T) { @@ -23,4 +24,4 @@ func TestRenderWorkflow(t *testing.T) { assert.NoError(t, err) assert.NotNil(t, b) fmt.Println(string(b)) -} \ No newline at end of file +} From b8908c14743cd44cd50e9259e59331a66ee4c313 Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Mon, 31 May 2021 22:59:45 -0700 Subject: [PATCH 04/16] Working nested graphs - v0 Signed-off-by: Ketan Umare Signed-off-by: Prafulla Mahindrakar --- .../subcommand/workflow/config_flags.go | 20 ++------- .../subcommand/workflow/config_flags_test.go | 44 +++++++++++++++++++ .../subcommand/workflow/workflow_config.go | 24 +++++++++- cmd/get/workflow.go | 25 ++++++++--- pkg/printer/outputformat_enumer.go | 13 +++--- pkg/printer/printer.go | 3 -- pkg/visualize/graphviz.go | 7 ++- pkg/visualize/graphviz_test.go | 7 +-- 8 files changed, 105 insertions(+), 38 deletions(-) diff --git a/cmd/config/subcommand/workflow/config_flags.go b/cmd/config/subcommand/workflow/config_flags.go index 61f6245c..97db2f9c 100755 --- a/cmd/config/subcommand/workflow/config_flags.go +++ b/cmd/config/subcommand/workflow/config_flags.go @@ -28,15 +28,6 @@ func (Config) elemValueOrNil(v interface{}) interface{} { return v } -func (Config) mustJsonMarshal(v interface{}) string { - raw, err := json.Marshal(v) - if err != nil { - panic(err) - } - - return string(raw) -} - func (Config) mustMarshalJSON(v json.Marshaler) string { raw, err := v.MarshalJSON() if err != nil { @@ -50,12 +41,9 @@ func (Config) mustMarshalJSON(v json.Marshaler) string { // flags is json-name.json-sub-name... etc. func (cfg Config) GetPFlagSet(prefix string) *pflag.FlagSet { cmdFlags := pflag.NewFlagSet("Config", pflag.ExitOnError) - cmdFlags.StringVar(&(DefaultConfig.Version), fmt.Sprintf("%v%v", prefix, "version"), *new(string), "version of the workflow to be fetched.") - cmdFlags.BoolVar(&(DefaultConfig.Latest), fmt.Sprintf("%v%v", prefix, "latest"), *new(bool), " flag to indicate to fetch the latest version, version flag will be ignored in this case") - cmdFlags.StringVar(&(DefaultConfig.Filter.FieldSelector), fmt.Sprintf("%v%v", prefix, "filter.field-selector"), *new(string), "Specifies the Field selector") - cmdFlags.StringVar((&DefaultConfig.Filter.SortBy), fmt.Sprintf("%v%v", prefix, "filter.sort-by"), *new(string), "Specifies which field to sort result by ") - cmdFlags.Int32Var((&DefaultConfig.Filter.Limit), fmt.Sprintf("%v%v", prefix, "filter.limit"), 100, "Specifies the limit") - cmdFlags.BoolVar((&DefaultConfig.Filter.Asc), fmt.Sprintf("%v%v", prefix, "filter.asc"), false, "Specifies the sorting order. By default flytectl sort result in descending order") - + cmdFlags.StringVar(&(DefaultConfig.Version), fmt.Sprintf("%v%v", prefix, "version"), DefaultConfig.Version, "version of the workflow to be fetched.") + cmdFlags.BoolVar(&(DefaultConfig.Latest), fmt.Sprintf("%v%v", prefix, "latest"), DefaultConfig.Latest, " flag to indicate to fetch the latest version, version flag will be ignored in this case") + cmdFlags.StringVar(&(DefaultConfig.Visualize), fmt.Sprintf("%v%v", prefix, "visualize"), DefaultConfig.Visualize, "optional flag to visualize a workflow as one of [png, dot, svg, jpg]") + cmdFlags.StringVar(&(DefaultConfig.OutputFile), fmt.Sprintf("%v%v", prefix, "output_file"), DefaultConfig.OutputFile, "path and a filename of where the output image should be dumped. This can only be used in concert with visualize") return cmdFlags } diff --git a/cmd/config/subcommand/workflow/config_flags_test.go b/cmd/config/subcommand/workflow/config_flags_test.go index 86998d77..fc7160d1 100755 --- a/cmd/config/subcommand/workflow/config_flags_test.go +++ b/cmd/config/subcommand/workflow/config_flags_test.go @@ -183,4 +183,48 @@ func TestConfig_SetFlags(t *testing.T) { } }) }) + t.Run("Test_visualize", func(t *testing.T) { + t.Run("DefaultValue", func(t *testing.T) { + // Test that default value is set properly + if vString, err := cmdFlags.GetString("visualize"); err == nil { + assert.Equal(t, string(DefaultConfig.Visualize), vString) + } else { + assert.FailNow(t, err.Error()) + } + }) + + t.Run("Override", func(t *testing.T) { + testValue := "1" + + cmdFlags.Set("visualize", testValue) + if vString, err := cmdFlags.GetString("visualize"); err == nil { + testDecodeJson_Config(t, fmt.Sprintf("%v", vString), &actual.Visualize) + + } else { + assert.FailNow(t, err.Error()) + } + }) + }) + t.Run("Test_output_file", func(t *testing.T) { + t.Run("DefaultValue", func(t *testing.T) { + // Test that default value is set properly + if vString, err := cmdFlags.GetString("output_file"); err == nil { + assert.Equal(t, string(DefaultConfig.OutputFile), vString) + } else { + assert.FailNow(t, err.Error()) + } + }) + + t.Run("Override", func(t *testing.T) { + testValue := "1" + + cmdFlags.Set("output_file", testValue) + if vString, err := cmdFlags.GetString("output_file"); err == nil { + testDecodeJson_Config(t, fmt.Sprintf("%v", vString), &actual.OutputFile) + + } else { + assert.FailNow(t, err.Error()) + } + }) + }) } diff --git a/cmd/config/subcommand/workflow/workflow_config.go b/cmd/config/subcommand/workflow/workflow_config.go index 8051d454..55b0a507 100644 --- a/cmd/config/subcommand/workflow/workflow_config.go +++ b/cmd/config/subcommand/workflow/workflow_config.go @@ -1,6 +1,11 @@ package workflow -import "github.com/flyteorg/flytectl/pkg/filters" +import ( + "fmt" + "github.com/flyteorg/flytectl/pkg/filters" + "github.com/goccy/go-graphviz" + "strings" +) //go:generate pflags Config --default-var DefaultConfig @@ -15,4 +20,21 @@ type Config struct { Version string `json:"version" pflag:",version of the workflow to be fetched."` Latest bool `json:"latest" pflag:", flag to indicate to fetch the latest version, version flag will be ignored in this case"` Filter filters.Filters `json:"filter" pflag:","` + Visualize string `json:"visualize" pflag:",optional flag to visualize a workflow as one of [png, dot, svg, jpg]"` + OutputFile string `json:"output_file" pflag:",path and a filename of where the output image should be dumped. This can only be used in concert with visualize"` } + +// GraphvizFormat returns the graphviz format based on the user-specified value or an error +func (cfg *Config) GraphvizFormat() (graphviz.Format, error) { + switch strings.ToLower(cfg.Visualize) { + case "png": + return graphviz.PNG, nil + case "svg": + return graphviz.SVG, nil + case "dot": + return graphviz.XDOT, nil + case "jpg": + return graphviz.JPG, nil + } + return graphviz.SVG, fmt.Errorf("unsupported visualization format [%s]. Supported [png, dot, svg, jpg]", cfg.Visualize) +} \ No newline at end of file diff --git a/cmd/get/workflow.go b/cmd/get/workflow.go index 9cfc35a6..e6c8c7a7 100644 --- a/cmd/get/workflow.go +++ b/cmd/get/workflow.go @@ -3,6 +3,10 @@ package get import ( "context" "fmt" + "io/fs" + "io/ioutil" + + "github.com/flyteorg/flytestdlib/errors" "github.com/flyteorg/flytectl/pkg/visualize" @@ -99,17 +103,26 @@ func getWorkflowFunc(ctx context.Context, args []string, cmdCtx cmdCore.CommandC return err } logger.Debugf(ctx, "Retrieved %v workflow", len(workflows)) - if config.GetConfig().MustOutputFormat() == printer.OutputFormatSVG { - b, err := visualize.RenderWorkflow(workflows[0].Closure.CompiledWorkflow) + err = adminPrinter.Print(config.GetConfig().MustOutputFormat(), workflowColumns, WorkflowToProtoMessages(workflows)...) + if err != nil { + return err + } + if len(workflows) > 0 && workflowconfig.DefaultConfig.Visualize != "" { + f, err := workflowconfig.DefaultConfig.GraphvizFormat() if err != nil { return err } - fmt.Println(string(b)) - } else { - err = adminPrinter.Print(config.GetConfig().MustOutputFormat(), workflowColumns, WorkflowToProtoMessages(workflows)...) + if workflowconfig.DefaultConfig.OutputFile == "" { + return fmt.Errorf("--visualize should be accompanied with a file-name using --output_file option") + } + b, err := visualize.RenderWorkflow(workflows[0].Closure.CompiledWorkflow, f) if err != nil { - return err + return errors.Wrapf("VisualizationError", err, "failed to visualize workflow") + } + if err := ioutil.WriteFile(workflowconfig.DefaultConfig.OutputFile, b, fs.ModePerm); err != nil { + return errors.Wrapf("FileWriteError", err, "failed to write visualization file") } + fmt.Printf("File [%s] written", workflowconfig.DefaultConfig.OutputFile) } return nil } diff --git a/pkg/printer/outputformat_enumer.go b/pkg/printer/outputformat_enumer.go index d1a3a27b..b10b4514 100644 --- a/pkg/printer/outputformat_enumer.go +++ b/pkg/printer/outputformat_enumer.go @@ -8,9 +8,9 @@ import ( "fmt" ) -const _OutputFormatName = "TABLEJSONYAMLSVG" +const _OutputFormatName = "TABLEJSONYAML" -var _OutputFormatIndex = [...]uint8{0, 5, 9, 13, 16} +var _OutputFormatIndex = [...]uint8{0, 5, 9, 13} func (i OutputFormat) String() string { if i >= OutputFormat(len(_OutputFormatIndex)-1) { @@ -19,13 +19,12 @@ func (i OutputFormat) String() string { return _OutputFormatName[_OutputFormatIndex[i]:_OutputFormatIndex[i+1]] } -var _OutputFormatValues = []OutputFormat{0, 1, 2, 3} +var _OutputFormatValues = []OutputFormat{0, 1, 2} var _OutputFormatNameToValueMap = map[string]OutputFormat{ - _OutputFormatName[0:5]: 0, - _OutputFormatName[5:9]: 1, - _OutputFormatName[9:13]: 2, - _OutputFormatName[13:16]: 3, + _OutputFormatName[0:5]: 0, + _OutputFormatName[5:9]: 1, + _OutputFormatName[9:13]: 2, } // OutputFormatString retrieves an enum value from the enum constants string name. diff --git a/pkg/printer/printer.go b/pkg/printer/printer.go index e7e47788..fa86127b 100644 --- a/pkg/printer/printer.go +++ b/pkg/printer/printer.go @@ -22,7 +22,6 @@ const ( OutputFormatTABLE OutputFormat = iota OutputFormatJSON OutputFormatYAML - OutputFormatSVG ) func OutputFormats() []string { @@ -116,8 +115,6 @@ func (p Printer) Print(format OutputFormat, columns []Column, messages ...proto. // Factory Method for all printer switch format { - case OutputFormatSVG: - return fmt.Errorf("OutputFormat SVG is only support for Workflow type of object currently") case OutputFormatJSON, OutputFormatYAML: // Print protobuf to json buf := new(bytes.Buffer) encoder := json.NewEncoder(buf) diff --git a/pkg/visualize/graphviz.go b/pkg/visualize/graphviz.go index 88b44698..76b3477d 100644 --- a/pkg/visualize/graphviz.go +++ b/pkg/visualize/graphviz.go @@ -119,6 +119,9 @@ func (gb *graphBuilder) constructNode(prefix string, graph *cgraph.Graph, n *cor } func (gb *graphBuilder) constructGraph(prefix string, graph *cgraph.Graph, w *core.CompiledWorkflow) error { + if w == nil || w.Template == nil { + return nil + } for _, n := range w.Template.Nodes { if _, err := gb.constructNode(prefix, graph, n); err != nil { return err @@ -182,7 +185,7 @@ func newGraphBuilder() *graphBuilder { } // RenderWorkflow Renders the workflow graph to the given file -func RenderWorkflow(w *core.CompiledWorkflowClosure) ([]byte, error) { +func RenderWorkflow(w *core.CompiledWorkflowClosure, o graphviz.Format) ([]byte, error) { g := graphviz.New() defer func() { if err := g.Close(); err != nil { @@ -201,7 +204,7 @@ func RenderWorkflow(w *core.CompiledWorkflowClosure) ([]byte, error) { }() var buf bytes.Buffer - if err := g.Render(graph, graphviz.SVG, &buf); err != nil { + if err := g.Render(graph, o, &buf); err != nil { return nil, err } return buf.Bytes(), nil diff --git a/pkg/visualize/graphviz_test.go b/pkg/visualize/graphviz_test.go index 5647bfca..df1b7f23 100644 --- a/pkg/visualize/graphviz_test.go +++ b/pkg/visualize/graphviz_test.go @@ -2,7 +2,8 @@ package visualize import ( "bytes" - "fmt" + "github.com/goccy/go-graphviz" + "io/fs" "io/ioutil" "testing" @@ -20,8 +21,8 @@ func TestRenderWorkflow(t *testing.T) { c := &core.CompiledWorkflowClosure{} err = jsonpb.Unmarshal(i, c) assert.NoError(t, err) - b, err := RenderWorkflow(c) + b, err := RenderWorkflow(c, graphviz.PNG) assert.NoError(t, err) assert.NotNil(t, b) - fmt.Println(string(b)) + ioutil.WriteFile("/tmp/image.png", b, fs.ModePerm) } From 6f05a5efd2b04a43f19c171410e9f38303305016 Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Mon, 31 May 2021 23:23:14 -0700 Subject: [PATCH 05/16] better graph Signed-off-by: Ketan Umare Signed-off-by: Prafulla Mahindrakar --- pkg/visualize/graphviz.go | 75 ++++++++++++++++++++++++++++++++++----- 1 file changed, 66 insertions(+), 9 deletions(-) diff --git a/pkg/visualize/graphviz.go b/pkg/visualize/graphviz.go index 76b3477d..8dfe72c4 100644 --- a/pkg/visualize/graphviz.go +++ b/pkg/visualize/graphviz.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "strings" "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" "github.com/flyteorg/flytestdlib/errors" @@ -12,6 +13,63 @@ import ( "github.com/goccy/go-graphviz/cgraph" ) +func constructStartNode(graph *cgraph.Graph) (*cgraph.Node, error) { + gn, err := graph.CreateNode("start-node") + if err != nil { + return nil, err + } + gn.SetLabel("start") + gn.SetShape(cgraph.CircleShape) + gn.SetColor("green") + return gn, nil +} + +func constructEndNode(graph *cgraph.Graph) (*cgraph.Node, error) { + gn, err := graph.CreateNode("end-node") + if err != nil { + return nil, err + } + gn.SetLabel("end") + gn.SetShape(cgraph.CircleShape) + gn.SetColor("red") + return gn, nil +} + +func constructTaskNode(name string, graph *cgraph.Graph, n *core.Node) (*cgraph.Node, error) { + gn, err := graph.CreateNode(name) + if err != nil { + return nil, err + } + if n.Metadata != nil && n.Metadata.Name != ""{ + v := strings.LastIndexAny(n.Metadata.Name, ".") + gn.SetLabel(n.Metadata.Name[v+1:]) + } + gn.SetShape(cgraph.BoxShape) + return gn, nil +} + +func constructErrorNode(name string, graph *cgraph.Graph, m string) (*cgraph.Node, error) { + gn, err := graph.CreateNode(name) + if err != nil { + return nil, err + } + gn.SetLabel(m) + gn.SetShape(cgraph.PentagonShape) + return gn, nil +} + +func constructBranchConditionNode(name string, graph *cgraph.Graph, n *core.Node) (*cgraph.Node, error) { + gn, err := graph.CreateNode(name) + if err != nil { + return nil, err + } + if n.Metadata != nil && n.Metadata.Name != ""{ + gn.SetLabel(n.Metadata.Name) + } + gn.SetShape(cgraph.DiamondShape) + return gn, nil +} + func getName(prefix, id string) string { if prefix != "" { return prefix + "-" + id @@ -38,13 +96,11 @@ func (gb *graphBuilder) addSubNodeEdge(graph *cgraph.Graph, parentNode, n *cgrap } func (gb *graphBuilder) constructBranchNode(prefix string, graph *cgraph.Graph, n *core.Node) (*cgraph.Node, error) { - parentBranchNodeName := getName(prefix, n.Id) - parentBranchNode, err := graph.CreateNode(parentBranchNodeName) + parentBranchNode, err := constructBranchConditionNode(getName(prefix, n.Id), graph, n) if err != nil { return nil, err } - parentBranchNode.SetLabel(n.Metadata.Name) - gb.graphNodes[parentBranchNodeName] = parentBranchNode + gb.graphNodes[parentBranchNode.Name()] = parentBranchNode if n.GetBranchNode().GetIfElse() == nil { return parentBranchNode, nil @@ -60,8 +116,7 @@ func (gb *graphBuilder) constructBranchNode(prefix string, graph *cgraph.Graph, if n.GetBranchNode().GetIfElse().GetError() != nil { name := fmt.Sprintf("%s-error", parentBranchNode.Name()) - subNode, err := graph.CreateNode(name) - subNode.SetLabel(n.GetBranchNode().GetIfElse().GetError().Message) + subNode, err := constructErrorNode(name, graph, n.GetBranchNode().GetIfElse().GetError().Message) if err != nil { return nil, err } @@ -98,12 +153,14 @@ func (gb *graphBuilder) constructNode(prefix string, graph *cgraph.Graph, n *cor var err error var gn *cgraph.Node - if n.Id == "start-node" || n.Id == "end-node" { - gn, err = graph.CreateNode(name) + if n.Id == "start-node"{ + gn, err = constructStartNode(graph) + } else if n.Id == "end-node" { + gn, err = constructEndNode(graph) } else { switch n.Target.(type) { case *core.Node_TaskNode: - gn, err = graph.CreateNode(name) + gn, err = constructTaskNode(name, graph, n) case *core.Node_BranchNode: branch := graph.SubGraph(fmt.Sprintf("cluster_"+n.Metadata.Name), 2) gn, err = gb.constructBranchNode(prefix, branch, n) From ad4d4c38044fdfe37c6077ab46a01dbae83e4dc4 Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Mon, 31 May 2021 23:27:19 -0700 Subject: [PATCH 06/16] update todos Signed-off-by: Ketan Umare Signed-off-by: Prafulla Mahindrakar --- pkg/visualize/graphviz.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/visualize/graphviz.go b/pkg/visualize/graphviz.go index 8dfe72c4..5d8b406b 100644 --- a/pkg/visualize/graphviz.go +++ b/pkg/visualize/graphviz.go @@ -35,7 +35,8 @@ func constructEndNode(graph *cgraph.Graph) (*cgraph.Node, error) { return gn, nil } -func constructTaskNode(name string, graph *cgraph.Graph, n *core.Node) (*cgraph.Node, error) { +func constructTaskNode(name string, graph *cgraph.Graph, n *core.Node, t *core.CompiledTask) (*cgraph.Node, error) { + // TODO Add task task type and other information from the task template gn, err := graph.CreateNode(name) if err != nil { return nil, err @@ -81,6 +82,7 @@ type graphBuilder struct { graphNodes map[string]*cgraph.Node graphEdges map[string]*cgraph.Edge subWf map[string]*cgraph.Graph + // TODO Add tasks } func (gb *graphBuilder) addSubNodeEdge(graph *cgraph.Graph, parentNode, n *cgraph.Node) error { @@ -160,7 +162,8 @@ func (gb *graphBuilder) constructNode(prefix string, graph *cgraph.Graph, n *cor } else { switch n.Target.(type) { case *core.Node_TaskNode: - gn, err = constructTaskNode(name, graph, n) + // TODO add task lookup + gn, err = constructTaskNode(name, graph, n, nil) case *core.Node_BranchNode: branch := graph.SubGraph(fmt.Sprintf("cluster_"+n.Metadata.Name), 2) gn, err = gb.constructBranchNode(prefix, branch, n) From 644ff2b0fe00d71cc9d682ebbaf30982db5a0f38 Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Tue, 1 Jun 2021 15:32:08 -0700 Subject: [PATCH 07/16] improved graph Signed-off-by: Ketan Umare Signed-off-by: Prafulla Mahindrakar --- pkg/visualize/graphviz.go | 152 +++++++++++++++++++++++++++----------- 1 file changed, 108 insertions(+), 44 deletions(-) diff --git a/pkg/visualize/graphviz.go b/pkg/visualize/graphviz.go index 5d8b406b..c7de8df8 100644 --- a/pkg/visualize/graphviz.go +++ b/pkg/visualize/graphviz.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "github.com/flyteorg/flyteidl/clients/go/coreutils" "strings" "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" @@ -13,58 +14,90 @@ import ( "github.com/goccy/go-graphviz/cgraph" ) -func constructStartNode(graph *cgraph.Graph) (*cgraph.Node, error) { +func operandToString(op *core.Operand) string { + if op.GetPrimitive() != nil { + l, err := coreutils.ExtractFromLiteral(&core.Literal{Value: &core.Literal_Scalar{ + Scalar: &core.Scalar{ + Value: &core.Scalar_Primitive{ + Primitive: op.GetPrimitive(), + }, + }, + }}) + if err != nil { + return err.Error() + } + return fmt.Sprintf("%v", l) + } + return op.GetVar() +} + +func comparisonToString(expr *core.ComparisonExpression) string { + return fmt.Sprintf("%s %s %s", operandToString(expr.LeftValue), expr.Operator.String(), operandToString(expr.RightValue)) +} + +func conjunctionToString(expr *core.ConjunctionExpression) string { + return fmt.Sprintf("(%s) %s (%s)", booleanExprToString(expr.LeftExpression), expr.Operator.String(), booleanExprToString(expr.RightExpression)) +} + +func booleanExprToString(expr *core.BooleanExpression) string { + if expr.GetConjunction() != nil { + return conjunctionToString(expr.GetConjunction()) + } + return comparisonToString(expr.GetComparison()) +} + +func constructStartNode(graph *cgraph.Graph) (*cgraph.Node, error) { gn, err := graph.CreateNode("start-node") if err != nil { return nil, err } gn.SetLabel("start") - gn.SetShape(cgraph.CircleShape) + gn.SetShape(cgraph.DoubleCircleShape) gn.SetColor("green") return gn, nil } -func constructEndNode(graph *cgraph.Graph) (*cgraph.Node, error) { +func constructEndNode(graph *cgraph.Graph) (*cgraph.Node, error) { gn, err := graph.CreateNode("end-node") if err != nil { return nil, err } gn.SetLabel("end") - gn.SetShape(cgraph.CircleShape) + gn.SetShape(cgraph.DoubleCircleShape) gn.SetColor("red") return gn, nil } -func constructTaskNode(name string, graph *cgraph.Graph, n *core.Node, t *core.CompiledTask) (*cgraph.Node, error) { - // TODO Add task task type and other information from the task template +func constructTaskNode(name string, graph *cgraph.Graph, n *core.Node, t *core.CompiledTask) (*cgraph.Node, error) { gn, err := graph.CreateNode(name) if err != nil { return nil, err } - if n.Metadata != nil && n.Metadata.Name != ""{ + if n.Metadata != nil && n.Metadata.Name != "" { v := strings.LastIndexAny(n.Metadata.Name, ".") - gn.SetLabel(n.Metadata.Name[v+1:]) + gn.SetLabel(fmt.Sprintf("%s [%s]", n.Metadata.Name[v+1:], t.Template.Type)) } gn.SetShape(cgraph.BoxShape) return gn, nil } -func constructErrorNode(name string, graph *cgraph.Graph, m string) (*cgraph.Node, error) { +func constructErrorNode(name string, graph *cgraph.Graph, m string) (*cgraph.Node, error) { gn, err := graph.CreateNode(name) if err != nil { return nil, err } gn.SetLabel(m) - gn.SetShape(cgraph.PentagonShape) + gn.SetShape(cgraph.BoxShape) + gn.SetColor("red") return gn, nil } -func constructBranchConditionNode(name string, graph *cgraph.Graph, n *core.Node) (*cgraph.Node, error) { +func constructBranchConditionNode(name string, graph *cgraph.Graph, n *core.Node) (*cgraph.Node, error) { gn, err := graph.CreateNode(name) if err != nil { return nil, err } - if n.Metadata != nil && n.Metadata.Name != ""{ + if n.Metadata != nil && n.Metadata.Name != "" { gn.SetLabel(n.Metadata.Name) } gn.SetShape(cgraph.DiamondShape) @@ -79,19 +112,30 @@ func getName(prefix, id string) string { } type graphBuilder struct { + // Mutated as graph is built graphNodes map[string]*cgraph.Node + // Mutated as graph is built. lookup table for all graphviz compiled edges. graphEdges map[string]*cgraph.Edge + // lookup table for all graphviz compiled subgraphs subWf map[string]*cgraph.Graph - // TODO Add tasks + // a lookup table for all tasks in the graph + tasks map[string]*core.CompiledTask + // a lookup for all node clusters. This is to remap the edges to the cluster itself (instead of the node) + // this is useful in the case of branchNodes and subworkflow nodes + nodeClusters map[string]*cgraph.Graph } -func (gb *graphBuilder) addSubNodeEdge(graph *cgraph.Graph, parentNode, n *cgraph.Node) error { +func (gb *graphBuilder) addBranchSubNodeEdge(graph *cgraph.Graph, parentNode, n *cgraph.Node, label string) error { edgeName := fmt.Sprintf("%s-%s", parentNode.Name(), n.Name()) if _, ok := gb.graphEdges[edgeName]; !ok { edge, err := graph.CreateEdge(edgeName, parentNode, n) if err != nil { return err } + edge.SetLabel(label) + if c, ok := gb.nodeClusters[n.Name()]; ok { + edge.SetLogicalHead(c.Name()) + } gb.graphEdges[edgeName] = edge } return nil @@ -112,7 +156,7 @@ func (gb *graphBuilder) constructBranchNode(prefix string, graph *cgraph.Graph, if err != nil { return nil, err } - if err := gb.addSubNodeEdge(graph, parentBranchNode, subNode); err != nil { + if err := gb.addBranchSubNodeEdge(graph, parentBranchNode, subNode, booleanExprToString(n.GetBranchNode().GetIfElse().Case.Condition)); err != nil { return nil, err } @@ -123,7 +167,7 @@ func (gb *graphBuilder) constructBranchNode(prefix string, graph *cgraph.Graph, return nil, err } gb.graphNodes[name] = subNode - if err := gb.addSubNodeEdge(graph, parentBranchNode, subNode); err != nil { + if err := gb.addBranchSubNodeEdge(graph, parentBranchNode, subNode, "orElse - Fail"); err != nil { return nil, err } } else { @@ -131,7 +175,7 @@ func (gb *graphBuilder) constructBranchNode(prefix string, graph *cgraph.Graph, if err != nil { return nil, err } - if err := gb.addSubNodeEdge(graph, parentBranchNode, subNode); err != nil { + if err := gb.addBranchSubNodeEdge(graph, parentBranchNode, subNode, "orElse"); err != nil { return nil, err } } @@ -142,7 +186,7 @@ func (gb *graphBuilder) constructBranchNode(prefix string, graph *cgraph.Graph, if err != nil { return nil, err } - if err := gb.addSubNodeEdge(graph, parentBranchNode, subNode); err != nil { + if err := gb.addBranchSubNodeEdge(graph, parentBranchNode, subNode, booleanExprToString(c.Condition)); err != nil { return nil, err } } @@ -155,18 +199,23 @@ func (gb *graphBuilder) constructNode(prefix string, graph *cgraph.Graph, n *cor var err error var gn *cgraph.Node - if n.Id == "start-node"{ + if n.Id == "start-node" { gn, err = constructStartNode(graph) } else if n.Id == "end-node" { gn, err = constructEndNode(graph) } else { switch n.Target.(type) { case *core.Node_TaskNode: - // TODO add task lookup - gn, err = constructTaskNode(name, graph, n, nil) + tID := n.GetTaskNode().GetReferenceId().String() + t, ok := gb.tasks[tID] + if !ok { + return nil, fmt.Errorf("failed to find task [%s] in closure", tID) + } + gn, err = constructTaskNode(name, graph, n, t) case *core.Node_BranchNode: branch := graph.SubGraph(fmt.Sprintf("cluster_"+n.Metadata.Name), 2) gn, err = gb.constructBranchNode(prefix, branch, n) + gb.nodeClusters[name] = branch case *core.Node_WorkflowNode: gn, err = graph.CreateNode(name) } @@ -178,6 +227,31 @@ func (gb *graphBuilder) constructNode(prefix string, graph *cgraph.Graph, n *cor return gn, nil } +func (gb *graphBuilder) addEdge(fromNodeName, toNodeName string, graph *cgraph.Graph) error { + toNode, toOk := gb.graphNodes[toNodeName] + fromNode, fromOk := gb.graphNodes[fromNodeName] + if !toOk || !fromOk { + return fmt.Errorf("nodes[%s] -> [%s] referenced before creation", fromNodeName, toNodeName) + } + edgeName := fmt.Sprintf("%s-%s", fromNodeName, toNodeName) + if _, ok := gb.graphEdges[edgeName]; !ok { + edge, err := graph.CreateEdge(edgeName, fromNode, toNode) + if err != nil { + return err + } + // Now lets check that the toNode or the fromNode is a cluster. If so then following this thread, + // https://stackoverflow.com/questions/2012036/graphviz-how-to-connect-subgraphs, we will connect the cluster + if c, ok := gb.nodeClusters[toNodeName]; ok { + edge.SetLogicalHead(c.Name()) + } + if c, ok := gb.nodeClusters[fromNodeName]; ok { + edge.SetLogicalTail(c.Name()) + } + gb.graphEdges[edgeName] = edge + } + return nil +} + func (gb *graphBuilder) constructGraph(prefix string, graph *cgraph.Graph, w *core.CompiledWorkflow) error { if w == nil || w.Template == nil { return nil @@ -188,38 +262,20 @@ func (gb *graphBuilder) constructGraph(prefix string, graph *cgraph.Graph, w *co } } - for name, node := range gb.graphNodes { + for name, _ := range gb.graphNodes { upstreamNodes := w.Connections.Upstream[name] downstreamNodes := w.Connections.Downstream[name] if downstreamNodes != nil { for _, n := range downstreamNodes.Ids { - dNode, ok := gb.graphNodes[n] - if !ok { - return fmt.Errorf("node[%s], downstream from[%s] referenced before creation", n, name) - } - edgeName := fmt.Sprintf("%s-%s", name, n) - if _, ok := gb.graphEdges[edgeName]; !ok { - edge, err := graph.CreateEdge(edgeName, node, dNode) - if err != nil { - return err - } - gb.graphEdges[edgeName] = edge + if err := gb.addEdge(name, n, graph); err != nil { + return err } } } if upstreamNodes != nil { for _, n := range upstreamNodes.Ids { - uNode, ok := gb.graphNodes[n] - if !ok { - return fmt.Errorf("node[%s], upstream from[%s] referenced before creation", n, name) - } - edgeName := fmt.Sprintf("%s-%s", n, name) - if _, ok := gb.graphEdges[edgeName]; !ok { - edge, err := graph.CreateEdge(edgeName, uNode, node) - if err != nil { - return err - } - gb.graphEdges[edgeName] = edge + if err := gb.addEdge(n, name, graph); err != nil { + return err } } } @@ -233,6 +289,13 @@ func (gb *graphBuilder) CompiledWorkflowClosureToGraph(g *graphviz.Graphviz, w * return nil, errors.Wrapf("GraphInitFailure", err, "failed to initialize graphviz") } + graph.SetCompound(true) + tLookup := make(map[string]*core.CompiledTask) + for _, t := range w.Tasks { + tLookup[t.Template.Id.String()] = t + } + gb.tasks = tLookup + return graph, gb.constructGraph("", graph, w.Primary) } @@ -241,6 +304,7 @@ func newGraphBuilder() *graphBuilder { graphNodes: make(map[string]*cgraph.Node), graphEdges: make(map[string]*cgraph.Edge), subWf: make(map[string]*cgraph.Graph), + nodeClusters: make(map[string]*cgraph.Graph), } } From d420faf8f95b45ba12d7af5fa9c1e174a0435979 Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Tue, 1 Jun 2021 16:33:15 -0700 Subject: [PATCH 08/16] Working with subworkflows Signed-off-by: Ketan Umare Signed-off-by: Prafulla Mahindrakar --- pkg/visualize/graphviz.go | 59 ++- pkg/visualize/graphviz_test.go | 17 +- .../testdata/compiled_subworkflows.json | 482 ++++++++++++++++++ 3 files changed, 540 insertions(+), 18 deletions(-) create mode 100644 pkg/visualize/testdata/compiled_subworkflows.json diff --git a/pkg/visualize/graphviz.go b/pkg/visualize/graphviz.go index c7de8df8..d5a5c844 100644 --- a/pkg/visualize/graphviz.go +++ b/pkg/visualize/graphviz.go @@ -46,8 +46,8 @@ func booleanExprToString(expr *core.BooleanExpression) string { return comparisonToString(expr.GetComparison()) } -func constructStartNode(graph *cgraph.Graph) (*cgraph.Node, error) { - gn, err := graph.CreateNode("start-node") +func constructStartNode(n string, graph *cgraph.Graph) (*cgraph.Node, error) { + gn, err := graph.CreateNode(n) if err != nil { return nil, err } @@ -57,8 +57,8 @@ func constructStartNode(graph *cgraph.Graph) (*cgraph.Node, error) { return gn, nil } -func constructEndNode(graph *cgraph.Graph) (*cgraph.Node, error) { - gn, err := graph.CreateNode("end-node") +func constructEndNode(n string, graph *cgraph.Graph) (*cgraph.Node, error) { + gn, err := graph.CreateNode(n) if err != nil { return nil, err } @@ -117,7 +117,7 @@ type graphBuilder struct { // Mutated as graph is built. lookup table for all graphviz compiled edges. graphEdges map[string]*cgraph.Edge // lookup table for all graphviz compiled subgraphs - subWf map[string]*cgraph.Graph + subWf map[string]*core.CompiledWorkflow // a lookup table for all tasks in the graph tasks map[string]*core.CompiledTask // a lookup for all node clusters. This is to remap the edges to the cluster itself (instead of the node) @@ -146,7 +146,7 @@ func (gb *graphBuilder) constructBranchNode(prefix string, graph *cgraph.Graph, if err != nil { return nil, err } - gb.graphNodes[parentBranchNode.Name()] = parentBranchNode + gb.graphNodes[n.Id] = parentBranchNode if n.GetBranchNode().GetIfElse() == nil { return parentBranchNode, nil @@ -200,9 +200,9 @@ func (gb *graphBuilder) constructNode(prefix string, graph *cgraph.Graph, n *cor var gn *cgraph.Node if n.Id == "start-node" { - gn, err = constructStartNode(graph) + gn, err = constructStartNode(name, graph) } else if n.Id == "end-node" { - gn, err = constructEndNode(graph) + gn, err = constructEndNode(name, graph) } else { switch n.Target.(type) { case *core.Node_TaskNode: @@ -217,13 +217,27 @@ func (gb *graphBuilder) constructNode(prefix string, graph *cgraph.Graph, n *cor gn, err = gb.constructBranchNode(prefix, branch, n) gb.nodeClusters[name] = branch case *core.Node_WorkflowNode: - gn, err = graph.CreateNode(name) + if n.GetWorkflowNode().GetLaunchplanRef() != nil { + gn, err = graph.CreateNode(name) + } else { + subWfGraph := graph.SubGraph("cluster_"+name, 2) + subGB := graphBuilderFromParent(gb) + swf, ok := gb.subWf[n.GetWorkflowNode().GetSubWorkflowRef().String()] + if !ok { + return nil, fmt.Errorf("subworkfow [%s] not found", n.GetWorkflowNode().GetSubWorkflowRef().String()) + } + if err := subGB.constructGraph(name, subWfGraph, swf); err != nil { + return nil, err + } + gn = subGB.graphNodes["start-node"] + gb.nodeClusters[gn.Name()] = subWfGraph + } } } if err != nil { return nil, err } - gb.graphNodes[name] = gn + gb.graphNodes[n.Id] = gn return gn, nil } @@ -233,7 +247,7 @@ func (gb *graphBuilder) addEdge(fromNodeName, toNodeName string, graph *cgraph.G if !toOk || !fromOk { return fmt.Errorf("nodes[%s] -> [%s] referenced before creation", fromNodeName, toNodeName) } - edgeName := fmt.Sprintf("%s-%s", fromNodeName, toNodeName) + edgeName := fmt.Sprintf("%s-%s", fromNode.Name(), toNode.Name()) if _, ok := gb.graphEdges[edgeName]; !ok { edge, err := graph.CreateEdge(edgeName, fromNode, toNode) if err != nil { @@ -241,12 +255,12 @@ func (gb *graphBuilder) addEdge(fromNodeName, toNodeName string, graph *cgraph.G } // Now lets check that the toNode or the fromNode is a cluster. If so then following this thread, // https://stackoverflow.com/questions/2012036/graphviz-how-to-connect-subgraphs, we will connect the cluster - if c, ok := gb.nodeClusters[toNodeName]; ok { - edge.SetLogicalHead(c.Name()) - } - if c, ok := gb.nodeClusters[fromNodeName]; ok { + if c, ok := gb.nodeClusters[fromNode.Name()]; ok { edge.SetLogicalTail(c.Name()) } + if c, ok := gb.nodeClusters[toNode.Name()]; ok { + edge.SetLogicalHead(c.Name()) + } gb.graphEdges[edgeName] = edge } return nil @@ -262,7 +276,7 @@ func (gb *graphBuilder) constructGraph(prefix string, graph *cgraph.Graph, w *co } } - for name, _ := range gb.graphNodes { + for name := range gb.graphNodes { upstreamNodes := w.Connections.Upstream[name] downstreamNodes := w.Connections.Downstream[name] if downstreamNodes != nil { @@ -295,6 +309,11 @@ func (gb *graphBuilder) CompiledWorkflowClosureToGraph(g *graphviz.Graphviz, w * tLookup[t.Template.Id.String()] = t } gb.tasks = tLookup + wLookup := make(map[string]*core.CompiledWorkflow) + for _, swf := range w.SubWorkflows { + wLookup[swf.Template.Id.String()] = swf + } + gb.subWf = wLookup return graph, gb.constructGraph("", graph, w.Primary) } @@ -303,11 +322,17 @@ func newGraphBuilder() *graphBuilder { return &graphBuilder{ graphNodes: make(map[string]*cgraph.Node), graphEdges: make(map[string]*cgraph.Edge), - subWf: make(map[string]*cgraph.Graph), nodeClusters: make(map[string]*cgraph.Graph), } } +func graphBuilderFromParent(gb *graphBuilder) *graphBuilder { + newGB := newGraphBuilder() + newGB.subWf = gb.subWf + newGB.tasks = gb.tasks + return newGB +} + // RenderWorkflow Renders the workflow graph to the given file func RenderWorkflow(w *core.CompiledWorkflowClosure, o graphviz.Format) ([]byte, error) { g := graphviz.New() diff --git a/pkg/visualize/graphviz_test.go b/pkg/visualize/graphviz_test.go index df1b7f23..675745e2 100644 --- a/pkg/visualize/graphviz_test.go +++ b/pkg/visualize/graphviz_test.go @@ -12,7 +12,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestRenderWorkflow(t *testing.T) { +func TestRenderWorkflowBranch(t *testing.T) { r, err := ioutil.ReadFile("testdata/compiled_closure_branch_nested.json") assert.NoError(t, err) @@ -26,3 +26,18 @@ func TestRenderWorkflow(t *testing.T) { assert.NotNil(t, b) ioutil.WriteFile("/tmp/image.png", b, fs.ModePerm) } + +func TestRenderWorkflowSubWorkflow(t *testing.T) { + r, err := ioutil.ReadFile("testdata/compiled_subworkflows.json") + assert.NoError(t, err) + + i := bytes.NewReader(r) + + c := &core.CompiledWorkflowClosure{} + err = jsonpb.Unmarshal(i, c) + assert.NoError(t, err) + b, err := RenderWorkflow(c, graphviz.PNG) + assert.NoError(t, err) + assert.NotNil(t, b) + ioutil.WriteFile("/tmp/image.png", b, fs.ModePerm) +} \ No newline at end of file diff --git a/pkg/visualize/testdata/compiled_subworkflows.json b/pkg/visualize/testdata/compiled_subworkflows.json new file mode 100644 index 00000000..8bbf4413 --- /dev/null +++ b/pkg/visualize/testdata/compiled_subworkflows.json @@ -0,0 +1,482 @@ +{ + "primary": { + "template": { + "id": { + "resourceType": "WORKFLOW", + "project": "flytesnacks", + "domain": "development", + "name": "core.control_flow.subworkflows.parent_wf", + "version": "d5fa3ecfaca02f9b83957c68fd5fe3c9082ccc59" + }, + "metadata": {}, + "interface": { + "inputs": { + "variables": { + "a": { + "type": { + "simple": "INTEGER" + }, + "description": "a" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "INTEGER" + }, + "description": "o0" + }, + "o1": { + "type": { + "simple": "STRING" + }, + "description": "o1" + }, + "o2": { + "type": { + "simple": "STRING" + }, + "description": "o2" + } + } + } + }, + "nodes": [ + { + "id": "start-node" + }, + { + "id": "end-node", + "inputs": [ + { + "var": "o0", + "binding": { + "promise": { + "nodeId": "node-t1-parent", + "var": "t1_int_output" + } + } + }, + { + "var": "o1", + "binding": { + "promise": { + "nodeId": "n1", + "var": "o0" + } + } + }, + { + "var": "o2", + "binding": { + "promise": { + "nodeId": "n1", + "var": "o1" + } + } + } + ] + }, + { + "id": "node-t1-parent", + "metadata": { + "name": "flytekit.core.python_function_task.core.control_flow.subworkflows.t1", + "retries": {}, + "interruptible": false + }, + "inputs": [ + { + "var": "a", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "a" + } + } + } + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "flytesnacks", + "domain": "development", + "name": "core.control_flow.subworkflows.t1", + "version": "d5fa3ecfaca02f9b83957c68fd5fe3c9082ccc59" + } + } + }, + { + "id": "n1", + "metadata": { + "name": "flytekit.core.workflow.core.control_flow.subworkflows.my_subwf", + "retries": {}, + "interruptible": false + }, + "inputs": [ + { + "var": "a", + "binding": { + "promise": { + "nodeId": "node-t1-parent", + "var": "t1_int_output" + } + } + } + ], + "upstreamNodeIds": [ + "node-t1-parent" + ], + "workflowNode": { + "subWorkflowRef": { + "resourceType": "WORKFLOW", + "project": "flytesnacks", + "domain": "development", + "name": "core.control_flow.subworkflows.my_subwf", + "version": "d5fa3ecfaca02f9b83957c68fd5fe3c9082ccc59" + } + } + } + ], + "outputs": [ + { + "var": "o0", + "binding": { + "promise": { + "nodeId": "node-t1-parent", + "var": "t1_int_output" + } + } + }, + { + "var": "o1", + "binding": { + "promise": { + "nodeId": "n1", + "var": "o0" + } + } + }, + { + "var": "o2", + "binding": { + "promise": { + "nodeId": "n1", + "var": "o1" + } + } + } + ], + "metadataDefaults": {} + }, + "connections": { + "downstream": { + "n1": { + "ids": [ + "end-node" + ] + }, + "node-t1-parent": { + "ids": [ + "end-node", + "n1" + ] + }, + "start-node": { + "ids": [ + "node-t1-parent" + ] + } + }, + "upstream": { + "end-node": { + "ids": [ + "n1", + "node-t1-parent" + ] + }, + "n1": { + "ids": [ + "node-t1-parent" + ] + }, + "node-t1-parent": { + "ids": [ + "start-node" + ] + } + } + } + }, + "subWorkflows": [ + { + "template": { + "id": { + "resourceType": "WORKFLOW", + "project": "flytesnacks", + "domain": "development", + "name": "core.control_flow.subworkflows.my_subwf", + "version": "d5fa3ecfaca02f9b83957c68fd5fe3c9082ccc59" + }, + "metadata": {}, + "interface": { + "inputs": { + "variables": { + "a": { + "type": { + "simple": "INTEGER" + }, + "description": "a" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "STRING" + }, + "description": "o0" + }, + "o1": { + "type": { + "simple": "STRING" + }, + "description": "o1" + } + } + } + }, + "nodes": [ + { + "id": "start-node" + }, + { + "id": "end-node", + "inputs": [ + { + "var": "o0", + "binding": { + "promise": { + "nodeId": "n0", + "var": "c" + } + } + }, + { + "var": "o1", + "binding": { + "promise": { + "nodeId": "n1", + "var": "c" + } + } + } + ] + }, + { + "id": "n0", + "metadata": { + "name": "flytekit.core.python_function_task.core.control_flow.subworkflows.t1", + "retries": {}, + "interruptible": false + }, + "inputs": [ + { + "var": "a", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "a" + } + } + } + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "flytesnacks", + "domain": "development", + "name": "core.control_flow.subworkflows.t1", + "version": "d5fa3ecfaca02f9b83957c68fd5fe3c9082ccc59" + } + } + }, + { + "id": "n1", + "metadata": { + "name": "flytekit.core.python_function_task.core.control_flow.subworkflows.t1", + "retries": {}, + "interruptible": false + }, + "inputs": [ + { + "var": "a", + "binding": { + "promise": { + "nodeId": "n0", + "var": "t1_int_output" + } + } + } + ], + "upstreamNodeIds": [ + "n0" + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "flytesnacks", + "domain": "development", + "name": "core.control_flow.subworkflows.t1", + "version": "d5fa3ecfaca02f9b83957c68fd5fe3c9082ccc59" + } + } + } + ], + "outputs": [ + { + "var": "o0", + "binding": { + "promise": { + "nodeId": "n0", + "var": "c" + } + } + }, + { + "var": "o1", + "binding": { + "promise": { + "nodeId": "n1", + "var": "c" + } + } + } + ], + "metadataDefaults": {} + }, + "connections": { + "downstream": { + "n0": { + "ids": [ + "end-node", + "n1" + ] + }, + "n1": { + "ids": [ + "end-node" + ] + }, + "start-node": { + "ids": [ + "n0" + ] + } + }, + "upstream": { + "end-node": { + "ids": [ + "n0", + "n1" + ] + }, + "n0": { + "ids": [ + "start-node" + ] + }, + "n1": { + "ids": [ + "n0" + ] + } + } + } + } + ], + "tasks": [ + { + "template": { + "id": { + "resourceType": "TASK", + "project": "flytesnacks", + "domain": "development", + "name": "core.control_flow.subworkflows.t1", + "version": "d5fa3ecfaca02f9b83957c68fd5fe3c9082ccc59" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.0.0+develop", + "flavor": "python" + }, + "retries": {}, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "a": { + "type": { + "simple": "INTEGER" + }, + "description": "a" + } + } + }, + "outputs": { + "variables": { + "c": { + "type": { + "simple": "STRING" + }, + "description": "c" + }, + "t1_int_output": { + "type": { + "simple": "INTEGER" + }, + "description": "t1_int_output" + } + } + } + }, + "container": { + "image": "flytecookbook:core-d5fa3ecfaca02f9b83957c68fd5fe3c9082ccc59", + "args": [ + "pyflyte-execute", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}", + "--resolver", + "flytekit.core.python_auto_container.default_task_resolver", + "--", + "task-module", + "core.control_flow.subworkflows", + "task-name", + "t1" + ], + "resources": {}, + "env": [ + { + "key": "FLYTE_INTERNAL_CONFIGURATION_PATH", + "value": "/root/sandbox.config" + }, + { + "key": "FLYTE_INTERNAL_IMAGE", + "value": "flytecookbook:core-d5fa3ecfaca02f9b83957c68fd5fe3c9082ccc59" + } + ] + } + } + } + ] +} + From 7dbb47bea2fa2d904c98c3b283f868d8c9bd71f0 Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Tue, 1 Jun 2021 16:49:36 -0700 Subject: [PATCH 09/16] subworkflows rendering Signed-off-by: Ketan Umare Signed-off-by: Prafulla Mahindrakar --- .../subcommand/workflow/workflow_config.go | 3 +- pkg/visualize/graphviz.go | 11 +- pkg/visualize/graphviz_test.go | 50 +++--- .../compiled_closure_branch_nested.svg | 160 ++++++++++++++++++ .../testdata/compiled_subworkflows.svg | 110 ++++++++++++ 5 files changed, 298 insertions(+), 36 deletions(-) create mode 100644 pkg/visualize/testdata/compiled_closure_branch_nested.svg create mode 100644 pkg/visualize/testdata/compiled_subworkflows.svg diff --git a/cmd/config/subcommand/workflow/workflow_config.go b/cmd/config/subcommand/workflow/workflow_config.go index 55b0a507..56f96e41 100644 --- a/cmd/config/subcommand/workflow/workflow_config.go +++ b/cmd/config/subcommand/workflow/workflow_config.go @@ -4,7 +4,6 @@ import ( "fmt" "github.com/flyteorg/flytectl/pkg/filters" "github.com/goccy/go-graphviz" - "strings" ) //go:generate pflags Config --default-var DefaultConfig @@ -37,4 +36,4 @@ func (cfg *Config) GraphvizFormat() (graphviz.Format, error) { return graphviz.JPG, nil } return graphviz.SVG, fmt.Errorf("unsupported visualization format [%s]. Supported [png, dot, svg, jpg]", cfg.Visualize) -} \ No newline at end of file +} diff --git a/pkg/visualize/graphviz.go b/pkg/visualize/graphviz.go index d5a5c844..8cb89aa6 100644 --- a/pkg/visualize/graphviz.go +++ b/pkg/visualize/graphviz.go @@ -4,9 +4,10 @@ import ( "bytes" "context" "fmt" - "github.com/flyteorg/flyteidl/clients/go/coreutils" "strings" + "github.com/flyteorg/flyteidl/clients/go/coreutils" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" "github.com/flyteorg/flytestdlib/errors" "github.com/flyteorg/flytestdlib/logger" @@ -117,9 +118,9 @@ type graphBuilder struct { // Mutated as graph is built. lookup table for all graphviz compiled edges. graphEdges map[string]*cgraph.Edge // lookup table for all graphviz compiled subgraphs - subWf map[string]*core.CompiledWorkflow + subWf map[string]*core.CompiledWorkflow // a lookup table for all tasks in the graph - tasks map[string]*core.CompiledTask + tasks map[string]*core.CompiledTask // a lookup for all node clusters. This is to remap the edges to the cluster itself (instead of the node) // this is useful in the case of branchNodes and subworkflow nodes nodeClusters map[string]*cgraph.Graph @@ -320,8 +321,8 @@ func (gb *graphBuilder) CompiledWorkflowClosureToGraph(g *graphviz.Graphviz, w * func newGraphBuilder() *graphBuilder { return &graphBuilder{ - graphNodes: make(map[string]*cgraph.Node), - graphEdges: make(map[string]*cgraph.Edge), + graphNodes: make(map[string]*cgraph.Node), + graphEdges: make(map[string]*cgraph.Edge), nodeClusters: make(map[string]*cgraph.Graph), } } diff --git a/pkg/visualize/graphviz_test.go b/pkg/visualize/graphviz_test.go index 675745e2..14efea63 100644 --- a/pkg/visualize/graphviz_test.go +++ b/pkg/visualize/graphviz_test.go @@ -2,42 +2,34 @@ package visualize import ( "bytes" - "github.com/goccy/go-graphviz" - "io/fs" + "fmt" "io/ioutil" "testing" + "github.com/goccy/go-graphviz" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" "github.com/golang/protobuf/jsonpb" "github.com/stretchr/testify/assert" ) func TestRenderWorkflowBranch(t *testing.T) { - r, err := ioutil.ReadFile("testdata/compiled_closure_branch_nested.json") - assert.NoError(t, err) - - i := bytes.NewReader(r) - - c := &core.CompiledWorkflowClosure{} - err = jsonpb.Unmarshal(i, c) - assert.NoError(t, err) - b, err := RenderWorkflow(c, graphviz.PNG) - assert.NoError(t, err) - assert.NotNil(t, b) - ioutil.WriteFile("/tmp/image.png", b, fs.ModePerm) + // Sadly we cannot compare the output of svg, as it slightly changes. + file := []string{"compiled_closure_branch_nested", "compiled_subworkflows"} + + for _, s := range file { + t.Run(s, func(t *testing.T) { + r, err := ioutil.ReadFile(fmt.Sprintf("testdata/%s.json", s)) + assert.NoError(t, err) + + i := bytes.NewReader(r) + + c := &core.CompiledWorkflowClosure{} + err = jsonpb.Unmarshal(i, c) + assert.NoError(t, err) + b, err := RenderWorkflow(c, graphviz.SVG) + assert.NoError(t, err) + assert.NotNil(t, b) + }) + } } - -func TestRenderWorkflowSubWorkflow(t *testing.T) { - r, err := ioutil.ReadFile("testdata/compiled_subworkflows.json") - assert.NoError(t, err) - - i := bytes.NewReader(r) - - c := &core.CompiledWorkflowClosure{} - err = jsonpb.Unmarshal(i, c) - assert.NoError(t, err) - b, err := RenderWorkflow(c, graphviz.PNG) - assert.NoError(t, err) - assert.NotNil(t, b) - ioutil.WriteFile("/tmp/image.png", b, fs.ModePerm) -} \ No newline at end of file diff --git a/pkg/visualize/testdata/compiled_closure_branch_nested.svg b/pkg/visualize/testdata/compiled_closure_branch_nested.svg new file mode 100644 index 00000000..66bd9ffe --- /dev/null +++ b/pkg/visualize/testdata/compiled_closure_branch_nested.svg @@ -0,0 +1,160 @@ + + + + + + + + +cluster_fractions + + + +cluster_inner_fractions + + + + +start-node + + +start + + + +n0 + +fractions + + + +start-node->n0 + + + + + +n0-n0 + +inner_fractions + + + +start-node->n0-n0 + + + + + +n0-n0-n0-n0 + +double [python-task] + + + +start-node->n0-n0-n0-n0 + + + + + +n0-n0-n0-n1 + +square [python-task] + + + +start-node->n0-n0-n0-n1 + + + + + +n0-n2 + +double [python-task] + + + +start-node->n0-n2 + + + + + +n0-n1 + +square [python-task] + + + +start-node->n0-n1 + + + + + +end-node + + +end + + + +n0->end-node + + + + + +n0->n0-n0 + + +(.my_input GT 0.1) AND (.my_input LT 1) + + + +n0->n0-n2 + + +orElse + + + +n0->n0-n1 + + +(.my_input GT 1) AND (.my_input LT 10) + + + +n0-n0->n0-n0-n0-n0 + + +.my_input LT 0.5 + + + +n0-n0-error + +Only <0.7 allowed + + + +n0-n0->n0-n0-error + + +orElse - Fail + + + +n0-n0->n0-n0-n0-n1 + + +(.my_input GT 0.5) AND (.my_input LT 0.7) + + + diff --git a/pkg/visualize/testdata/compiled_subworkflows.svg b/pkg/visualize/testdata/compiled_subworkflows.svg new file mode 100644 index 00000000..2bb2457a --- /dev/null +++ b/pkg/visualize/testdata/compiled_subworkflows.svg @@ -0,0 +1,110 @@ + + + + + + + + +cluster_n1 + + + + +start-node + + +start + + + +node-t1-parent + +t1 [python-task] + + + +start-node->node-t1-parent + + + + + +end-node + + +end + + + +node-t1-parent->end-node + + + + + +n1-start-node + + +start + + + +node-t1-parent->n1-start-node + + + + + +n1-start-node->end-node + + + + + +n1-n0 + +t1 [python-task] + + + +n1-start-node->n1-n0 + + + + + +n1-end-node + + +end + + + +n1-n0->n1-end-node + + + + + +n1-n1 + +t1 [python-task] + + + +n1-n0->n1-n1 + + + + + +n1-n1->n1-end-node + + + + + From 517f5390ad945325cdeda485a50b5c7ebc1c3088 Mon Sep 17 00:00:00 2001 From: pmahindrakar-oss <77798312+pmahindrakar-oss@users.noreply.github.com> Date: Thu, 17 Jun 2021 09:19:53 +0530 Subject: [PATCH 10/16] Using go graphviz library to print dot file (#97) * Using go graphviz library to print dot file Signed-off-by: Prafulla Mahindrakar * Rebased Signed-off-by: Prafulla Mahindrakar --- .../subcommand/workflow/workflow_config.go | 19 +- cmd/get/workflow.go | 10 +- go.mod | 1 + go.sum | 2 + pkg/visualize/graphviz.go | 213 ++++++++---------- 5 files changed, 101 insertions(+), 144 deletions(-) diff --git a/cmd/config/subcommand/workflow/workflow_config.go b/cmd/config/subcommand/workflow/workflow_config.go index 56f96e41..1b79c50b 100644 --- a/cmd/config/subcommand/workflow/workflow_config.go +++ b/cmd/config/subcommand/workflow/workflow_config.go @@ -1,9 +1,7 @@ package workflow import ( - "fmt" "github.com/flyteorg/flytectl/pkg/filters" - "github.com/goccy/go-graphviz" ) //go:generate pflags Config --default-var DefaultConfig @@ -21,19 +19,4 @@ type Config struct { Filter filters.Filters `json:"filter" pflag:","` Visualize string `json:"visualize" pflag:",optional flag to visualize a workflow as one of [png, dot, svg, jpg]"` OutputFile string `json:"output_file" pflag:",path and a filename of where the output image should be dumped. This can only be used in concert with visualize"` -} - -// GraphvizFormat returns the graphviz format based on the user-specified value or an error -func (cfg *Config) GraphvizFormat() (graphviz.Format, error) { - switch strings.ToLower(cfg.Visualize) { - case "png": - return graphviz.PNG, nil - case "svg": - return graphviz.SVG, nil - case "dot": - return graphviz.XDOT, nil - case "jpg": - return graphviz.JPG, nil - } - return graphviz.SVG, fmt.Errorf("unsupported visualization format [%s]. Supported [png, dot, svg, jpg]", cfg.Visualize) -} +} \ No newline at end of file diff --git a/cmd/get/workflow.go b/cmd/get/workflow.go index e6c8c7a7..7511b130 100644 --- a/cmd/get/workflow.go +++ b/cmd/get/workflow.go @@ -108,14 +108,14 @@ func getWorkflowFunc(ctx context.Context, args []string, cmdCtx cmdCore.CommandC return err } if len(workflows) > 0 && workflowconfig.DefaultConfig.Visualize != "" { - f, err := workflowconfig.DefaultConfig.GraphvizFormat() - if err != nil { - return err - } + //f, err := workflowconfig.DefaultConfig.GraphvizFormat() + //if err != nil { + // return err + //} if workflowconfig.DefaultConfig.OutputFile == "" { return fmt.Errorf("--visualize should be accompanied with a file-name using --output_file option") } - b, err := visualize.RenderWorkflow(workflows[0].Closure.CompiledWorkflow, f) + b, err := visualize.RenderWorkflow(workflows[0].Closure.CompiledWorkflow) if err != nil { return errors.Wrapf("VisualizationError", err, "failed to visualize workflow") } diff --git a/go.mod b/go.mod index 9523d88d..2a46c5f4 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.13 require ( github.com/Microsoft/go-winio v0.5.0 // indirect + github.com/awalterschulze/gographviz v2.0.3+incompatible github.com/containerd/containerd v1.5.2 // indirect github.com/docker/docker v20.10.7+incompatible github.com/docker/go-connections v0.4.0 diff --git a/go.sum b/go.sum index 07f80f64..3369537b 100644 --- a/go.sum +++ b/go.sum @@ -122,6 +122,8 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/awalterschulze/gographviz v2.0.3+incompatible h1:9sVEXJBJLwGX7EQVhLm2elIKCm7P2YHFC8v6096G09E= +github.com/awalterschulze/gographviz v2.0.3+incompatible/go.mod h1:GEV5wmg4YquNw7v1kkyoX9etIk8yVmXj+AkDHuuETHs= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/aws/aws-sdk-go v1.23.4/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= diff --git a/pkg/visualize/graphviz.go b/pkg/visualize/graphviz.go index 8cb89aa6..e5828917 100644 --- a/pkg/visualize/graphviz.go +++ b/pkg/visualize/graphviz.go @@ -1,18 +1,13 @@ package visualize import ( - "bytes" - "context" "fmt" "strings" "github.com/flyteorg/flyteidl/clients/go/coreutils" + graphviz "github.com/awalterschulze/gographviz" "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" - "github.com/flyteorg/flytestdlib/errors" - "github.com/flyteorg/flytestdlib/logger" - "github.com/goccy/go-graphviz" - "github.com/goccy/go-graphviz/cgraph" ) func operandToString(op *core.Operand) string { @@ -47,103 +42,82 @@ func booleanExprToString(expr *core.BooleanExpression) string { return comparisonToString(expr.GetComparison()) } -func constructStartNode(n string, graph *cgraph.Graph) (*cgraph.Node, error) { - gn, err := graph.CreateNode(n) - if err != nil { - return nil, err - } - gn.SetLabel("start") - gn.SetShape(cgraph.DoubleCircleShape) - gn.SetColor("green") - return gn, nil +func constructStartNode(parentGraph string, n string, graph *graphviz.Graph) (*graphviz.Node, error) { + attrs := map[string]string {"shape" : "doublecircle", "color" : "green"} + attrs["label"] = "start" + err := graph.AddNode(parentGraph, n, attrs) + return graph.Nodes.Lookup[n], err } -func constructEndNode(n string, graph *cgraph.Graph) (*cgraph.Node, error) { - gn, err := graph.CreateNode(n) - if err != nil { - return nil, err - } - gn.SetLabel("end") - gn.SetShape(cgraph.DoubleCircleShape) - gn.SetColor("red") - return gn, nil +func constructEndNode(parentGraph string,n string, graph *graphviz.Graph) (*graphviz.Node, error) { + attrs := map[string]string {"shape" : "doublecircle", "color" : "red"} + attrs["label"] = "end" + err := graph.AddNode(parentGraph, n, attrs) + return graph.Nodes.Lookup[n], err } -func constructTaskNode(name string, graph *cgraph.Graph, n *core.Node, t *core.CompiledTask) (*cgraph.Node, error) { - gn, err := graph.CreateNode(name) - if err != nil { - return nil, err - } +func constructTaskNode(parentGraph string, name string, graph *graphviz.Graph, n *core.Node, t *core.CompiledTask) (*graphviz.Node, error) { + attrs := map[string]string {"shape" : "box"} if n.Metadata != nil && n.Metadata.Name != "" { v := strings.LastIndexAny(n.Metadata.Name, ".") - gn.SetLabel(fmt.Sprintf("%s [%s]", n.Metadata.Name[v+1:], t.Template.Type)) + attrs["label"] = fmt.Sprintf("\"%s [%s]\"", n.Metadata.Name[v+1:], t.Template.Type) } - gn.SetShape(cgraph.BoxShape) - return gn, nil + tName := strings.ReplaceAll(name, "-", "_") + err := graph.AddNode(parentGraph, tName, attrs) + return graph.Nodes.Lookup[tName], err } -func constructErrorNode(name string, graph *cgraph.Graph, m string) (*cgraph.Node, error) { - gn, err := graph.CreateNode(name) - if err != nil { - return nil, err - } - gn.SetLabel(m) - gn.SetShape(cgraph.BoxShape) - gn.SetColor("red") - return gn, nil +func constructErrorNode(parentGraph string, name string, graph *graphviz.Graph, m string) (*graphviz.Node, error) { + attrs := map[string]string {"shape" : "box", "color" : "red", "label" : m} + eName := strings.ReplaceAll(name, "-", "_") + err := graph.AddNode(parentGraph, eName, attrs) + return graph.Nodes.Lookup[eName], err } -func constructBranchConditionNode(name string, graph *cgraph.Graph, n *core.Node) (*cgraph.Node, error) { - gn, err := graph.CreateNode(name) - if err != nil { - return nil, err - } +func constructBranchConditionNode(parentGraph string, name string, graph *graphviz.Graph, n *core.Node) (*graphviz.Node, error) { + attrs := map[string]string {"shape" : "diamond"} if n.Metadata != nil && n.Metadata.Name != "" { - gn.SetLabel(n.Metadata.Name) + attrs["label"] = n.Metadata.Name } - gn.SetShape(cgraph.DiamondShape) - return gn, nil + cName := strings.ReplaceAll(name, "-", "_") + err := graph.AddNode(parentGraph, cName, attrs) + return graph.Nodes.Lookup[cName], err } func getName(prefix, id string) string { if prefix != "" { - return prefix + "-" + id + return prefix + "_" + id } return id } type graphBuilder struct { // Mutated as graph is built - graphNodes map[string]*cgraph.Node + graphNodes map[string]*graphviz.Node // Mutated as graph is built. lookup table for all graphviz compiled edges. - graphEdges map[string]*cgraph.Edge + graphEdges map[string]*graphviz.Edge // lookup table for all graphviz compiled subgraphs subWf map[string]*core.CompiledWorkflow // a lookup table for all tasks in the graph tasks map[string]*core.CompiledTask - // a lookup for all node clusters. This is to remap the edges to the cluster itself (instead of the node) - // this is useful in the case of branchNodes and subworkflow nodes - nodeClusters map[string]*cgraph.Graph } -func (gb *graphBuilder) addBranchSubNodeEdge(graph *cgraph.Graph, parentNode, n *cgraph.Node, label string) error { - edgeName := fmt.Sprintf("%s-%s", parentNode.Name(), n.Name()) +func (gb *graphBuilder) addBranchSubNodeEdge(graph *graphviz.Graph, parentNode, n *graphviz.Node, label string) error { + edgeName := fmt.Sprintf("%s-%s", parentNode.Name, n.Name) if _, ok := gb.graphEdges[edgeName]; !ok { - edge, err := graph.CreateEdge(edgeName, parentNode, n) + attrs := map[string]string {} + attrs["label"] = fmt.Sprintf("\"%s\"", label) + err := graph.AddEdge(parentNode.Name, n.Name, true, attrs) if err != nil { return err } - edge.SetLabel(label) - if c, ok := gb.nodeClusters[n.Name()]; ok { - edge.SetLogicalHead(c.Name()) - } - gb.graphEdges[edgeName] = edge + gb.graphEdges[edgeName] = graph.Edges.SrcToDsts[parentNode.Name][n.Name][0] } return nil } -func (gb *graphBuilder) constructBranchNode(prefix string, graph *cgraph.Graph, n *core.Node) (*cgraph.Node, error) { - parentBranchNode, err := constructBranchConditionNode(getName(prefix, n.Id), graph, n) +func (gb *graphBuilder) constructBranchNode(parentGraph string, prefix string, graph *graphviz.Graph, n *core.Node) (*graphviz.Node, error) { + parentBranchNode, err := constructBranchConditionNode(parentGraph, getName(prefix, n.Id), graph, n) if err != nil { return nil, err } @@ -153,7 +127,7 @@ func (gb *graphBuilder) constructBranchNode(prefix string, graph *cgraph.Graph, return parentBranchNode, nil } - subNode, err := gb.constructNode(prefix, graph, n.GetBranchNode().GetIfElse().Case.ThenNode) + subNode, err := gb.constructNode(parentGraph, prefix, graph, n.GetBranchNode().GetIfElse().Case.ThenNode) if err != nil { return nil, err } @@ -162,8 +136,8 @@ func (gb *graphBuilder) constructBranchNode(prefix string, graph *cgraph.Graph, } if n.GetBranchNode().GetIfElse().GetError() != nil { - name := fmt.Sprintf("%s-error", parentBranchNode.Name()) - subNode, err := constructErrorNode(name, graph, n.GetBranchNode().GetIfElse().GetError().Message) + name := fmt.Sprintf("%s-error", parentBranchNode.Name) + subNode, err := constructErrorNode(prefix, name, graph, n.GetBranchNode().GetIfElse().GetError().Message) if err != nil { return nil, err } @@ -172,7 +146,7 @@ func (gb *graphBuilder) constructBranchNode(prefix string, graph *cgraph.Graph, return nil, err } } else { - subNode, err := gb.constructNode(prefix, graph, n.GetBranchNode().GetIfElse().GetElseNode()) + subNode, err := gb.constructNode(parentGraph, prefix, graph, n.GetBranchNode().GetIfElse().GetElseNode()) if err != nil { return nil, err } @@ -183,7 +157,7 @@ func (gb *graphBuilder) constructBranchNode(prefix string, graph *cgraph.Graph, if n.GetBranchNode().GetIfElse().GetOther() != nil { for _, c := range n.GetBranchNode().GetIfElse().GetOther() { - subNode, err := gb.constructNode(prefix, graph, c.ThenNode) + subNode, err := gb.constructNode(parentGraph, prefix, graph, c.ThenNode) if err != nil { return nil, err } @@ -195,15 +169,15 @@ func (gb *graphBuilder) constructBranchNode(prefix string, graph *cgraph.Graph, return parentBranchNode, nil } -func (gb *graphBuilder) constructNode(prefix string, graph *cgraph.Graph, n *core.Node) (*cgraph.Node, error) { +func (gb *graphBuilder) constructNode(parentGraphName string, prefix string, graph *graphviz.Graph, n *core.Node) (*graphviz.Node, error) { name := getName(prefix, n.Id) var err error - var gn *cgraph.Node + var gn *graphviz.Node if n.Id == "start-node" { - gn, err = constructStartNode(name, graph) + gn, err = constructStartNode(parentGraphName, strings.ReplaceAll(name, "-", "_"), graph) } else if n.Id == "end-node" { - gn, err = constructEndNode(name, graph) + gn, err = constructEndNode(parentGraphName, strings.ReplaceAll(name, "-", "_"), graph) } else { switch n.Target.(type) { case *core.Node_TaskNode: @@ -212,26 +186,40 @@ func (gb *graphBuilder) constructNode(prefix string, graph *cgraph.Graph, n *cor if !ok { return nil, fmt.Errorf("failed to find task [%s] in closure", tID) } - gn, err = constructTaskNode(name, graph, n, t) + gn, err = constructTaskNode(parentGraphName, name, graph, n, t) + if err != nil { + return nil, err + } case *core.Node_BranchNode: - branch := graph.SubGraph(fmt.Sprintf("cluster_"+n.Metadata.Name), 2) - gn, err = gb.constructBranchNode(prefix, branch, n) - gb.nodeClusters[name] = branch + err := graph.AddSubGraph(parentGraphName, fmt.Sprintf("cluster_"+n.Metadata.Name), nil) + if err != nil { + return nil, err + } + gn, err = gb.constructBranchNode(fmt.Sprintf("cluster_"+n.Metadata.Name), prefix, graph, n) + if err != nil { + return nil, err + } case *core.Node_WorkflowNode: if n.GetWorkflowNode().GetLaunchplanRef() != nil { - gn, err = graph.CreateNode(name) + attrs := map[string]string {} + err := graph.AddNode(parentGraphName, name, attrs) + if err != nil { + return nil, err + } } else { - subWfGraph := graph.SubGraph("cluster_"+name, 2) + err := graph.AddSubGraph(parentGraphName, "cluster_"+name, nil) + if err != nil { + return nil, err + } subGB := graphBuilderFromParent(gb) swf, ok := gb.subWf[n.GetWorkflowNode().GetSubWorkflowRef().String()] if !ok { return nil, fmt.Errorf("subworkfow [%s] not found", n.GetWorkflowNode().GetSubWorkflowRef().String()) } - if err := subGB.constructGraph(name, subWfGraph, swf); err != nil { + if err := subGB.constructGraph("cluster_"+name, name, graph, swf); err != nil { return nil, err } gn = subGB.graphNodes["start-node"] - gb.nodeClusters[gn.Name()] = subWfGraph } } } @@ -242,37 +230,37 @@ func (gb *graphBuilder) constructNode(prefix string, graph *cgraph.Graph, n *cor return gn, nil } -func (gb *graphBuilder) addEdge(fromNodeName, toNodeName string, graph *cgraph.Graph) error { +func (gb *graphBuilder) addEdge(fromNodeName, toNodeName string, graph *graphviz.Graph) error { toNode, toOk := gb.graphNodes[toNodeName] fromNode, fromOk := gb.graphNodes[fromNodeName] if !toOk || !fromOk { return fmt.Errorf("nodes[%s] -> [%s] referenced before creation", fromNodeName, toNodeName) } - edgeName := fmt.Sprintf("%s-%s", fromNode.Name(), toNode.Name()) - if _, ok := gb.graphEdges[edgeName]; !ok { - edge, err := graph.CreateEdge(edgeName, fromNode, toNode) + if _,ok := graph.Edges.SrcToDsts[fromNode.Name][toNode.Name]; !ok { + attrs := map[string]string {} + err := graph.AddEdge(fromNode.Name, toNode.Name, true, attrs) if err != nil { return err } // Now lets check that the toNode or the fromNode is a cluster. If so then following this thread, // https://stackoverflow.com/questions/2012036/graphviz-how-to-connect-subgraphs, we will connect the cluster - if c, ok := gb.nodeClusters[fromNode.Name()]; ok { - edge.SetLogicalTail(c.Name()) + + if c, ok := graph.Nodes.Lookup[fromNode.Name]; ok { + attrs["ltail"] = c.Name } - if c, ok := gb.nodeClusters[toNode.Name()]; ok { - edge.SetLogicalHead(c.Name()) + if c, ok := graph.Nodes.Lookup[toNode.Name]; ok { + attrs["lhead"] = c.Name } - gb.graphEdges[edgeName] = edge } return nil } -func (gb *graphBuilder) constructGraph(prefix string, graph *cgraph.Graph, w *core.CompiledWorkflow) error { +func (gb *graphBuilder) constructGraph(parentGraphName string, prefix string, graph *graphviz.Graph, w *core.CompiledWorkflow) error { if w == nil || w.Template == nil { return nil } for _, n := range w.Template.Nodes { - if _, err := gb.constructNode(prefix, graph, n); err != nil { + if _, err := gb.constructNode(parentGraphName, prefix, graph, n); err != nil { return err } } @@ -298,13 +286,11 @@ func (gb *graphBuilder) constructGraph(prefix string, graph *cgraph.Graph, w *co return nil } -func (gb *graphBuilder) CompiledWorkflowClosureToGraph(g *graphviz.Graphviz, w *core.CompiledWorkflowClosure) (*cgraph.Graph, error) { - graph, err := g.Graph(graphviz.Directed) - if err != nil { - return nil, errors.Wrapf("GraphInitFailure", err, "failed to initialize graphviz") - } +func (gb *graphBuilder) CompiledWorkflowClosureToGraph(w *core.CompiledWorkflowClosure) (*graphviz.Graph, error) { + dotGraph := graphviz.NewGraph() + _ = dotGraph.SetDir(true) + _ = dotGraph.SetStrict(true) - graph.SetCompound(true) tLookup := make(map[string]*core.CompiledTask) for _, t := range w.Tasks { tLookup[t.Template.Id.String()] = t @@ -316,14 +302,13 @@ func (gb *graphBuilder) CompiledWorkflowClosureToGraph(g *graphviz.Graphviz, w * } gb.subWf = wLookup - return graph, gb.constructGraph("", graph, w.Primary) + return dotGraph, gb.constructGraph("", "", dotGraph, w.Primary) } func newGraphBuilder() *graphBuilder { return &graphBuilder{ - graphNodes: make(map[string]*cgraph.Node), - graphEdges: make(map[string]*cgraph.Edge), - nodeClusters: make(map[string]*cgraph.Graph), + graphNodes: make(map[string]*graphviz.Node), + graphEdges: make(map[string]*graphviz.Edge), } } @@ -335,27 +320,13 @@ func graphBuilderFromParent(gb *graphBuilder) *graphBuilder { } // RenderWorkflow Renders the workflow graph to the given file -func RenderWorkflow(w *core.CompiledWorkflowClosure, o graphviz.Format) ([]byte, error) { - g := graphviz.New() - defer func() { - if err := g.Close(); err != nil { - logger.Fatalf(context.TODO(), "failed to close graphviz. err: %s", err) - } - }() +func RenderWorkflow(w *core.CompiledWorkflowClosure) ([]byte, error) { gb := newGraphBuilder() - graph, err := gb.CompiledWorkflowClosureToGraph(g, w) + graph, err := gb.CompiledWorkflowClosureToGraph(w) if err != nil { return nil, err } - defer func() { - if err := graph.Close(); err != nil { - logger.Fatalf(context.TODO(), "Failed to close the graphviz Graph. err: %s", err) - } - }() - var buf bytes.Buffer - if err := g.Render(graph, o, &buf); err != nil { - return nil, err - } - return buf.Bytes(), nil + fmt.Printf(graph.String()) + return []byte(graph.String()), nil } From b347c0cc99629fb79029b2cf44e3a97df9c62cde Mon Sep 17 00:00:00 2001 From: pmahindrakar-oss <77798312+pmahindrakar-oss@users.noreply.github.com> Date: Thu, 17 Jun 2021 14:01:35 +0530 Subject: [PATCH 11/16] Added output option for dot and doturl (#101) Signed-off-by: Prafulla Mahindrakar --- cmd/get/workflow.go | 43 ++--- go.mod | 1 + pkg/printer/outputformat_enumer.go | 14 +- pkg/printer/printer.go | 32 ++++ pkg/visualize/graphviz.go | 123 +++++++++----- pkg/visualize/graphviz_test.go | 4 +- .../compiled_closure_branch_nested.svg | 160 ------------------ .../testdata/compiled_subworkflows.svg | 110 ------------ 8 files changed, 137 insertions(+), 350 deletions(-) delete mode 100644 pkg/visualize/testdata/compiled_closure_branch_nested.svg delete mode 100644 pkg/visualize/testdata/compiled_subworkflows.svg diff --git a/cmd/get/workflow.go b/cmd/get/workflow.go index 7511b130..777b43c4 100644 --- a/cmd/get/workflow.go +++ b/cmd/get/workflow.go @@ -2,14 +2,6 @@ package get import ( "context" - "fmt" - "io/fs" - "io/ioutil" - - "github.com/flyteorg/flytestdlib/errors" - - "github.com/flyteorg/flytectl/pkg/visualize" - workflowconfig "github.com/flyteorg/flytectl/cmd/config/subcommand/workflow" "github.com/flyteorg/flytectl/pkg/ext" "github.com/flyteorg/flytestdlib/logger" @@ -75,6 +67,18 @@ Retrieves all the workflow within project and domain in json format. flytectl get workflow -p flytesnacks -d development -o json +Visualize the graph for a workflow within project and domain in dot format. + +:: + + flytectl get workflow -p flytesnacks -d development -o dot + +Visualize the graph for a workflow within project and domain in a dot content render. + +:: + + flytectl get workflow -p flytesnacks -d development -o dot-url + Usage ` ) @@ -103,28 +107,7 @@ func getWorkflowFunc(ctx context.Context, args []string, cmdCtx cmdCore.CommandC return err } logger.Debugf(ctx, "Retrieved %v workflow", len(workflows)) - err = adminPrinter.Print(config.GetConfig().MustOutputFormat(), workflowColumns, WorkflowToProtoMessages(workflows)...) - if err != nil { - return err - } - if len(workflows) > 0 && workflowconfig.DefaultConfig.Visualize != "" { - //f, err := workflowconfig.DefaultConfig.GraphvizFormat() - //if err != nil { - // return err - //} - if workflowconfig.DefaultConfig.OutputFile == "" { - return fmt.Errorf("--visualize should be accompanied with a file-name using --output_file option") - } - b, err := visualize.RenderWorkflow(workflows[0].Closure.CompiledWorkflow) - if err != nil { - return errors.Wrapf("VisualizationError", err, "failed to visualize workflow") - } - if err := ioutil.WriteFile(workflowconfig.DefaultConfig.OutputFile, b, fs.ModePerm); err != nil { - return errors.Wrapf("FileWriteError", err, "failed to write visualization file") - } - fmt.Printf("File [%s] written", workflowconfig.DefaultConfig.OutputFile) - } - return nil + return adminPrinter.Print(config.GetConfig().MustOutputFormat(), workflowColumns, WorkflowToProtoMessages(workflows)...) } workflows, err = cmdCtx.AdminFetcherExt().FetchAllVerOfWorkflow(ctx, "", config.GetConfig().Project, config.GetConfig().Domain, workflowconfig.DefaultConfig.Filter) diff --git a/go.mod b/go.mod index 2a46c5f4..15c5c396 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,7 @@ require ( github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect + github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 github.com/sirupsen/logrus v1.8.0 github.com/spf13/cobra v1.1.3 github.com/spf13/pflag v1.0.5 diff --git a/pkg/printer/outputformat_enumer.go b/pkg/printer/outputformat_enumer.go index b10b4514..ab56b2a0 100644 --- a/pkg/printer/outputformat_enumer.go +++ b/pkg/printer/outputformat_enumer.go @@ -8,9 +8,9 @@ import ( "fmt" ) -const _OutputFormatName = "TABLEJSONYAML" +const _OutputFormatName = "TABLEJSONYAMLDOTDOTURL" -var _OutputFormatIndex = [...]uint8{0, 5, 9, 13} +var _OutputFormatIndex = [...]uint8{0, 5, 9, 13, 16, 22} func (i OutputFormat) String() string { if i >= OutputFormat(len(_OutputFormatIndex)-1) { @@ -19,12 +19,14 @@ func (i OutputFormat) String() string { return _OutputFormatName[_OutputFormatIndex[i]:_OutputFormatIndex[i+1]] } -var _OutputFormatValues = []OutputFormat{0, 1, 2} +var _OutputFormatValues = []OutputFormat{0, 1, 2, 3, 4} var _OutputFormatNameToValueMap = map[string]OutputFormat{ - _OutputFormatName[0:5]: 0, - _OutputFormatName[5:9]: 1, - _OutputFormatName[9:13]: 2, + _OutputFormatName[0:5]: 0, + _OutputFormatName[5:9]: 1, + _OutputFormatName[9:13]: 2, + _OutputFormatName[13:16]: 3, + _OutputFormatName[16:22]: 4, } // OutputFormatString retrieves an enum value from the enum constants string name. diff --git a/pkg/printer/printer.go b/pkg/printer/printer.go index fa86127b..a25ccbeb 100644 --- a/pkg/printer/printer.go +++ b/pkg/printer/printer.go @@ -4,9 +4,14 @@ import ( "bytes" "encoding/json" "fmt" + "github.com/pkg/browser" + "net/url" "os" + "github.com/flyteorg/flytectl/pkg/visualize" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" "github.com/flyteorg/flytestdlib/errors" + "github.com/ghodss/yaml" "github.com/golang/protobuf/jsonpb" "github.com/golang/protobuf/proto" @@ -22,8 +27,12 @@ const ( OutputFormatTABLE OutputFormat = iota OutputFormatJSON OutputFormatYAML + OutputFormatDOT + OutputFormatDOTURL ) +const GraphVisualizationServiceUrl = "http://graph.flyte.org/#" + func OutputFormats() []string { var v []string for _, o := range OutputFormatValues() { @@ -137,6 +146,29 @@ func (p Printer) Print(format OutputFormat, columns []Column, messages ...proto. } fmt.Println(string(v)) } + case OutputFormatDOT, OutputFormatDOTURL: + var workflows []*admin.Workflow + for _, m := range messages { + if w, ok := m.(*admin.Workflow); ok { + workflows = append(workflows, w) + } else { + return fmt.Errorf("visualization is only supported on workflows") + } + } + if len(workflows) == 0 { + return fmt.Errorf("atleast one workflow required for visualization") + } + workflow := workflows[0] + graphStr, err := visualize.RenderWorkflow(workflow.Closure.CompiledWorkflow) + if err != nil { + return errors.Wrapf("VisualizationError", err, "failed to visualize workflow") + } + if format == OutputFormatDOTURL { + urlToOpen := GraphVisualizationServiceUrl + url.PathEscape(graphStr) + fmt.Println("Opening the browser at "+ urlToOpen) + return browser.OpenURL(urlToOpen) + } + fmt.Println(graphStr) default: // Print table rows, err := json.Marshal(printableMessages) if err != nil { diff --git a/pkg/visualize/graphviz.go b/pkg/visualize/graphviz.go index e5828917..4bc1b606 100644 --- a/pkg/visualize/graphviz.go +++ b/pkg/visualize/graphviz.go @@ -10,6 +10,36 @@ import ( "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" ) +const ( + // node identifiers + StartNode string = "start-node" + EndNode string = "end-node" + + // subgraph attributes + SubgraphPrefix string = "cluster_" + + // shape attributes + DoubleCircleShape string = "doublecircle" + BoxShape string = "box" + DiamondShape string = "diamond" + ShapeType string = "shape" + + // color attributes + ColorAttr string = "color" + Red string = "red" + Green string = "green" + + // structural attributes + LabelAttr string = "label" + LHeadAttr string = "lhead" + LTailAttr string = "ltail" + + // conditional + ElseFail string = "orElse - Fail" + Else string = "orElse" + +) + func operandToString(op *core.Operand) string { if op.GetPrimitive() != nil { l, err := coreutils.ExtractFromLiteral(&core.Literal{Value: &core.Literal_Scalar{ @@ -43,24 +73,24 @@ func booleanExprToString(expr *core.BooleanExpression) string { } func constructStartNode(parentGraph string, n string, graph *graphviz.Graph) (*graphviz.Node, error) { - attrs := map[string]string {"shape" : "doublecircle", "color" : "green"} - attrs["label"] = "start" + attrs := map[string]string{ShapeType: DoubleCircleShape, ColorAttr: Green} + attrs[LabelAttr] = "start" err := graph.AddNode(parentGraph, n, attrs) return graph.Nodes.Lookup[n], err } -func constructEndNode(parentGraph string,n string, graph *graphviz.Graph) (*graphviz.Node, error) { - attrs := map[string]string {"shape" : "doublecircle", "color" : "red"} - attrs["label"] = "end" +func constructEndNode(parentGraph string, n string, graph *graphviz.Graph) (*graphviz.Node, error) { + attrs := map[string]string{ShapeType: DoubleCircleShape, ColorAttr: Red} + attrs[LabelAttr] = "end" err := graph.AddNode(parentGraph, n, attrs) return graph.Nodes.Lookup[n], err } func constructTaskNode(parentGraph string, name string, graph *graphviz.Graph, n *core.Node, t *core.CompiledTask) (*graphviz.Node, error) { - attrs := map[string]string {"shape" : "box"} + attrs := map[string]string{ShapeType: BoxShape} if n.Metadata != nil && n.Metadata.Name != "" { v := strings.LastIndexAny(n.Metadata.Name, ".") - attrs["label"] = fmt.Sprintf("\"%s [%s]\"", n.Metadata.Name[v+1:], t.Template.Type) + attrs[LabelAttr] = fmt.Sprintf("\"%s [%s]\"", n.Metadata.Name[v+1:], t.Template.Type) } tName := strings.ReplaceAll(name, "-", "_") err := graph.AddNode(parentGraph, tName, attrs) @@ -68,16 +98,16 @@ func constructTaskNode(parentGraph string, name string, graph *graphviz.Graph, n } func constructErrorNode(parentGraph string, name string, graph *graphviz.Graph, m string) (*graphviz.Node, error) { - attrs := map[string]string {"shape" : "box", "color" : "red", "label" : m} + attrs := map[string]string{ShapeType: BoxShape, ColorAttr: Red, LabelAttr: m} eName := strings.ReplaceAll(name, "-", "_") err := graph.AddNode(parentGraph, eName, attrs) return graph.Nodes.Lookup[eName], err } func constructBranchConditionNode(parentGraph string, name string, graph *graphviz.Graph, n *core.Node) (*graphviz.Node, error) { - attrs := map[string]string {"shape" : "diamond"} + attrs := map[string]string{ShapeType: DiamondShape} if n.Metadata != nil && n.Metadata.Name != "" { - attrs["label"] = n.Metadata.Name + attrs[LabelAttr] = fmt.Sprintf("\"[%s]\"", n.Metadata.Name) } cName := strings.ReplaceAll(name, "-", "_") err := graph.AddNode(parentGraph, cName, attrs) @@ -100,13 +130,19 @@ type graphBuilder struct { subWf map[string]*core.CompiledWorkflow // a lookup table for all tasks in the graph tasks map[string]*core.CompiledTask + // a lookup for all node clusters. This is to remap the edges to the cluster itself (instead of the node) + // this is useful in the case of branchNodes and subworkflow nodes + nodeClusters map[string]string } func (gb *graphBuilder) addBranchSubNodeEdge(graph *graphviz.Graph, parentNode, n *graphviz.Node, label string) error { edgeName := fmt.Sprintf("%s-%s", parentNode.Name, n.Name) if _, ok := gb.graphEdges[edgeName]; !ok { - attrs := map[string]string {} - attrs["label"] = fmt.Sprintf("\"%s\"", label) + attrs := map[string]string{} + if c, ok := gb.nodeClusters[n.Name]; ok { + attrs[LHeadAttr] = c + } + attrs[LabelAttr] = fmt.Sprintf("\"%s\"", label) err := graph.AddEdge(parentNode.Name, n.Name, true, attrs) if err != nil { return err @@ -142,7 +178,7 @@ func (gb *graphBuilder) constructBranchNode(parentGraph string, prefix string, g return nil, err } gb.graphNodes[name] = subNode - if err := gb.addBranchSubNodeEdge(graph, parentBranchNode, subNode, "orElse - Fail"); err != nil { + if err := gb.addBranchSubNodeEdge(graph, parentBranchNode, subNode, ElseFail); err != nil { return nil, err } } else { @@ -150,7 +186,7 @@ func (gb *graphBuilder) constructBranchNode(parentGraph string, prefix string, g if err != nil { return nil, err } - if err := gb.addBranchSubNodeEdge(graph, parentBranchNode, subNode, "orElse"); err != nil { + if err := gb.addBranchSubNodeEdge(graph, parentBranchNode, subNode, Else); err != nil { return nil, err } } @@ -174,10 +210,12 @@ func (gb *graphBuilder) constructNode(parentGraphName string, prefix string, gra var err error var gn *graphviz.Node - if n.Id == "start-node" { + if n.Id == StartNode { gn, err = constructStartNode(parentGraphName, strings.ReplaceAll(name, "-", "_"), graph) - } else if n.Id == "end-node" { + gb.nodeClusters[name] = parentGraphName + } else if n.Id == EndNode { gn, err = constructEndNode(parentGraphName, strings.ReplaceAll(name, "-", "_"), graph) + gb.nodeClusters[name] = parentGraphName } else { switch n.Target.(type) { case *core.Node_TaskNode: @@ -190,24 +228,28 @@ func (gb *graphBuilder) constructNode(parentGraphName string, prefix string, gra if err != nil { return nil, err } + gb.nodeClusters[name] = parentGraphName case *core.Node_BranchNode: - err := graph.AddSubGraph(parentGraphName, fmt.Sprintf("cluster_"+n.Metadata.Name), nil) + branchSubGraphName := SubgraphPrefix + n.Metadata.Name + err := graph.AddSubGraph(parentGraphName, branchSubGraphName, nil) if err != nil { return nil, err } - gn, err = gb.constructBranchNode(fmt.Sprintf("cluster_"+n.Metadata.Name), prefix, graph, n) + gn, err = gb.constructBranchNode(branchSubGraphName, prefix, graph, n) if err != nil { return nil, err } + gb.nodeClusters[name] = branchSubGraphName case *core.Node_WorkflowNode: if n.GetWorkflowNode().GetLaunchplanRef() != nil { - attrs := map[string]string {} + attrs := map[string]string{} err := graph.AddNode(parentGraphName, name, attrs) if err != nil { return nil, err } } else { - err := graph.AddSubGraph(parentGraphName, "cluster_"+name, nil) + subGraphName := SubgraphPrefix + name + err := graph.AddSubGraph(parentGraphName, subGraphName, nil) if err != nil { return nil, err } @@ -216,10 +258,11 @@ func (gb *graphBuilder) constructNode(parentGraphName string, prefix string, gra if !ok { return nil, fmt.Errorf("subworkfow [%s] not found", n.GetWorkflowNode().GetSubWorkflowRef().String()) } - if err := subGB.constructGraph("cluster_"+name, name, graph, swf); err != nil { + if err := subGB.constructGraph(subGraphName, name, graph, swf); err != nil { return nil, err } - gn = subGB.graphNodes["start-node"] + gn = subGB.graphNodes[StartNode] + gb.nodeClusters[gn.Name] = subGraphName } } } @@ -236,20 +279,19 @@ func (gb *graphBuilder) addEdge(fromNodeName, toNodeName string, graph *graphviz if !toOk || !fromOk { return fmt.Errorf("nodes[%s] -> [%s] referenced before creation", fromNodeName, toNodeName) } - if _,ok := graph.Edges.SrcToDsts[fromNode.Name][toNode.Name]; !ok { - attrs := map[string]string {} - err := graph.AddEdge(fromNode.Name, toNode.Name, true, attrs) - if err != nil { - return err - } + if _, ok := graph.Edges.SrcToDsts[fromNode.Name][toNode.Name]; !ok { + attrs := map[string]string{} // Now lets check that the toNode or the fromNode is a cluster. If so then following this thread, // https://stackoverflow.com/questions/2012036/graphviz-how-to-connect-subgraphs, we will connect the cluster - - if c, ok := graph.Nodes.Lookup[fromNode.Name]; ok { - attrs["ltail"] = c.Name + if c, ok := gb.nodeClusters[fromNode.Name]; ok { + attrs[LTailAttr] = c } - if c, ok := graph.Nodes.Lookup[toNode.Name]; ok { - attrs["lhead"] = c.Name + if c, ok := gb.nodeClusters[toNode.Name]; ok { + attrs[LHeadAttr] = c + } + err := graph.AddEdge(fromNode.Name, toNode.Name, true, attrs) + if err != nil { + return err } } return nil @@ -307,8 +349,9 @@ func (gb *graphBuilder) CompiledWorkflowClosureToGraph(w *core.CompiledWorkflowC func newGraphBuilder() *graphBuilder { return &graphBuilder{ - graphNodes: make(map[string]*graphviz.Node), - graphEdges: make(map[string]*graphviz.Edge), + graphNodes: make(map[string]*graphviz.Node), + graphEdges: make(map[string]*graphviz.Edge), + nodeClusters: make(map[string]string), } } @@ -319,14 +362,12 @@ func graphBuilderFromParent(gb *graphBuilder) *graphBuilder { return newGB } -// RenderWorkflow Renders the workflow graph to the given file -func RenderWorkflow(w *core.CompiledWorkflowClosure) ([]byte, error) { +// RenderWorkflow Renders the workflow graph on the console +func RenderWorkflow(w *core.CompiledWorkflowClosure) (string, error) { gb := newGraphBuilder() graph, err := gb.CompiledWorkflowClosureToGraph(w) if err != nil { - return nil, err + return "", err } - - fmt.Printf(graph.String()) - return []byte(graph.String()), nil + return graph.String(), nil } diff --git a/pkg/visualize/graphviz_test.go b/pkg/visualize/graphviz_test.go index 14efea63..1a2619fb 100644 --- a/pkg/visualize/graphviz_test.go +++ b/pkg/visualize/graphviz_test.go @@ -6,8 +6,6 @@ import ( "io/ioutil" "testing" - "github.com/goccy/go-graphviz" - "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" "github.com/golang/protobuf/jsonpb" "github.com/stretchr/testify/assert" @@ -27,7 +25,7 @@ func TestRenderWorkflowBranch(t *testing.T) { c := &core.CompiledWorkflowClosure{} err = jsonpb.Unmarshal(i, c) assert.NoError(t, err) - b, err := RenderWorkflow(c, graphviz.SVG) + b, err := RenderWorkflow(c) assert.NoError(t, err) assert.NotNil(t, b) }) diff --git a/pkg/visualize/testdata/compiled_closure_branch_nested.svg b/pkg/visualize/testdata/compiled_closure_branch_nested.svg deleted file mode 100644 index 66bd9ffe..00000000 --- a/pkg/visualize/testdata/compiled_closure_branch_nested.svg +++ /dev/null @@ -1,160 +0,0 @@ - - - - - - - - -cluster_fractions - - - -cluster_inner_fractions - - - - -start-node - - -start - - - -n0 - -fractions - - - -start-node->n0 - - - - - -n0-n0 - -inner_fractions - - - -start-node->n0-n0 - - - - - -n0-n0-n0-n0 - -double [python-task] - - - -start-node->n0-n0-n0-n0 - - - - - -n0-n0-n0-n1 - -square [python-task] - - - -start-node->n0-n0-n0-n1 - - - - - -n0-n2 - -double [python-task] - - - -start-node->n0-n2 - - - - - -n0-n1 - -square [python-task] - - - -start-node->n0-n1 - - - - - -end-node - - -end - - - -n0->end-node - - - - - -n0->n0-n0 - - -(.my_input GT 0.1) AND (.my_input LT 1) - - - -n0->n0-n2 - - -orElse - - - -n0->n0-n1 - - -(.my_input GT 1) AND (.my_input LT 10) - - - -n0-n0->n0-n0-n0-n0 - - -.my_input LT 0.5 - - - -n0-n0-error - -Only <0.7 allowed - - - -n0-n0->n0-n0-error - - -orElse - Fail - - - -n0-n0->n0-n0-n0-n1 - - -(.my_input GT 0.5) AND (.my_input LT 0.7) - - - diff --git a/pkg/visualize/testdata/compiled_subworkflows.svg b/pkg/visualize/testdata/compiled_subworkflows.svg deleted file mode 100644 index 2bb2457a..00000000 --- a/pkg/visualize/testdata/compiled_subworkflows.svg +++ /dev/null @@ -1,110 +0,0 @@ - - - - - - - - -cluster_n1 - - - - -start-node - - -start - - - -node-t1-parent - -t1 [python-task] - - - -start-node->node-t1-parent - - - - - -end-node - - -end - - - -node-t1-parent->end-node - - - - - -n1-start-node - - -start - - - -node-t1-parent->n1-start-node - - - - - -n1-start-node->end-node - - - - - -n1-n0 - -t1 [python-task] - - - -n1-start-node->n1-n0 - - - - - -n1-end-node - - -end - - - -n1-n0->n1-end-node - - - - - -n1-n1 - -t1 [python-task] - - - -n1-n0->n1-n1 - - - - - -n1-n1->n1-end-node - - - - - From af5fc126b210d7e9cda4caf57c3474aa338a43ab Mon Sep 17 00:00:00 2001 From: pmahindrakar-oss Date: Fri, 18 Jun 2021 13:39:37 +0530 Subject: [PATCH 12/16] Graphviz interfaces and unit tests (#106) * Linter fixes Signed-off-by: Prafulla Mahindrakar * Added some coverage Signed-off-by: Prafulla Mahindrakar * Added interface for the mehtods used from graphviz Signed-off-by: Prafulla Mahindrakar * Fixed the workflow config flag file and bug fixes Signed-off-by: Prafulla Mahindrakar (cherry picked from commit 77be561b2047ccddb98b7d2d46f6f28d2df4adae) * Added unit tests Signed-off-by: Prafulla Mahindrakar --- .../subcommand/workflow/config_flags.go | 19 +- .../subcommand/workflow/config_flags_test.go | 44 --- .../subcommand/workflow/workflow_config.go | 6 +- cmd/get/workflow.go | 1 + pkg/printer/printer.go | 8 +- pkg/printer/printer_test.go | 134 ++++++- pkg/visualize/graphviz.go | 62 ++-- pkg/visualize/graphviz_test.go | 339 ++++++++++++++++++ pkg/visualize/graphvizer.go | 35 ++ pkg/visualize/mocks/graphvizer.go | 273 ++++++++++++++ 10 files changed, 835 insertions(+), 86 deletions(-) create mode 100644 pkg/visualize/graphvizer.go create mode 100644 pkg/visualize/mocks/graphvizer.go diff --git a/cmd/config/subcommand/workflow/config_flags.go b/cmd/config/subcommand/workflow/config_flags.go index 97db2f9c..5765379f 100755 --- a/cmd/config/subcommand/workflow/config_flags.go +++ b/cmd/config/subcommand/workflow/config_flags.go @@ -28,6 +28,15 @@ func (Config) elemValueOrNil(v interface{}) interface{} { return v } +func (Config) mustJsonMarshal(v interface{}) string { + raw, err := json.Marshal(v) + if err != nil { + panic(err) + } + + return string(raw) +} + func (Config) mustMarshalJSON(v json.Marshaler) string { raw, err := v.MarshalJSON() if err != nil { @@ -41,9 +50,11 @@ func (Config) mustMarshalJSON(v json.Marshaler) string { // flags is json-name.json-sub-name... etc. func (cfg Config) GetPFlagSet(prefix string) *pflag.FlagSet { cmdFlags := pflag.NewFlagSet("Config", pflag.ExitOnError) - cmdFlags.StringVar(&(DefaultConfig.Version), fmt.Sprintf("%v%v", prefix, "version"), DefaultConfig.Version, "version of the workflow to be fetched.") - cmdFlags.BoolVar(&(DefaultConfig.Latest), fmt.Sprintf("%v%v", prefix, "latest"), DefaultConfig.Latest, " flag to indicate to fetch the latest version, version flag will be ignored in this case") - cmdFlags.StringVar(&(DefaultConfig.Visualize), fmt.Sprintf("%v%v", prefix, "visualize"), DefaultConfig.Visualize, "optional flag to visualize a workflow as one of [png, dot, svg, jpg]") - cmdFlags.StringVar(&(DefaultConfig.OutputFile), fmt.Sprintf("%v%v", prefix, "output_file"), DefaultConfig.OutputFile, "path and a filename of where the output image should be dumped. This can only be used in concert with visualize") + cmdFlags.StringVar(&DefaultConfig.Version, fmt.Sprintf("%v%v", prefix, "version"), DefaultConfig.Version, "version of the workflow to be fetched.") + cmdFlags.BoolVar(&DefaultConfig.Latest, fmt.Sprintf("%v%v", prefix, "latest"), DefaultConfig.Latest, " flag to indicate to fetch the latest version, version flag will be ignored in this case") + cmdFlags.StringVar(&DefaultConfig.Filter.FieldSelector, fmt.Sprintf("%v%v", prefix, "filter.field-selector"), DefaultConfig.Filter.FieldSelector, "Specifies the Field selector") + cmdFlags.StringVar(&DefaultConfig.Filter.SortBy, fmt.Sprintf("%v%v", prefix, "filter.sort-by"), DefaultConfig.Filter.SortBy, "Specifies which field to sort results ") + cmdFlags.Int32Var(&DefaultConfig.Filter.Limit, fmt.Sprintf("%v%v", prefix, "filter.limit"), DefaultConfig.Filter.Limit, "Specifies the limit") + cmdFlags.BoolVar(&DefaultConfig.Filter.Asc, fmt.Sprintf("%v%v", prefix, "filter.asc"), DefaultConfig.Filter.Asc, "Specifies the sorting order. By default flytectl sort result in descending order") return cmdFlags } diff --git a/cmd/config/subcommand/workflow/config_flags_test.go b/cmd/config/subcommand/workflow/config_flags_test.go index fc7160d1..86998d77 100755 --- a/cmd/config/subcommand/workflow/config_flags_test.go +++ b/cmd/config/subcommand/workflow/config_flags_test.go @@ -183,48 +183,4 @@ func TestConfig_SetFlags(t *testing.T) { } }) }) - t.Run("Test_visualize", func(t *testing.T) { - t.Run("DefaultValue", func(t *testing.T) { - // Test that default value is set properly - if vString, err := cmdFlags.GetString("visualize"); err == nil { - assert.Equal(t, string(DefaultConfig.Visualize), vString) - } else { - assert.FailNow(t, err.Error()) - } - }) - - t.Run("Override", func(t *testing.T) { - testValue := "1" - - cmdFlags.Set("visualize", testValue) - if vString, err := cmdFlags.GetString("visualize"); err == nil { - testDecodeJson_Config(t, fmt.Sprintf("%v", vString), &actual.Visualize) - - } else { - assert.FailNow(t, err.Error()) - } - }) - }) - t.Run("Test_output_file", func(t *testing.T) { - t.Run("DefaultValue", func(t *testing.T) { - // Test that default value is set properly - if vString, err := cmdFlags.GetString("output_file"); err == nil { - assert.Equal(t, string(DefaultConfig.OutputFile), vString) - } else { - assert.FailNow(t, err.Error()) - } - }) - - t.Run("Override", func(t *testing.T) { - testValue := "1" - - cmdFlags.Set("output_file", testValue) - if vString, err := cmdFlags.GetString("output_file"); err == nil { - testDecodeJson_Config(t, fmt.Sprintf("%v", vString), &actual.OutputFile) - - } else { - assert.FailNow(t, err.Error()) - } - }) - }) } diff --git a/cmd/config/subcommand/workflow/workflow_config.go b/cmd/config/subcommand/workflow/workflow_config.go index 1b79c50b..34049e08 100644 --- a/cmd/config/subcommand/workflow/workflow_config.go +++ b/cmd/config/subcommand/workflow/workflow_config.go @@ -4,7 +4,7 @@ import ( "github.com/flyteorg/flytectl/pkg/filters" ) -//go:generate pflags Config --default-var DefaultConfig +//go:generate pflags Config --default-var DefaultConfig --bind-default-var var ( DefaultConfig = &Config{ @@ -17,6 +17,4 @@ type Config struct { Version string `json:"version" pflag:",version of the workflow to be fetched."` Latest bool `json:"latest" pflag:", flag to indicate to fetch the latest version, version flag will be ignored in this case"` Filter filters.Filters `json:"filter" pflag:","` - Visualize string `json:"visualize" pflag:",optional flag to visualize a workflow as one of [png, dot, svg, jpg]"` - OutputFile string `json:"output_file" pflag:",path and a filename of where the output image should be dumped. This can only be used in concert with visualize"` -} \ No newline at end of file +} diff --git a/cmd/get/workflow.go b/cmd/get/workflow.go index 777b43c4..45bd7711 100644 --- a/cmd/get/workflow.go +++ b/cmd/get/workflow.go @@ -2,6 +2,7 @@ package get import ( "context" + workflowconfig "github.com/flyteorg/flytectl/cmd/config/subcommand/workflow" "github.com/flyteorg/flytectl/pkg/ext" "github.com/flyteorg/flytestdlib/logger" diff --git a/pkg/printer/printer.go b/pkg/printer/printer.go index a25ccbeb..98282590 100644 --- a/pkg/printer/printer.go +++ b/pkg/printer/printer.go @@ -4,7 +4,6 @@ import ( "bytes" "encoding/json" "fmt" - "github.com/pkg/browser" "net/url" "os" @@ -17,6 +16,7 @@ import ( "github.com/golang/protobuf/proto" "github.com/kataras/tablewriter" "github.com/landoop/tableprinter" + "github.com/pkg/browser" "github.com/yalp/jsonpath" ) @@ -31,7 +31,7 @@ const ( OutputFormatDOTURL ) -const GraphVisualizationServiceUrl = "http://graph.flyte.org/#" +const GraphVisualizationServiceURL = "http://graph.flyte.org/#" func OutputFormats() []string { var v []string @@ -164,8 +164,8 @@ func (p Printer) Print(format OutputFormat, columns []Column, messages ...proto. return errors.Wrapf("VisualizationError", err, "failed to visualize workflow") } if format == OutputFormatDOTURL { - urlToOpen := GraphVisualizationServiceUrl + url.PathEscape(graphStr) - fmt.Println("Opening the browser at "+ urlToOpen) + urlToOpen := GraphVisualizationServiceURL + url.PathEscape(graphStr) + fmt.Println("Opening the browser at " + urlToOpen) return browser.OpenURL(urlToOpen) } fmt.Println(graphStr) diff --git a/pkg/printer/printer_test.go b/pkg/printer/printer_test.go index 5db91054..08ed10c9 100644 --- a/pkg/printer/printer_test.go +++ b/pkg/printer/printer_test.go @@ -2,6 +2,7 @@ package printer import ( "encoding/json" + "errors" "fmt" "testing" "time" @@ -27,6 +28,14 @@ func LaunchplanToProtoMessages(l []*admin.LaunchPlan) []proto.Message { return messages } +func WorkflowToProtoMessages(l []*admin.Workflow) []proto.Message { + messages := make([]proto.Message, 0, len(l)) + for _, m := range l { + messages = append(messages, m) + } + return messages +} + // TODO Convert this to a Testable Example. For some reason the comparison fails func TestJSONToTable(t *testing.T) { d := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) @@ -57,9 +66,9 @@ func TestJSONToTable(t *testing.T) { } func TestOutputFormats(t *testing.T) { - expected := []string{"TABLE", "JSON", "YAML"} + expected := []string{"TABLE", "JSON", "YAML", "DOT", "DOTURL"} outputs := OutputFormats() - assert.Equal(t, 3, len(outputs)) + assert.Equal(t, 5, len(outputs)) assert.Equal(t, expected, outputs) } @@ -77,7 +86,7 @@ func TestOutputFormatStringErr(t *testing.T) { } func TestIsAOutputFormat(t *testing.T) { - o := OutputFormat(4) + o := OutputFormat(5) check := o.IsAOutputFormat() assert.Equal(t, false, check) @@ -123,4 +132,123 @@ func TestPrint(t *testing.T) { assert.Nil(t, err) err = p.Print(OutputFormat(2), lp, LaunchplanToProtoMessages(launchPlans)...) assert.Nil(t, err) + err = p.Print(OutputFormat(3), lp, LaunchplanToProtoMessages(launchPlans)...) + assert.NotNil(t, err) + err = p.Print(OutputFormat(4), lp, LaunchplanToProtoMessages(launchPlans)...) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("visualization is only supported on workflows"), err) + + sortedListLiteralType := core.Variable{ + Type: &core.LiteralType{ + Type: &core.LiteralType_CollectionType{ + CollectionType: &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_INTEGER, + }, + }, + }, + }, + } + variableMap := map[string]*core.Variable{ + "sorted_list1": &sortedListLiteralType, + "sorted_list2": &sortedListLiteralType, + } + + var compiledTasks []*core.CompiledTask + compiledTasks = append(compiledTasks, &core.CompiledTask{ + Template: &core.TaskTemplate{ + Id: &core.Identifier{ + Project: "dummyProject", + Domain: "dummyDomain", + Name: "dummyName", + Version: "dummyVersion", + }, + Interface: &core.TypedInterface{ + Inputs: &core.VariableMap{ + Variables: variableMap, + }, + }, + }, + }) + + workflow1 := &admin.Workflow{ + Id: &core.Identifier{ + Name: "task1", + Version: "v1", + }, + Closure: &admin.WorkflowClosure{ + CreatedAt: ×tamppb.Timestamp{Seconds: 1, Nanos: 0}, + CompiledWorkflow: &core.CompiledWorkflowClosure{ + Tasks: compiledTasks, + }, + }, + } + + workflows := []*admin.Workflow{workflow1} + + err = p.Print(OutputFormat(3), lp, WorkflowToProtoMessages(workflows)...) + assert.Nil(t, err) + workflows = []*admin.Workflow{} + err = p.Print(OutputFormat(3), lp, WorkflowToProtoMessages(workflows)...) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("atleast one workflow required for visualization"), err) + var badCompiledTasks []*core.CompiledTask + badCompiledTasks = append(badCompiledTasks, &core.CompiledTask{ + Template: &core.TaskTemplate{}, + }) + badWorkflow := &admin.Workflow{ + Id: &core.Identifier{ + Name: "task1", + Version: "v1", + }, + Closure: &admin.WorkflowClosure{ + CreatedAt: ×tamppb.Timestamp{Seconds: 1, Nanos: 0}, + CompiledWorkflow: &core.CompiledWorkflowClosure{ + Tasks: badCompiledTasks, + }, + }, + } + workflows = []*admin.Workflow{badWorkflow} + err = p.Print(OutputFormat(3), lp, WorkflowToProtoMessages(workflows)...) + assert.NotNil(t, err) + + assert.Equal(t, fmt.Errorf("no template found in the workflow task template:<> "), errors.Unwrap(err)) + + badWorkflow2 := &admin.Workflow{ + Id: &core.Identifier{ + Name: "task1", + Version: "v1", + }, + Closure: &admin.WorkflowClosure{ + CreatedAt: ×tamppb.Timestamp{Seconds: 1, Nanos: 0}, + CompiledWorkflow: nil, + }, + } + workflows = []*admin.Workflow{badWorkflow2} + err = p.Print(OutputFormat(3), lp, WorkflowToProtoMessages(workflows)...) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("empty workflow closure"), errors.Unwrap(err)) + + var badSubWorkflow []*core.CompiledWorkflow + badSubWorkflow = append(badSubWorkflow, &core.CompiledWorkflow{ + Template: &core.WorkflowTemplate{}, + }) + + badWorkflow3 := &admin.Workflow{ + Id: &core.Identifier{ + Name: "task1", + Version: "v1", + }, + Closure: &admin.WorkflowClosure{ + CreatedAt: ×tamppb.Timestamp{Seconds: 1, Nanos: 0}, + CompiledWorkflow: &core.CompiledWorkflowClosure{ + Tasks: compiledTasks, + SubWorkflows: badSubWorkflow, + }, + }, + } + workflows = []*admin.Workflow{badWorkflow3} + err = p.Print(OutputFormat(3), lp, WorkflowToProtoMessages(workflows)...) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("no template found in the sub workflow template:<> "), errors.Unwrap(err)) } diff --git a/pkg/visualize/graphviz.go b/pkg/visualize/graphviz.go index 4bc1b606..314c3251 100644 --- a/pkg/visualize/graphviz.go +++ b/pkg/visualize/graphviz.go @@ -36,8 +36,7 @@ const ( // conditional ElseFail string = "orElse - Fail" - Else string = "orElse" - + Else string = "orElse" ) func operandToString(op *core.Operand) string { @@ -72,21 +71,21 @@ func booleanExprToString(expr *core.BooleanExpression) string { return comparisonToString(expr.GetComparison()) } -func constructStartNode(parentGraph string, n string, graph *graphviz.Graph) (*graphviz.Node, error) { +func constructStartNode(parentGraph string, n string, graph Graphvizer) (*graphviz.Node, error) { attrs := map[string]string{ShapeType: DoubleCircleShape, ColorAttr: Green} attrs[LabelAttr] = "start" err := graph.AddNode(parentGraph, n, attrs) - return graph.Nodes.Lookup[n], err + return graph.GetNode(n), err } -func constructEndNode(parentGraph string, n string, graph *graphviz.Graph) (*graphviz.Node, error) { +func constructEndNode(parentGraph string, n string, graph Graphvizer) (*graphviz.Node, error) { attrs := map[string]string{ShapeType: DoubleCircleShape, ColorAttr: Red} attrs[LabelAttr] = "end" err := graph.AddNode(parentGraph, n, attrs) - return graph.Nodes.Lookup[n], err + return graph.GetNode(n), err } -func constructTaskNode(parentGraph string, name string, graph *graphviz.Graph, n *core.Node, t *core.CompiledTask) (*graphviz.Node, error) { +func constructTaskNode(parentGraph string, name string, graph Graphvizer, n *core.Node, t *core.CompiledTask) (*graphviz.Node, error) { attrs := map[string]string{ShapeType: BoxShape} if n.Metadata != nil && n.Metadata.Name != "" { v := strings.LastIndexAny(n.Metadata.Name, ".") @@ -94,24 +93,24 @@ func constructTaskNode(parentGraph string, name string, graph *graphviz.Graph, n } tName := strings.ReplaceAll(name, "-", "_") err := graph.AddNode(parentGraph, tName, attrs) - return graph.Nodes.Lookup[tName], err + return graph.GetNode(tName), err } -func constructErrorNode(parentGraph string, name string, graph *graphviz.Graph, m string) (*graphviz.Node, error) { - attrs := map[string]string{ShapeType: BoxShape, ColorAttr: Red, LabelAttr: m} +func constructErrorNode(parentGraph string, name string, graph Graphvizer, m string) (*graphviz.Node, error) { + attrs := map[string]string{ShapeType: BoxShape, ColorAttr: Red, LabelAttr: fmt.Sprintf("\"%s\"", m)} eName := strings.ReplaceAll(name, "-", "_") err := graph.AddNode(parentGraph, eName, attrs) - return graph.Nodes.Lookup[eName], err + return graph.GetNode(eName), err } -func constructBranchConditionNode(parentGraph string, name string, graph *graphviz.Graph, n *core.Node) (*graphviz.Node, error) { +func constructBranchConditionNode(parentGraph string, name string, graph Graphvizer, n *core.Node) (*graphviz.Node, error) { attrs := map[string]string{ShapeType: DiamondShape} if n.Metadata != nil && n.Metadata.Name != "" { attrs[LabelAttr] = fmt.Sprintf("\"[%s]\"", n.Metadata.Name) } cName := strings.ReplaceAll(name, "-", "_") err := graph.AddNode(parentGraph, cName, attrs) - return graph.Nodes.Lookup[cName], err + return graph.GetNode(cName), err } func getName(prefix, id string) string { @@ -135,24 +134,24 @@ type graphBuilder struct { nodeClusters map[string]string } -func (gb *graphBuilder) addBranchSubNodeEdge(graph *graphviz.Graph, parentNode, n *graphviz.Node, label string) error { +func (gb *graphBuilder) addBranchSubNodeEdge(graph Graphvizer, parentNode, n *graphviz.Node, label string) error { edgeName := fmt.Sprintf("%s-%s", parentNode.Name, n.Name) if _, ok := gb.graphEdges[edgeName]; !ok { attrs := map[string]string{} if c, ok := gb.nodeClusters[n.Name]; ok { - attrs[LHeadAttr] = c + attrs[LHeadAttr] = fmt.Sprintf("\"%s\"", c) } attrs[LabelAttr] = fmt.Sprintf("\"%s\"", label) err := graph.AddEdge(parentNode.Name, n.Name, true, attrs) if err != nil { return err } - gb.graphEdges[edgeName] = graph.Edges.SrcToDsts[parentNode.Name][n.Name][0] + gb.graphEdges[edgeName] = graph.GetEdge(parentNode.Name, n.Name) } return nil } -func (gb *graphBuilder) constructBranchNode(parentGraph string, prefix string, graph *graphviz.Graph, n *core.Node) (*graphviz.Node, error) { +func (gb *graphBuilder) constructBranchNode(parentGraph string, prefix string, graph Graphvizer, n *core.Node) (*graphviz.Node, error) { parentBranchNode, err := constructBranchConditionNode(parentGraph, getName(prefix, n.Id), graph, n) if err != nil { return nil, err @@ -205,7 +204,7 @@ func (gb *graphBuilder) constructBranchNode(parentGraph string, prefix string, g return parentBranchNode, nil } -func (gb *graphBuilder) constructNode(parentGraphName string, prefix string, graph *graphviz.Graph, n *core.Node) (*graphviz.Node, error) { +func (gb *graphBuilder) constructNode(parentGraphName string, prefix string, graph Graphvizer, n *core.Node) (*graphviz.Node, error) { name := getName(prefix, n.Id) var err error var gn *graphviz.Node @@ -273,21 +272,21 @@ func (gb *graphBuilder) constructNode(parentGraphName string, prefix string, gra return gn, nil } -func (gb *graphBuilder) addEdge(fromNodeName, toNodeName string, graph *graphviz.Graph) error { +func (gb *graphBuilder) addEdge(fromNodeName, toNodeName string, graph Graphvizer) error { toNode, toOk := gb.graphNodes[toNodeName] fromNode, fromOk := gb.graphNodes[fromNodeName] if !toOk || !fromOk { return fmt.Errorf("nodes[%s] -> [%s] referenced before creation", fromNodeName, toNodeName) } - if _, ok := graph.Edges.SrcToDsts[fromNode.Name][toNode.Name]; !ok { + if !graph.DoesEdgeExist(fromNode.Name, toNode.Name) { attrs := map[string]string{} // Now lets check that the toNode or the fromNode is a cluster. If so then following this thread, // https://stackoverflow.com/questions/2012036/graphviz-how-to-connect-subgraphs, we will connect the cluster if c, ok := gb.nodeClusters[fromNode.Name]; ok { - attrs[LTailAttr] = c + attrs[LTailAttr] = fmt.Sprintf("\"%s\"", c) } if c, ok := gb.nodeClusters[toNode.Name]; ok { - attrs[LHeadAttr] = c + attrs[LHeadAttr] = fmt.Sprintf("\"%s\"", c) } err := graph.AddEdge(fromNode.Name, toNode.Name, true, attrs) if err != nil { @@ -297,7 +296,7 @@ func (gb *graphBuilder) addEdge(fromNodeName, toNodeName string, graph *graphviz return nil } -func (gb *graphBuilder) constructGraph(parentGraphName string, prefix string, graph *graphviz.Graph, w *core.CompiledWorkflow) error { +func (gb *graphBuilder) constructGraph(parentGraphName string, prefix string, graph Graphvizer, w *core.CompiledWorkflow) error { if w == nil || w.Template == nil { return nil } @@ -328,18 +327,24 @@ func (gb *graphBuilder) constructGraph(parentGraphName string, prefix string, gr return nil } -func (gb *graphBuilder) CompiledWorkflowClosureToGraph(w *core.CompiledWorkflowClosure) (*graphviz.Graph, error) { - dotGraph := graphviz.NewGraph() +func (gb *graphBuilder) CompiledWorkflowClosureToGraph(w *core.CompiledWorkflowClosure) (FlyteGraph, error) { + dotGraph := FlyteGraph{graphviz.NewGraph()} _ = dotGraph.SetDir(true) _ = dotGraph.SetStrict(true) tLookup := make(map[string]*core.CompiledTask) for _, t := range w.Tasks { + if t.Template == nil || t.Template.Id == nil { + return FlyteGraph{}, fmt.Errorf("no template found in the workflow task %v", t) + } tLookup[t.Template.Id.String()] = t } gb.tasks = tLookup wLookup := make(map[string]*core.CompiledWorkflow) for _, swf := range w.SubWorkflows { + if swf.Template == nil || swf.Template.Id == nil { + return FlyteGraph{}, fmt.Errorf("no template found in the sub workflow %v", swf) + } wLookup[swf.Template.Id.String()] = swf } gb.subWf = wLookup @@ -349,8 +354,8 @@ func (gb *graphBuilder) CompiledWorkflowClosureToGraph(w *core.CompiledWorkflowC func newGraphBuilder() *graphBuilder { return &graphBuilder{ - graphNodes: make(map[string]*graphviz.Node), - graphEdges: make(map[string]*graphviz.Edge), + graphNodes: make(map[string]*graphviz.Node), + graphEdges: make(map[string]*graphviz.Edge), nodeClusters: make(map[string]string), } } @@ -364,6 +369,9 @@ func graphBuilderFromParent(gb *graphBuilder) *graphBuilder { // RenderWorkflow Renders the workflow graph on the console func RenderWorkflow(w *core.CompiledWorkflowClosure) (string, error) { + if w == nil { + return "", fmt.Errorf("empty workflow closure") + } gb := newGraphBuilder() graph, err := gb.CompiledWorkflowClosureToGraph(w) if err != nil { diff --git a/pkg/visualize/graphviz_test.go b/pkg/visualize/graphviz_test.go index 1a2619fb..51571339 100644 --- a/pkg/visualize/graphviz_test.go +++ b/pkg/visualize/graphviz_test.go @@ -6,9 +6,13 @@ import ( "io/ioutil" "testing" + "github.com/flyteorg/flytectl/pkg/visualize/mocks" "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" + + graphviz "github.com/awalterschulze/gographviz" "github.com/golang/protobuf/jsonpb" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" ) func TestRenderWorkflowBranch(t *testing.T) { @@ -26,8 +30,343 @@ func TestRenderWorkflowBranch(t *testing.T) { err = jsonpb.Unmarshal(i, c) assert.NoError(t, err) b, err := RenderWorkflow(c) + fmt.Println(b) assert.NoError(t, err) assert.NotNil(t, b) }) } } + +func TestAddBranchSubNodeEdge(t *testing.T) { + attrs := map[string]string{} + attrs[LHeadAttr] = fmt.Sprintf("\"%s\"", "innerGraph") + attrs[LabelAttr] = fmt.Sprintf("\"%s\"", "label") + t.Run("Successful", func(t *testing.T) { + gb := newGraphBuilder() + gb.nodeClusters["n"] = "innerGraph" + parentNode := &graphviz.Node{Name: "parentNode", Attrs: nil} + n := &graphviz.Node{Name: "n"} + + mockGraph := &mocks.Graphvizer{} + // Verify the attributes + mockGraph.OnAddEdgeMatch(mock.Anything, mock.Anything, mock.Anything, attrs).Return(nil) + mockGraph.OnGetEdgeMatch(mock.Anything, mock.Anything).Return(&graphviz.Edge{}) + err := gb.addBranchSubNodeEdge(mockGraph, parentNode, n, "label") + assert.NoError(t, err) + }) + + t.Run("Error", func(t *testing.T) { + gb := newGraphBuilder() + gb.nodeClusters["n"] = "innerGraph" + parentNode := &graphviz.Node{Name: "parentNode", Attrs: nil} + n := &graphviz.Node{Name: "n"} + + mockGraph := &mocks.Graphvizer{} + // Verify the attributes + mockGraph.OnAddEdgeMatch(mock.Anything, mock.Anything, mock.Anything, attrs).Return(fmt.Errorf("error adding edge")) + err := gb.addBranchSubNodeEdge(mockGraph, parentNode, n, "label") + assert.NotNil(t, err) + }) +} + +func TestConstructBranchNode(t *testing.T) { + attrs := map[string]string{} + attrs[LabelAttr] = fmt.Sprintf("\"[%s]\"", "nodeMetadata") + attrs[ShapeType] = DiamondShape + t.Run("Successful", func(t *testing.T) { + gb := newGraphBuilder() + mockGraph := &mocks.Graphvizer{} + expectedGraphvizNode := &graphviz.Node{ + Name: "brancheName_id", + Attrs: map[graphviz.Attr]string{"label": fmt.Sprintf("\"[%s]\"", "nodeMetadata"), + "shape": "diamond"}, + } + // Verify the attributes + mockGraph.OnAddNodeMatch(mock.Anything, mock.Anything, attrs).Return(nil) + mockGraph.OnGetNodeMatch(mock.Anything).Return(expectedGraphvizNode) + flyteNode := &core.Node{ + Id: "id", + Metadata: &core.NodeMetadata{ + Name: "nodeMetadata", + }, + Target: &core.Node_BranchNode{ + BranchNode: &core.BranchNode{}, + }, + } + resultBranchNode, err := gb.constructBranchNode("parentGraph", "branchName", mockGraph, flyteNode) + assert.NoError(t, err) + assert.NotNil(t, resultBranchNode) + assert.Equal(t, expectedGraphvizNode, resultBranchNode) + }) + + t.Run("Add Node Error", func(t *testing.T) { + gb := newGraphBuilder() + mockGraph := &mocks.Graphvizer{} + // Verify the attributes + mockGraph.OnAddNodeMatch(mock.Anything, mock.Anything, attrs).Return(fmt.Errorf("unable to add node")) + mockGraph.OnGetNodeMatch(mock.Anything).Return(nil) + flyteNode := &core.Node{ + Id: "id", + Metadata: &core.NodeMetadata{ + Name: "nodeMetadata", + }, + Target: &core.Node_BranchNode{ + BranchNode: &core.BranchNode{}, + }, + } + resultBranchNode, err := gb.constructBranchNode("parentGraph", "branchName", mockGraph, flyteNode) + assert.NotNil(t, err) + assert.Nil(t, resultBranchNode) + }) +} + +func TestConstructNode(t *testing.T) { + + t.Run("Start-Node", func(t *testing.T) { + attrs := map[string]string{} + attrs[LabelAttr] = "start" + attrs[ShapeType] = DoubleCircleShape + attrs[ColorAttr] = Green + gb := newGraphBuilder() + mockGraph := &mocks.Graphvizer{} + expectedGraphvizNode := &graphviz.Node{ + Name: "start-node", + Attrs: map[graphviz.Attr]string{"label": "start", "shape": "doublecircle", "color": "green"}, + } + // Verify the attributes + mockGraph.OnAddNodeMatch(mock.Anything, mock.Anything, attrs).Return(nil) + mockGraph.OnGetNodeMatch(mock.Anything).Return(expectedGraphvizNode) + flyteNode := &core.Node{ + Id: "start-node", + } + resultNode, err := gb.constructNode("", "", mockGraph, flyteNode) + assert.NoError(t, err) + assert.NotNil(t, resultNode) + assert.Equal(t, expectedGraphvizNode, resultNode) + }) + + t.Run("End-Node", func(t *testing.T) { + attrs := map[string]string{} + attrs[LabelAttr] = "end" + attrs[ShapeType] = DoubleCircleShape + attrs[ColorAttr] = Red + gb := newGraphBuilder() + mockGraph := &mocks.Graphvizer{} + expectedGraphvizNode := &graphviz.Node{ + Name: "end-node", + Attrs: map[graphviz.Attr]string{"label": "end", "shape": "doublecircle", "color": "red"}, + } + // Verify the attributes + mockGraph.OnAddNodeMatch(mock.Anything, mock.Anything, attrs).Return(nil) + mockGraph.OnGetNodeMatch(mock.Anything).Return(expectedGraphvizNode) + flyteNode := &core.Node{ + Id: "end-node", + } + resultNode, err := gb.constructNode("", "", mockGraph, flyteNode) + assert.NoError(t, err) + assert.NotNil(t, resultNode) + assert.Equal(t, expectedGraphvizNode, resultNode) + }) + + t.Run("Task-Node-Error", func(t *testing.T) { + gb := newGraphBuilder() + mockGraph := &mocks.Graphvizer{} + flyteNode := &core.Node{ + Id: "id", + Metadata: &core.NodeMetadata{ + Name: "nodeMetadata", + }, + Target: &core.Node_TaskNode{ + TaskNode: &core.TaskNode{ + Reference: &core.TaskNode_ReferenceId{ + ReferenceId: &core.Identifier{ + Project: "dummyProject", + Domain: "dummyDomain", + Name: "dummyName", + Version: "dummyVersion", + }, + }, + }, + }, + } + resultNode, err := gb.constructNode("", "", mockGraph, flyteNode) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("failed to find task [%s] in closure", + flyteNode.GetTaskNode().GetReferenceId().String()), err) + assert.Nil(t, resultNode) + }) + + t.Run("Branch-Node-SubGraph-Error", func(t *testing.T) { + gb := newGraphBuilder() + mockGraph := &mocks.Graphvizer{} + // Verify the attributes + mockGraph.OnAddSubGraphMatch("", SubgraphPrefix+"nodeMetadata", + mock.Anything).Return(fmt.Errorf("unable to create subgraph")) + flyteNode := &core.Node{ + Id: "id", + Metadata: &core.NodeMetadata{ + Name: "nodeMetadata", + }, + Target: &core.Node_BranchNode{ + BranchNode: &core.BranchNode{}, + }, + } + resultBranchNode, err := gb.constructNode("", "", mockGraph, flyteNode) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("unable to create subgraph"), err) + assert.Nil(t, resultBranchNode) + }) + + t.Run("Branch-Node-Add-Error", func(t *testing.T) { + attrs := map[string]string{} + attrs[LabelAttr] = fmt.Sprintf("\"[%s]\"", "nodeMetadata") + attrs[ShapeType] = DiamondShape + gb := newGraphBuilder() + mockGraph := &mocks.Graphvizer{} + + // Verify the attributes + mockGraph.OnAddSubGraphMatch(mock.Anything, SubgraphPrefix+"nodeMetadata", mock.Anything).Return(nil) + mockGraph.OnAddNodeMatch(mock.Anything, mock.Anything, attrs).Return(fmt.Errorf("unable to add node")) + mockGraph.OnGetNodeMatch(mock.Anything).Return(nil) + + flyteNode := &core.Node{ + Id: "id", + Metadata: &core.NodeMetadata{ + Name: "nodeMetadata", + }, + Target: &core.Node_BranchNode{ + BranchNode: &core.BranchNode{}, + }, + } + resultBranchNode, err := gb.constructNode("", "", mockGraph, flyteNode) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("unable to add node"), err) + assert.Nil(t, resultBranchNode) + }) + + t.Run("Workflow-Node-Add-Error", func(t *testing.T) { + attrs := map[string]string{} + gb := newGraphBuilder() + mockGraph := &mocks.Graphvizer{} + + // Verify the attributes + mockGraph.OnAddNodeMatch(mock.Anything, mock.Anything, attrs).Return(fmt.Errorf("unable to add node")) + mockGraph.OnGetNodeMatch(mock.Anything).Return(nil) + + flyteNode := &core.Node{ + Id: "id", + Metadata: &core.NodeMetadata{ + Name: "nodeMetadata", + }, + Target: &core.Node_WorkflowNode{ + WorkflowNode: &core.WorkflowNode{ + Reference: &core.WorkflowNode_LaunchplanRef{ + LaunchplanRef: &core.Identifier{ + Project: "dummyProject", + Domain: "dummyDomain", + Name: "dummyName", + Version: "dummyVersion", + }, + }, + }, + }, + } + resultWorkflowNode, err := gb.constructNode("", "", mockGraph, flyteNode) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("unable to add node"), err) + assert.Nil(t, resultWorkflowNode) + }) + + t.Run("Workflow-Node-SubGraph-Error", func(t *testing.T) { + gb := newGraphBuilder() + mockGraph := &mocks.Graphvizer{} + // Verify the attributes + mockGraph.OnAddSubGraphMatch("", SubgraphPrefix+"id", + mock.Anything).Return(fmt.Errorf("unable to create subgraph")) + flyteNode := &core.Node{ + Id: "id", + Metadata: &core.NodeMetadata{ + Name: "nodeMetadata", + }, + Target: &core.Node_WorkflowNode{ + WorkflowNode: &core.WorkflowNode{}, + }, + } + resultWorkflowNode, err := gb.constructNode("", "", mockGraph, flyteNode) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("unable to create subgraph"), err) + assert.Nil(t, resultWorkflowNode) + }) + t.Run("Workflow-Node-Subworkflow-NotFound-Error", func(t *testing.T) { + gb := newGraphBuilder() + mockGraph := &mocks.Graphvizer{} + + // Verify the attributes + mockGraph.OnAddSubGraphMatch(mock.Anything, SubgraphPrefix+"id", mock.Anything).Return(nil) + + flyteNode := &core.Node{ + Id: "id", + Metadata: &core.NodeMetadata{ + Name: "nodeMetadata", + }, + Target: &core.Node_WorkflowNode{ + WorkflowNode: &core.WorkflowNode{ + Reference: &core.WorkflowNode_SubWorkflowRef{ + SubWorkflowRef: &core.Identifier{ + Project: "dummyProject", + Domain: "dummyDomain", + Name: "dummyName", + Version: "dummyVersion", + }, + }, + }, + }, + } + resultWorkflowNode, err := gb.constructNode("", "", mockGraph, flyteNode) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("subworkfow [project:\"dummyProject\" domain:\"dummyDomain\" name:\"dummyName\" version:\"dummyVersion\" ] not found"), err) + assert.Nil(t, resultWorkflowNode) + }) + + t.Run("Workflow-Node-Subworkflow-Graph-Create-Error", func(t *testing.T) { + gb := newGraphBuilder() + mockGraph := &mocks.Graphvizer{} + + // Verify the attributes + mockGraph.OnAddSubGraphMatch(mock.Anything, SubgraphPrefix+"id", mock.Anything).Return(nil) + mockGraph.OnAddNodeMatch(mock.Anything, mock.Anything, mock.Anything).Return(fmt.Errorf("unable to add node")) + mockGraph.OnGetNodeMatch(mock.Anything).Return(nil) + + gb.subWf = make(map[string]*core.CompiledWorkflow) + subwfNode := &core.Node{ + Id: "start-node", + } + sbwfNodes := []*core.Node{subwfNode} + + gb.subWf["project:\"dummyProject\" domain:\"dummyDomain\" name:\"dummyName\" version:\"dummyVersion\" "] = + &core.CompiledWorkflow{Template: &core.WorkflowTemplate{Nodes: sbwfNodes}} + flyteNode := &core.Node{ + Id: "id", + Metadata: &core.NodeMetadata{ + Name: "nodeMetadata", + }, + Target: &core.Node_WorkflowNode{ + WorkflowNode: &core.WorkflowNode{ + Reference: &core.WorkflowNode_SubWorkflowRef{ + SubWorkflowRef: &core.Identifier{ + Project: "dummyProject", + Domain: "dummyDomain", + Name: "dummyName", + Version: "dummyVersion", + }, + }, + }, + }, + } + resultWorkflowNode, err := gb.constructNode("", "", mockGraph, flyteNode) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("unable to add node"), err) + assert.Nil(t, resultWorkflowNode) + }) + +} diff --git a/pkg/visualize/graphvizer.go b/pkg/visualize/graphvizer.go new file mode 100644 index 00000000..7fcb24ee --- /dev/null +++ b/pkg/visualize/graphvizer.go @@ -0,0 +1,35 @@ +package visualize + +import graphviz "github.com/awalterschulze/gographviz" + +//go:generate mockery -all -case=underscore + +type Graphvizer interface { + AddEdge(src, dst string, directed bool, attrs map[string]string) error + AddNode(parentGraph string, name string, attrs map[string]string) error + AddSubGraph(parentGraph string, name string, attrs map[string]string) error + AddAttr(parentGraph string, field string, value string) error + SetName(name string) error + GetEdge(src, dest string) *graphviz.Edge + GetNode(key string) *graphviz.Node + DoesEdgeExist(src, dest string) bool +} + +type FlyteGraph struct { + *graphviz.Graph +} + +// GetNode given the key to the node +func (g FlyteGraph) GetNode(key string) *graphviz.Node { + return g.Nodes.Lookup[key] +} + +// GetEdge gets the edge in the graph from src to dest +func (g FlyteGraph) GetEdge(src, dest string) *graphviz.Edge { + return g.Edges.SrcToDsts[src][dest][0] +} + +// DoesEdgeExist checks if an edge exists in the graph from src to dest +func (g FlyteGraph) DoesEdgeExist(src, dest string) bool { + return g.Edges.SrcToDsts[src][dest] != nil +} diff --git a/pkg/visualize/mocks/graphvizer.go b/pkg/visualize/mocks/graphvizer.go new file mode 100644 index 00000000..e36204ed --- /dev/null +++ b/pkg/visualize/mocks/graphvizer.go @@ -0,0 +1,273 @@ +// Code generated by mockery v1.0.1. DO NOT EDIT. + +package mocks + +import ( + gographviz "github.com/awalterschulze/gographviz" + mock "github.com/stretchr/testify/mock" +) + +// Graphvizer is an autogenerated mock type for the Graphvizer type +type Graphvizer struct { + mock.Mock +} + +type Graphvizer_AddAttr struct { + *mock.Call +} + +func (_m Graphvizer_AddAttr) Return(_a0 error) *Graphvizer_AddAttr { + return &Graphvizer_AddAttr{Call: _m.Call.Return(_a0)} +} + +func (_m *Graphvizer) OnAddAttr(parentGraph string, field string, value string) *Graphvizer_AddAttr { + c := _m.On("AddAttr", parentGraph, field, value) + return &Graphvizer_AddAttr{Call: c} +} + +func (_m *Graphvizer) OnAddAttrMatch(matchers ...interface{}) *Graphvizer_AddAttr { + c := _m.On("AddAttr", matchers...) + return &Graphvizer_AddAttr{Call: c} +} + +// AddAttr provides a mock function with given fields: parentGraph, field, value +func (_m *Graphvizer) AddAttr(parentGraph string, field string, value string) error { + ret := _m.Called(parentGraph, field, value) + + var r0 error + if rf, ok := ret.Get(0).(func(string, string, string) error); ok { + r0 = rf(parentGraph, field, value) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +type Graphvizer_AddEdge struct { + *mock.Call +} + +func (_m Graphvizer_AddEdge) Return(_a0 error) *Graphvizer_AddEdge { + return &Graphvizer_AddEdge{Call: _m.Call.Return(_a0)} +} + +func (_m *Graphvizer) OnAddEdge(src string, dst string, directed bool, attrs map[string]string) *Graphvizer_AddEdge { + c := _m.On("AddEdge", src, dst, directed, attrs) + return &Graphvizer_AddEdge{Call: c} +} + +func (_m *Graphvizer) OnAddEdgeMatch(matchers ...interface{}) *Graphvizer_AddEdge { + c := _m.On("AddEdge", matchers...) + return &Graphvizer_AddEdge{Call: c} +} + +// AddEdge provides a mock function with given fields: src, dst, directed, attrs +func (_m *Graphvizer) AddEdge(src string, dst string, directed bool, attrs map[string]string) error { + ret := _m.Called(src, dst, directed, attrs) + + var r0 error + if rf, ok := ret.Get(0).(func(string, string, bool, map[string]string) error); ok { + r0 = rf(src, dst, directed, attrs) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +type Graphvizer_AddNode struct { + *mock.Call +} + +func (_m Graphvizer_AddNode) Return(_a0 error) *Graphvizer_AddNode { + return &Graphvizer_AddNode{Call: _m.Call.Return(_a0)} +} + +func (_m *Graphvizer) OnAddNode(parentGraph string, name string, attrs map[string]string) *Graphvizer_AddNode { + c := _m.On("AddNode", parentGraph, name, attrs) + return &Graphvizer_AddNode{Call: c} +} + +func (_m *Graphvizer) OnAddNodeMatch(matchers ...interface{}) *Graphvizer_AddNode { + c := _m.On("AddNode", matchers...) + return &Graphvizer_AddNode{Call: c} +} + +// AddNode provides a mock function with given fields: parentGraph, name, attrs +func (_m *Graphvizer) AddNode(parentGraph string, name string, attrs map[string]string) error { + ret := _m.Called(parentGraph, name, attrs) + + var r0 error + if rf, ok := ret.Get(0).(func(string, string, map[string]string) error); ok { + r0 = rf(parentGraph, name, attrs) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +type Graphvizer_AddSubGraph struct { + *mock.Call +} + +func (_m Graphvizer_AddSubGraph) Return(_a0 error) *Graphvizer_AddSubGraph { + return &Graphvizer_AddSubGraph{Call: _m.Call.Return(_a0)} +} + +func (_m *Graphvizer) OnAddSubGraph(parentGraph string, name string, attrs map[string]string) *Graphvizer_AddSubGraph { + c := _m.On("AddSubGraph", parentGraph, name, attrs) + return &Graphvizer_AddSubGraph{Call: c} +} + +func (_m *Graphvizer) OnAddSubGraphMatch(matchers ...interface{}) *Graphvizer_AddSubGraph { + c := _m.On("AddSubGraph", matchers...) + return &Graphvizer_AddSubGraph{Call: c} +} + +// AddSubGraph provides a mock function with given fields: parentGraph, name, attrs +func (_m *Graphvizer) AddSubGraph(parentGraph string, name string, attrs map[string]string) error { + ret := _m.Called(parentGraph, name, attrs) + + var r0 error + if rf, ok := ret.Get(0).(func(string, string, map[string]string) error); ok { + r0 = rf(parentGraph, name, attrs) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +type Graphvizer_DoesEdgeExist struct { + *mock.Call +} + +func (_m Graphvizer_DoesEdgeExist) Return(_a0 bool) *Graphvizer_DoesEdgeExist { + return &Graphvizer_DoesEdgeExist{Call: _m.Call.Return(_a0)} +} + +func (_m *Graphvizer) OnDoesEdgeExist(src string, dest string) *Graphvizer_DoesEdgeExist { + c := _m.On("DoesEdgeExist", src, dest) + return &Graphvizer_DoesEdgeExist{Call: c} +} + +func (_m *Graphvizer) OnDoesEdgeExistMatch(matchers ...interface{}) *Graphvizer_DoesEdgeExist { + c := _m.On("DoesEdgeExist", matchers...) + return &Graphvizer_DoesEdgeExist{Call: c} +} + +// DoesEdgeExist provides a mock function with given fields: src, dest +func (_m *Graphvizer) DoesEdgeExist(src string, dest string) bool { + ret := _m.Called(src, dest) + + var r0 bool + if rf, ok := ret.Get(0).(func(string, string) bool); ok { + r0 = rf(src, dest) + } else { + r0 = ret.Get(0).(bool) + } + + return r0 +} + +type Graphvizer_GetEdge struct { + *mock.Call +} + +func (_m Graphvizer_GetEdge) Return(_a0 *gographviz.Edge) *Graphvizer_GetEdge { + return &Graphvizer_GetEdge{Call: _m.Call.Return(_a0)} +} + +func (_m *Graphvizer) OnGetEdge(src string, dest string) *Graphvizer_GetEdge { + c := _m.On("GetEdge", src, dest) + return &Graphvizer_GetEdge{Call: c} +} + +func (_m *Graphvizer) OnGetEdgeMatch(matchers ...interface{}) *Graphvizer_GetEdge { + c := _m.On("GetEdge", matchers...) + return &Graphvizer_GetEdge{Call: c} +} + +// GetEdge provides a mock function with given fields: src, dest +func (_m *Graphvizer) GetEdge(src string, dest string) *gographviz.Edge { + ret := _m.Called(src, dest) + + var r0 *gographviz.Edge + if rf, ok := ret.Get(0).(func(string, string) *gographviz.Edge); ok { + r0 = rf(src, dest) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*gographviz.Edge) + } + } + + return r0 +} + +type Graphvizer_GetNode struct { + *mock.Call +} + +func (_m Graphvizer_GetNode) Return(_a0 *gographviz.Node) *Graphvizer_GetNode { + return &Graphvizer_GetNode{Call: _m.Call.Return(_a0)} +} + +func (_m *Graphvizer) OnGetNode(key string) *Graphvizer_GetNode { + c := _m.On("GetNode", key) + return &Graphvizer_GetNode{Call: c} +} + +func (_m *Graphvizer) OnGetNodeMatch(matchers ...interface{}) *Graphvizer_GetNode { + c := _m.On("GetNode", matchers...) + return &Graphvizer_GetNode{Call: c} +} + +// GetNode provides a mock function with given fields: key +func (_m *Graphvizer) GetNode(key string) *gographviz.Node { + ret := _m.Called(key) + + var r0 *gographviz.Node + if rf, ok := ret.Get(0).(func(string) *gographviz.Node); ok { + r0 = rf(key) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*gographviz.Node) + } + } + + return r0 +} + +type Graphvizer_SetName struct { + *mock.Call +} + +func (_m Graphvizer_SetName) Return(_a0 error) *Graphvizer_SetName { + return &Graphvizer_SetName{Call: _m.Call.Return(_a0)} +} + +func (_m *Graphvizer) OnSetName(name string) *Graphvizer_SetName { + c := _m.On("SetName", name) + return &Graphvizer_SetName{Call: c} +} + +func (_m *Graphvizer) OnSetNameMatch(matchers ...interface{}) *Graphvizer_SetName { + c := _m.On("SetName", matchers...) + return &Graphvizer_SetName{Call: c} +} + +// SetName provides a mock function with given fields: name +func (_m *Graphvizer) SetName(name string) error { + ret := _m.Called(name) + + var r0 error + if rf, ok := ret.Get(0).(func(string) error); ok { + r0 = rf(name) + } else { + r0 = ret.Error(0) + } + + return r0 +} From f4711afb199a6a91b7ff24381b256d685233e0f5 Mon Sep 17 00:00:00 2001 From: Prafulla Mahindrakar Date: Fri, 18 Jun 2021 15:37:21 +0530 Subject: [PATCH 13/16] Added more coverage Signed-off-by: Prafulla Mahindrakar --- pkg/visualize/graphviz_test.go | 98 ++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/pkg/visualize/graphviz_test.go b/pkg/visualize/graphviz_test.go index 51571339..56b8621e 100644 --- a/pkg/visualize/graphviz_test.go +++ b/pkg/visualize/graphviz_test.go @@ -118,6 +118,104 @@ func TestConstructBranchNode(t *testing.T) { assert.NotNil(t, err) assert.Nil(t, resultBranchNode) }) + + t.Run("Add ThenNode Error", func(t *testing.T) { + gb := newGraphBuilder() + mockGraph := &mocks.Graphvizer{} + attrs := map[string]string{} + attrs[LabelAttr] = fmt.Sprintf("\"[%s]\"", "nodeMetadata") + attrs[ShapeType] = DiamondShape + + // Verify the attributes + mockGraph.OnAddNodeMatch(mock.Anything, "branchName_id", attrs).Return(nil) + mockGraph.OnAddNodeMatch(mock.Anything, "branchName_start_node", mock.Anything).Return(fmt.Errorf("unable to add node")) + mockGraph.OnGetNodeMatch(mock.Anything).Return(nil) + flyteNode := &core.Node{ + Id: "id", + Metadata: &core.NodeMetadata{ + Name: "nodeMetadata", + }, + Target: &core.Node_BranchNode{ + BranchNode: &core.BranchNode{ + IfElse: &core.IfElseBlock{ + Case: &core.IfBlock{ + Condition: &core.BooleanExpression{}, + ThenNode: &core.Node{ + Id: "start-node", + }, + }, + }, + }, + }, + } + resultBranchNode, err := gb.constructBranchNode("parentGraph", "branchName", mockGraph, flyteNode) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("unable to add node"), err) + assert.Nil(t, resultBranchNode) + }) + + t.Run("Add Condition Node Edge Error", func(t *testing.T) { + gb := newGraphBuilder() + mockGraph := &mocks.Graphvizer{} + attrs := map[string]string{} + attrs[LabelAttr] = fmt.Sprintf("\"[%s]\"", "nodeMetadata") + attrs[ShapeType] = DiamondShape + + parentNode := &graphviz.Node{Name: "parentNode", Attrs: nil} + thenBranchStartNode := &graphviz.Node{Name: "branchName_start_node", Attrs: nil} + + mockGraph.OnAddNodeMatch(mock.Anything, "branchName_id", attrs).Return(nil) + mockGraph.OnAddNodeMatch(mock.Anything, "branchName_start_node", mock.Anything).Return(nil) + mockGraph.OnGetNodeMatch("branchName_id").Return(parentNode) + mockGraph.OnGetNodeMatch("branchName_start_node").Return(thenBranchStartNode) + mockGraph.OnAddEdgeMatch(mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(fmt.Errorf("unable to add edge")) + flyteNode := &core.Node{ + Id: "id", + Metadata: &core.NodeMetadata{ + Name: "nodeMetadata", + }, + Target: &core.Node_BranchNode{ + BranchNode: &core.BranchNode{ + IfElse: &core.IfElseBlock{ + Case: &core.IfBlock{ + Condition: &core.BooleanExpression{ + Expr: &core.BooleanExpression_Comparison{ + Comparison: &core.ComparisonExpression{ + Operator: core.ComparisonExpression_EQ, + LeftValue: &core.Operand{ + Val: &core.Operand_Primitive{ + Primitive: &core.Primitive{ + Value: &core.Primitive_Integer{ + Integer: 40, + }, + }, + }, + }, + RightValue: &core.Operand{ + Val: &core.Operand_Primitive{ + Primitive: &core.Primitive{ + Value: &core.Primitive_Integer{ + Integer: 50, + }, + }, + }, + }, + }, + }, + }, + ThenNode: &core.Node{ + Id: "start-node", + }, + }, + }, + }, + }, + } + resultBranchNode, err := gb.constructBranchNode("parentGraph", "branchName", mockGraph, flyteNode) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("unable to add edge"), err) + assert.Nil(t, resultBranchNode) + }) } func TestConstructNode(t *testing.T) { From 731bcc60c0177adcfae9d696b4e0042d3e2f7ba0 Mon Sep 17 00:00:00 2001 From: Prafulla Mahindrakar Date: Fri, 18 Jun 2021 18:20:08 +0530 Subject: [PATCH 14/16] Fixed the doc Signed-off-by: Prafulla Mahindrakar --- cmd/get/workflow.go | 2 +- cmd/root.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/get/workflow.go b/cmd/get/workflow.go index 45bd7711..1766713c 100644 --- a/cmd/get/workflow.go +++ b/cmd/get/workflow.go @@ -78,7 +78,7 @@ Visualize the graph for a workflow within project and domain in a dot content re :: - flytectl get workflow -p flytesnacks -d development -o dot-url + flytectl get workflow -p flytesnacks -d development -o doturl Usage ` diff --git a/cmd/root.go b/cmd/root.go index 6d43702a..40bc92ed 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -53,7 +53,7 @@ func newRootCmd() *cobra.Command { // --root.project, this adds a convenience on top to allow --project to be used rootCmd.PersistentFlags().StringVarP(&(config.GetConfig().Project), "project", "p", "", "Specifies the Flyte project.") rootCmd.PersistentFlags().StringVarP(&(config.GetConfig().Domain), "domain", "d", "", "Specifies the Flyte project's domain.") - rootCmd.PersistentFlags().StringVarP(&(config.GetConfig().Output), "output", "o", printer.OutputFormatTABLE.String(), fmt.Sprintf("Specifies the output type - supported formats %s. NOTE: SVG image is only supported for Workflow", printer.OutputFormats())) + rootCmd.PersistentFlags().StringVarP(&(config.GetConfig().Output), "output", "o", printer.OutputFormatTABLE.String(), fmt.Sprintf("Specifies the output type - supported formats %s. NOTE: dot, doturl are only supported for Workflow", printer.OutputFormats())) rootCmd.AddCommand(viper.GetConfigCommand()) rootCmd.AddCommand(get.CreateGetCommand()) rootCmd.AddCommand(create.RemoteCreateCommand()) From f85d6b67d547924bb90f0418bcc9d2ce39718e91 Mon Sep 17 00:00:00 2001 From: Prafulla Mahindrakar Date: Fri, 18 Jun 2021 18:21:53 +0530 Subject: [PATCH 15/16] Generated the docs Signed-off-by: Prafulla Mahindrakar --- docs/source/gen/flytectl.rst | 5 +- docs/source/gen/flytectl_config.rst | 4 +- docs/source/gen/flytectl_config_discover.rst | 4 +- docs/source/gen/flytectl_config_validate.rst | 4 +- docs/source/gen/flytectl_create.rst | 4 +- docs/source/gen/flytectl_create_execution.rst | 4 +- docs/source/gen/flytectl_create_project.rst | 4 +- docs/source/gen/flytectl_delete.rst | 4 +- ...ectl_delete_cluster-resource-attribute.rst | 4 +- ...lytectl_delete_execution-cluster-label.rst | 4 +- ...tectl_delete_execution-queue-attribute.rst | 4 +- docs/source/gen/flytectl_delete_execution.rst | 4 +- .../gen/flytectl_delete_plugin-override.rst | 4 +- ...lytectl_delete_task-resource-attribute.rst | 4 +- docs/source/gen/flytectl_get.rst | 4 +- ...lytectl_get_cluster-resource-attribute.rst | 4 +- .../flytectl_get_execution-cluster-label.rst | 4 +- ...flytectl_get_execution-queue-attribute.rst | 4 +- docs/source/gen/flytectl_get_execution.rst | 14 ++- docs/source/gen/flytectl_get_launchplan.rst | 10 +-- .../gen/flytectl_get_plugin-override.rst | 4 +- docs/source/gen/flytectl_get_project.rst | 6 +- .../flytectl_get_task-resource-attribute.rst | 4 +- docs/source/gen/flytectl_get_task.rst | 6 +- docs/source/gen/flytectl_get_workflow.rst | 20 ++++- docs/source/gen/flytectl_register.rst | 5 +- .../source/gen/flytectl_register_examples.rst | 87 +++++++++++++++++++ docs/source/gen/flytectl_register_files.rst | 16 ++-- docs/source/gen/flytectl_sandbox.rst | 82 +++++++++++++++++ docs/source/gen/flytectl_sandbox_start.rst | 80 +++++++++++++++++ docs/source/gen/flytectl_sandbox_teardown.rst | 86 ++++++++++++++++++ docs/source/gen/flytectl_update.rst | 4 +- ...ectl_update_cluster-resource-attribute.rst | 4 +- ...lytectl_update_execution-cluster-label.rst | 4 +- ...tectl_update_execution-queue-attribute.rst | 4 +- .../source/gen/flytectl_update_launchplan.rst | 4 +- .../gen/flytectl_update_plugin-override.rst | 4 +- docs/source/gen/flytectl_update_project.rst | 4 +- ...lytectl_update_task-resource-attribute.rst | 4 +- docs/source/gen/flytectl_update_task.rst | 4 +- docs/source/gen/flytectl_update_workflow.rst | 4 +- docs/source/gen/flytectl_version.rst | 4 +- 42 files changed, 442 insertions(+), 95 deletions(-) create mode 100644 docs/source/gen/flytectl_register_examples.rst create mode 100644 docs/source/gen/flytectl_sandbox.rst create mode 100644 docs/source/gen/flytectl_sandbox_start.rst create mode 100644 docs/source/gen/flytectl_sandbox_teardown.rst diff --git a/docs/source/gen/flytectl.rst b/docs/source/gen/flytectl.rst index 35598ae7..446c0f39 100644 --- a/docs/source/gen/flytectl.rst +++ b/docs/source/gen/flytectl.rst @@ -30,14 +30,14 @@ Options --admin.scopes strings List of scopes to request --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. - --config string config file (default is $HOME/.flyte/config.yaml) + -c, --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. -h, --help help for flytectl --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. --logger.show-source Includes source code location in logs. - -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML DOT DOTURL]. NOTE: dot, doturl are only supported for Workflow (default "TABLE") -p, --project string Specifies the Flyte project. --root.domain string Specified the domain to work on. --root.output string Specified the output type. @@ -64,6 +64,7 @@ SEE ALSO * :doc:`flytectl_delete` - Used for terminating/deleting various flyte resources including tasks/workflows/launchplans/executions/project. * :doc:`flytectl_get` - Used for fetching various flyte resources including tasks/workflows/launchplans/executions/project. * :doc:`flytectl_register` - Registers tasks/workflows/launchplans from list of generated serialized files. +* :doc:`flytectl_sandbox` - Used for testing flyte sandbox. * :doc:`flytectl_update` - Used for updating flyte resources eg: project. * :doc:`flytectl_version` - Used for fetching flyte version diff --git a/docs/source/gen/flytectl_config.rst b/docs/source/gen/flytectl_config.rst index 5039c54e..918923db 100644 --- a/docs/source/gen/flytectl_config.rst +++ b/docs/source/gen/flytectl_config.rst @@ -39,13 +39,13 @@ Options inherited from parent commands --admin.scopes strings List of scopes to request --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. - --config string config file (default is $HOME/.flyte/config.yaml) + -c, --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. --logger.show-source Includes source code location in logs. - -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML DOT DOTURL]. NOTE: dot, doturl are only supported for Workflow (default "TABLE") -p, --project string Specifies the Flyte project. --root.domain string Specified the domain to work on. --root.output string Specified the output type. diff --git a/docs/source/gen/flytectl_config_discover.rst b/docs/source/gen/flytectl_config_discover.rst index 932addfa..c8c73a03 100644 --- a/docs/source/gen/flytectl_config_discover.rst +++ b/docs/source/gen/flytectl_config_discover.rst @@ -41,7 +41,7 @@ Options inherited from parent commands --admin.scopes strings List of scopes to request --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. - --config string config file (default is $HOME/.flyte/config.yaml) + -c, --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --file stringArray Passes the config file to load. If empty, it'll first search for the config file path then, if found, will load config from there. @@ -49,7 +49,7 @@ Options inherited from parent commands --logger.level int Sets the minimum logging level. (default 4) --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. --logger.show-source Includes source code location in logs. - -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML DOT DOTURL]. NOTE: dot, doturl are only supported for Workflow (default "TABLE") -p, --project string Specifies the Flyte project. --root.domain string Specified the domain to work on. --root.output string Specified the output type. diff --git a/docs/source/gen/flytectl_config_validate.rst b/docs/source/gen/flytectl_config_validate.rst index 984231c5..e0fe692b 100644 --- a/docs/source/gen/flytectl_config_validate.rst +++ b/docs/source/gen/flytectl_config_validate.rst @@ -43,7 +43,7 @@ Options inherited from parent commands --admin.scopes strings List of scopes to request --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. - --config string config file (default is $HOME/.flyte/config.yaml) + -c, --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --file stringArray Passes the config file to load. If empty, it'll first search for the config file path then, if found, will load config from there. @@ -51,7 +51,7 @@ Options inherited from parent commands --logger.level int Sets the minimum logging level. (default 4) --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. --logger.show-source Includes source code location in logs. - -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML DOT DOTURL]. NOTE: dot, doturl are only supported for Workflow (default "TABLE") -p, --project string Specifies the Flyte project. --root.domain string Specified the domain to work on. --root.output string Specified the output type. diff --git a/docs/source/gen/flytectl_create.rst b/docs/source/gen/flytectl_create.rst index b2463ade..dea24a33 100644 --- a/docs/source/gen/flytectl_create.rst +++ b/docs/source/gen/flytectl_create.rst @@ -42,13 +42,13 @@ Options inherited from parent commands --admin.scopes strings List of scopes to request --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. - --config string config file (default is $HOME/.flyte/config.yaml) + -c, --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. --logger.show-source Includes source code location in logs. - -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML DOT DOTURL]. NOTE: dot, doturl are only supported for Workflow (default "TABLE") -p, --project string Specifies the Flyte project. --root.domain string Specified the domain to work on. --root.output string Specified the output type. diff --git a/docs/source/gen/flytectl_create_execution.rst b/docs/source/gen/flytectl_create_execution.rst index dc646888..d2f3f7e8 100644 --- a/docs/source/gen/flytectl_create_execution.rst +++ b/docs/source/gen/flytectl_create_execution.rst @@ -160,13 +160,13 @@ Options inherited from parent commands --admin.scopes strings List of scopes to request --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. - --config string config file (default is $HOME/.flyte/config.yaml) + -c, --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. --logger.show-source Includes source code location in logs. - -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML DOT DOTURL]. NOTE: dot, doturl are only supported for Workflow (default "TABLE") -p, --project string Specifies the Flyte project. --root.domain string Specified the domain to work on. --root.output string Specified the output type. diff --git a/docs/source/gen/flytectl_create_project.rst b/docs/source/gen/flytectl_create_project.rst index a0670e5f..e7f6580e 100644 --- a/docs/source/gen/flytectl_create_project.rst +++ b/docs/source/gen/flytectl_create_project.rst @@ -66,13 +66,13 @@ Options inherited from parent commands --admin.scopes strings List of scopes to request --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. - --config string config file (default is $HOME/.flyte/config.yaml) + -c, --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. --logger.show-source Includes source code location in logs. - -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML DOT DOTURL]. NOTE: dot, doturl are only supported for Workflow (default "TABLE") -p, --project string Specifies the Flyte project. --root.domain string Specified the domain to work on. --root.output string Specified the output type. diff --git a/docs/source/gen/flytectl_delete.rst b/docs/source/gen/flytectl_delete.rst index 9cfd95ed..42881783 100644 --- a/docs/source/gen/flytectl_delete.rst +++ b/docs/source/gen/flytectl_delete.rst @@ -42,13 +42,13 @@ Options inherited from parent commands --admin.scopes strings List of scopes to request --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. - --config string config file (default is $HOME/.flyte/config.yaml) + -c, --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. --logger.show-source Includes source code location in logs. - -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML DOT DOTURL]. NOTE: dot, doturl are only supported for Workflow (default "TABLE") -p, --project string Specifies the Flyte project. --root.domain string Specified the domain to work on. --root.output string Specified the output type. diff --git a/docs/source/gen/flytectl_delete_cluster-resource-attribute.rst b/docs/source/gen/flytectl_delete_cluster-resource-attribute.rst index 17eae9ec..29aac116 100644 --- a/docs/source/gen/flytectl_delete_cluster-resource-attribute.rst +++ b/docs/source/gen/flytectl_delete_cluster-resource-attribute.rst @@ -78,13 +78,13 @@ Options inherited from parent commands --admin.scopes strings List of scopes to request --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. - --config string config file (default is $HOME/.flyte/config.yaml) + -c, --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. --logger.show-source Includes source code location in logs. - -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML DOT DOTURL]. NOTE: dot, doturl are only supported for Workflow (default "TABLE") -p, --project string Specifies the Flyte project. --root.domain string Specified the domain to work on. --root.output string Specified the output type. diff --git a/docs/source/gen/flytectl_delete_execution-cluster-label.rst b/docs/source/gen/flytectl_delete_execution-cluster-label.rst index 2524bc63..daa247a4 100644 --- a/docs/source/gen/flytectl_delete_execution-cluster-label.rst +++ b/docs/source/gen/flytectl_delete_execution-cluster-label.rst @@ -76,13 +76,13 @@ Options inherited from parent commands --admin.scopes strings List of scopes to request --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. - --config string config file (default is $HOME/.flyte/config.yaml) + -c, --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. --logger.show-source Includes source code location in logs. - -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML DOT DOTURL]. NOTE: dot, doturl are only supported for Workflow (default "TABLE") -p, --project string Specifies the Flyte project. --root.domain string Specified the domain to work on. --root.output string Specified the output type. diff --git a/docs/source/gen/flytectl_delete_execution-queue-attribute.rst b/docs/source/gen/flytectl_delete_execution-queue-attribute.rst index c9935a0f..5e6e0352 100644 --- a/docs/source/gen/flytectl_delete_execution-queue-attribute.rst +++ b/docs/source/gen/flytectl_delete_execution-queue-attribute.rst @@ -80,13 +80,13 @@ Options inherited from parent commands --admin.scopes strings List of scopes to request --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. - --config string config file (default is $HOME/.flyte/config.yaml) + -c, --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. --logger.show-source Includes source code location in logs. - -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML DOT DOTURL]. NOTE: dot, doturl are only supported for Workflow (default "TABLE") -p, --project string Specifies the Flyte project. --root.domain string Specified the domain to work on. --root.output string Specified the output type. diff --git a/docs/source/gen/flytectl_delete_execution.rst b/docs/source/gen/flytectl_delete_execution.rst index f87a1747..2a12a2b3 100644 --- a/docs/source/gen/flytectl_delete_execution.rst +++ b/docs/source/gen/flytectl_delete_execution.rst @@ -85,13 +85,13 @@ Options inherited from parent commands --admin.scopes strings List of scopes to request --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. - --config string config file (default is $HOME/.flyte/config.yaml) + -c, --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. --logger.show-source Includes source code location in logs. - -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML DOT DOTURL]. NOTE: dot, doturl are only supported for Workflow (default "TABLE") -p, --project string Specifies the Flyte project. --root.domain string Specified the domain to work on. --root.output string Specified the output type. diff --git a/docs/source/gen/flytectl_delete_plugin-override.rst b/docs/source/gen/flytectl_delete_plugin-override.rst index 03b9f01b..e69560d9 100644 --- a/docs/source/gen/flytectl_delete_plugin-override.rst +++ b/docs/source/gen/flytectl_delete_plugin-override.rst @@ -81,13 +81,13 @@ Options inherited from parent commands --admin.scopes strings List of scopes to request --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. - --config string config file (default is $HOME/.flyte/config.yaml) + -c, --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. --logger.show-source Includes source code location in logs. - -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML DOT DOTURL]. NOTE: dot, doturl are only supported for Workflow (default "TABLE") -p, --project string Specifies the Flyte project. --root.domain string Specified the domain to work on. --root.output string Specified the output type. diff --git a/docs/source/gen/flytectl_delete_task-resource-attribute.rst b/docs/source/gen/flytectl_delete_task-resource-attribute.rst index 2d782bb6..17cb7003 100644 --- a/docs/source/gen/flytectl_delete_task-resource-attribute.rst +++ b/docs/source/gen/flytectl_delete_task-resource-attribute.rst @@ -81,13 +81,13 @@ Options inherited from parent commands --admin.scopes strings List of scopes to request --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. - --config string config file (default is $HOME/.flyte/config.yaml) + -c, --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. --logger.show-source Includes source code location in logs. - -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML DOT DOTURL]. NOTE: dot, doturl are only supported for Workflow (default "TABLE") -p, --project string Specifies the Flyte project. --root.domain string Specified the domain to work on. --root.output string Specified the output type. diff --git a/docs/source/gen/flytectl_get.rst b/docs/source/gen/flytectl_get.rst index 82dcafb4..8442a4e4 100644 --- a/docs/source/gen/flytectl_get.rst +++ b/docs/source/gen/flytectl_get.rst @@ -42,13 +42,13 @@ Options inherited from parent commands --admin.scopes strings List of scopes to request --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. - --config string config file (default is $HOME/.flyte/config.yaml) + -c, --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. --logger.show-source Includes source code location in logs. - -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML DOT DOTURL]. NOTE: dot, doturl are only supported for Workflow (default "TABLE") -p, --project string Specifies the Flyte project. --root.domain string Specified the domain to work on. --root.output string Specified the output type. diff --git a/docs/source/gen/flytectl_get_cluster-resource-attribute.rst b/docs/source/gen/flytectl_get_cluster-resource-attribute.rst index a713afac..7cf9ff55 100644 --- a/docs/source/gen/flytectl_get_cluster-resource-attribute.rst +++ b/docs/source/gen/flytectl_get_cluster-resource-attribute.rst @@ -87,13 +87,13 @@ Options inherited from parent commands --admin.scopes strings List of scopes to request --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. - --config string config file (default is $HOME/.flyte/config.yaml) + -c, --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. --logger.show-source Includes source code location in logs. - -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML DOT DOTURL]. NOTE: dot, doturl are only supported for Workflow (default "TABLE") -p, --project string Specifies the Flyte project. --root.domain string Specified the domain to work on. --root.output string Specified the output type. diff --git a/docs/source/gen/flytectl_get_execution-cluster-label.rst b/docs/source/gen/flytectl_get_execution-cluster-label.rst index 0f043d8c..58abf2ab 100644 --- a/docs/source/gen/flytectl_get_execution-cluster-label.rst +++ b/docs/source/gen/flytectl_get_execution-cluster-label.rst @@ -85,13 +85,13 @@ Options inherited from parent commands --admin.scopes strings List of scopes to request --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. - --config string config file (default is $HOME/.flyte/config.yaml) + -c, --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. --logger.show-source Includes source code location in logs. - -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML DOT DOTURL]. NOTE: dot, doturl are only supported for Workflow (default "TABLE") -p, --project string Specifies the Flyte project. --root.domain string Specified the domain to work on. --root.output string Specified the output type. diff --git a/docs/source/gen/flytectl_get_execution-queue-attribute.rst b/docs/source/gen/flytectl_get_execution-queue-attribute.rst index 1f0593a1..4b92076d 100644 --- a/docs/source/gen/flytectl_get_execution-queue-attribute.rst +++ b/docs/source/gen/flytectl_get_execution-queue-attribute.rst @@ -89,13 +89,13 @@ Options inherited from parent commands --admin.scopes strings List of scopes to request --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. - --config string config file (default is $HOME/.flyte/config.yaml) + -c, --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. --logger.show-source Includes source code location in logs. - -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML DOT DOTURL]. NOTE: dot, doturl are only supported for Workflow (default "TABLE") -p, --project string Specifies the Flyte project. --root.domain string Specified the domain to work on. --root.output string Specified the output type. diff --git a/docs/source/gen/flytectl_get_execution.rst b/docs/source/gen/flytectl_get_execution.rst index 1726008f..4040492e 100644 --- a/docs/source/gen/flytectl_get_execution.rst +++ b/docs/source/gen/flytectl_get_execution.rst @@ -21,15 +21,11 @@ Retrieves execution by name within project and domain. bin/flytectl get execution -p flytesnacks -d development oeh94k9r2r -Retrieves all the execution with filters. +Retrieves all the executions with filters. :: bin/flytectl get execution -p flytesnacks -d development --filter.field-selector="execution.phase in (FAILED;SUCCEEDED),execution.duration<200" - -Retrieve specific execution with filters. -:: - - bin/flytectl get execution -p flytesnacks -d development y8n2wtuspj --filter.field-selector="execution.phase in (FAILED),execution.duration<200" + Retrieves all the execution with limit and sorting. :: @@ -64,7 +60,7 @@ Options --filter.asc Specifies the sorting order. By default flytectl sort result in descending order --filter.field-selector string Specifies the Field selector --filter.limit int32 Specifies the limit (default 100) - --filter.sort-by string Specifies which field to sort results (default "created_at") + --filter.sort-by string Specifies which field to sort result by -h, --help help for execution Options inherited from parent commands @@ -86,13 +82,13 @@ Options inherited from parent commands --admin.scopes strings List of scopes to request --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. - --config string config file (default is $HOME/.flyte/config.yaml) + -c, --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. --logger.show-source Includes source code location in logs. - -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML DOT DOTURL]. NOTE: dot, doturl are only supported for Workflow (default "TABLE") -p, --project string Specifies the Flyte project. --root.domain string Specified the domain to work on. --root.output string Specified the output type. diff --git a/docs/source/gen/flytectl_get_launchplan.rst b/docs/source/gen/flytectl_get_launchplan.rst index 23b06925..37f81617 100644 --- a/docs/source/gen/flytectl_get_launchplan.rst +++ b/docs/source/gen/flytectl_get_launchplan.rst @@ -39,7 +39,7 @@ Retrieves all the launch plans with filters. bin/flytectl get launchplan -p flytesnacks -d development --filter.field-selector="name=core.basic.lp.go_greet" -Retrieves specific launch plans with filters. +Retrieves launch plans entity search across all versions with filters. :: bin/flytectl get launchplan -p flytesnacks -d development k8s_spark.dataframe_passing.my_smart_schema --filter.field-selector="version=v1" @@ -67,7 +67,7 @@ Retrieves a launch plans within project and domain for a version and generate th :: - flytectl get launchplan -d development -p flytectldemo core.advanced.run_merge_sort.merge_sort --execFile execution_spec.yam + flytectl get launchplan -d development -p flytectldemo core.advanced.run_merge_sort.merge_sort --execFile execution_spec.yaml The generated file would look similar to this @@ -103,7 +103,7 @@ Options --filter.asc Specifies the sorting order. By default flytectl sort result in descending order --filter.field-selector string Specifies the Field selector --filter.limit int32 Specifies the limit (default 100) - --filter.sort-by string Specifies which field to sort results (default "created_at") + --filter.sort-by string Specifies which field to sort result by -h, --help help for launchplan --latest flag to indicate to fetch the latest version, version flag will be ignored in this case --version string version of the launchplan to be fetched. @@ -127,13 +127,13 @@ Options inherited from parent commands --admin.scopes strings List of scopes to request --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. - --config string config file (default is $HOME/.flyte/config.yaml) + -c, --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. --logger.show-source Includes source code location in logs. - -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML DOT DOTURL]. NOTE: dot, doturl are only supported for Workflow (default "TABLE") -p, --project string Specifies the Flyte project. --root.domain string Specified the domain to work on. --root.output string Specified the output type. diff --git a/docs/source/gen/flytectl_get_plugin-override.rst b/docs/source/gen/flytectl_get_plugin-override.rst index fcc8e5c9..f608e680 100644 --- a/docs/source/gen/flytectl_get_plugin-override.rst +++ b/docs/source/gen/flytectl_get_plugin-override.rst @@ -109,13 +109,13 @@ Options inherited from parent commands --admin.scopes strings List of scopes to request --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. - --config string config file (default is $HOME/.flyte/config.yaml) + -c, --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. --logger.show-source Includes source code location in logs. - -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML DOT DOTURL]. NOTE: dot, doturl are only supported for Workflow (default "TABLE") -p, --project string Specifies the Flyte project. --root.domain string Specified the domain to work on. --root.output string Specified the output type. diff --git a/docs/source/gen/flytectl_get_project.rst b/docs/source/gen/flytectl_get_project.rst index ba86d53c..2f7e6224 100644 --- a/docs/source/gen/flytectl_get_project.rst +++ b/docs/source/gen/flytectl_get_project.rst @@ -58,7 +58,7 @@ Options --filter.asc Specifies the sorting order. By default flytectl sort result in descending order --filter.field-selector string Specifies the Field selector --filter.limit int32 Specifies the limit (default 100) - --filter.sort-by string Specifies which field to sort results (default "created_at") + --filter.sort-by string Specifies which field to sort result by -h, --help help for project Options inherited from parent commands @@ -80,13 +80,13 @@ Options inherited from parent commands --admin.scopes strings List of scopes to request --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. - --config string config file (default is $HOME/.flyte/config.yaml) + -c, --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. --logger.show-source Includes source code location in logs. - -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML DOT DOTURL]. NOTE: dot, doturl are only supported for Workflow (default "TABLE") -p, --project string Specifies the Flyte project. --root.domain string Specified the domain to work on. --root.output string Specified the output type. diff --git a/docs/source/gen/flytectl_get_task-resource-attribute.rst b/docs/source/gen/flytectl_get_task-resource-attribute.rst index dfd44d3c..fb180a8a 100644 --- a/docs/source/gen/flytectl_get_task-resource-attribute.rst +++ b/docs/source/gen/flytectl_get_task-resource-attribute.rst @@ -91,13 +91,13 @@ Options inherited from parent commands --admin.scopes strings List of scopes to request --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. - --config string config file (default is $HOME/.flyte/config.yaml) + -c, --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. --logger.show-source Includes source code location in logs. - -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML DOT DOTURL]. NOTE: dot, doturl are only supported for Workflow (default "TABLE") -p, --project string Specifies the Flyte project. --root.domain string Specified the domain to work on. --root.output string Specified the output type. diff --git a/docs/source/gen/flytectl_get_task.rst b/docs/source/gen/flytectl_get_task.rst index f4950156..dab41e7f 100644 --- a/docs/source/gen/flytectl_get_task.rst +++ b/docs/source/gen/flytectl_get_task.rst @@ -99,7 +99,7 @@ Options --filter.asc Specifies the sorting order. By default flytectl sort result in descending order --filter.field-selector string Specifies the Field selector --filter.limit int32 Specifies the limit (default 100) - --filter.sort-by string Specifies which field to sort results (default "created_at") + --filter.sort-by string Specifies which field to sort result by -h, --help help for task --latest flag to indicate to fetch the latest version, version flag will be ignored in this case --version string version of the task to be fetched. @@ -123,13 +123,13 @@ Options inherited from parent commands --admin.scopes strings List of scopes to request --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. - --config string config file (default is $HOME/.flyte/config.yaml) + -c, --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. --logger.show-source Includes source code location in logs. - -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML DOT DOTURL]. NOTE: dot, doturl are only supported for Workflow (default "TABLE") -p, --project string Specifies the Flyte project. --root.domain string Specified the domain to work on. --root.output string Specified the output type. diff --git a/docs/source/gen/flytectl_get_workflow.rst b/docs/source/gen/flytectl_get_workflow.rst index 846afb96..e5ea0090 100644 --- a/docs/source/gen/flytectl_get_workflow.rst +++ b/docs/source/gen/flytectl_get_workflow.rst @@ -60,6 +60,18 @@ Retrieves all the workflow within project and domain in json format. flytectl get workflow -p flytesnacks -d development -o json +Visualize the graph for a workflow within project and domain in dot format. + +:: + + flytectl get workflow -p flytesnacks -d development -o dot + +Visualize the graph for a workflow within project and domain in a dot content render. + +:: + + flytectl get workflow -p flytesnacks -d development -o doturl + Usage @@ -75,8 +87,10 @@ Options --filter.asc Specifies the sorting order. By default flytectl sort result in descending order --filter.field-selector string Specifies the Field selector --filter.limit int32 Specifies the limit (default 100) - --filter.sort-by string Specifies which field to sort results (default "created_at") + --filter.sort-by string Specifies which field to sort results -h, --help help for workflow + --latest flag to indicate to fetch the latest version, version flag will be ignored in this case + --version string version of the workflow to be fetched. Options inherited from parent commands ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -97,13 +111,13 @@ Options inherited from parent commands --admin.scopes strings List of scopes to request --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. - --config string config file (default is $HOME/.flyte/config.yaml) + -c, --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. --logger.show-source Includes source code location in logs. - -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML DOT DOTURL]. NOTE: dot, doturl are only supported for Workflow (default "TABLE") -p, --project string Specifies the Flyte project. --root.domain string Specified the domain to work on. --root.output string Specified the output type. diff --git a/docs/source/gen/flytectl_register.rst b/docs/source/gen/flytectl_register.rst index 98d8c02b..0417174a 100644 --- a/docs/source/gen/flytectl_register.rst +++ b/docs/source/gen/flytectl_register.rst @@ -42,13 +42,13 @@ Options inherited from parent commands --admin.scopes strings List of scopes to request --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. - --config string config file (default is $HOME/.flyte/config.yaml) + -c, --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. --logger.show-source Includes source code location in logs. - -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML DOT DOTURL]. NOTE: dot, doturl are only supported for Workflow (default "TABLE") -p, --project string Specifies the Flyte project. --root.domain string Specified the domain to work on. --root.output string Specified the output type. @@ -71,5 +71,6 @@ SEE ALSO ~~~~~~~~ * :doc:`flytectl` - flyetcl CLI tool +* :doc:`flytectl_register_examples` - Registers flytesnack example * :doc:`flytectl_register_files` - Registers file resources diff --git a/docs/source/gen/flytectl_register_examples.rst b/docs/source/gen/flytectl_register_examples.rst new file mode 100644 index 00000000..e6df378e --- /dev/null +++ b/docs/source/gen/flytectl_register_examples.rst @@ -0,0 +1,87 @@ +.. _flytectl_register_examples: + +flytectl register examples +-------------------------- + +Registers flytesnack example + +Synopsis +~~~~~~~~ + + + +Registers all latest flytesnacks example +:: + + bin/flytectl register examples -d development -p flytesnacks + + +Usage + + +:: + + flytectl register examples [flags] + +Options +~~~~~~~ + +:: + + -a, --archive pass in archive file either an http link or local path. + -i, --assumableIamRole string Custom assumable iam auth role to register launch plans with. + --continueOnError continue on error when registering files. + -h, --help help for examples + -k, --k8ServiceAccount string custom kubernetes service account auth role to register launch plans with. + -l, --outputLocationPrefix string custom output location prefix for offloaded types (files/schemas). + -v, --version string version of the entity to be registered with flyte. (default "v1") + +Options inherited from parent commands +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + --admin.authorizationHeader string Custom metadata header to pass JWT + --admin.authorizationServerUrl string This is the URL to your IdP's authorization server. It'll default to Endpoint + --admin.clientId string Client ID (default "flytepropeller") + --admin.clientSecretLocation string File containing the client secret (default "/etc/secrets/client_secret") + --admin.endpoint string For admin types, specify where the uri of the service is located. + --admin.insecure Use insecure connection. + --admin.maxBackoffDelay string Max delay for grpc backoff (default "8s") + --admin.maxRetries int Max number of gRPC retries (default 4) + --admin.perRetryTimeout string gRPC per retry timeout (default "15s") + --admin.pkceConfig.refreshTime string (default "5m0s") + --admin.pkceConfig.timeout string (default "15s") + --admin.scopes strings List of scopes to request + --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. + --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. + -c, --config string config file (default is $HOME/.flyte/config.yaml) + -d, --domain string Specifies the Flyte project's domain. + --logger.formatter.type string Sets logging format type. (default "json") + --logger.level int Sets the minimum logging level. (default 4) + --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. + --logger.show-source Includes source code location in logs. + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML DOT DOTURL]. NOTE: dot, doturl are only supported for Workflow (default "TABLE") + -p, --project string Specifies the Flyte project. + --root.domain string Specified the domain to work on. + --root.output string Specified the output type. + --root.project string Specifies the project to work on. + --storage.cache.max_size_mbs int Maximum size of the cache where the Blob store data is cached in-memory. If not specified or set to 0, cache is not used + --storage.cache.target_gc_percent int Sets the garbage collection target percentage. + --storage.connection.access-key string Access key to use. Only required when authtype is set to accesskey. + --storage.connection.auth-type string Auth Type to use [iam, accesskey]. (default "iam") + --storage.connection.disable-ssl Disables SSL connection. Should only be used for development. + --storage.connection.endpoint string URL for storage client to connect to. + --storage.connection.region string Region to connect to. (default "us-east-1") + --storage.connection.secret-key string Secret to use when accesskey is set. + --storage.container string Initial container to create -if it doesn't exist-.' + --storage.defaultHttpClient.timeout string Sets time out on the http client. (default "0s") + --storage.enable-multicontainer If this is true, then the container argument is overlooked and redundant. This config will automatically open new connections to new containers/buckets as they are encountered + --storage.limits.maxDownloadMBs int Maximum allowed download size (in MBs) per call. (default 2) + --storage.type string Sets the type of storage to configure [s3/minio/local/mem/stow]. (default "s3") + +SEE ALSO +~~~~~~~~ + +* :doc:`flytectl_register` - Registers tasks/workflows/launchplans from list of generated serialized files. + diff --git a/docs/source/gen/flytectl_register_files.rst b/docs/source/gen/flytectl_register_files.rst index 8d79a9df..ada5a928 100644 --- a/docs/source/gen/flytectl_register_files.rst +++ b/docs/source/gen/flytectl_register_files.rst @@ -39,7 +39,7 @@ the continueOnError flag. Using short format of continueOnError flag :: - bin/flytectl register file _pb_output/* -d development -p flytesnacks -c + bin/flytectl register file _pb_output/* -d development -p flytesnacks --continueOnError Overriding the default version v1 using version string. :: @@ -50,25 +50,25 @@ Change the o/p format has not effect on registration. The O/p is currently avail :: - bin/flytectl register file _pb_output/* -d development -p flytesnacks -c -o yaml + bin/flytectl register file _pb_output/* -d development -p flytesnacks --continueOnError -o yaml Override IamRole during registration. :: - bin/flytectl register file _pb_output/* -d development -p flytesnacks -c -v v2 -i "arn:aws:iam::123456789:role/dummy" + bin/flytectl register file _pb_output/* -d development -p flytesnacks --continueOnError -v v2 -i "arn:aws:iam::123456789:role/dummy" Override Kubernetes service account during registration. :: - bin/flytectl register file _pb_output/* -d development -p flytesnacks -c -v v2 -k "kubernetes-service-account" + bin/flytectl register file _pb_output/* -d development -p flytesnacks --continueOnError -v v2 -k "kubernetes-service-account" Override Output location prefix during registration. :: - bin/flytectl register file _pb_output/* -d development -p flytesnacks -c -v v2 -l "s3://dummy/prefix" + bin/flytectl register file _pb_output/* -d development -p flytesnacks --continueOnError -v v2 -l "s3://dummy/prefix" Usage @@ -84,7 +84,7 @@ Options -a, --archive pass in archive file either an http link or local path. -i, --assumableIamRole string Custom assumable iam auth role to register launch plans with. - -c, --continueOnError continue on error when registering files. + --continueOnError continue on error when registering files. -h, --help help for files -k, --k8ServiceAccount string custom kubernetes service account auth role to register launch plans with. -l, --outputLocationPrefix string custom output location prefix for offloaded types (files/schemas). @@ -109,13 +109,13 @@ Options inherited from parent commands --admin.scopes strings List of scopes to request --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. - --config string config file (default is $HOME/.flyte/config.yaml) + -c, --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. --logger.show-source Includes source code location in logs. - -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML DOT DOTURL]. NOTE: dot, doturl are only supported for Workflow (default "TABLE") -p, --project string Specifies the Flyte project. --root.domain string Specified the domain to work on. --root.output string Specified the output type. diff --git a/docs/source/gen/flytectl_sandbox.rst b/docs/source/gen/flytectl_sandbox.rst new file mode 100644 index 00000000..c33cfcc8 --- /dev/null +++ b/docs/source/gen/flytectl_sandbox.rst @@ -0,0 +1,82 @@ +.. _flytectl_sandbox: + +flytectl sandbox +---------------- + +Used for testing flyte sandbox. + +Synopsis +~~~~~~~~ + + + +Example Create sandbox cluster. +:: + + bin/flytectl sandbox start + + +Example Remove sandbox cluster. +:: + + bin/flytectl sandbox teardown + + +Options +~~~~~~~ + +:: + + -h, --help help for sandbox + +Options inherited from parent commands +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + --admin.authorizationHeader string Custom metadata header to pass JWT + --admin.authorizationServerUrl string This is the URL to your IdP's authorization server. It'll default to Endpoint + --admin.clientId string Client ID (default "flytepropeller") + --admin.clientSecretLocation string File containing the client secret (default "/etc/secrets/client_secret") + --admin.endpoint string For admin types, specify where the uri of the service is located. + --admin.insecure Use insecure connection. + --admin.maxBackoffDelay string Max delay for grpc backoff (default "8s") + --admin.maxRetries int Max number of gRPC retries (default 4) + --admin.perRetryTimeout string gRPC per retry timeout (default "15s") + --admin.pkceConfig.refreshTime string (default "5m0s") + --admin.pkceConfig.timeout string (default "15s") + --admin.scopes strings List of scopes to request + --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. + --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. + -c, --config string config file (default is $HOME/.flyte/config.yaml) + -d, --domain string Specifies the Flyte project's domain. + --logger.formatter.type string Sets logging format type. (default "json") + --logger.level int Sets the minimum logging level. (default 4) + --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. + --logger.show-source Includes source code location in logs. + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML DOT DOTURL]. NOTE: dot, doturl are only supported for Workflow (default "TABLE") + -p, --project string Specifies the Flyte project. + --root.domain string Specified the domain to work on. + --root.output string Specified the output type. + --root.project string Specifies the project to work on. + --storage.cache.max_size_mbs int Maximum size of the cache where the Blob store data is cached in-memory. If not specified or set to 0, cache is not used + --storage.cache.target_gc_percent int Sets the garbage collection target percentage. + --storage.connection.access-key string Access key to use. Only required when authtype is set to accesskey. + --storage.connection.auth-type string Auth Type to use [iam, accesskey]. (default "iam") + --storage.connection.disable-ssl Disables SSL connection. Should only be used for development. + --storage.connection.endpoint string URL for storage client to connect to. + --storage.connection.region string Region to connect to. (default "us-east-1") + --storage.connection.secret-key string Secret to use when accesskey is set. + --storage.container string Initial container to create -if it doesn't exist-.' + --storage.defaultHttpClient.timeout string Sets time out on the http client. (default "0s") + --storage.enable-multicontainer If this is true, then the container argument is overlooked and redundant. This config will automatically open new connections to new containers/buckets as they are encountered + --storage.limits.maxDownloadMBs int Maximum allowed download size (in MBs) per call. (default 2) + --storage.type string Sets the type of storage to configure [s3/minio/local/mem/stow]. (default "s3") + +SEE ALSO +~~~~~~~~ + +* :doc:`flytectl` - flyetcl CLI tool +* :doc:`flytectl_sandbox_start` - Start the flyte sandbox +* :doc:`flytectl_sandbox_teardown` - Teardown will cleanup the sandbox environment + diff --git a/docs/source/gen/flytectl_sandbox_start.rst b/docs/source/gen/flytectl_sandbox_start.rst new file mode 100644 index 00000000..56716a9f --- /dev/null +++ b/docs/source/gen/flytectl_sandbox_start.rst @@ -0,0 +1,80 @@ +.. _flytectl_sandbox_start: + +flytectl sandbox start +---------------------- + +Start the flyte sandbox + +Synopsis +~~~~~~~~ + + + +Start will run the flyte sandbox cluster inside a docker container and setup the config that is required +:: + + bin/flytectl start + +Usage + + +:: + + flytectl sandbox start [flags] + +Options +~~~~~~~ + +:: + + -h, --help help for start + +Options inherited from parent commands +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + --admin.authorizationHeader string Custom metadata header to pass JWT + --admin.authorizationServerUrl string This is the URL to your IdP's authorization server. It'll default to Endpoint + --admin.clientId string Client ID (default "flytepropeller") + --admin.clientSecretLocation string File containing the client secret (default "/etc/secrets/client_secret") + --admin.endpoint string For admin types, specify where the uri of the service is located. + --admin.insecure Use insecure connection. + --admin.maxBackoffDelay string Max delay for grpc backoff (default "8s") + --admin.maxRetries int Max number of gRPC retries (default 4) + --admin.perRetryTimeout string gRPC per retry timeout (default "15s") + --admin.pkceConfig.refreshTime string (default "5m0s") + --admin.pkceConfig.timeout string (default "15s") + --admin.scopes strings List of scopes to request + --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. + --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. + -c, --config string config file (default is $HOME/.flyte/config.yaml) + -d, --domain string Specifies the Flyte project's domain. + --logger.formatter.type string Sets logging format type. (default "json") + --logger.level int Sets the minimum logging level. (default 4) + --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. + --logger.show-source Includes source code location in logs. + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML DOT DOTURL]. NOTE: dot, doturl are only supported for Workflow (default "TABLE") + -p, --project string Specifies the Flyte project. + --root.domain string Specified the domain to work on. + --root.output string Specified the output type. + --root.project string Specifies the project to work on. + --storage.cache.max_size_mbs int Maximum size of the cache where the Blob store data is cached in-memory. If not specified or set to 0, cache is not used + --storage.cache.target_gc_percent int Sets the garbage collection target percentage. + --storage.connection.access-key string Access key to use. Only required when authtype is set to accesskey. + --storage.connection.auth-type string Auth Type to use [iam, accesskey]. (default "iam") + --storage.connection.disable-ssl Disables SSL connection. Should only be used for development. + --storage.connection.endpoint string URL for storage client to connect to. + --storage.connection.region string Region to connect to. (default "us-east-1") + --storage.connection.secret-key string Secret to use when accesskey is set. + --storage.container string Initial container to create -if it doesn't exist-.' + --storage.defaultHttpClient.timeout string Sets time out on the http client. (default "0s") + --storage.enable-multicontainer If this is true, then the container argument is overlooked and redundant. This config will automatically open new connections to new containers/buckets as they are encountered + --storage.limits.maxDownloadMBs int Maximum allowed download size (in MBs) per call. (default 2) + --storage.type string Sets the type of storage to configure [s3/minio/local/mem/stow]. (default "s3") + +SEE ALSO +~~~~~~~~ + +* :doc:`flytectl_sandbox` - Used for testing flyte sandbox. + diff --git a/docs/source/gen/flytectl_sandbox_teardown.rst b/docs/source/gen/flytectl_sandbox_teardown.rst new file mode 100644 index 00000000..f7fccfd8 --- /dev/null +++ b/docs/source/gen/flytectl_sandbox_teardown.rst @@ -0,0 +1,86 @@ +.. _flytectl_sandbox_teardown: + +flytectl sandbox teardown +------------------------- + +Teardown will cleanup the sandbox environment + +Synopsis +~~~~~~~~ + + + +Teardown will remove docker container and all the flyte config +:: + + bin/flytectl sandbox teardown + +Stop will remove docker container and all the flyte config +:: + + bin/flytectl sandbox stop + + +Usage + + +:: + + flytectl sandbox teardown [flags] + +Options +~~~~~~~ + +:: + + -h, --help help for teardown + +Options inherited from parent commands +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + --admin.authorizationHeader string Custom metadata header to pass JWT + --admin.authorizationServerUrl string This is the URL to your IdP's authorization server. It'll default to Endpoint + --admin.clientId string Client ID (default "flytepropeller") + --admin.clientSecretLocation string File containing the client secret (default "/etc/secrets/client_secret") + --admin.endpoint string For admin types, specify where the uri of the service is located. + --admin.insecure Use insecure connection. + --admin.maxBackoffDelay string Max delay for grpc backoff (default "8s") + --admin.maxRetries int Max number of gRPC retries (default 4) + --admin.perRetryTimeout string gRPC per retry timeout (default "15s") + --admin.pkceConfig.refreshTime string (default "5m0s") + --admin.pkceConfig.timeout string (default "15s") + --admin.scopes strings List of scopes to request + --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. + --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. + -c, --config string config file (default is $HOME/.flyte/config.yaml) + -d, --domain string Specifies the Flyte project's domain. + --logger.formatter.type string Sets logging format type. (default "json") + --logger.level int Sets the minimum logging level. (default 4) + --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. + --logger.show-source Includes source code location in logs. + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML DOT DOTURL]. NOTE: dot, doturl are only supported for Workflow (default "TABLE") + -p, --project string Specifies the Flyte project. + --root.domain string Specified the domain to work on. + --root.output string Specified the output type. + --root.project string Specifies the project to work on. + --storage.cache.max_size_mbs int Maximum size of the cache where the Blob store data is cached in-memory. If not specified or set to 0, cache is not used + --storage.cache.target_gc_percent int Sets the garbage collection target percentage. + --storage.connection.access-key string Access key to use. Only required when authtype is set to accesskey. + --storage.connection.auth-type string Auth Type to use [iam, accesskey]. (default "iam") + --storage.connection.disable-ssl Disables SSL connection. Should only be used for development. + --storage.connection.endpoint string URL for storage client to connect to. + --storage.connection.region string Region to connect to. (default "us-east-1") + --storage.connection.secret-key string Secret to use when accesskey is set. + --storage.container string Initial container to create -if it doesn't exist-.' + --storage.defaultHttpClient.timeout string Sets time out on the http client. (default "0s") + --storage.enable-multicontainer If this is true, then the container argument is overlooked and redundant. This config will automatically open new connections to new containers/buckets as they are encountered + --storage.limits.maxDownloadMBs int Maximum allowed download size (in MBs) per call. (default 2) + --storage.type string Sets the type of storage to configure [s3/minio/local/mem/stow]. (default "s3") + +SEE ALSO +~~~~~~~~ + +* :doc:`flytectl_sandbox` - Used for testing flyte sandbox. + diff --git a/docs/source/gen/flytectl_update.rst b/docs/source/gen/flytectl_update.rst index 17ba3972..9a513303 100644 --- a/docs/source/gen/flytectl_update.rst +++ b/docs/source/gen/flytectl_update.rst @@ -44,13 +44,13 @@ Options inherited from parent commands --admin.scopes strings List of scopes to request --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. - --config string config file (default is $HOME/.flyte/config.yaml) + -c, --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. --logger.show-source Includes source code location in logs. - -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML DOT DOTURL]. NOTE: dot, doturl are only supported for Workflow (default "TABLE") -p, --project string Specifies the Flyte project. --root.domain string Specified the domain to work on. --root.output string Specified the output type. diff --git a/docs/source/gen/flytectl_update_cluster-resource-attribute.rst b/docs/source/gen/flytectl_update_cluster-resource-attribute.rst index b8000b4d..5f50268a 100644 --- a/docs/source/gen/flytectl_update_cluster-resource-attribute.rst +++ b/docs/source/gen/flytectl_update_cluster-resource-attribute.rst @@ -83,13 +83,13 @@ Options inherited from parent commands --admin.scopes strings List of scopes to request --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. - --config string config file (default is $HOME/.flyte/config.yaml) + -c, --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. --logger.show-source Includes source code location in logs. - -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML DOT DOTURL]. NOTE: dot, doturl are only supported for Workflow (default "TABLE") -p, --project string Specifies the Flyte project. --root.domain string Specified the domain to work on. --root.output string Specified the output type. diff --git a/docs/source/gen/flytectl_update_execution-cluster-label.rst b/docs/source/gen/flytectl_update_execution-cluster-label.rst index 2431eedd..957aeff7 100644 --- a/docs/source/gen/flytectl_update_execution-cluster-label.rst +++ b/docs/source/gen/flytectl_update_execution-cluster-label.rst @@ -76,13 +76,13 @@ Options inherited from parent commands --admin.scopes strings List of scopes to request --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. - --config string config file (default is $HOME/.flyte/config.yaml) + -c, --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. --logger.show-source Includes source code location in logs. - -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML DOT DOTURL]. NOTE: dot, doturl are only supported for Workflow (default "TABLE") -p, --project string Specifies the Flyte project. --root.domain string Specified the domain to work on. --root.output string Specified the output type. diff --git a/docs/source/gen/flytectl_update_execution-queue-attribute.rst b/docs/source/gen/flytectl_update_execution-queue-attribute.rst index a32e711f..e6ed4192 100644 --- a/docs/source/gen/flytectl_update_execution-queue-attribute.rst +++ b/docs/source/gen/flytectl_update_execution-queue-attribute.rst @@ -87,13 +87,13 @@ Options inherited from parent commands --admin.scopes strings List of scopes to request --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. - --config string config file (default is $HOME/.flyte/config.yaml) + -c, --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. --logger.show-source Includes source code location in logs. - -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML DOT DOTURL]. NOTE: dot, doturl are only supported for Workflow (default "TABLE") -p, --project string Specifies the Flyte project. --root.domain string Specified the domain to work on. --root.output string Specified the output type. diff --git a/docs/source/gen/flytectl_update_launchplan.rst b/docs/source/gen/flytectl_update_launchplan.rst index aeb20a9f..88b569d1 100644 --- a/docs/source/gen/flytectl_update_launchplan.rst +++ b/docs/source/gen/flytectl_update_launchplan.rst @@ -61,13 +61,13 @@ Options inherited from parent commands --admin.scopes strings List of scopes to request --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. - --config string config file (default is $HOME/.flyte/config.yaml) + -c, --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. --logger.show-source Includes source code location in logs. - -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML DOT DOTURL]. NOTE: dot, doturl are only supported for Workflow (default "TABLE") -p, --project string Specifies the Flyte project. --root.domain string Specified the domain to work on. --root.output string Specified the output type. diff --git a/docs/source/gen/flytectl_update_plugin-override.rst b/docs/source/gen/flytectl_update_plugin-override.rst index 657efbfb..bea5a1c6 100644 --- a/docs/source/gen/flytectl_update_plugin-override.rst +++ b/docs/source/gen/flytectl_update_plugin-override.rst @@ -89,13 +89,13 @@ Options inherited from parent commands --admin.scopes strings List of scopes to request --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. - --config string config file (default is $HOME/.flyte/config.yaml) + -c, --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. --logger.show-source Includes source code location in logs. - -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML DOT DOTURL]. NOTE: dot, doturl are only supported for Workflow (default "TABLE") -p, --project string Specifies the Flyte project. --root.domain string Specified the domain to work on. --root.output string Specified the output type. diff --git a/docs/source/gen/flytectl_update_project.rst b/docs/source/gen/flytectl_update_project.rst index 742db693..74252c4f 100644 --- a/docs/source/gen/flytectl_update_project.rst +++ b/docs/source/gen/flytectl_update_project.rst @@ -86,13 +86,13 @@ Options inherited from parent commands --admin.scopes strings List of scopes to request --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. - --config string config file (default is $HOME/.flyte/config.yaml) + -c, --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. --logger.show-source Includes source code location in logs. - -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML DOT DOTURL]. NOTE: dot, doturl are only supported for Workflow (default "TABLE") -p, --project string Specifies the Flyte project. --root.domain string Specified the domain to work on. --root.output string Specified the output type. diff --git a/docs/source/gen/flytectl_update_task-resource-attribute.rst b/docs/source/gen/flytectl_update_task-resource-attribute.rst index d4ae5db9..4f75f37e 100644 --- a/docs/source/gen/flytectl_update_task-resource-attribute.rst +++ b/docs/source/gen/flytectl_update_task-resource-attribute.rst @@ -89,13 +89,13 @@ Options inherited from parent commands --admin.scopes strings List of scopes to request --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. - --config string config file (default is $HOME/.flyte/config.yaml) + -c, --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. --logger.show-source Includes source code location in logs. - -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML DOT DOTURL]. NOTE: dot, doturl are only supported for Workflow (default "TABLE") -p, --project string Specifies the Flyte project. --root.domain string Specified the domain to work on. --root.output string Specified the output type. diff --git a/docs/source/gen/flytectl_update_task.rst b/docs/source/gen/flytectl_update_task.rst index 26d92378..a119d011 100644 --- a/docs/source/gen/flytectl_update_task.rst +++ b/docs/source/gen/flytectl_update_task.rst @@ -61,13 +61,13 @@ Options inherited from parent commands --admin.scopes strings List of scopes to request --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. - --config string config file (default is $HOME/.flyte/config.yaml) + -c, --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. --logger.show-source Includes source code location in logs. - -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML DOT DOTURL]. NOTE: dot, doturl are only supported for Workflow (default "TABLE") -p, --project string Specifies the Flyte project. --root.domain string Specified the domain to work on. --root.output string Specified the output type. diff --git a/docs/source/gen/flytectl_update_workflow.rst b/docs/source/gen/flytectl_update_workflow.rst index b79c8197..347ecb17 100644 --- a/docs/source/gen/flytectl_update_workflow.rst +++ b/docs/source/gen/flytectl_update_workflow.rst @@ -61,13 +61,13 @@ Options inherited from parent commands --admin.scopes strings List of scopes to request --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. - --config string config file (default is $HOME/.flyte/config.yaml) + -c, --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. --logger.show-source Includes source code location in logs. - -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML DOT DOTURL]. NOTE: dot, doturl are only supported for Workflow (default "TABLE") -p, --project string Specifies the Flyte project. --root.domain string Specified the domain to work on. --root.output string Specified the output type. diff --git a/docs/source/gen/flytectl_version.rst b/docs/source/gen/flytectl_version.rst index 7d125dcf..8a73ae4a 100644 --- a/docs/source/gen/flytectl_version.rst +++ b/docs/source/gen/flytectl_version.rst @@ -46,13 +46,13 @@ Options inherited from parent commands --admin.scopes strings List of scopes to request --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. - --config string config file (default is $HOME/.flyte/config.yaml) + -c, --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. --logger.show-source Includes source code location in logs. - -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML DOT DOTURL]. NOTE: dot, doturl are only supported for Workflow (default "TABLE") -p, --project string Specifies the Flyte project. --root.domain string Specified the domain to work on. --root.output string Specified the output type. From 9d1c85a7b12ac90bfbb5ec841722432496afe9f8 Mon Sep 17 00:00:00 2001 From: Prafulla Mahindrakar Date: Fri, 18 Jun 2021 18:25:19 +0530 Subject: [PATCH 16/16] Fixed the docs Signed-off-by: Prafulla Mahindrakar --- cmd/get/workflow.go | 4 ++-- docs/source/gen/flytectl_get_workflow.rst | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/get/workflow.go b/cmd/get/workflow.go index 1766713c..4174f96c 100644 --- a/cmd/get/workflow.go +++ b/cmd/get/workflow.go @@ -72,13 +72,13 @@ Visualize the graph for a workflow within project and domain in dot format. :: - flytectl get workflow -p flytesnacks -d development -o dot + flytectl get workflow -p flytesnacks -d development core.flyte_basics.basic_workflow.my_wf --latest -o dot Visualize the graph for a workflow within project and domain in a dot content render. :: - flytectl get workflow -p flytesnacks -d development -o doturl + flytectl get workflow -p flytesnacks -d development core.flyte_basics.basic_workflow.my_wf --latest -o doturl Usage ` diff --git a/docs/source/gen/flytectl_get_workflow.rst b/docs/source/gen/flytectl_get_workflow.rst index e5ea0090..081ae99b 100644 --- a/docs/source/gen/flytectl_get_workflow.rst +++ b/docs/source/gen/flytectl_get_workflow.rst @@ -64,13 +64,13 @@ Visualize the graph for a workflow within project and domain in dot format. :: - flytectl get workflow -p flytesnacks -d development -o dot + flytectl get workflow -p flytesnacks -d development core.flyte_basics.basic_workflow.my_wf --latest -o dot Visualize the graph for a workflow within project and domain in a dot content render. :: - flytectl get workflow -p flytesnacks -d development -o doturl + flytectl get workflow -p flytesnacks -d development core.flyte_basics.basic_workflow.my_wf --latest -o doturl Usage