Auto rendering of JSON entities using path (#13)

flytectl/cmd/get/named_entity.go

@@ -0,0 +1,11 @@
+package get
+
+import (
+	"github.com/lyft/flytectl/printer"
+)
+
+var entityColumns = []printer.Column{
+	{"Domain", "$.domain"},
+	{"Name", "$.name"},
+	{"Project", "$.project"},
+}

flytectl/cmd/get/project.go

@@ -2,7 +2,6 @@ package get
 
 import (
 	"context"
-	"encoding/json"
 
 	"github.com/lyft/flyteidl/gen/pb-go/flyteidl/admin"
 	"github.com/lyft/flytestdlib/logger"
@@ -12,24 +11,10 @@ import (
 	"github.com/lyft/flytectl/printer"
 )
 
-type PrintableProject struct {
-	ID          string `header:"Id"`
-	Name        string `header:"Name"`
-	Description string `header:"Description"`
-}
-
-var tableStructure = map[string]string{
-	"ID":          "$.id",
-	"Name":        "$.name",
-	"Description": "$.description",
-}
-
-func transformProject(jsonbody []byte) (interface{}, error) {
-	results := PrintableProject{}
-	if err := json.Unmarshal(jsonbody, &results); err != nil {
-		return results, err
-	}
-	return results, nil
+var projectColumns = []printer.Column{
+	{"ID", "$.id"},
+	{"Name", "$.name"},
+	{"Description", "$.description"},
 }
 
 func getProjectsFunc(ctx context.Context, args []string, cmdCtx cmdCore.CommandContext) error {
@@ -44,7 +29,7 @@ func getProjectsFunc(ctx context.Context, args []string, cmdCtx cmdCore.CommandC
 		logger.Debugf(ctx, "Retrieved %v projects", len(projects.Projects))
 		for _, v := range projects.Projects {
 			if v.Name == name {
-				err := adminPrinter.Print(config.GetConfig().MustOutputFormat(), v, tableStructure, transformProject)
+				err := adminPrinter.Print(config.GetConfig().MustOutputFormat(), v, projectColumns)
 				if err != nil {
 					return err
 				}
@@ -58,5 +43,5 @@ func getProjectsFunc(ctx context.Context, args []string, cmdCtx cmdCore.CommandC
 		return err
 	}
 	logger.Debugf(ctx, "Retrieved %v projects", len(projects.Projects))
-	return adminPrinter.Print(config.GetConfig().MustOutputFormat(), projects.Projects, tableStructure, transformProject)
+	return adminPrinter.Print(config.GetConfig().MustOutputFormat(), projects.Projects, projectColumns)
 }

flytectl/cmd/get/task.go

@@ -2,7 +2,6 @@ package get
 
 import (
 	"context"
-	"encoding/json"
 
 	"github.com/lyft/flytestdlib/logger"
 
@@ -15,28 +14,12 @@ import (
 	"github.com/lyft/flyteidl/gen/pb-go/flyteidl/admin"
 )
 
-type PrintableTask struct {
-	Version          string `header:"Version"`
-	Name             string `header:"Name"`
-	Type             string `header:"Type"`
-	Discoverable     bool   `header:"Discoverable"`
-	DiscoveryVersion string `header:"DiscoveryVersion"`
-}
-
-var taskStructure = map[string]string{
-	"Version":          "$.id.version",
-	"Name":             "$.id.name",
-	"Type":             "$.closure.compiledTask.template.type",
-	"Discoverable":     "$.closure.compiledTask.template.metadata.discoverable",
-	"DiscoveryVersion": "$.closure.compiledTask.template.metadata.discovery_version",
-}
-
-var transformTask = func(jsonbody []byte) (interface{}, error) {
-	results := PrintableTask{}
-	if err := json.Unmarshal(jsonbody, &results); err != nil {
-		return results, err
-	}
-	return results, nil
+var taskColumns = []printer.Column{
+	{"Version", "$.id.version"},
+	{"Name", "$.id.name"},
+	{"Type", "$.closure.compiledTask.template.type"},
+	{"Discoverable", "$.closure.compiledTask.template.metadata.discoverable"},
+	{"DiscoveryVersion", "$.closure.compiledTask.template.metadata.discovery_version"},
 }
 
 func getTaskFunc(ctx context.Context, args []string, cmdCtx cmdCore.CommandContext) error {
@@ -57,12 +40,12 @@ func getTaskFunc(ctx context.Context, args []string, cmdCtx cmdCore.CommandConte
 		}
 		logger.Debugf(ctx, "Retrieved Task", task.Tasks)
 
-		return taskPrinter.Print(config.GetConfig().MustOutputFormat(), task.Tasks, taskStructure, transformTask)
+		return taskPrinter.Print(config.GetConfig().MustOutputFormat(), task.Tasks, taskColumns)
 	}
 	tasks, err := adminutils.GetAllNamedEntities(ctx, cmdCtx.AdminClient().ListTaskIds, adminutils.ListRequest{Project: config.GetConfig().Project, Domain: config.GetConfig().Domain})
 	if err != nil {
 		return err
 	}
 	logger.Debugf(ctx, "Retrieved %v Task", len(tasks))
-	return taskPrinter.Print(config.GetConfig().MustOutputFormat(), tasks, entityStructure, transformTaskEntity)
+	return taskPrinter.Print(config.GetConfig().MustOutputFormat(), tasks, entityColumns)
 }

flytectl/cmd/get/workflow.go

@@ -2,7 +2,6 @@ package get
 
 import (
 	"context"
-	"encoding/json"
 
 	"github.com/lyft/flytestdlib/logger"
 
@@ -14,22 +13,9 @@ import (
 	"github.com/lyft/flyteidl/gen/pb-go/flyteidl/admin"
 )
 
-var workflowStructure = map[string]string{
-	"Version": "$.id.version",
-	"Name":    "$.id.name",
-}
-
-type PrintableWorkflow struct {
-	Name    string `header:"Name"`
-	Version string `header:"Version"`
-}
-
-var transformWorkflow = func(jsonbody []byte) (interface{}, error) {
-	results := PrintableWorkflow{}
-	if err := json.Unmarshal(jsonbody, &results); err != nil {
-		return results, err
-	}
-	return results, nil
+var workflowColumns = []printer.Column{
+	{"Version", "$.id.version"},
+	{"Name", "$.id.name"},
 }
 
 func getWorkflowFunc(ctx context.Context, args []string, cmdCtx cmdCore.CommandContext) error {
@@ -48,13 +34,13 @@ func getWorkflowFunc(ctx context.Context, args []string, cmdCtx cmdCore.CommandC
 		}
 		logger.Debugf(ctx, "Retrieved %v workflows", len(workflows.Workflows))
 
-		return adminPrinter.Print(config.GetConfig().MustOutputFormat(), workflows.Workflows, workflowStructure, transformWorkflow)
+		return adminPrinter.Print(config.GetConfig().MustOutputFormat(), workflows.Workflows, workflowColumns)
 	}
 
 	workflows, err := adminutils.GetAllNamedEntities(ctx, cmdCtx.AdminClient().ListWorkflowIds, adminutils.ListRequest{Project: config.GetConfig().Project, Domain: config.GetConfig().Domain})
 	if err != nil {
 		return err
 	}
 	logger.Debugf(ctx, "Retrieved %v workflows", len(workflows))
-	return adminPrinter.Print(config.GetConfig().MustOutputFormat(), workflows, entityStructure, transformTaskEntity)
+	return adminPrinter.Print(config.GetConfig().MustOutputFormat(), workflows, entityColumns)
 }

flytectl/config.yaml

@@ -1,8 +1,8 @@
 admin:
   # For GRPC endpoints you might want to use dns:///flyte.myexample.com
-  endpoint: http://localhost:30082
-  # endpoint: dns:///flyte.lyft.net
-  insecure: true
+  # endpoint: http://localhost:30082
+  endpoint: dns:///flyte.lyft.net
+  insecure: false
 logger:
   show-source: true
   level: 1

flytectl/go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/ghodss/yaml v1.0.0
 	github.com/golang/protobuf v1.3.2
 	github.com/kataras/tablewriter v0.0.0-20180708051242-e063d29b7c23
-	github.com/landoop/tableprinter v0.0.0-20200104100433-ae9249991eb1
+	github.com/landoop/tableprinter v0.0.0-20180806200924-8bd8c2576d27
 	github.com/lyft/flyteidl v0.18.1
 	github.com/lyft/flytestdlib v0.3.10-0.20200619054107-45f341b716fa
 	github.com/mattn/go-runewidth v0.0.9 // indirect

flytectl/go.sum

@@ -186,6 +186,8 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/landoop/tableprinter v0.0.0-20180806200924-8bd8c2576d27 h1:O664tckOIC4smyHDDJPXAh/YBYYc0Y1O8S5wmZDm3d8=
+github.com/landoop/tableprinter v0.0.0-20180806200924-8bd8c2576d27/go.mod h1:f0X1c0za3TbET/rl5ThtCSel0+G3/yZ8iuU9BxnyVK0=
 github.com/landoop/tableprinter v0.0.0-20200104100433-ae9249991eb1 h1:SH30nioTpP0VaCYafgUw+iS1EfJTDbtAeBcByEFqFrY=
 github.com/landoop/tableprinter v0.0.0-20200104100433-ae9249991eb1/go.mod h1:f0X1c0za3TbET/rl5ThtCSel0+G3/yZ8iuU9BxnyVK0=
 github.com/lyft/flyteidl v0.18.1 h1:COKkZi5k6bQvUYOk5gE70+FJX9/NUn0WOQ1uMrw3Qio=

flytectl/printer/printer.go

@@ -29,39 +29,81 @@ func OutputFormats() []string {
 	return v
 }
 
+type Column struct {
+	Header   string
+	JSONPath string
+}
+
 type Printer struct{}
 
 const (
 	empty = ""
 	tab   = "\t"
 )
 
-func (p Printer) projectColumns(input []interface{}, column map[string]string, printTransform func(data []byte) (interface{}, error)) ([]interface{}, error) {
-	responses := make([]interface{}, 0, len(input))
-	for _, data := range input {
-		tableData := make(map[string]interface{})
-
-		for k := range column {
-			out, err := jsonpath.Read(data, column[k])
-			if err != nil {
-				out = nil
-			}
-			tableData[k] = out
-		}
-		jsonbody, err := json.Marshal(tableData)
-		if err != nil {
-			return responses, err
-		}
-		response, err := printTransform(jsonbody)
+// Projects the columns in one row of data from the given JSON using the []Column map
+func extractRow(data interface{}, columns []Column) []string {
+	if columns == nil || data == nil {
+		return nil
+	}
+	tableData := make([]string, 0, len(columns))
+
+	for _, c := range columns {
+		out, err := jsonpath.Read(data, c.JSONPath)
 		if err != nil {
-			return responses, err
+			out = ""
 		}
-		responses = append(responses, response)
+		tableData = append(tableData, fmt.Sprintf("%s", out))
+	}
+	return tableData
+}
+
+// Projects the columns from the given list of JSON elements using the []Column map
+// Potential performance problem, as it returns all the rows in memory.
+// We could use the render row, but that may lead to misalignment.
+// TODO figure out a more optimal way
+func projectColumns(input []interface{}, column []Column) ([][]string, error) {
+	responses := make([][]string, 0, len(input))
+	for _, data := range input {
+		responses = append(responses, extractRow(data, column))
 	}
 	return responses, nil
 }
 
-func (p Printer) Print(format OutputFormat, i interface{}, column map[string]string, printTransform func(data []byte) (interface{}, error)) error {
+func JSONToTable(b []byte, columns []Column) error {
+	var jsonRows []interface{}
+	err := json.Unmarshal(b, &jsonRows)
+	if err != nil {
+		return err
+	}
+	if jsonRows == nil {
+		return nil
+	}
+	rows, err := projectColumns(jsonRows, columns)
+	if err != nil {
+		return err
+	}
+	printer := tableprinter.New(os.Stdout)
+	// TODO make this configurable
+	printer.AutoWrapText = false
+	printer.BorderLeft = true
+	printer.BorderRight = true
+	printer.ColumnSeparator = "|"
+	printer.HeaderBgColor = tablewriter.BgHiWhiteColor
+	headers := make([]string, 0, len(columns))
+	positions := make([]int, 0, len(columns))
+	for _, c := range columns {
+		headers = append(headers, c.Header)
+		positions = append(positions, 30)
+	}
+	if r := printer.Render(headers, rows, positions, true); r == -1 {
+		return fmt.Errorf("failed to render table")
+	}
+	fmt.Printf("%d rows\n", len(rows))
+	return nil
+}
+
+func (p Printer) Print(format OutputFormat, i interface{}, columns []Column) error {
 
 	buf := new(bytes.Buffer)
 	encoder := json.NewEncoder(buf)
@@ -83,28 +125,7 @@ func (p Printer) Print(format OutputFormat, i interface{}, column map[string]str
 		}
 		fmt.Println(string(v))
 	default: // Print table
-		var rows []interface{}
-		err := json.Unmarshal(buf.Bytes(), &rows)
-		if err != nil {
-			return err
-		}
-		if rows == nil {
-			return nil
-		}
-		response, err := p.projectColumns(rows, column, printTransform)
-		if err != nil {
-			return err
-		}
-		printer := tableprinter.New(os.Stdout)
-		printer.AutoWrapText = false
-		printer.BorderLeft = true
-		printer.BorderRight = true
-		printer.ColumnSeparator = "|"
-		printer.HeaderBgColor = tablewriter.BgHiWhiteColor
-		if printer.Print(response) == -1 {
-			return fmt.Errorf("failed to print table data")
-		}
-		fmt.Printf("%d rows\n", len(rows))
+		return JSONToTable(buf.Bytes(), columns)
 	}
 	return nil
 }