Skip to content

Commit

Permalink
Add config docs generate command (flyteorg#103)
Browse files Browse the repository at this point in the history 
* Added show command

Signed-off-by: Kevin Su <[email protected]>

* Added tests

Signed-off-by: Kevin Su <[email protected]>

* Added tests

Signed-off-by: Kevin Su <[email protected]>

* Added link

Signed-off-by: Kevin Su <[email protected]>

* Config docs

Signed-off-by: Kevin Su <[email protected]>

* Address comments

Signed-off-by: Kevin Su <[email protected]>

* Added tests

Signed-off-by: Kevin Su <[email protected]>

* Fixed lint

Signed-off-by: Kevin Su <[email protected]>

* Address comments

Signed-off-by: Kevin Su <[email protected]>

* Fixed lint

Signed-off-by: Kevin Su <[email protected]>

* Updated tests

Signed-off-by: Kevin Su <[email protected]>

* Updated tests

Signed-off-by: Kevin Su <[email protected]>

* Fixed lint

Signed-off-by: Kevin Su <[email protected]>

* Updated tests

Signed-off-by: Kevin Su <[email protected]>

* Updated tests

Signed-off-by: Kevin Su <[email protected]>
  • Loading branch information
@pingsutw
pingsutw authored Nov 6, 2021
1 parent 1177784 commit 31ed146
Show file tree
Hide file tree
Showing 2 changed files with 270 additions and 1 deletion.
212 changes: 211 additions & 1 deletion config/config_cmd.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,14 @@ package config

import (
"context"
"fmt"
"os"
"reflect"
"strings"
"unsafe"

"github.com/ghodss/yaml"
"k8s.io/apimachinery/pkg/util/sets"

"github.com/fatih/color"
"github.com/spf13/cobra"
Expand All @@ -14,6 +20,7 @@ const (
StrictModeFlag = "strict"
CommandValidate = "validate"
CommandDiscover = "discover"
CommandDocs = "docs"
)

type AccessorProvider func(options Options) Accessor
Expand All @@ -28,7 +35,7 @@ func NewConfigCommand(accessorProvider AccessorProvider) *cobra.Command {
rootCmd := &cobra.Command{
Use: "config",
Short: "Runs various config commands, look at the help of this command to get a list of available commands..",
ValidArgs: []string{CommandValidate, CommandDiscover},
ValidArgs: []string{CommandValidate, CommandDiscover, CommandDocs},
}

validateCmd := &cobra.Command{
Expand All @@ -47,12 +54,34 @@ func NewConfigCommand(accessorProvider AccessorProvider) *cobra.Command {
},
}

docsCmd := &cobra.Command{
Use: "docs",
Short: "Generate configuration documetation in rst format",
RunE: func(cmd *cobra.Command, args []string) error {
sections := GetRootSection().GetSections()
orderedSectionKeys := sets.NewString()
for s := range sections {
orderedSectionKeys.Insert(s)
}
printToc(orderedSectionKeys)
visitedSection := map[string]bool{}
visitedType := map[reflect.Type]bool{}
for _, sectionKey := range orderedSectionKeys.List() {
if canPrint(sections[sectionKey].GetConfig()) {
printDocs(sectionKey, false, sections[sectionKey], visitedSection, visitedType)
}
}
return nil
},
}

// Configure Root Command
rootCmd.PersistentFlags().StringArrayVar(&opts.SearchPaths, PathFlag, []string{}, `Passes the config file to load.
If empty, it'll first search for the config file path then, if found, will load config from there.`)

rootCmd.AddCommand(validateCmd)
rootCmd.AddCommand(discoverCmd)
rootCmd.AddCommand(docsCmd)

// Configure Validate Command
validateCmd.Flags().BoolVar(&opts.StrictMode, StrictModeFlag, false, `Validates that all keys in loaded config
Expand All @@ -75,6 +104,187 @@ func redirectStdOut() (old, new *os.File) {
return
}

func printDocs(title string, isSubsection bool, section Section, visitedSection map[string]bool, visitedType map[reflect.Type]bool) {
printTitle(title, isSubsection)
val := reflect.Indirect(reflect.ValueOf(section.GetConfig()))
if val.Kind() == reflect.Slice {
val = reflect.Indirect(reflect.ValueOf(val.Index(0).Interface()))
}

subsections := make(map[string]interface{})
for i := 0; i < val.Type().NumField(); i++ {
field := val.Type().Field(i)
tagType := field.Type
if tagType.Kind() == reflect.Ptr {
tagType = field.Type.Elem()
}

fieldName := getFieldNameFromJSONTag(field)
fieldTypeString := getFieldTypeString(tagType)
fieldDefaultValue := getDefaultValue(fmt.Sprintf("%v", reflect.Indirect(val.Field(i))))
fieldDescription := getFieldDescriptionFromPflag(field)

subVal := val.Field(i)
if tagType.Kind() == reflect.Struct {
// In order to get value from unexported field in struct
if subVal.Kind() == reflect.Ptr {
subVal = reflect.NewAt(subVal.Type(), unsafe.Pointer(subVal.UnsafeAddr())).Elem()
} else {
subVal = reflect.NewAt(subVal.Type(), unsafe.Pointer(subVal.UnsafeAddr()))
}
}

if tagType.Kind() == reflect.Map || tagType.Kind() == reflect.Slice || tagType.Kind() == reflect.Struct {
fieldDefaultValue = getDefaultValue(subVal.Interface())
}

if tagType.Kind() == reflect.Struct {
if canPrint(subVal.Interface()) {
addSubsection(subVal.Interface(), subsections, fieldName, &fieldTypeString, tagType, visitedSection, visitedType)
}
}
printSection(fieldName, fieldTypeString, fieldDefaultValue, fieldDescription)
}

if section != nil {
sections := section.GetSections()
orderedSectionKeys := sets.NewString()
for s := range sections {
orderedSectionKeys.Insert(s)
}
for _, sectionKey := range orderedSectionKeys.List() {
fieldName := sectionKey
fieldType := reflect.TypeOf(sections[sectionKey].GetConfig())
fieldTypeString := getFieldTypeString(fieldType)
fieldDefaultValue := getDefaultValue(sections[sectionKey].GetConfig())

addSubsection(sections[sectionKey].GetConfig(), subsections, fieldName, &fieldTypeString, fieldType, visitedSection, visitedType)
printSection(fieldName, fieldTypeString, fieldDefaultValue, "")
}
}
orderedSectionKeys := sets.NewString()
for s := range subsections {
orderedSectionKeys.Insert(s)
}

for _, sectionKey := range orderedSectionKeys.List() {
printDocs(sectionKey, true, NewSection(subsections[sectionKey], nil), visitedSection, visitedType)
}
}

// Print Table of contents
func printToc(orderedSectionKeys sets.String) {
for _, sectionKey := range orderedSectionKeys.List() {
fmt.Printf("- `%s <#section-%s>`_\n\n", sectionKey, sectionKey)
}
}

func printTitle(title string, isSubsection bool) {
if isSubsection {
fmt.Println(title)
fmt.Println(strings.Repeat("-", 80))
} else {
fmt.Println("Section:", title)
fmt.Println(strings.Repeat("=", 80))
}
fmt.Println()
}

func printSection(name string, dataType string, defaultValue string, description string) {
c := "-"

fmt.Printf("%s ", name)
fmt.Printf("(%s)\n", dataType)
fmt.Println(strings.Repeat(c, 80))
fmt.Println()
if description != "" {
fmt.Printf("%s\n\n", description)
}
if defaultValue != "" {
val := strings.Replace(defaultValue, "\n", "\n ", -1)
val = ".. code-block:: yaml\n\n " + val
fmt.Printf("**Default Value**: \n\n%s\n", val)
}
fmt.Println()
}

func addSubsection(val interface{}, subsections map[string]interface{}, fieldName string,
fieldTypeString *string, fieldType reflect.Type, visitedSection map[string]bool, visitedType map[reflect.Type]bool) {

if visitedSection[*fieldTypeString] {
if !visitedType[fieldType] {
// Some types have the same name, but they are different type.
// Add field name at the end to tell the difference between them.
*fieldTypeString = fmt.Sprintf("%s (%s)", *fieldTypeString, fieldName)
subsections[*fieldTypeString] = val

}
} else {
visitedSection[*fieldTypeString] = true
subsections[*fieldTypeString] = val
}
*fieldTypeString = fmt.Sprintf("`%s`_", *fieldTypeString)
visitedType[fieldType] = true
}

func getDefaultValue(val interface{}) string {
defaultValue, err := yaml.Marshal(val)
if err != nil {
return ""
}
DefaultValue := string(defaultValue)
return DefaultValue
}

func getFieldTypeString(tagType reflect.Type) string {
kind := tagType.Kind()
if kind == reflect.Ptr {
tagType = tagType.Elem()
kind = tagType.Kind()
}

FieldTypeString := kind.String()
if kind == reflect.Map || kind == reflect.Slice || kind == reflect.Struct {
FieldTypeString = tagType.String()
}
return FieldTypeString
}

func getFieldDescriptionFromPflag(field reflect.StructField) string {
if pFlag := field.Tag.Get("pflag"); len(pFlag) > 0 && !strings.HasPrefix(pFlag, "-") {
var commaIdx int
if commaIdx = strings.Index(pFlag, ","); commaIdx < 0 {
commaIdx = -1
}
if strippedDescription := pFlag[commaIdx+1:]; len(strippedDescription) > 0 {
return strings.TrimPrefix(strippedDescription, " ")
}
}
return ""
}

func getFieldNameFromJSONTag(field reflect.StructField) string {
if jsonTag := field.Tag.Get("json"); len(jsonTag) > 0 && !strings.HasPrefix(jsonTag, "-") {
var commaIdx int
if commaIdx = strings.Index(jsonTag, ","); commaIdx < 0 {
commaIdx = len(jsonTag)
}
if strippedName := jsonTag[:commaIdx]; len(strippedName) > 0 {
return strippedName
}
}
return field.Name
}

// Print out config docs if and only if the section type is struct or slice
func canPrint(b interface{}) bool {
val := reflect.Indirect(reflect.ValueOf(b))
if val.Kind() == reflect.Struct || val.Kind() == reflect.Slice {
return true
}
return false
}

func validate(accessor Accessor, p printer) error {
// Redirect stdout
old, n := redirectStdOut()
Expand Down
59 changes: 59 additions & 0 deletions config/config_cmd_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,18 @@ import (
"bytes"
"context"
"flag"
"reflect"
"testing"

"github.com/spf13/cobra"
"github.com/spf13/pflag"
"github.com/stretchr/testify/assert"
)

var redisConfig = "mockRedis"
var resourceManagerConfig = ResourceManagerConfig{"mockType", 100, &redisConfig,
[]int{1, 2, 3}, InnerConfig{"hello"}, &InnerConfig{"world"}}

type MockAccessor struct {
}

Expand Down Expand Up @@ -61,4 +66,58 @@ func TestNewConfigCommand(t *testing.T) {
output, err = executeCommandC(cmd, CommandValidate)
assert.NoError(t, err)
assert.Contains(t, output, "test")

section, err := GetRootSection().RegisterSection("root", &resourceManagerConfig)
assert.NoError(t, err)
section.MustRegisterSection("subsection", &resourceManagerConfig)
_, err = executeCommandC(cmd, CommandDocs)
assert.NoError(t, err)
}

type InnerConfig struct {
InnerType string `json:"type" pflag:"noop,Which resource manager to use"`
}

type ResourceManagerConfig struct {
Type string `json:"type" pflag:"noop,Which resource manager to use"`
ResourceMaxQuota int `json:"resourceMaxQuota" pflag:",Global limit for concurrent Qubole queries"`
RedisConfig *string `json:"" pflag:",Config for Redis resource manager."`
ListConfig []int `json:"" pflag:","`
InnerConfig InnerConfig
InnerConfig1 *InnerConfig
}

func TestGetDefaultValue(t *testing.T) {
val := getDefaultValue(resourceManagerConfig)
res := "InnerConfig:\n type: hello\nInnerConfig1:\n type: world\nListConfig:\n- 1\n- 2\n- 3\nRedisConfig: mockRedis\nresourceMaxQuota: 100\ntype: mockType\n"
assert.Equal(t, res, val)
}

func TestGetFieldTypeString(t *testing.T) {
val := reflect.ValueOf(resourceManagerConfig)
assert.Equal(t, "config.ResourceManagerConfig", getFieldTypeString(val.Type()))
assert.Equal(t, "string", getFieldTypeString(val.Field(0).Type()))
assert.Equal(t, "int", getFieldTypeString(val.Field(1).Type()))
assert.Equal(t, "string", getFieldTypeString(val.Field(2).Type()))
}

func TestGetFieldDescriptionFromPflag(t *testing.T) {
val := reflect.ValueOf(resourceManagerConfig)
assert.Equal(t, "Which resource manager to use", getFieldDescriptionFromPflag(val.Type().Field(0)))
assert.Equal(t, "Global limit for concurrent Qubole queries", getFieldDescriptionFromPflag(val.Type().Field(1)))
assert.Equal(t, "Config for Redis resource manager.", getFieldDescriptionFromPflag(val.Type().Field(2)))
}

func TestGetFieldNameFromJSONTag(t *testing.T) {
val := reflect.ValueOf(resourceManagerConfig)
assert.Equal(t, "type", getFieldNameFromJSONTag(val.Type().Field(0)))
assert.Equal(t, "resourceMaxQuota", getFieldNameFromJSONTag(val.Type().Field(1)))
assert.Equal(t, "RedisConfig", getFieldNameFromJSONTag(val.Type().Field(2)))
}

func TestCanPrint(t *testing.T) {
assert.True(t, canPrint(resourceManagerConfig))
assert.True(t, canPrint(&resourceManagerConfig))
assert.True(t, canPrint([]ResourceManagerConfig{resourceManagerConfig}))
assert.False(t, canPrint(map[string]ResourceManagerConfig{"config": resourceManagerConfig}))
}

0 comments on commit 31ed146

Please sign in to comment.