From 31ed14600cb23f80dcddd1711f6580c35f309ff9 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Sat, 6 Nov 2021 08:44:48 +0800 Subject: [PATCH] Add config docs generate command (#103) * Added show command Signed-off-by: Kevin Su * Added tests Signed-off-by: Kevin Su * Added tests Signed-off-by: Kevin Su * Added link Signed-off-by: Kevin Su * Config docs Signed-off-by: Kevin Su * Address comments Signed-off-by: Kevin Su * Added tests Signed-off-by: Kevin Su * Fixed lint Signed-off-by: Kevin Su * Address comments Signed-off-by: Kevin Su * Fixed lint Signed-off-by: Kevin Su * Updated tests Signed-off-by: Kevin Su * Updated tests Signed-off-by: Kevin Su * Fixed lint Signed-off-by: Kevin Su * Updated tests Signed-off-by: Kevin Su * Updated tests Signed-off-by: Kevin Su --- config/config_cmd.go | 212 +++++++++++++++++++++++++++++++++++++- config/config_cmd_test.go | 59 +++++++++++ 2 files changed, 270 insertions(+), 1 deletion(-) diff --git a/config/config_cmd.go b/config/config_cmd.go index a5023ceb..cadb5efa 100644 --- a/config/config_cmd.go +++ b/config/config_cmd.go @@ -2,8 +2,14 @@ package config import ( "context" + "fmt" "os" + "reflect" "strings" + "unsafe" + + "github.com/ghodss/yaml" + "k8s.io/apimachinery/pkg/util/sets" "github.com/fatih/color" "github.com/spf13/cobra" @@ -14,6 +20,7 @@ const ( StrictModeFlag = "strict" CommandValidate = "validate" CommandDiscover = "discover" + CommandDocs = "docs" ) type AccessorProvider func(options Options) Accessor @@ -28,7 +35,7 @@ func NewConfigCommand(accessorProvider AccessorProvider) *cobra.Command { rootCmd := &cobra.Command{ Use: "config", Short: "Runs various config commands, look at the help of this command to get a list of available commands..", - ValidArgs: []string{CommandValidate, CommandDiscover}, + ValidArgs: []string{CommandValidate, CommandDiscover, CommandDocs}, } validateCmd := &cobra.Command{ @@ -47,12 +54,34 @@ func NewConfigCommand(accessorProvider AccessorProvider) *cobra.Command { }, } + docsCmd := &cobra.Command{ + Use: "docs", + Short: "Generate configuration documetation in rst format", + RunE: func(cmd *cobra.Command, args []string) error { + sections := GetRootSection().GetSections() + orderedSectionKeys := sets.NewString() + for s := range sections { + orderedSectionKeys.Insert(s) + } + printToc(orderedSectionKeys) + visitedSection := map[string]bool{} + visitedType := map[reflect.Type]bool{} + for _, sectionKey := range orderedSectionKeys.List() { + if canPrint(sections[sectionKey].GetConfig()) { + printDocs(sectionKey, false, sections[sectionKey], visitedSection, visitedType) + } + } + return nil + }, + } + // Configure Root Command rootCmd.PersistentFlags().StringArrayVar(&opts.SearchPaths, PathFlag, []string{}, `Passes the config file to load. If empty, it'll first search for the config file path then, if found, will load config from there.`) rootCmd.AddCommand(validateCmd) rootCmd.AddCommand(discoverCmd) + rootCmd.AddCommand(docsCmd) // Configure Validate Command validateCmd.Flags().BoolVar(&opts.StrictMode, StrictModeFlag, false, `Validates that all keys in loaded config @@ -75,6 +104,187 @@ func redirectStdOut() (old, new *os.File) { return } +func printDocs(title string, isSubsection bool, section Section, visitedSection map[string]bool, visitedType map[reflect.Type]bool) { + printTitle(title, isSubsection) + val := reflect.Indirect(reflect.ValueOf(section.GetConfig())) + if val.Kind() == reflect.Slice { + val = reflect.Indirect(reflect.ValueOf(val.Index(0).Interface())) + } + + subsections := make(map[string]interface{}) + for i := 0; i < val.Type().NumField(); i++ { + field := val.Type().Field(i) + tagType := field.Type + if tagType.Kind() == reflect.Ptr { + tagType = field.Type.Elem() + } + + fieldName := getFieldNameFromJSONTag(field) + fieldTypeString := getFieldTypeString(tagType) + fieldDefaultValue := getDefaultValue(fmt.Sprintf("%v", reflect.Indirect(val.Field(i)))) + fieldDescription := getFieldDescriptionFromPflag(field) + + subVal := val.Field(i) + if tagType.Kind() == reflect.Struct { + // In order to get value from unexported field in struct + if subVal.Kind() == reflect.Ptr { + subVal = reflect.NewAt(subVal.Type(), unsafe.Pointer(subVal.UnsafeAddr())).Elem() + } else { + subVal = reflect.NewAt(subVal.Type(), unsafe.Pointer(subVal.UnsafeAddr())) + } + } + + if tagType.Kind() == reflect.Map || tagType.Kind() == reflect.Slice || tagType.Kind() == reflect.Struct { + fieldDefaultValue = getDefaultValue(subVal.Interface()) + } + + if tagType.Kind() == reflect.Struct { + if canPrint(subVal.Interface()) { + addSubsection(subVal.Interface(), subsections, fieldName, &fieldTypeString, tagType, visitedSection, visitedType) + } + } + printSection(fieldName, fieldTypeString, fieldDefaultValue, fieldDescription) + } + + if section != nil { + sections := section.GetSections() + orderedSectionKeys := sets.NewString() + for s := range sections { + orderedSectionKeys.Insert(s) + } + for _, sectionKey := range orderedSectionKeys.List() { + fieldName := sectionKey + fieldType := reflect.TypeOf(sections[sectionKey].GetConfig()) + fieldTypeString := getFieldTypeString(fieldType) + fieldDefaultValue := getDefaultValue(sections[sectionKey].GetConfig()) + + addSubsection(sections[sectionKey].GetConfig(), subsections, fieldName, &fieldTypeString, fieldType, visitedSection, visitedType) + printSection(fieldName, fieldTypeString, fieldDefaultValue, "") + } + } + orderedSectionKeys := sets.NewString() + for s := range subsections { + orderedSectionKeys.Insert(s) + } + + for _, sectionKey := range orderedSectionKeys.List() { + printDocs(sectionKey, true, NewSection(subsections[sectionKey], nil), visitedSection, visitedType) + } +} + +// Print Table of contents +func printToc(orderedSectionKeys sets.String) { + for _, sectionKey := range orderedSectionKeys.List() { + fmt.Printf("- `%s <#section-%s>`_\n\n", sectionKey, sectionKey) + } +} + +func printTitle(title string, isSubsection bool) { + if isSubsection { + fmt.Println(title) + fmt.Println(strings.Repeat("-", 80)) + } else { + fmt.Println("Section:", title) + fmt.Println(strings.Repeat("=", 80)) + } + fmt.Println() +} + +func printSection(name string, dataType string, defaultValue string, description string) { + c := "-" + + fmt.Printf("%s ", name) + fmt.Printf("(%s)\n", dataType) + fmt.Println(strings.Repeat(c, 80)) + fmt.Println() + if description != "" { + fmt.Printf("%s\n\n", description) + } + if defaultValue != "" { + val := strings.Replace(defaultValue, "\n", "\n ", -1) + val = ".. code-block:: yaml\n\n " + val + fmt.Printf("**Default Value**: \n\n%s\n", val) + } + fmt.Println() +} + +func addSubsection(val interface{}, subsections map[string]interface{}, fieldName string, + fieldTypeString *string, fieldType reflect.Type, visitedSection map[string]bool, visitedType map[reflect.Type]bool) { + + if visitedSection[*fieldTypeString] { + if !visitedType[fieldType] { + // Some types have the same name, but they are different type. + // Add field name at the end to tell the difference between them. + *fieldTypeString = fmt.Sprintf("%s (%s)", *fieldTypeString, fieldName) + subsections[*fieldTypeString] = val + + } + } else { + visitedSection[*fieldTypeString] = true + subsections[*fieldTypeString] = val + } + *fieldTypeString = fmt.Sprintf("`%s`_", *fieldTypeString) + visitedType[fieldType] = true +} + +func getDefaultValue(val interface{}) string { + defaultValue, err := yaml.Marshal(val) + if err != nil { + return "" + } + DefaultValue := string(defaultValue) + return DefaultValue +} + +func getFieldTypeString(tagType reflect.Type) string { + kind := tagType.Kind() + if kind == reflect.Ptr { + tagType = tagType.Elem() + kind = tagType.Kind() + } + + FieldTypeString := kind.String() + if kind == reflect.Map || kind == reflect.Slice || kind == reflect.Struct { + FieldTypeString = tagType.String() + } + return FieldTypeString +} + +func getFieldDescriptionFromPflag(field reflect.StructField) string { + if pFlag := field.Tag.Get("pflag"); len(pFlag) > 0 && !strings.HasPrefix(pFlag, "-") { + var commaIdx int + if commaIdx = strings.Index(pFlag, ","); commaIdx < 0 { + commaIdx = -1 + } + if strippedDescription := pFlag[commaIdx+1:]; len(strippedDescription) > 0 { + return strings.TrimPrefix(strippedDescription, " ") + } + } + return "" +} + +func getFieldNameFromJSONTag(field reflect.StructField) string { + if jsonTag := field.Tag.Get("json"); len(jsonTag) > 0 && !strings.HasPrefix(jsonTag, "-") { + var commaIdx int + if commaIdx = strings.Index(jsonTag, ","); commaIdx < 0 { + commaIdx = len(jsonTag) + } + if strippedName := jsonTag[:commaIdx]; len(strippedName) > 0 { + return strippedName + } + } + return field.Name +} + +// Print out config docs if and only if the section type is struct or slice +func canPrint(b interface{}) bool { + val := reflect.Indirect(reflect.ValueOf(b)) + if val.Kind() == reflect.Struct || val.Kind() == reflect.Slice { + return true + } + return false +} + func validate(accessor Accessor, p printer) error { // Redirect stdout old, n := redirectStdOut() diff --git a/config/config_cmd_test.go b/config/config_cmd_test.go index 03546ea9..0974ca36 100644 --- a/config/config_cmd_test.go +++ b/config/config_cmd_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "flag" + "reflect" "testing" "github.com/spf13/cobra" @@ -11,6 +12,10 @@ import ( "github.com/stretchr/testify/assert" ) +var redisConfig = "mockRedis" +var resourceManagerConfig = ResourceManagerConfig{"mockType", 100, &redisConfig, + []int{1, 2, 3}, InnerConfig{"hello"}, &InnerConfig{"world"}} + type MockAccessor struct { } @@ -61,4 +66,58 @@ func TestNewConfigCommand(t *testing.T) { output, err = executeCommandC(cmd, CommandValidate) assert.NoError(t, err) assert.Contains(t, output, "test") + + section, err := GetRootSection().RegisterSection("root", &resourceManagerConfig) + assert.NoError(t, err) + section.MustRegisterSection("subsection", &resourceManagerConfig) + _, err = executeCommandC(cmd, CommandDocs) + assert.NoError(t, err) +} + +type InnerConfig struct { + InnerType string `json:"type" pflag:"noop,Which resource manager to use"` +} + +type ResourceManagerConfig struct { + Type string `json:"type" pflag:"noop,Which resource manager to use"` + ResourceMaxQuota int `json:"resourceMaxQuota" pflag:",Global limit for concurrent Qubole queries"` + RedisConfig *string `json:"" pflag:",Config for Redis resource manager."` + ListConfig []int `json:"" pflag:","` + InnerConfig InnerConfig + InnerConfig1 *InnerConfig +} + +func TestGetDefaultValue(t *testing.T) { + val := getDefaultValue(resourceManagerConfig) + res := "InnerConfig:\n type: hello\nInnerConfig1:\n type: world\nListConfig:\n- 1\n- 2\n- 3\nRedisConfig: mockRedis\nresourceMaxQuota: 100\ntype: mockType\n" + assert.Equal(t, res, val) +} + +func TestGetFieldTypeString(t *testing.T) { + val := reflect.ValueOf(resourceManagerConfig) + assert.Equal(t, "config.ResourceManagerConfig", getFieldTypeString(val.Type())) + assert.Equal(t, "string", getFieldTypeString(val.Field(0).Type())) + assert.Equal(t, "int", getFieldTypeString(val.Field(1).Type())) + assert.Equal(t, "string", getFieldTypeString(val.Field(2).Type())) +} + +func TestGetFieldDescriptionFromPflag(t *testing.T) { + val := reflect.ValueOf(resourceManagerConfig) + assert.Equal(t, "Which resource manager to use", getFieldDescriptionFromPflag(val.Type().Field(0))) + assert.Equal(t, "Global limit for concurrent Qubole queries", getFieldDescriptionFromPflag(val.Type().Field(1))) + assert.Equal(t, "Config for Redis resource manager.", getFieldDescriptionFromPflag(val.Type().Field(2))) +} + +func TestGetFieldNameFromJSONTag(t *testing.T) { + val := reflect.ValueOf(resourceManagerConfig) + assert.Equal(t, "type", getFieldNameFromJSONTag(val.Type().Field(0))) + assert.Equal(t, "resourceMaxQuota", getFieldNameFromJSONTag(val.Type().Field(1))) + assert.Equal(t, "RedisConfig", getFieldNameFromJSONTag(val.Type().Field(2))) +} + +func TestCanPrint(t *testing.T) { + assert.True(t, canPrint(resourceManagerConfig)) + assert.True(t, canPrint(&resourceManagerConfig)) + assert.True(t, canPrint([]ResourceManagerConfig{resourceManagerConfig})) + assert.False(t, canPrint(map[string]ResourceManagerConfig{"config": resourceManagerConfig})) }