From ae4ebc2f328d713537a50a86a6869949467aa251 Mon Sep 17 00:00:00 2001 From: Haytham AbuelFutuh Date: Wed, 11 Dec 2019 10:23:52 -0800 Subject: [PATCH] Set config field json names (#37) --- .../tasks/pluginmachinery/catalog/config.go | 4 +- .../pluginmachinery/catalog/config_flags.go | 12 ++--- .../catalog/config_flags_test.go | 48 +++++++++---------- .../tasks/plugins/array/core/phase_enumer.go | 15 +++--- 4 files changed, 40 insertions(+), 39 deletions(-) diff --git a/flyteplugins/go/tasks/pluginmachinery/catalog/config.go b/flyteplugins/go/tasks/pluginmachinery/catalog/config.go index bc8e86e741..d3b7609e51 100644 --- a/flyteplugins/go/tasks/pluginmachinery/catalog/config.go +++ b/flyteplugins/go/tasks/pluginmachinery/catalog/config.go @@ -10,8 +10,8 @@ import ( var cfgSection = config.MustRegisterSubSection("catalogCache", defaultConfig) type Config struct { - ReaderWorkqueueConfig workqueue.Config - WriterWorkqueueConfig workqueue.Config + ReaderWorkqueueConfig workqueue.Config `json:"reader" pflag:",Catalog reader workqueue config. Make sure the index cache must be big enough to accommodate the biggest array task allowed to run on the system."` + WriterWorkqueueConfig workqueue.Config `json:"writer" pflag:",Catalog writer workqueue config. Make sure the index cache must be big enough to accommodate the biggest array task allowed to run on the system."` } var defaultConfig = &Config{ diff --git a/flyteplugins/go/tasks/pluginmachinery/catalog/config_flags.go b/flyteplugins/go/tasks/pluginmachinery/catalog/config_flags.go index 1a7a06b8b9..f7bd0561fa 100755 --- a/flyteplugins/go/tasks/pluginmachinery/catalog/config_flags.go +++ b/flyteplugins/go/tasks/pluginmachinery/catalog/config_flags.go @@ -41,11 +41,11 @@ func (Config) mustMarshalJSON(v json.Marshaler) string { // flags is json-name.json-sub-name... etc. func (cfg Config) GetPFlagSet(prefix string) *pflag.FlagSet { cmdFlags := pflag.NewFlagSet("Config", pflag.ExitOnError) - cmdFlags.Int(fmt.Sprintf("%v%v", prefix, "ReaderWorkqueueConfig.workers"), defaultConfig.ReaderWorkqueueConfig.Workers, "Number of concurrent workers to start processing the queue.") - cmdFlags.Int(fmt.Sprintf("%v%v", prefix, "ReaderWorkqueueConfig.maxRetries"), defaultConfig.ReaderWorkqueueConfig.MaxRetries, "Maximum number of retries per item.") - cmdFlags.Int(fmt.Sprintf("%v%v", prefix, "ReaderWorkqueueConfig.maxItems"), defaultConfig.ReaderWorkqueueConfig.IndexCacheMaxItems, "Maximum number of entries to keep in the index.") - cmdFlags.Int(fmt.Sprintf("%v%v", prefix, "WriterWorkqueueConfig.workers"), defaultConfig.WriterWorkqueueConfig.Workers, "Number of concurrent workers to start processing the queue.") - cmdFlags.Int(fmt.Sprintf("%v%v", prefix, "WriterWorkqueueConfig.maxRetries"), defaultConfig.WriterWorkqueueConfig.MaxRetries, "Maximum number of retries per item.") - cmdFlags.Int(fmt.Sprintf("%v%v", prefix, "WriterWorkqueueConfig.maxItems"), defaultConfig.WriterWorkqueueConfig.IndexCacheMaxItems, "Maximum number of entries to keep in the index.") + cmdFlags.Int(fmt.Sprintf("%v%v", prefix, "reader.workers"), defaultConfig.ReaderWorkqueueConfig.Workers, "Number of concurrent workers to start processing the queue.") + cmdFlags.Int(fmt.Sprintf("%v%v", prefix, "reader.maxRetries"), defaultConfig.ReaderWorkqueueConfig.MaxRetries, "Maximum number of retries per item.") + cmdFlags.Int(fmt.Sprintf("%v%v", prefix, "reader.maxItems"), defaultConfig.ReaderWorkqueueConfig.IndexCacheMaxItems, "Maximum number of entries to keep in the index.") + cmdFlags.Int(fmt.Sprintf("%v%v", prefix, "writer.workers"), defaultConfig.WriterWorkqueueConfig.Workers, "Number of concurrent workers to start processing the queue.") + cmdFlags.Int(fmt.Sprintf("%v%v", prefix, "writer.maxRetries"), defaultConfig.WriterWorkqueueConfig.MaxRetries, "Maximum number of retries per item.") + cmdFlags.Int(fmt.Sprintf("%v%v", prefix, "writer.maxItems"), defaultConfig.WriterWorkqueueConfig.IndexCacheMaxItems, "Maximum number of entries to keep in the index.") return cmdFlags } diff --git a/flyteplugins/go/tasks/pluginmachinery/catalog/config_flags_test.go b/flyteplugins/go/tasks/pluginmachinery/catalog/config_flags_test.go index e623482282..0b9b1efd91 100755 --- a/flyteplugins/go/tasks/pluginmachinery/catalog/config_flags_test.go +++ b/flyteplugins/go/tasks/pluginmachinery/catalog/config_flags_test.go @@ -99,10 +99,10 @@ func TestConfig_SetFlags(t *testing.T) { cmdFlags := actual.GetPFlagSet("") assert.True(t, cmdFlags.HasFlags()) - t.Run("Test_ReaderWorkqueueConfig.workers", func(t *testing.T) { + t.Run("Test_reader.workers", func(t *testing.T) { t.Run("DefaultValue", func(t *testing.T) { // Test that default value is set properly - if vInt, err := cmdFlags.GetInt("ReaderWorkqueueConfig.workers"); err == nil { + if vInt, err := cmdFlags.GetInt("reader.workers"); err == nil { assert.Equal(t, int(defaultConfig.ReaderWorkqueueConfig.Workers), vInt) } else { assert.FailNow(t, err.Error()) @@ -112,8 +112,8 @@ func TestConfig_SetFlags(t *testing.T) { t.Run("Override", func(t *testing.T) { testValue := "1" - cmdFlags.Set("ReaderWorkqueueConfig.workers", testValue) - if vInt, err := cmdFlags.GetInt("ReaderWorkqueueConfig.workers"); err == nil { + cmdFlags.Set("reader.workers", testValue) + if vInt, err := cmdFlags.GetInt("reader.workers"); err == nil { testDecodeJson_Config(t, fmt.Sprintf("%v", vInt), &actual.ReaderWorkqueueConfig.Workers) } else { @@ -121,10 +121,10 @@ func TestConfig_SetFlags(t *testing.T) { } }) }) - t.Run("Test_ReaderWorkqueueConfig.maxRetries", func(t *testing.T) { + t.Run("Test_reader.maxRetries", func(t *testing.T) { t.Run("DefaultValue", func(t *testing.T) { // Test that default value is set properly - if vInt, err := cmdFlags.GetInt("ReaderWorkqueueConfig.maxRetries"); err == nil { + if vInt, err := cmdFlags.GetInt("reader.maxRetries"); err == nil { assert.Equal(t, int(defaultConfig.ReaderWorkqueueConfig.MaxRetries), vInt) } else { assert.FailNow(t, err.Error()) @@ -134,8 +134,8 @@ func TestConfig_SetFlags(t *testing.T) { t.Run("Override", func(t *testing.T) { testValue := "1" - cmdFlags.Set("ReaderWorkqueueConfig.maxRetries", testValue) - if vInt, err := cmdFlags.GetInt("ReaderWorkqueueConfig.maxRetries"); err == nil { + cmdFlags.Set("reader.maxRetries", testValue) + if vInt, err := cmdFlags.GetInt("reader.maxRetries"); err == nil { testDecodeJson_Config(t, fmt.Sprintf("%v", vInt), &actual.ReaderWorkqueueConfig.MaxRetries) } else { @@ -143,10 +143,10 @@ func TestConfig_SetFlags(t *testing.T) { } }) }) - t.Run("Test_ReaderWorkqueueConfig.maxItems", func(t *testing.T) { + t.Run("Test_reader.maxItems", func(t *testing.T) { t.Run("DefaultValue", func(t *testing.T) { // Test that default value is set properly - if vInt, err := cmdFlags.GetInt("ReaderWorkqueueConfig.maxItems"); err == nil { + if vInt, err := cmdFlags.GetInt("reader.maxItems"); err == nil { assert.Equal(t, int(defaultConfig.ReaderWorkqueueConfig.IndexCacheMaxItems), vInt) } else { assert.FailNow(t, err.Error()) @@ -156,8 +156,8 @@ func TestConfig_SetFlags(t *testing.T) { t.Run("Override", func(t *testing.T) { testValue := "1" - cmdFlags.Set("ReaderWorkqueueConfig.maxItems", testValue) - if vInt, err := cmdFlags.GetInt("ReaderWorkqueueConfig.maxItems"); err == nil { + cmdFlags.Set("reader.maxItems", testValue) + if vInt, err := cmdFlags.GetInt("reader.maxItems"); err == nil { testDecodeJson_Config(t, fmt.Sprintf("%v", vInt), &actual.ReaderWorkqueueConfig.IndexCacheMaxItems) } else { @@ -165,10 +165,10 @@ func TestConfig_SetFlags(t *testing.T) { } }) }) - t.Run("Test_WriterWorkqueueConfig.workers", func(t *testing.T) { + t.Run("Test_writer.workers", func(t *testing.T) { t.Run("DefaultValue", func(t *testing.T) { // Test that default value is set properly - if vInt, err := cmdFlags.GetInt("WriterWorkqueueConfig.workers"); err == nil { + if vInt, err := cmdFlags.GetInt("writer.workers"); err == nil { assert.Equal(t, int(defaultConfig.WriterWorkqueueConfig.Workers), vInt) } else { assert.FailNow(t, err.Error()) @@ -178,8 +178,8 @@ func TestConfig_SetFlags(t *testing.T) { t.Run("Override", func(t *testing.T) { testValue := "1" - cmdFlags.Set("WriterWorkqueueConfig.workers", testValue) - if vInt, err := cmdFlags.GetInt("WriterWorkqueueConfig.workers"); err == nil { + cmdFlags.Set("writer.workers", testValue) + if vInt, err := cmdFlags.GetInt("writer.workers"); err == nil { testDecodeJson_Config(t, fmt.Sprintf("%v", vInt), &actual.WriterWorkqueueConfig.Workers) } else { @@ -187,10 +187,10 @@ func TestConfig_SetFlags(t *testing.T) { } }) }) - t.Run("Test_WriterWorkqueueConfig.maxRetries", func(t *testing.T) { + t.Run("Test_writer.maxRetries", func(t *testing.T) { t.Run("DefaultValue", func(t *testing.T) { // Test that default value is set properly - if vInt, err := cmdFlags.GetInt("WriterWorkqueueConfig.maxRetries"); err == nil { + if vInt, err := cmdFlags.GetInt("writer.maxRetries"); err == nil { assert.Equal(t, int(defaultConfig.WriterWorkqueueConfig.MaxRetries), vInt) } else { assert.FailNow(t, err.Error()) @@ -200,8 +200,8 @@ func TestConfig_SetFlags(t *testing.T) { t.Run("Override", func(t *testing.T) { testValue := "1" - cmdFlags.Set("WriterWorkqueueConfig.maxRetries", testValue) - if vInt, err := cmdFlags.GetInt("WriterWorkqueueConfig.maxRetries"); err == nil { + cmdFlags.Set("writer.maxRetries", testValue) + if vInt, err := cmdFlags.GetInt("writer.maxRetries"); err == nil { testDecodeJson_Config(t, fmt.Sprintf("%v", vInt), &actual.WriterWorkqueueConfig.MaxRetries) } else { @@ -209,10 +209,10 @@ func TestConfig_SetFlags(t *testing.T) { } }) }) - t.Run("Test_WriterWorkqueueConfig.maxItems", func(t *testing.T) { + t.Run("Test_writer.maxItems", func(t *testing.T) { t.Run("DefaultValue", func(t *testing.T) { // Test that default value is set properly - if vInt, err := cmdFlags.GetInt("WriterWorkqueueConfig.maxItems"); err == nil { + if vInt, err := cmdFlags.GetInt("writer.maxItems"); err == nil { assert.Equal(t, int(defaultConfig.WriterWorkqueueConfig.IndexCacheMaxItems), vInt) } else { assert.FailNow(t, err.Error()) @@ -222,8 +222,8 @@ func TestConfig_SetFlags(t *testing.T) { t.Run("Override", func(t *testing.T) { testValue := "1" - cmdFlags.Set("WriterWorkqueueConfig.maxItems", testValue) - if vInt, err := cmdFlags.GetInt("WriterWorkqueueConfig.maxItems"); err == nil { + cmdFlags.Set("writer.maxItems", testValue) + if vInt, err := cmdFlags.GetInt("writer.maxItems"); err == nil { testDecodeJson_Config(t, fmt.Sprintf("%v", vInt), &actual.WriterWorkqueueConfig.IndexCacheMaxItems) } else { diff --git a/flyteplugins/go/tasks/plugins/array/core/phase_enumer.go b/flyteplugins/go/tasks/plugins/array/core/phase_enumer.go index 6d8f750a77..b582c6fffd 100644 --- a/flyteplugins/go/tasks/plugins/array/core/phase_enumer.go +++ b/flyteplugins/go/tasks/plugins/array/core/phase_enumer.go @@ -7,9 +7,9 @@ import ( "fmt" ) -const _PhaseName = "PhaseStartPhasePreLaunchPhaseLaunchPhaseWaitingForResourcesPhaseCheckingSubTaskExecutionsPhaseAssembleFinalOutputPhaseWriteToDiscoveryPhaseSuccessPhaseAssembleFinalErrorPhaseRetryableFailurePhasePermanentFailure" +const _PhaseName = "PhaseStartPhasePreLaunchPhaseLaunchPhaseWaitingForResourcesPhaseCheckingSubTaskExecutionsPhaseAssembleFinalOutputPhaseWriteToDiscoveryPhaseWriteToDiscoveryThenFailPhaseSuccessPhaseAssembleFinalErrorPhaseRetryableFailurePhasePermanentFailure" -var _PhaseIndex = [...]uint8{0, 10, 24, 35, 59, 89, 113, 134, 146, 169, 190, 211} +var _PhaseIndex = [...]uint8{0, 10, 24, 35, 59, 89, 113, 134, 163, 175, 198, 219, 240} func (i Phase) String() string { if i >= Phase(len(_PhaseIndex)-1) { @@ -18,7 +18,7 @@ func (i Phase) String() string { return _PhaseName[_PhaseIndex[i]:_PhaseIndex[i+1]] } -var _PhaseValues = []Phase{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10} +var _PhaseValues = []Phase{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11} var _PhaseNameToValueMap = map[string]Phase{ _PhaseName[0:10]: 0, @@ -28,10 +28,11 @@ var _PhaseNameToValueMap = map[string]Phase{ _PhaseName[59:89]: 4, _PhaseName[89:113]: 5, _PhaseName[113:134]: 6, - _PhaseName[134:146]: 7, - _PhaseName[146:169]: 8, - _PhaseName[169:190]: 9, - _PhaseName[190:211]: 10, + _PhaseName[134:163]: 7, + _PhaseName[163:175]: 8, + _PhaseName[175:198]: 9, + _PhaseName[198:219]: 10, + _PhaseName[219:240]: 11, } // PhaseString retrieves an enum value from the enum constants string name.