From 18f6328284b9386861d6393f4c06fa70affc98d7 Mon Sep 17 00:00:00 2001 From: Seth Pollack Date: Fri, 15 Mar 2019 14:28:18 -0400 Subject: [PATCH 1/2] add secret and configmap generator plugins --- k8sdeps/configmapandsecret/basefactory.go | 28 ++++++++- .../configmapandsecret/basefactory_test.go | 54 +++++++++++++++- .../configmapandsecret/configmapfactory.go | 7 ++- .../configmapfactory_test.go | 5 +- .../configmapandsecret/secretfactory_test.go | 5 +- k8sdeps/kunstruct/factory.go | 5 +- k8sdeps/kv/plugin/goplugin.go | 63 +++++++++++++++++++ k8sdeps/kv/plugin/plugin.go | 32 ++++++++++ k8sdeps/kv/plugin/registry.go | 54 ++++++++++++++++ k8sdeps/kv/plugin/testonly.go | 43 +++++++++++++ pkg/target/configmaps_test.go | 28 +++++++++ pkg/types/kustomization.go | 10 +++ 12 files changed, 325 insertions(+), 9 deletions(-) create mode 100644 k8sdeps/kv/plugin/goplugin.go create mode 100644 k8sdeps/kv/plugin/plugin.go create mode 100644 k8sdeps/kv/plugin/registry.go create mode 100644 k8sdeps/kv/plugin/testonly.go diff --git a/k8sdeps/configmapandsecret/basefactory.go b/k8sdeps/configmapandsecret/basefactory.go index 3266016738..dd1a4d64e0 100644 --- a/k8sdeps/configmapandsecret/basefactory.go +++ b/k8sdeps/configmapandsecret/basefactory.go @@ -24,6 +24,7 @@ import ( "github.com/pkg/errors" "k8s.io/apimachinery/pkg/util/validation" "sigs.k8s.io/kustomize/k8sdeps/kv" + "sigs.k8s.io/kustomize/k8sdeps/kv/plugin" "sigs.k8s.io/kustomize/pkg/ifc" "sigs.k8s.io/kustomize/pkg/types" ) @@ -32,11 +33,20 @@ import ( type baseFactory struct { ldr ifc.Loader options *types.GeneratorOptions + reg plugin.Registry } func (bf baseFactory) loadKvPairs( args types.GeneratorArgs) (all []kv.Pair, err error) { - pairs, err := bf.keyValuesFromEnvFile(args.EnvSource) + pairs, err := bf.keyValuesFromPlugins(args.KVSources) + if err != nil { + return nil, errors.Wrap(err, fmt.Sprintf( + "plugins: %s", + args.EnvSource)) + } + all = append(all, pairs...) + + pairs, err = bf.keyValuesFromEnvFile(args.EnvSource) if err != nil { return nil, errors.Wrap(err, fmt.Sprintf( "env source file: %s", @@ -81,6 +91,22 @@ func keyValuesFromLiteralSources(sources []string) ([]kv.Pair, error) { return kvs, nil } +func (bf baseFactory) keyValuesFromPlugins(sources []types.KVSource) ([]kv.Pair, error) { + var allKvs []kv.Pair + for _, s := range sources { + plug, err := bf.reg.Load(s.PluginType, s.Name) + if err != nil { + return nil, err + } + kvs, err := plug.Get(bf.reg.Root(), s.Args) + if err != nil { + return nil, err + } + allKvs = append(allKvs, kvs...) + } + return allKvs, nil +} + func (bf baseFactory) keyValuesFromFileSources(sources []string) ([]kv.Pair, error) { var kvs []kv.Pair for _, s := range sources { diff --git a/k8sdeps/configmapandsecret/basefactory_test.go b/k8sdeps/configmapandsecret/basefactory_test.go index 022fa93609..d96092fc1a 100644 --- a/k8sdeps/configmapandsecret/basefactory_test.go +++ b/k8sdeps/configmapandsecret/basefactory_test.go @@ -21,8 +21,10 @@ import ( "testing" "sigs.k8s.io/kustomize/k8sdeps/kv" + "sigs.k8s.io/kustomize/k8sdeps/kv/plugin" "sigs.k8s.io/kustomize/pkg/fs" "sigs.k8s.io/kustomize/pkg/loader" + "sigs.k8s.io/kustomize/pkg/types" ) func TestKeyValuesFromFileSources(t *testing.T) { @@ -45,7 +47,9 @@ func TestKeyValuesFromFileSources(t *testing.T) { fSys := fs.MakeFakeFS() fSys.WriteFile("/files/app-init.ini", []byte("FOO=bar")) - bf := baseFactory{loader.NewFileLoaderAtRoot(fSys), nil} + ldr := loader.NewFileLoaderAtRoot(fSys) + reg := plugin.NewRegistry(ldr) + bf := baseFactory{loader.NewFileLoaderAtRoot(fSys), nil, reg} for _, tc := range tests { kvs, err := bf.keyValuesFromFileSources(tc.sources) if err != nil { @@ -56,3 +60,51 @@ func TestKeyValuesFromFileSources(t *testing.T) { } } } + +func TestKeyValuesFromPlugins(t *testing.T) { + tests := []struct { + description string + sources []types.KVSource + expected []kv.Pair + }{ + { + description: "Create kv.Pairs from plugin", + sources: []types.KVSource{ + { + PluginType: "testonly", + Name: "testonly", + Args: []string{"FOO", "BAR", "BAZ"}, + }, + }, + expected: []kv.Pair{ + { + Key: "k_FOO", + Value: "v_FOO", + }, + { + Key: "k_BAR", + Value: "v_BAR", + }, + { + Key: "k_BAZ", + Value: "v_BAZ", + }, + }, + }, + } + + fSys := fs.MakeFakeFS() + ldr := loader.NewFileLoaderAtRoot(fSys) + reg := plugin.NewRegistry(ldr) + bf := baseFactory{ldr, nil, reg} + + for _, tc := range tests { + kvs, err := bf.keyValuesFromPlugins(tc.sources) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !reflect.DeepEqual(kvs, tc.expected) { + t.Fatalf("in testcase: %q updated:\n%#v\ndoesn't match expected:\n%#v\n", tc.description, kvs, tc.expected) + } + } +} diff --git a/k8sdeps/configmapandsecret/configmapfactory.go b/k8sdeps/configmapandsecret/configmapfactory.go index 03f7abc74a..a407922585 100644 --- a/k8sdeps/configmapandsecret/configmapfactory.go +++ b/k8sdeps/configmapandsecret/configmapfactory.go @@ -21,8 +21,9 @@ import ( "fmt" "unicode/utf8" - "k8s.io/api/core/v1" corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" + "sigs.k8s.io/kustomize/k8sdeps/kv/plugin" "sigs.k8s.io/kustomize/pkg/ifc" "sigs.k8s.io/kustomize/pkg/types" ) @@ -34,8 +35,8 @@ type Factory struct { // NewFactory returns a new Factory. func NewFactory( - l ifc.Loader, o *types.GeneratorOptions) *Factory { - return &Factory{baseFactory{ldr: l, options: o}} + l ifc.Loader, o *types.GeneratorOptions, reg plugin.Registry) *Factory { + return &Factory{baseFactory{ldr: l, options: o, reg: reg}} } func makeFreshConfigMap( diff --git a/k8sdeps/configmapandsecret/configmapfactory_test.go b/k8sdeps/configmapandsecret/configmapfactory_test.go index ba0b528f33..08d41b1fbf 100644 --- a/k8sdeps/configmapandsecret/configmapfactory_test.go +++ b/k8sdeps/configmapandsecret/configmapfactory_test.go @@ -22,6 +22,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/kustomize/k8sdeps/kv/plugin" "sigs.k8s.io/kustomize/pkg/fs" "sigs.k8s.io/kustomize/pkg/loader" "sigs.k8s.io/kustomize/pkg/types" @@ -141,8 +142,10 @@ func TestConstructConfigMap(t *testing.T) { fSys.WriteFile("/configmap/app.env", []byte("DB_USERNAME=admin\nDB_PASSWORD=somepw\n")) fSys.WriteFile("/configmap/app-init.ini", []byte("FOO=bar\nBAR=baz\n")) fSys.WriteFile("/configmap/app.bin", []byte{0xff, 0xfd}) + ldr := loader.NewFileLoaderAtRoot(fSys) + reg := plugin.NewRegistry(ldr) for _, tc := range testCases { - f := NewFactory(loader.NewFileLoaderAtRoot(fSys), tc.options) + f := NewFactory(ldr, tc.options, reg) cm, err := f.MakeConfigMap(&tc.input) if err != nil { t.Fatalf("unexpected error: %v", err) diff --git a/k8sdeps/configmapandsecret/secretfactory_test.go b/k8sdeps/configmapandsecret/secretfactory_test.go index 45505510d1..281aaff405 100644 --- a/k8sdeps/configmapandsecret/secretfactory_test.go +++ b/k8sdeps/configmapandsecret/secretfactory_test.go @@ -22,6 +22,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/kustomize/k8sdeps/kv/plugin" "sigs.k8s.io/kustomize/pkg/fs" "sigs.k8s.io/kustomize/pkg/loader" "sigs.k8s.io/kustomize/pkg/types" @@ -138,8 +139,10 @@ func TestConstructSecret(t *testing.T) { fSys := fs.MakeFakeFS() fSys.WriteFile("/secret/app.env", []byte("DB_USERNAME=admin\nDB_PASSWORD=somepw\n")) fSys.WriteFile("/secret/app-init.ini", []byte("FOO=bar\nBAR=baz\n")) + ldr := loader.NewFileLoaderAtRoot(fSys) + reg := plugin.NewRegistry(ldr) for _, tc := range testCases { - f := NewFactory(loader.NewFileLoaderAtRoot(fSys), tc.options) + f := NewFactory(ldr, tc.options, reg) cm, err := f.MakeSecret(&tc.input) if err != nil { t.Fatalf("unexpected error: %v", err) diff --git a/k8sdeps/kunstruct/factory.go b/k8sdeps/kunstruct/factory.go index f17a454df9..3c3644c3f2 100644 --- a/k8sdeps/kunstruct/factory.go +++ b/k8sdeps/kunstruct/factory.go @@ -25,6 +25,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/yaml" "sigs.k8s.io/kustomize/k8sdeps/configmapandsecret" + "sigs.k8s.io/kustomize/k8sdeps/kv/plugin" "sigs.k8s.io/kustomize/pkg/ifc" "sigs.k8s.io/kustomize/pkg/types" ) @@ -81,7 +82,7 @@ func (kf *KunstructuredFactoryImpl) MakeConfigMap( ldr ifc.Loader, options *types.GeneratorOptions, args *types.ConfigMapArgs) (ifc.Kunstructured, error) { - o, err := configmapandsecret.NewFactory(ldr, options).MakeConfigMap(args) + o, err := configmapandsecret.NewFactory(ldr, options, plugin.NewRegistry(ldr)).MakeConfigMap(args) if err != nil { return nil, err } @@ -93,7 +94,7 @@ func (kf *KunstructuredFactoryImpl) MakeSecret( ldr ifc.Loader, options *types.GeneratorOptions, args *types.SecretArgs) (ifc.Kunstructured, error) { - o, err := configmapandsecret.NewFactory(ldr, options).MakeSecret(args) + o, err := configmapandsecret.NewFactory(ldr, options, plugin.NewRegistry(ldr)).MakeSecret(args) if err != nil { return nil, err } diff --git a/k8sdeps/kv/plugin/goplugin.go b/k8sdeps/kv/plugin/goplugin.go new file mode 100644 index 0000000000..d316a76cd3 --- /dev/null +++ b/k8sdeps/kv/plugin/goplugin.go @@ -0,0 +1,63 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plugin + +import ( + "fmt" + "os" + "plugin" +) + +var _ Factory = &goFactory{} + +const ( + dir = "$HOME/.config/kustomize/plugins/kvsource" +) + +func newGoFactory() *goFactory { + return &goFactory{ + plugins: make(map[string]KVSource), + } +} + +type goFactory struct { + plugins map[string]KVSource +} + +func (p *goFactory) load(name string) (KVSource, error) { + if plug, ok := p.plugins[name]; ok { + return plug, nil + } + + goPlugin, err := plugin.Open(fmt.Sprintf("%s/kustomize-%s.so", os.ExpandEnv(dir), name)) + if err != nil { + return nil, err + } + + symbol, err := goPlugin.Lookup("Plugin") + if err != nil { + return nil, err + } + + plug, ok := symbol.(KVSource) + if !ok { + return nil, fmt.Errorf("plugin %s not found", name) + } + + p.plugins[name] = plug + return plug, nil +} diff --git a/k8sdeps/kv/plugin/plugin.go b/k8sdeps/kv/plugin/plugin.go new file mode 100644 index 0000000000..e3e40a9de4 --- /dev/null +++ b/k8sdeps/kv/plugin/plugin.go @@ -0,0 +1,32 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package plugin provides a plugin abstraction layer. +package plugin + +import ( + "sigs.k8s.io/kustomize/k8sdeps/kv" +) + +// KVSource is the interface for kv source plugins. +type KVSource interface { + Get(root string, args []string) ([]kv.Pair, error) +} + +// Factory is the interface for new kv source plugin implementations. +type Factory interface { + load(string) (KVSource, error) +} diff --git a/k8sdeps/kv/plugin/registry.go b/k8sdeps/kv/plugin/registry.go new file mode 100644 index 0000000000..bf6c8ae0a1 --- /dev/null +++ b/k8sdeps/kv/plugin/registry.go @@ -0,0 +1,54 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plugin + +import ( + "fmt" + + "sigs.k8s.io/kustomize/pkg/ifc" +) + +// Registry holds all the plugin factories. +type Registry struct { + factories map[string]Factory + ldr ifc.Loader +} + +// NewRegistry returns a new Registry loaded with all the factories. +func NewRegistry(ldr ifc.Loader) Registry { + return Registry{ + ldr: ldr, + factories: map[string]Factory{ + "go": newGoFactory(), + "testonly": newTestonlyFactory(), + }, + } +} + +// Load returns a plugin by type and name, +func (r *Registry) Load(pluginType, name string) (KVSource, error) { + factory, exists := r.factories[pluginType] + if !exists { + return nil, fmt.Errorf("%s is not a valid plugin type", pluginType) + } + return factory.load(name) +} + +// Root returns the root of the plugins kustomization file. +func (r *Registry) Root() string { + return r.ldr.Root() +} diff --git a/k8sdeps/kv/plugin/testonly.go b/k8sdeps/kv/plugin/testonly.go new file mode 100644 index 0000000000..a06da588a6 --- /dev/null +++ b/k8sdeps/kv/plugin/testonly.go @@ -0,0 +1,43 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// testonly is temporary until we have builtin plugins to use in the tests. + +package plugin + +import ( + "sigs.k8s.io/kustomize/k8sdeps/kv" +) + +var _ Factory = &testonlyFactory{} + +func newTestonlyFactory() *testonlyFactory { + return &testonlyFactory{} +} + +type testonlyFactory struct{} + +func (p testonlyFactory) Get(_ string, args []string) ([]kv.Pair, error) { + var kvs []kv.Pair + for _, arg := range args { + kvs = append(kvs, kv.Pair{Key: "k_" + arg, Value: "v_" + arg}) + } + return kvs, nil +} + +func (p *testonlyFactory) load(name string) (KVSource, error) { + return p, nil +} diff --git a/pkg/target/configmaps_test.go b/pkg/target/configmaps_test.go index dd69adb3b8..3ba0545d87 100644 --- a/pkg/target/configmaps_test.go +++ b/pkg/target/configmaps_test.go @@ -240,3 +240,31 @@ metadata: name: p2-com2-c4b8md75k9 `) } + +func TestGeneratorPlugins(t *testing.T) { + th := NewKustTestHarness(t, "/app") + th.writeK("/app", ` +secretGenerator: +- name: bob + kvSources: + - pluginType: testonly + name: testonly + args: + - FRUIT + - VEGETABLE +`) + m, err := th.makeKustTarget().MakeCustomizedResMap() + if err != nil { + t.Fatalf("Err: %v", err) + } + th.assertActualEqualsExpected(m, ` +apiVersion: v1 +data: + k_FRUIT: dl9GUlVJVA== + k_VEGETABLE: dl9WRUdFVEFCTEU= +kind: Secret +metadata: + name: bob-cb9mhbh9gg +type: Opaque +`) +} diff --git a/pkg/types/kustomization.go b/pkg/types/kustomization.go index 12d09820f0..c928674223 100644 --- a/pkg/types/kustomization.go +++ b/pkg/types/kustomization.go @@ -189,6 +189,9 @@ type GeneratorArgs struct { // DataSources for the generator. DataSources `json:",inline,omitempty" yaml:",inline,omitempty"` + + // KVSources for the generator. + KVSources []KVSource `json:",inline,omitempty" yaml:",inline,omitempty"` } // ConfigMapArgs contains the metadata of how to generate a configmap. @@ -248,3 +251,10 @@ type GeneratorOptions struct { // resource contents. DisableNameSuffixHash bool `json:"disableNameSuffixHash,omitempty" yaml:"disableNameSuffixHash,omitempty"` } + +// KVSource represents a KV plugin backend. +type KVSource struct { + PluginType string `json:"pluginType,omitempty" yaml:"pluginType,omitempty"` + Name string `json:"name,omitempty" yaml:"name,omitempty"` + Args []string `json:"args,omitempty" yaml:"args,omitempty"` +} From 56965a0046854d4a6eb0b1b4721cc3b27492b62e Mon Sep 17 00:00:00 2001 From: Seth Pollack Date: Fri, 15 Mar 2019 17:33:17 -0400 Subject: [PATCH 2/2] fix test --- k8sdeps/kv/plugin/testonly.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8sdeps/kv/plugin/testonly.go b/k8sdeps/kv/plugin/testonly.go index a06da588a6..ecbc36cc8b 100644 --- a/k8sdeps/kv/plugin/testonly.go +++ b/k8sdeps/kv/plugin/testonly.go @@ -38,6 +38,6 @@ func (p testonlyFactory) Get(_ string, args []string) ([]kv.Pair, error) { return kvs, nil } -func (p *testonlyFactory) load(name string) (KVSource, error) { +func (p *testonlyFactory) load(_ string) (KVSource, error) { return p, nil }