Skip to content

Commit

Permalink
feat(cmd/run): resource option refactoring
Browse files Browse the repository at this point in the history 
* Provide support for configmap, secret or local file
* Refactoring the structure of mount points to distinguish clearly between config and resource options

Ref apache#2003
  • Loading branch information
@squakez
squakez committed Jun 16, 2021
1 parent 52aaa41 commit cd5c0f5
Show file tree
Hide file tree
Showing 11 changed files with 175 additions and 91 deletions.
11 changes: 7 additions & 4 deletions pkg/apis/camel/v1/common_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,9 @@ import (

// ConfigurationSpec --
type ConfigurationSpec struct {
Type string `json:"type"`
Value string `json:"value"`
Type string `json:"type"`
Value string `json:"value"`
ResourceType string `json:"resourceType,omitempty"`
}

// Artifact --
Expand Down Expand Up @@ -186,9 +187,11 @@ type ResourceSpec struct {
}

const (
// ResourceTypeData --
// ResourceTypeData represents a generic data resource
ResourceTypeData ResourceType = "data"
// ResourceTypeOpenAPI --
// ResourceTypeConfig represents a config resource known to runtime
ResourceTypeConfig ResourceType = "config"
// ResourceTypeOpenAPI represents an OpenAPI config resource
ResourceTypeOpenAPI ResourceType = "openapi"
)

Expand Down
9 changes: 9 additions & 0 deletions pkg/apis/camel/v1/integration_types_support.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,15 @@ func (in *IntegrationSpec) AddConfiguration(confType string, confValue string) {
})
}

// AddConfigurationAsResourceType will set a configuration specified with a resource type
func (in *IntegrationSpec) AddConfigurationAsResourceType(confType string, confValue string, resourceType string) {
in.Configuration = append(in.Configuration, ConfigurationSpec{
Type: confType,
Value: confValue,
ResourceType: resourceType,
})
}

// AddDependency --
func (in *IntegrationSpec) AddDependency(dependency string) {
if in.Dependencies == nil {
Expand Down
46 changes: 7 additions & 39 deletions pkg/cmd/run.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ func newCmdRun(rootCmdOptions *RootCmdOptions) (*cobra.Command, *runCmdOptions)
cmd.Flags().StringArrayP("property", "p", nil, "Add a runtime property or properties file (syntax: [my-key=my-value|file:/path/to/my-conf.properties])")
cmd.Flags().StringArray("build-property", nil, "Add a build time property or properties file (syntax: [my-key=my-value|file:/path/to/my-conf.properties])")
cmd.Flags().StringArray("config", nil, "Add runtime configuration from a Configmap, a Secret or a file (syntax: [configmap|secret|file]:name)")
cmd.Flags().StringArray("resource", nil, "Add runtime resource from a Configmap, a Secret or a file (syntax: [configmap|secret|file]:name)")
cmd.Flags().StringArray("configmap", nil, "[Deprecated] Add a ConfigMap")
cmd.Flags().StringArray("secret", nil, "[Deprecated] Add a Secret")
cmd.Flags().StringArray("maven-repository", nil, "Add a maven repository")
Expand All @@ -92,7 +93,6 @@ func newCmdRun(rootCmdOptions *RootCmdOptions) (*cobra.Command, *runCmdOptions)
cmd.Flags().StringArray("logging-level", nil, "Configure the logging level. e.g. \"--logging-level org.apache.camel=DEBUG\"")
cmd.Flags().StringP("output", "o", "", "Output format. One of: json|yaml")
cmd.Flags().Bool("compression", false, "Enable storage of sources and resources as a compressed binary blobs")
cmd.Flags().StringArray("resource", nil, "Add a resource")
cmd.Flags().StringArray("open-api", nil, "Add an OpenAPI v2 spec")
cmd.Flags().StringArrayP("volume", "v", nil, "Mount a volume into the integration container. E.g \"-v pvcname:/container/path\"")
cmd.Flags().StringArrayP("env", "e", nil, "Set an environment variable in the integration container. E.g \"-e MY_VAR=my-value\"")
Expand Down Expand Up @@ -537,16 +537,13 @@ func (o *runCmdOptions) updateIntegrationCode(c client.Client, sources []string,
}

for _, resource := range o.Resources {
rawData, contentType, err := loadRawContent(resource)
if err != nil {
return nil, err
}

resourceSpec, err := binaryOrTextResource(path.Base(resource), rawData, contentType, o.Compression)
if err != nil {
return nil, err
if config, parseErr := ParseConfigOption(resource); parseErr == nil {
if applyResourceOptionErr := ApplyResourceOption(config, &integration.Spec, c, namespace, o.Compression); applyResourceOptionErr != nil {
return nil, applyResourceOptionErr
}
} else {
return nil, parseErr
}
integration.Spec.AddResources(resourceSpec)
}

for _, resource := range o.OpenAPIs {
Expand Down Expand Up @@ -710,35 +707,6 @@ func keyValueProps(value string) (*properties.Properties, error) {
return properties.Load([]byte(value), properties.UTF8)
}

func binaryOrTextResource(fileName string, data []byte, contentType string, base64Compression bool) (v1.ResourceSpec, error) {
resourceSpec := v1.ResourceSpec{
DataSpec: v1.DataSpec{
Name: fileName,
ContentKey: fileName,
ContentType: contentType,
Compression: false,
},
Type: v1.ResourceTypeData,
}

if !base64Compression && isBinary(contentType) {
resourceSpec.RawContent = data
return resourceSpec, nil
}
// either is a text resource or base64 compression is enabled
if base64Compression {
content, err := compressToString(data)
if err != nil {
return resourceSpec, err
}
resourceSpec.Content = content
resourceSpec.Compression = true
} else {
resourceSpec.Content = string(data)
}
return resourceSpec, nil
}

func (o *runCmdOptions) GetIntegrationName(sources []string) string {
name := ""
if o.IntegrationName != "" {
Expand Down
55 changes: 47 additions & 8 deletions pkg/cmd/run_help.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,43 +76,82 @@ func ParseConfigOption(item string) (*RunConfigOption, error) {
return newRunConfigOption(cot, groups[2]), nil
}

// ApplyConfigOption will set the proper option behavior to the IntegrationSpec
func ApplyConfigOption(config *RunConfigOption, integrationSpec *v1.IntegrationSpec, c client.Client, namespace string, enableCompression bool) error {
func applyOption(config *RunConfigOption, integrationSpec *v1.IntegrationSpec,
c client.Client, namespace string, enableCompression bool, resourceType v1.ResourceType) error {
switch config.ConfigType {
case ConfigOptionTypeConfigmap:
cm := kubernetes.LookupConfigmap(context.Background(), c, namespace, config.Value)
if cm == nil {
fmt.Printf("Warn: %s Configmap not found in %s namespace, make sure to provide it before the Integration can run\n",
config.Value, namespace)
} else if cm.BinaryData != nil {
} else if resourceType != v1.ResourceTypeData && cm.BinaryData != nil {
return fmt.Errorf("you cannot provide a binary config, use a text file instead")
}
integrationSpec.AddConfiguration(string(config.ConfigType), config.Value)
integrationSpec.AddConfigurationAsResourceType(string(config.ConfigType), config.Value, string(resourceType))
case ConfigOptionTypeSecret:
secret := kubernetes.LookupSecret(context.Background(), c, namespace, config.Value)
if secret == nil {
fmt.Printf("Warn: %s Secret not found in %s namespace, make sure to provide it before the Integration can run\n",
config.Value, namespace)
}
integrationSpec.AddConfiguration(string(config.ConfigType), config.Value)
integrationSpec.AddConfigurationAsResourceType(string(config.ConfigType), config.Value, string(resourceType))
case ConfigOptionTypeFile:
// Don't allow a binary non compressed resource
rawData, contentType, err := loadRawContent(config.Value)
if err != nil {
return err
}
if !enableCompression && isBinary(contentType) {
if resourceType != v1.ResourceTypeData && !enableCompression && isBinary(contentType) {
return fmt.Errorf("you cannot provide a binary config, use a text file or check --resource flag instead")
}
resourceSpec, err := binaryOrTextResource(path.Base(config.Value), rawData, contentType, enableCompression)
resourceSpec, err := binaryOrTextResource(path.Base(config.Value), rawData, contentType, enableCompression, resourceType)
if err != nil {
return err
}
integrationSpec.AddResources(resourceSpec)
default:
// Should never reach this
return fmt.Errorf("invalid config option type %s", config.ConfigType)
return fmt.Errorf("invalid option type %s", config.ConfigType)
}

return nil
}

// ApplyConfigOption will set the proper --config option behavior to the IntegrationSpec
func ApplyConfigOption(config *RunConfigOption, integrationSpec *v1.IntegrationSpec, c client.Client, namespace string, enableCompression bool) error {
return applyOption(config, integrationSpec, c, namespace, enableCompression, v1.ResourceTypeConfig)
}

// ApplyResourceOption will set the proper --resource option behavior to the IntegrationSpec
func ApplyResourceOption(config *RunConfigOption, integrationSpec *v1.IntegrationSpec, c client.Client, namespace string, enableCompression bool) error {
return applyOption(config, integrationSpec, c, namespace, enableCompression, v1.ResourceTypeData)
}

func binaryOrTextResource(fileName string, data []byte, contentType string, base64Compression bool, resourceType v1.ResourceType) (v1.ResourceSpec, error) {
resourceSpec := v1.ResourceSpec{
DataSpec: v1.DataSpec{
Name: fileName,
ContentKey: fileName,
ContentType: contentType,
Compression: false,
},
Type: resourceType,
}

if !base64Compression && isBinary(contentType) {
resourceSpec.RawContent = data
return resourceSpec, nil
}
// either is a text resource or base64 compression is enabled
if base64Compression {
content, err := compressToString(data)
if err != nil {
return resourceSpec, err
}
resourceSpec.Content = content
resourceSpec.Compression = true
} else {
resourceSpec.Content = string(data)
}
return resourceSpec, nil
}
8 changes: 4 additions & 4 deletions pkg/cmd/run_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -485,7 +485,7 @@ func TestRunWithSavedValues(t *testing.T) {
}*/

func TestRunBinaryResource(t *testing.T) {
binaryResourceSpec, err := binaryOrTextResource("file.ext", []byte{1, 2, 3, 4}, "application/octet-stream", false)
binaryResourceSpec, err := binaryOrTextResource("file.ext", []byte{1, 2, 3, 4}, "application/octet-stream", false, v1.ResourceTypeData)
assert.Nil(t, err)
assert.Equal(t, "", binaryResourceSpec.Content)
assert.NotNil(t, binaryResourceSpec.RawContent)
Expand All @@ -497,7 +497,7 @@ func TestRunBinaryResource(t *testing.T) {
func TestRunBinaryCompressedResource(t *testing.T) {
data := []byte{1, 2, 3, 4}
base64Compressed, _ := compressToString(data)
binaryResourceSpec, err := binaryOrTextResource("file.ext", data, "application/octet-stream", true)
binaryResourceSpec, err := binaryOrTextResource("file.ext", data, "application/octet-stream", true, v1.ResourceTypeData)
assert.Nil(t, err)
assert.Equal(t, base64Compressed, binaryResourceSpec.Content)
assert.Nil(t, binaryResourceSpec.RawContent)
Expand All @@ -507,7 +507,7 @@ func TestRunBinaryCompressedResource(t *testing.T) {
}

func TestRunTextResource(t *testing.T) {
textResourceSpec, err := binaryOrTextResource("file.ext", []byte("hello world"), "text/plain", false)
textResourceSpec, err := binaryOrTextResource("file.ext", []byte("hello world"), "text/plain", false, v1.ResourceTypeData)
assert.Nil(t, err)
assert.Equal(t, "hello world", textResourceSpec.Content)
assert.Nil(t, textResourceSpec.RawContent)
Expand All @@ -519,7 +519,7 @@ func TestRunTextResource(t *testing.T) {
func TestRunTextCompressedResource(t *testing.T) {
data := []byte("hello horld")
base64Compressed, _ := compressToString(data)
textResourceSpec, err := binaryOrTextResource("file.ext", []byte("hello horld"), "text/plain", true)
textResourceSpec, err := binaryOrTextResource("file.ext", []byte("hello horld"), "text/plain", true, v1.ResourceTypeData)
assert.Nil(t, err)
assert.Equal(t, base64Compressed, textResourceSpec.Content)
assert.Nil(t, textResourceSpec.RawContent)
Expand Down
4 changes: 2 additions & 2 deletions pkg/trait/environment.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,8 +72,8 @@ func (t *environmentTrait) Apply(e *Environment) error {
envvar.SetVal(&e.EnvVars, envVarCamelKIntegration, e.Integration.Name)
}
envvar.SetVal(&e.EnvVars, envVarCamelKRuntimeVersion, e.RuntimeVersion)
envvar.SetVal(&e.EnvVars, envVarMountPathConfigMaps, configMapsMountPath)
envvar.SetVal(&e.EnvVars, envVarMountPathSecrets, secretsMountPath)
envvar.SetVal(&e.EnvVars, envVarMountPathConfigMaps, configConfigmapsMountPath)
envvar.SetVal(&e.EnvVars, envVarMountPathSecrets, configSecretsMountPath)

if util.IsNilOrTrue(t.ContainerMeta) {
envvar.SetValFrom(&e.EnvVars, envVarNamespace, "metadata.namespace")
Expand Down
1 change: 0 additions & 1 deletion pkg/trait/jvm.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,6 @@ func (t *jvmTrait) Apply(e *Environment) error {

classpath := strset.New()

classpath.Add(resourcesMountPath)
classpath.Add("./resources")

for _, artifact := range kit.Status.Artifacts {
Expand Down
8 changes: 4 additions & 4 deletions pkg/trait/jvm_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,12 +103,12 @@ func TestApplyJvmTraitWithDeploymentResource(t *testing.T) {

assert.Nil(t, err)

cp := strset.New("/etc/camel/resources", "./resources", "/mount/path").List()
cp := strset.New("./resources", "/mount/path").List()
sort.Strings(cp)

assert.Equal(t, []string{
"-cp",
"./resources:/etc/camel/resources:/mount/path",
"./resources:/mount/path",
"io.quarkus.bootstrap.runner.QuarkusEntryPoint",
}, d.Spec.Template.Spec.Containers[0].Args)
}
Expand All @@ -135,12 +135,12 @@ func TestApplyJvmTraitWithKNativeResource(t *testing.T) {

assert.Nil(t, err)

cp := strset.New("/etc/camel/resources", "./resources", "/mount/path").List()
cp := strset.New("./resources", "/mount/path").List()
sort.Strings(cp)

assert.Equal(t, []string{
"-cp",
"./resources:/etc/camel/resources:/mount/path",
"./resources:/mount/path",
"io.quarkus.bootstrap.runner.QuarkusEntryPoint",
}, s.Spec.Template.Spec.Containers[0].Args)
}
Expand Down
24 changes: 13 additions & 11 deletions pkg/trait/trait_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -253,12 +253,14 @@ func TestConfigureVolumesAndMountsTextResourcesAndProperties(t *testing.T) {
Value: "a=b",
},
{
Type: "configmap",
Value: "test-configmap",
Type: "configmap",
Value: "test-configmap",
ResourceType: "config",
},
{
Type: "secret",
Value: "test-secret",
Type: "secret",
Value: "test-secret",
ResourceType: "config",
},
{
Type: "volume",
Expand Down Expand Up @@ -313,7 +315,7 @@ func TestConfigureVolumesAndMountsTextResourcesAndProperties(t *testing.T) {

m = findVVolumeMount(mnts, func(m corev1.VolumeMount) bool { return m.Name == "i-resource-001" })
assert.NotNil(t, m)
assert.Equal(t, "/etc/camel/resources/res2.txt", m.MountPath)
assert.Equal(t, "/etc/camel/data/resources/res2.txt", m.MountPath)

v = findVolume(vols, func(v corev1.Volume) bool { return v.ConfigMap.Name == TestDeploymentName+"-resource-002" })
assert.NotNil(t, v)
Expand All @@ -324,7 +326,7 @@ func TestConfigureVolumesAndMountsTextResourcesAndProperties(t *testing.T) {

m = findVVolumeMount(mnts, func(m corev1.VolumeMount) bool { return m.Name == "i-resource-002" })
assert.NotNil(t, m)
assert.Equal(t, "/etc/camel/resources/res3.txt", m.MountPath)
assert.Equal(t, "/etc/camel/data/resources/res3.txt", m.MountPath)

v = findVolume(vols, func(v corev1.Volume) bool { return v.ConfigMap.Name == TestDeploymentName+"-resource-003" })
assert.NotNil(t, v)
Expand All @@ -335,7 +337,7 @@ func TestConfigureVolumesAndMountsTextResourcesAndProperties(t *testing.T) {

m = findVVolumeMount(mnts, func(m corev1.VolumeMount) bool { return m.Name == "i-resource-003" })
assert.NotNil(t, m)
assert.Equal(t, "/etc/camel/resources/res4.txt", m.MountPath)
assert.Equal(t, "/etc/camel/data/resources/res4.txt", m.MountPath)

v = findVolume(vols, func(v corev1.Volume) bool { return v.ConfigMap.Name == "test-configmap" })
assert.NotNil(t, v)
Expand All @@ -345,7 +347,7 @@ func TestConfigureVolumesAndMountsTextResourcesAndProperties(t *testing.T) {

m = findVVolumeMount(mnts, func(m corev1.VolumeMount) bool { return m.Name == "test-configmap" })
assert.NotNil(t, m)
assert.Equal(t, path.Join(configMapsMountPath, "test-configmap"), m.MountPath)
assert.Equal(t, path.Join(configConfigmapsMountPath, "test-configmap"), m.MountPath)

v = findVolume(vols, func(v corev1.Volume) bool { return v.Name == "test-secret" })
assert.NotNil(t, v)
Expand All @@ -354,7 +356,7 @@ func TestConfigureVolumesAndMountsTextResourcesAndProperties(t *testing.T) {

m = findVVolumeMount(mnts, func(m corev1.VolumeMount) bool { return m.Name == "test-secret" })
assert.NotNil(t, m)
assert.Equal(t, path.Join(secretsMountPath, "test-secret"), m.MountPath)
assert.Equal(t, path.Join(configSecretsMountPath, "test-secret"), m.MountPath)

v = findVolume(vols, func(v corev1.Volume) bool { return v.Name == "testvolume-data" })
assert.NotNil(t, v)
Expand Down Expand Up @@ -478,7 +480,7 @@ func TestConfigureVolumesAndMountsBinaryAndTextResources(t *testing.T) {

m := findVVolumeMount(mnts, func(m corev1.VolumeMount) bool { return m.Name == v.Name })
assert.NotNil(t, m)
assert.Equal(t, "/etc/camel/resources/res1.bin", m.MountPath)
assert.Equal(t, "/etc/camel/data/resources/res1.bin", m.MountPath)

v = findVolume(vols, func(v corev1.Volume) bool { return v.ConfigMap.Name == "my-cm2" })
assert.NotNil(t, v)
Expand All @@ -489,7 +491,7 @@ func TestConfigureVolumesAndMountsBinaryAndTextResources(t *testing.T) {

m = findVVolumeMount(mnts, func(m corev1.VolumeMount) bool { return m.Name == v.Name })
assert.NotNil(t, m)
assert.Equal(t, "/etc/camel/resources/res2.txt", m.MountPath)
assert.Equal(t, "/etc/camel/data/resources/res2.txt", m.MountPath)
}

func TestOnlySomeTraitsInfluenceBuild(t *testing.T) {
Expand Down
Loading

0 comments on commit cd5c0f5

Please sign in to comment.