✨ Support image builder (#172)

.gitignore

@@ -190,3 +190,5 @@ conf/ximager-dev.yaml
 *.db
 
 pkg/**/*.html
+
+.vscode/

.golangci.yml

@@ -17,7 +17,7 @@ linters-settings:
   whitespace:
     multi-func: true
   cyclop:
-    max-complexity: 30
+    max-complexity: 50
     package-average: 30
     skip-tests: true
   gci:

cmd/builder/builder.go

@@ -52,7 +52,7 @@ const (
 	cacheOut                = "/opt/cache_out"
 	knownHosts              = "known_hosts"
 	privateKey              = "private_key"
-	dockerConfig            = "docker_config.json"
+	dockerConfig            = "config.json"
 	buildkitdConfigFilename = "buildkitd.toml"
 	workspace               = "/code"
 	compressedCache         = "cache.tgz"
@@ -173,35 +173,33 @@ func (b Builder) initToken() error {
 		defer func() {
 			_ = privateKeyObj.Close() // nolint: errcheck
 		}()
-		if !utils.IsFile(path.Join(homeSigma, privateKey)) {
-			_, err = privateKeyObj.WriteString(b.ScmSshKey)
-			if err != nil {
-				return fmt.Errorf("Write private key failed: %v", err)
-			}
+		_, err = privateKeyObj.WriteString(b.ScmSshKey)
+		if err != nil {
+			return fmt.Errorf("Write private key failed: %v", err)
 		}
 	}
-	if b.OciRegistryPassword != "" && b.OciRegistryUsername != "" {
+	if len(b.OciRegistryDomain) != 0 {
 		dockerConfigObj, err := os.Create(path.Join(homeSigma, dockerConfig))
 		if err != nil {
 			return fmt.Errorf("Create file failed: %v", dockerConfigObj)
 		}
 		defer func() {
 			_ = dockerConfigObj.Close() // nolint: errcheck
 		}()
-		cf := configfile.ConfigFile{
-			AuthConfigs: map[string]dockertypes.AuthConfig{
-				b.OciRegistryDomain: {
-					Username: b.OciRegistryUsername,
-					Password: b.OciRegistryPassword,
-				},
-			},
-		}
-		if !utils.IsFile(path.Join(homeSigma, dockerConfig)) {
-			err = cf.SaveToWriter(dockerConfigObj)
-			if err != nil {
-				return fmt.Errorf("Save docker config failed: %v", err)
+		cf := configfile.ConfigFile{}
+		cf.AuthConfigs = make(map[string]dockertypes.AuthConfig)
+		for index, domain := range b.OciRegistryDomain {
+			if len(b.OciRegistryUsername[index]) != 0 || len(b.OciRegistryPassword[index]) != 0 {
+				cf.AuthConfigs[domain] = dockertypes.AuthConfig{
+					Username: b.OciRegistryUsername[index],
+					Password: b.OciRegistryPassword[index],
+				}
 			}
 		}
+		err = cf.SaveToWriter(dockerConfigObj)
+		if err != nil {
+			return fmt.Errorf("Save docker config failed: %v", err)
+		}
 	}
 	var btConfig buildkitdconfig.Config
 	if len(b.BuildkitInsecureRegistries) > 0 {
@@ -251,7 +249,7 @@ func (b Builder) gitClone() error {
 	if b.ScmDepth != 0 {
 		cmd.Args = append(cmd.Args, "--depth", strconv.Itoa(b.ScmDepth))
 	}
-	if b.ScmSubModule {
+	if b.ScmSubmodule {
 		cmd.Args = append(cmd.Args, "--recurse-submodules")
 	}
 	if b.ScmCredentialType == enums.ScmCredentialTypeSsh {
@@ -307,7 +305,7 @@ func (b Builder) build() error {
 		cmd.Args = append(cmd.Args, "--opt", fmt.Sprintf("platform=%s", strings.Join(platforms, ",")))
 	}
 	cmd.Args = append(cmd.Args, "--frontend", "gateway.v0", "--opt", "source=docker/dockerfile")                         // TODO: set frontend
-	cmd.Args = append(cmd.Args, "--output", fmt.Sprintf("type=image,name=%s,push=false", b.OciName))                     // TODO: set output push true
+	cmd.Args = append(cmd.Args, "--output", fmt.Sprintf("type=image,name=%s,push=true", b.OciName))                      // TODO: set output push true
 	cmd.Args = append(cmd.Args, "--export-cache", fmt.Sprintf("type=local,mode=max,compression=gzip,dest=%s", cacheOut)) // TODO: set cache volume
 	cmd.Args = append(cmd.Args, "--import-cache", fmt.Sprintf("type=local,src=%s", cacheIn))                             // TODO: set cache volume
 
@@ -316,6 +314,7 @@ func (b Builder) build() error {
 		buildkitdFlags += fmt.Sprintf("--config=%s", path.Join(homeSigma, buildkitdConfigFilename))
 	}
 	cmd.Env = append(os.Environ(), fmt.Sprintf("BUILDKITD_FLAGS=%s", buildkitdFlags))
+	cmd.Env = append(cmd.Env, fmt.Sprintf("DOCKER_CONFIG=%s", homeSigma))
 
 	log.Info().Str("command", cmd.String()).Strs("env", cmd.Env).Msg("Building image")
 	cmd.Stdout = os.Stdout

cmd/builder/checker.go

@@ -33,7 +33,7 @@ func (b *Builder) checker() error {
 		return fmt.Errorf("SCM_SSH_KEY should be set, if SCM_CREDENTIAL_TYPE is 'ssh'")
 	}
 	if b.ScmSshKey != "" {
-		b.ScmSshKey, err = crypt.Decrypt(b.ID, b.ScmSshKey)
+		b.ScmSshKey, err = crypt.Decrypt(fmt.Sprintf("%d-%d", b.ID, b.RunnerID), b.ScmSshKey)
 		if err != nil {
 			return fmt.Errorf("Decrypt ssh key failed: %v", err)
 		}
@@ -43,7 +43,7 @@ func (b *Builder) checker() error {
 		return fmt.Errorf("SCM_TOKEN should be set, if SCM_CREDENTIAL_TYPE is 'token'")
 	}
 	if b.ScmToken != "" {
-		b.ScmToken, err = crypt.Decrypt(b.ID, b.ScmToken)
+		b.ScmToken, err = crypt.Decrypt(fmt.Sprintf("%d-%d", b.ID, b.RunnerID), b.ScmToken)
 		if err != nil {
 			return fmt.Errorf("Decrypt scm token failed: %v", err)
 		}
@@ -56,7 +56,7 @@ func (b *Builder) checker() error {
 		return fmt.Errorf("SCM_USERNAME and SCM_PASSWORD should be set, if SCM_CREDENTIAL_TYPE is 'username'")
 	}
 	if b.ScmPassword != "" {
-		b.ScmPassword, err = crypt.Decrypt(b.ID, b.ScmPassword)
+		b.ScmPassword, err = crypt.Decrypt(fmt.Sprintf("%d-%d", b.ID, b.RunnerID), b.ScmPassword)
 		if err != nil {
 			return fmt.Errorf("Decrypt scm password failed: %v", err)
 		}
@@ -71,8 +71,12 @@ func (b *Builder) checker() error {
 		}
 	}
 
-	if b.OciRegistryPassword != "" {
-		b.OciRegistryPassword, err = crypt.Decrypt(b.ID, b.OciRegistryPassword)
+	if len(b.OciRegistryDomain) != len(b.OciRegistryUsername) || len(b.OciRegistryDomain) != len(b.OciRegistryPassword) {
+		return fmt.Errorf("OCI_REGISTRY_DOMAIN length should equal OCI_REGISTRY_USERNAME and OCI_REGISTRY_PASSWORD")
+	}
+
+	for index, password := range b.OciRegistryPassword {
+		b.OciRegistryPassword[index], err = crypt.Decrypt(fmt.Sprintf("%d-%d", b.ID, b.RunnerID), password)
 		if err != nil {
 			return fmt.Errorf("Decrypt oci registry password failed: %v", err)
 		}

cmd/imports/apis.go

@@ -17,6 +17,7 @@ package imports
 import (
 	_ "github.com/go-sigma/sigma/pkg/handlers/apidocs"
 	_ "github.com/go-sigma/sigma/pkg/handlers/artifacts"
+	_ "github.com/go-sigma/sigma/pkg/handlers/builders"
 	_ "github.com/go-sigma/sigma/pkg/handlers/daemons"
 	_ "github.com/go-sigma/sigma/pkg/handlers/namespaces"
 	_ "github.com/go-sigma/sigma/pkg/handlers/oauth2"

cmd/imports/builder.go

@@ -0,0 +1,23 @@
+// Copyright 2023 sigma
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package imports
+
+import (
+	_ "github.com/go-sigma/sigma/pkg/builder/docker"
+	_ "github.com/go-sigma/sigma/pkg/builder/k8s"
+	_ "github.com/go-sigma/sigma/pkg/builder/logger/database"
+	_ "github.com/go-sigma/sigma/pkg/builder/logger/obs"
+	_ "github.com/go-sigma/sigma/pkg/builder/podman"
+)

cmd/imports/daemon.go

@@ -15,6 +15,7 @@
 package imports
 
 import (
+	_ "github.com/go-sigma/sigma/pkg/daemon/builder"
 	_ "github.com/go-sigma/sigma/pkg/daemon/gc"
 	_ "github.com/go-sigma/sigma/pkg/daemon/sbom"
 	_ "github.com/go-sigma/sigma/pkg/daemon/vulnerability"

cmd/root.go

@@ -23,6 +23,8 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
+	"github.com/go-sigma/sigma/pkg/configs"
+
 	_ "github.com/go-sigma/sigma/cmd/imports"
 )
 
@@ -66,4 +68,6 @@ func initConfig() {
 	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
 
 	cobra.CheckErr(viper.ReadInConfig())
+
+	cobra.CheckErr(viper.Unmarshal(configs.GetConfiguration()))
 }

conf/config-dev.yaml

@@ -22,31 +22,31 @@ database:
     dbname: ximager
     sslmode: disable
 
-# deploy avaliable: single, replica
+# deploy available: single, replica
 # replica should use external redis
 deploy: single
 
 redis:
-  # redis type avaliable: internal, external
+  # redis type available: internal, external
   type: internal
   url: redis://:sigma@localhost:6379/0
 
 cache:
-  # the cache type avaliable is: redis
+  # the cache type available is: redis
   type: redis
-  # please attation in multi
+  # please attention in multi
   inmemory: {}
   redis: {}
 
 workqueue:
-  # the workqueue type avaliable: redis
+  # the workqueue type available: redis
   type: redis
   redis: {}
 
 namespace:
   # push image to registry, if namespace not exist, it will be created automatically
   autoCreate: true
-  # the automatic created namespace visibility, avaliable: public, private
+  # the automatic created namespace visibility, available: public, private
   visibility: public
 
 http:

conf/config.yaml

@@ -22,31 +22,31 @@ database:
     dbname: ximager
     sslmode: disable
 
-# deploy avaliable: single, replica
+# deploy available: single, replica
 # replica should use external redis
 deploy: single
 
 redis:
-  # redis type avaliable: internal, external
+  # redis type available: internal, external
   type: internal
   url: redis://:sigma@localhost:6379/0
 
 cache:
-  # the cache type avaliable is: redis
+  # the cache type available is: redis
   type: redis
-  # please attation in multi
+  # please attention in multi
   inmemory: {}
   redis: {}
 
 workqueue:
-  # the workqueue type avaliable: redis
+  # the workqueue type available: redis
   type: redis
   redis: {}
 
 namespace:
   # push image to registry, if namespace not exist, it will be created automatically
   autoCreate: false
-  # the automatic created namespace visibility, avaliable: public, private
+  # the automatic created namespace visibility, available: public, private
   visibility: public
 
 http:

pkg/builder/builder.go

@@ -0,0 +1,115 @@
+// Copyright 2023 sigma
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package builder
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"strings"
+
+	builderlogger "github.com/go-sigma/sigma/pkg/builder/logger"
+	"github.com/go-sigma/sigma/pkg/types"
+	"github.com/go-sigma/sigma/pkg/utils"
+	"github.com/go-sigma/sigma/pkg/utils/crypt"
+)
+
+// Builder ...
+type Builder interface {
+	// Start start a container to build oci image and push to registry
+	Start(ctx context.Context, builderConfig BuilderConfig) error
+	// Stop stop the container
+	Stop(ctx context.Context, builderID, runnerID int64) error
+	// Restart wrap stop and start
+	Restart(ctx context.Context, builderConfig BuilderConfig) error
+	// LogStream get the real time log stream
+	LogStream(ctx context.Context, builderID, runnerID int64, writer io.Writer) error
+}
+
+type BuilderConfig struct {
+	types.Builder
+}
+
+// Driver is the builder driver, maybe implement by docker, podman, k8s, etc.
+var Driver Builder
+
+// Factory is the interface for the builder driver factory
+type Factory interface {
+	New() (Builder, error)
+}
+
+// DriverFactories ...
+var DriverFactories = make(map[string]Factory)
+
+func Initialize() error {
+	typ := "docker"
+	factory, ok := DriverFactories[typ]
+	if !ok {
+		return fmt.Errorf("builder driver %q not registered", typ)
+	}
+	var err error
+	Driver, err = factory.New()
+	if err != nil {
+		return err
+	}
+	return builderlogger.Initialize()
+}
+
+// BuildEnv ...
+func BuildEnv(builderConfig BuilderConfig) []string {
+	buildConfigEnvs := []string{
+		fmt.Sprintf("ID=%d", builderConfig.ID),
+		fmt.Sprintf("RUNNER_ID=%d", builderConfig.RunnerID),
+
+		fmt.Sprintf("SCM_CREDENTIAL_TYPE=%s", builderConfig.ScmCredentialType.String()),
+		fmt.Sprintf("SCM_USERNAME=%s", builderConfig.ScmUsername),
+		fmt.Sprintf("SCM_PROVIDER=%s", builderConfig.ScmProvider.String()),
+		fmt.Sprintf("SCM_REPOSITORY=%s", builderConfig.ScmRepository),
+		fmt.Sprintf("SCM_BRANCH=%s", builderConfig.ScmBranch),
+		fmt.Sprintf("SCM_DEPTH=%d", builderConfig.ScmDepth),
+		fmt.Sprintf("SCM_SUBMODULE=%t", builderConfig.ScmSubmodule),
+
+		fmt.Sprintf("OCI_REGISTRY_DOMAIN=%s", strings.Join(builderConfig.OciRegistryDomain, ",")),
+		fmt.Sprintf("OCI_REGISTRY_USERNAME=%s", strings.Join(builderConfig.OciRegistryUsername, ",")),
+		fmt.Sprintf("OCI_NAME=%s", builderConfig.OciName),
+
+		fmt.Sprintf("BUILDKIT_INSECURE_REGISTRIES=%s", strings.Join(builderConfig.BuildkitInsecureRegistries, ",")),
+		fmt.Sprintf("BUILDKIT_CACHE_DIR=%s", builderConfig.BuildkitCacheDir),
+		fmt.Sprintf("BUILDKIT_CONTEXT=%s", builderConfig.BuildkitContext),
+		fmt.Sprintf("BUILDKIT_DOCKERFILE=%s", builderConfig.BuildkitDockerfile),
+		fmt.Sprintf("BUILDKIT_PLATFORMS=%s", utils.StringsJoin(builderConfig.BuildkitPlatforms, ",")),
+	}
+	if builderConfig.ScmPassword != "" {
+		buildConfigEnvs = append(buildConfigEnvs, fmt.Sprintf("SCM_PASSWORD=%s", crypt.MustEncrypt(
+			fmt.Sprintf("%d-%d", builderConfig.ID, builderConfig.RunnerID), builderConfig.ScmPassword)))
+	}
+	if builderConfig.ScmSshKey != "" {
+		buildConfigEnvs = append(buildConfigEnvs, fmt.Sprintf("SCM_SSH_KEY=%s", crypt.MustEncrypt(
+			fmt.Sprintf("%d-%d", builderConfig.ID, builderConfig.RunnerID), builderConfig.ScmSshKey)))
+	}
+	if builderConfig.ScmToken != "" {
+		buildConfigEnvs = append(buildConfigEnvs, fmt.Sprintf("SCM_TOKEN=%s", crypt.MustEncrypt(
+			fmt.Sprintf("%d-%d", builderConfig.ID, builderConfig.RunnerID), builderConfig.ScmToken)))
+	}
+	if len(builderConfig.OciRegistryPassword) != 0 {
+		var passwords []string
+		for _, p := range builderConfig.OciRegistryPassword {
+			passwords = append(passwords, crypt.MustEncrypt(fmt.Sprintf("%d-%d", builderConfig.ID, builderConfig.RunnerID), p))
+		}
+		buildConfigEnvs = append(buildConfigEnvs, fmt.Sprintf("OCI_REGISTRY_PASSWORD=%s", strings.Join(passwords, ",")))
+	}
+
+	return buildConfigEnvs
+}