✨ Runner list

cmd/builder/builder.go

@@ -139,31 +139,31 @@ func (b Builder) gitClone() error {
 		return fmt.Errorf("git not found: %v", err)
 	}
 	cmd := exec.Command(git, "clone", "--branch", ptr.To(b.ScmBranch))
-	if b.ScmDepth != 0 {
-		cmd.Args = append(cmd.Args, "--depth", strconv.Itoa(b.ScmDepth))
+	if ptr.To(b.ScmDepth) != 0 {
+		cmd.Args = append(cmd.Args, "--depth", strconv.Itoa(ptr.To(b.ScmDepth)))
 	}
-	if b.ScmSubmodule {
+	if ptr.To(b.ScmSubmodule) {
 		cmd.Args = append(cmd.Args, "--recurse-submodules")
 	}
-	if b.ScmCredentialType == enums.ScmCredentialTypeSsh {
+	if ptr.To(b.ScmCredentialType) == enums.ScmCredentialTypeSsh {
 		cmd.Args = append(cmd.Args, "-i", path.Join(homeSigma, privateKey))
 		cmd.Env = append(os.Environ(), fmt.Sprintf("SSH_KNOWN_HOSTS=%s", path.Join(homeSigma, knownHosts)))
 	}
-	repository := b.ScmRepository
-	if b.ScmCredentialType == enums.ScmCredentialTypeToken {
+	repository := ptr.To(b.ScmRepository)
+	if ptr.To(b.ScmCredentialType) == enums.ScmCredentialTypeToken {
 		u, err := url.Parse(repository)
 		if err != nil {
 			return fmt.Errorf("SCM_REPOSITORY parse with url failed: %v", err)
 		}
-		repository = fmt.Sprintf("%s//%s@%s/%s", u.Scheme, b.ScmToken, u.Host, u.Path)
+		repository = fmt.Sprintf("%s//%s@%s/%s", u.Scheme, ptr.To(b.ScmToken), u.Host, u.Path)
 	}
-	if b.ScmCredentialType == enums.ScmCredentialTypeUsername {
+	if ptr.To(b.ScmCredentialType) == enums.ScmCredentialTypeUsername {
 		endpoint, err := transport.NewEndpoint(repository)
 		if err != nil {
 			return fmt.Errorf("transport.NewEndpoint failed: %v", err)
 		}
-		endpoint.User = b.ScmUsername
-		endpoint.Password = b.ScmPassword
+		endpoint.User = ptr.To(b.ScmUsername)
+		endpoint.Password = ptr.To(b.ScmPassword)
 		repository = endpoint.String()
 	}
 	cmd.Args = append(cmd.Args, repository, workspace)

cmd/builder/checker.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/go-sigma/sigma/pkg/types/enums"
 	"github.com/go-sigma/sigma/pkg/utils/crypt"
+	"github.com/go-sigma/sigma/pkg/utils/ptr"
 )
 
 func (b *Builder) checker() error {
@@ -29,37 +30,41 @@ func (b *Builder) checker() error {
 		return fmt.Errorf("SCM_CREDENTIAL_TYPE should be one of 'ssh', 'token' or 'none', but got '%s'", b.ScmCredentialType.String())
 	}
 
-	if b.ScmCredentialType == enums.ScmCredentialTypeSsh && b.ScmSshKey == "" {
+	if b.ScmCredentialType != nil && ptr.To(b.ScmCredentialType) == enums.ScmCredentialTypeSsh && ptr.To(b.ScmSshKey) == "" {
 		return fmt.Errorf("SCM_SSH_KEY should be set, if SCM_CREDENTIAL_TYPE is 'ssh'")
 	}
-	if b.ScmSshKey != "" {
-		b.ScmSshKey, err = crypt.Decrypt(fmt.Sprintf("%d-%d", b.BuilderID, b.RunnerID), b.ScmSshKey)
+	if ptr.To(b.ScmSshKey) != "" {
+		scmSshKey, err := crypt.Decrypt(fmt.Sprintf("%d-%d", b.BuilderID, b.RunnerID), ptr.To(b.ScmSshKey))
 		if err != nil {
 			return fmt.Errorf("Decrypt ssh key failed: %v", err)
 		}
+		b.ScmSshKey = ptr.Of(scmSshKey)
 	}
 
-	if b.ScmCredentialType == enums.ScmCredentialTypeToken && b.ScmToken == "" {
+	if b.ScmCredentialType != nil && ptr.To(b.ScmCredentialType) == enums.ScmCredentialTypeToken && ptr.To(b.ScmToken) == "" {
 		return fmt.Errorf("SCM_TOKEN should be set, if SCM_CREDENTIAL_TYPE is 'token'")
 	}
-	if b.ScmToken != "" {
-		b.ScmToken, err = crypt.Decrypt(fmt.Sprintf("%d-%d", b.BuilderID, b.RunnerID), b.ScmToken)
+	if ptr.To(b.ScmToken) != "" {
+		scmToken, err := crypt.Decrypt(fmt.Sprintf("%d-%d", b.BuilderID, b.RunnerID), ptr.To(b.ScmToken))
 		if err != nil {
 			return fmt.Errorf("Decrypt scm token failed: %v", err)
 		}
+		b.ScmToken = ptr.Of(scmToken)
 	}
 
-	if b.ScmCredentialType == enums.ScmCredentialTypeToken && (!strings.HasPrefix(b.ScmRepository, "http://") && !strings.HasPrefix(b.ScmRepository, "https://")) {
+	if b.ScmCredentialType != nil && ptr.To(b.ScmCredentialType) == enums.ScmCredentialTypeToken &&
+		(!strings.HasPrefix(ptr.To(b.ScmRepository), "http://") && !strings.HasPrefix(ptr.To(b.ScmRepository), "https://")) {
 		return fmt.Errorf("SCM_REPOSITORY should be started with 'http://' or 'https://', if SCM_CREDENTIAL_TYPE is 'token'")
 	}
-	if b.ScmCredentialType == enums.ScmCredentialTypeUsername && (b.ScmUsername == "" || b.ScmPassword == "") {
+	if b.ScmCredentialType != nil && ptr.To(b.ScmCredentialType) == enums.ScmCredentialTypeUsername && (ptr.To(b.ScmUsername) == "" || ptr.To(b.ScmPassword) == "") {
 		return fmt.Errorf("SCM_USERNAME and SCM_PASSWORD should be set, if SCM_CREDENTIAL_TYPE is 'username'")
 	}
-	if b.ScmPassword != "" {
-		b.ScmPassword, err = crypt.Decrypt(fmt.Sprintf("%d-%d", b.BuilderID, b.RunnerID), b.ScmPassword)
+	if ptr.To(b.ScmPassword) != "" {
+		scmPassword, err := crypt.Decrypt(fmt.Sprintf("%d-%d", b.BuilderID, b.RunnerID), ptr.To(b.ScmPassword))
 		if err != nil {
 			return fmt.Errorf("Decrypt scm password failed: %v", err)
 		}
+		b.ScmPassword = ptr.Of(scmPassword)
 	}
 
 	if !b.ScmProvider.IsValid() {

cmd/builder/init.go

@@ -37,12 +37,12 @@ import (
 
 // initToken init git clone token and buildkit push token
 func (b Builder) initToken() error {
-	if b.ScmCredentialType == enums.ScmCredentialTypeSsh {
+	if b.ScmCredentialType != nil && ptr.To(b.ScmCredentialType) == enums.ScmCredentialTypeSsh {
 		keyScan, err := exec.LookPath("ssh-keyscan")
 		if err != nil {
 			return fmt.Errorf("ssh-keyscan binary not found in path: %v", err)
 		}
-		endpoint, err := transport.NewEndpoint(b.ScmRepository)
+		endpoint, err := transport.NewEndpoint(ptr.To(b.ScmRepository))
 		if err != nil {
 			return fmt.Errorf("transport.NewEndpoint failed: %v", err)
 		}
@@ -80,7 +80,7 @@ func (b Builder) initToken() error {
 		defer func() {
 			_ = privateKeyObj.Close() // nolint: errcheck
 		}()
-		_, err = privateKeyObj.WriteString(b.ScmSshKey)
+		_, err = privateKeyObj.WriteString(ptr.To(b.ScmSshKey))
 		if err != nil {
 			return fmt.Errorf("Write private key failed: %v", err)
 		}

go.mod

@@ -33,6 +33,7 @@ require (
 	github.com/golang-migrate/migrate/v4 v4.16.2
 	github.com/google/go-github/v53 v53.2.0
 	github.com/google/uuid v1.3.1
+	github.com/hako/durafmt v0.0.0-20210608085754-5c1018a4e16b
 	github.com/hashicorp/golang-lru/v2 v2.0.6
 	github.com/hibiken/asynq v0.24.1
 	github.com/jackc/pgx/v4 v4.18.1

go.sum

@@ -477,6 +477,8 @@ github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfre
 github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
 github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
+github.com/hako/durafmt v0.0.0-20210608085754-5c1018a4e16b h1:wDUNC2eKiL35DbLvsDhiblTUXHxcOPwQSCzi7xpQUN4=
+github.com/hako/durafmt v0.0.0-20210608085754-5c1018a4e16b/go.mod h1:VzxiSdG6j1pi7rwGm/xYI5RbtpBgM8sARDXlvEvxlu0=
 github.com/hashicorp/consul/api v1.11.0/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M=
 github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=

pkg/builder/builder.go

@@ -22,7 +22,7 @@ import (
 
 	corev1 "k8s.io/api/core/v1"
 
-	builderlogger "github.com/go-sigma/sigma/pkg/builder/logger"
+	"github.com/go-sigma/sigma/pkg/builder/logger"
 	"github.com/go-sigma/sigma/pkg/configs"
 	"github.com/go-sigma/sigma/pkg/types"
 	"github.com/go-sigma/sigma/pkg/utils"
@@ -68,23 +68,15 @@ func Initialize() error {
 	if err != nil {
 		return err
 	}
-	return builderlogger.Initialize()
+	return logger.Initialize()
 }
 
 // BuildEnv ...
 func BuildEnv(builderConfig BuilderConfig) []string {
 	buildConfigEnvs := []string{
-		fmt.Sprintf("ID=%d", builderConfig.BuilderID),
+		fmt.Sprintf("BUILDER_ID=%d", builderConfig.BuilderID),
 		fmt.Sprintf("RUNNER_ID=%d", builderConfig.RunnerID),
 
-		fmt.Sprintf("SCM_CREDENTIAL_TYPE=%s", builderConfig.ScmCredentialType.String()),
-		fmt.Sprintf("SCM_USERNAME=%s", builderConfig.ScmUsername),
-		fmt.Sprintf("SCM_PROVIDER=%s", builderConfig.ScmProvider.String()),
-		fmt.Sprintf("SCM_REPOSITORY=%s", builderConfig.ScmRepository),
-		fmt.Sprintf("SCM_BRANCH=%s", ptr.To(builderConfig.ScmBranch)),
-		fmt.Sprintf("SCM_DEPTH=%d", builderConfig.ScmDepth),
-		fmt.Sprintf("SCM_SUBMODULE=%t", builderConfig.ScmSubmodule),
-
 		fmt.Sprintf("OCI_REGISTRY_DOMAIN=%s", strings.Join(builderConfig.OciRegistryDomain, ",")),
 		fmt.Sprintf("OCI_REGISTRY_USERNAME=%s", strings.Join(builderConfig.OciRegistryUsername, ",")),
 		fmt.Sprintf("OCI_NAME=%s", builderConfig.OciName),
@@ -94,18 +86,43 @@ func BuildEnv(builderConfig BuilderConfig) []string {
 		fmt.Sprintf("BUILDKIT_CONTEXT=%s", builderConfig.BuildkitContext),
 		fmt.Sprintf("BUILDKIT_DOCKERFILE=%s", builderConfig.BuildkitDockerfile),
 		fmt.Sprintf("BUILDKIT_PLATFORMS=%s", utils.StringsJoin(builderConfig.BuildkitPlatforms, ",")),
+		fmt.Sprintf("BUILDKIT_BUILD_ARGS=%s", strings.Join(builderConfig.BuildkitBuildArgs, ",")),
+	}
+	if builderConfig.Dockerfile != nil {
+		buildConfigEnvs = append(buildConfigEnvs, fmt.Sprintf("DOCKERFILE=%s", ptr.To(builderConfig.Dockerfile)))
+	}
+	if builderConfig.ScmCredentialType != nil {
+		buildConfigEnvs = append(buildConfigEnvs, fmt.Sprintf("SCM_CREDENTIAL_TYPE=%s", builderConfig.ScmCredentialType.String()))
+	}
+	if builderConfig.ScmProvider != nil {
+		buildConfigEnvs = append(buildConfigEnvs, fmt.Sprintf("SCM_PROVIDER=%s", builderConfig.ScmProvider.String()))
+	}
+	if builderConfig.ScmRepository != nil {
+		buildConfigEnvs = append(buildConfigEnvs, fmt.Sprintf("SCM_REPOSITORY=%s", ptr.To(builderConfig.ScmRepository)))
+	}
+	if builderConfig.ScmBranch != nil {
+		buildConfigEnvs = append(buildConfigEnvs, fmt.Sprintf("SCM_BRANCH=%s", ptr.To(builderConfig.ScmBranch)))
+	}
+	if builderConfig.ScmDepth != nil {
+		buildConfigEnvs = append(buildConfigEnvs, fmt.Sprintf("SCM_DEPTH=%d", ptr.To(builderConfig.ScmDepth)))
+	}
+	if builderConfig.ScmSubmodule != nil {
+		buildConfigEnvs = append(buildConfigEnvs, fmt.Sprintf("SCM_SUBMODULE=%t", ptr.To(builderConfig.ScmSubmodule)))
+	}
+	if builderConfig.ScmUsername != nil {
+		buildConfigEnvs = append(buildConfigEnvs, fmt.Sprintf("SCM_USERNAME=%s", ptr.To(builderConfig.ScmUsername)))
 	}
-	if builderConfig.ScmPassword != "" {
+	if builderConfig.ScmPassword != nil {
 		buildConfigEnvs = append(buildConfigEnvs, fmt.Sprintf("SCM_PASSWORD=%s", crypt.MustEncrypt(
-			fmt.Sprintf("%d-%d", builderConfig.BuilderID, builderConfig.RunnerID), builderConfig.ScmPassword)))
+			fmt.Sprintf("%d-%d", builderConfig.BuilderID, builderConfig.RunnerID), ptr.To(builderConfig.ScmPassword))))
 	}
-	if builderConfig.ScmSshKey != "" {
+	if builderConfig.ScmSshKey != nil {
 		buildConfigEnvs = append(buildConfigEnvs, fmt.Sprintf("SCM_SSH_KEY=%s", crypt.MustEncrypt(
-			fmt.Sprintf("%d-%d", builderConfig.BuilderID, builderConfig.RunnerID), builderConfig.ScmSshKey)))
+			fmt.Sprintf("%d-%d", builderConfig.BuilderID, builderConfig.RunnerID), ptr.To(builderConfig.ScmSshKey))))
 	}
-	if builderConfig.ScmToken != "" {
+	if builderConfig.ScmToken != nil {
 		buildConfigEnvs = append(buildConfigEnvs, fmt.Sprintf("SCM_TOKEN=%s", crypt.MustEncrypt(
-			fmt.Sprintf("%d-%d", builderConfig.BuilderID, builderConfig.RunnerID), builderConfig.ScmToken)))
+			fmt.Sprintf("%d-%d", builderConfig.BuilderID, builderConfig.RunnerID), ptr.To(builderConfig.ScmToken))))
 	}
 	if len(builderConfig.OciRegistryPassword) != 0 {
 		var passwords []string

pkg/daemon/builder/builder.go

@@ -22,13 +22,12 @@ import (
 	"github.com/hibiken/asynq"
 	"github.com/rs/zerolog/log"
 
-	builderdriver "github.com/go-sigma/sigma/pkg/builder"
+	"github.com/go-sigma/sigma/pkg/builder"
 	"github.com/go-sigma/sigma/pkg/daemon"
 	"github.com/go-sigma/sigma/pkg/dal/dao"
 	"github.com/go-sigma/sigma/pkg/types"
 	"github.com/go-sigma/sigma/pkg/types/enums"
 	"github.com/go-sigma/sigma/pkg/utils"
-	"github.com/go-sigma/sigma/pkg/utils/ptr"
 )
 
 func init() {
@@ -41,19 +40,19 @@ func builderRunner(ctx context.Context, task *asynq.Task) error {
 	if err != nil {
 		return fmt.Errorf("Unmarshal payload failed: %v", err)
 	}
-	b := builder{
+	b := runner{
 		builderServiceFactory: dao.NewBuilderServiceFactory(),
 	}
 	return b.runner(ctx, payload)
 }
 
-type builder struct {
+type runner struct {
 	builderServiceFactory dao.BuilderServiceFactory
 }
 
-func (b builder) runner(ctx context.Context, payload types.DaemonBuilderPayload) error {
+func (b runner) runner(ctx context.Context, payload types.DaemonBuilderPayload) error {
 	if payload.Action == enums.DaemonBuilderActionStop {
-		return builderdriver.Driver.Stop(ctx, payload.BuilderID, payload.RunnerID)
+		return builder.Driver.Stop(ctx, payload.BuilderID, payload.RunnerID)
 	}
 	builderService := b.builderServiceFactory.New()
 	builderObj, err := builderService.GetByRepositoryID(ctx, payload.RepositoryID)
@@ -78,7 +77,7 @@ func (b builder) runner(ctx context.Context, payload types.DaemonBuilderPayload)
 	// 	return fmt.Errorf("Create builder runner record failed: %v", err)
 	// }
 
-	buildConfig := builderdriver.BuilderConfig{
+	buildConfig := builder.BuilderConfig{
 		Builder: types.Builder{
 			BuilderID: payload.BuilderID,
 			RunnerID:  runnerObj.ID,
@@ -91,7 +90,7 @@ func (b builder) runner(ctx context.Context, payload types.DaemonBuilderPayload)
 			// ScmPassword:       builderObj.ScmPassword,
 			// ScmRepository:     builderObj.ScmRepository,
 			ScmBranch: runnerObj.ScmBranch,
-			ScmDepth:  ptr.To(builderObj.ScmDepth),
+			ScmDepth:  builderObj.ScmDepth,
 			// ScmSubmodule: builderObj.ScmSubmodule,
 
 			OciRegistryDomain:   []string{"192.168.31.114:3000"},
@@ -103,9 +102,9 @@ func (b builder) runner(ctx context.Context, payload types.DaemonBuilderPayload)
 		},
 	}
 	if payload.Action == enums.DaemonBuilderActionStart { // nolint: gocritic
-		err = builderdriver.Driver.Start(ctx, buildConfig)
+		err = builder.Driver.Start(ctx, buildConfig)
 	} else if payload.Action == enums.DaemonBuilderActionRestart {
-		err = builderdriver.Driver.Start(ctx, buildConfig)
+		err = builder.Driver.Start(ctx, buildConfig)
 	} else {
 		log.Error().Err(err).Str("action", payload.Action.String()).Msg("Daemon builder action not found")
 		return fmt.Errorf("Daemon builder action not found")

pkg/dal/migrations/mysql/0001_initialize.up.sql

@@ -396,6 +396,7 @@ CREATE TABLE IF NOT EXISTS `builder_runners` (
   `scm_branch` varchar(30),
   `started_at` timestamp,
   `ended_at` timestamp,
+  `duration` bigint,
   -- other fields
   `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
   `updated_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,

pkg/dal/migrations/postgresql/0001_initialize.up.sql

@@ -465,6 +465,7 @@ CREATE TABLE IF NOT EXISTS "builder_runners" (
   "scm_branch" varchar(30),
   "started_at" timestamp,
   "ended_at" timestamp,
+  "duration" bigint,
   -- other fields
   "created_at" timestamp NOT NULL,
   "updated_at" timestamp NOT NULL,

pkg/dal/migrations/sqlite3/0001_initialize.up.sql

@@ -378,14 +378,14 @@ CREATE TABLE IF NOT EXISTS `builders` (
   -- other fields
   `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
   `updated_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
-  `deleted_at` bigint NOT NULL DEFAULT 0,
+  `deleted_at` integer NOT NULL DEFAULT 0,
   FOREIGN KEY (`repository_id`) REFERENCES `repositories` (`id`),
   CONSTRAINT `builders_unique_with_repository` UNIQUE (`repository_id`, `deleted_at`)
 );
 
 CREATE TABLE IF NOT EXISTS `builder_runners` (
   `id` integer PRIMARY KEY AUTOINCREMENT,
-  `builder_id` bigint NOT NULL,
+  `builder_id` integer NOT NULL,
   `log` BLOB,
   `status` text CHECK (`status` IN ('Success', 'Failed', 'Pending', 'Scheduling', 'Building')) NOT NULL DEFAULT 'Pending',
   -- common settings
@@ -394,10 +394,11 @@ CREATE TABLE IF NOT EXISTS `builder_runners` (
   `scm_branch` varchar(30),
   `started_at` timestamp,
   `ended_at` timestamp,
+  `duration` integer,
   -- other fields
   `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
   `updated_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
-  `deleted_at` bigint NOT NULL DEFAULT 0,
+  `deleted_at` integer NOT NULL DEFAULT 0,
   FOREIGN KEY (`builder_id`) REFERENCES `builders` (`id`)
 );
 

pkg/dal/models/builder.go

@@ -88,6 +88,7 @@ type BuilderRunner struct {
 
 	StartedAt *time.Time
 	EndedAt   *time.Time
+	Duration  *int64
 
 	Builder Builder
 }