Skip to content

Commit

Permalink
Merge pull request #2988 from buildkite/json-env-file
Browse files Browse the repository at this point in the history 
Add JSON-format env file, allow annotations from pre-bootstrap
  • Loading branch information
@DrJosh9000
DrJosh9000 authored Sep 16, 2024
2 parents f807ffd + 3c9f0c0 commit eb40d04
Show file tree
Hide file tree
Showing 4 changed files with 99 additions and 48 deletions.
82 changes: 60 additions & 22 deletions agent/job_runner.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package agent

import (
"context"
"encoding/json"
"fmt"
"io"
"os"
Expand Down Expand Up @@ -156,8 +157,9 @@ type JobRunner struct {
// A lock to protect concurrent calls to cancel
cancelLock sync.Mutex

// File containing a copy of the job env
envFile *os.File
// Files containing a copy of the job env
envShellFile *os.File
envJSONFile *os.File
}

type jobAPI interface {
Expand Down Expand Up @@ -225,12 +227,19 @@ func NewJobRunner(ctx context.Context, l logger.Logger, apiClient APIClient, con
}

// Prepare a file to receive the given job environment
if file, err := os.CreateTemp(tempDir, fmt.Sprintf("job-env-%s", r.conf.Job.ID)); err != nil {
file, err := os.CreateTemp(tempDir, fmt.Sprintf("job-env-%s", r.conf.Job.ID))
if err != nil {
return r, err
} else {
r.agentLogger.Debug("[JobRunner] Created env file: %s", file.Name())
r.envFile = file
}
r.agentLogger.Debug("[JobRunner] Created env file (shell format): %s", file.Name())
r.envShellFile = file

file, err = os.CreateTemp(tempDir, fmt.Sprintf("job-env-json-%s", r.conf.Job.ID))
if err != nil {
return r, err
}
r.agentLogger.Debug("[JobRunner] Created env file (JSON format): %s", file.Name())
r.envJSONFile = file

env, err := r.createEnvironment(ctx)
if err != nil {
Expand Down Expand Up @@ -420,19 +429,36 @@ func (r *JobRunner) createEnvironment(ctx context.Context) ([]string, error) {
// The agent registration token should never make it into the job environment
delete(env, "BUILDKITE_AGENT_TOKEN")

// Write out the job environment to a file, in k="v" format, with newlines escaped
// Write out the job environment to file:
// - envShellFile: in k="v" format, with newlines escaped
// - envJSONFile: as a single JSON object {"k":"v",...}, escaped appropriately for JSON.
// We present only the clean environment - i.e only variables configured
// on the job upstream - and expose the path in another environment variable.
if r.envFile != nil {
if r.envShellFile != nil {
for key, value := range env {
if _, err := r.envFile.WriteString(fmt.Sprintf("%s=%q\n", key, value)); err != nil {
if _, err := r.envShellFile.WriteString(fmt.Sprintf("%s=%q\n", key, value)); err != nil {
return nil, err
}
}
if err := r.envFile.Close(); err != nil {
if err := r.envShellFile.Close(); err != nil {
return nil, err
}
env["BUILDKITE_ENV_FILE"] = r.envFile.Name()
}
if r.envJSONFile != nil {
if err := json.NewEncoder(r.envJSONFile).Encode(env); err != nil {
return nil, err
}
if err := r.envJSONFile.Close(); err != nil {
return nil, err
}
}
// Now that the env files have been written, we can add their corresponding
// paths to the job env.
if r.envShellFile != nil {
env["BUILDKITE_ENV_FILE"] = r.envShellFile.Name()
}
if r.envJSONFile != nil {
env["BUILDKITE_ENV_JSON_FILE"] = r.envJSONFile.Name()
}

var ignoredEnv []string
Expand All @@ -458,10 +484,11 @@ func (r *JobRunner) createEnvironment(ctx context.Context) ([]string, error) {
apiConfig := r.apiClient.Config()
env["BUILDKITE_AGENT_ENDPOINT"] = apiConfig.Endpoint
env["BUILDKITE_AGENT_ACCESS_TOKEN"] = apiConfig.Token
env["BUILDKITE_NO_HTTP2"] = fmt.Sprint(apiConfig.DisableHTTP2)

// Add agent environment variables
env["BUILDKITE_AGENT_DEBUG"] = fmt.Sprintf("%t", r.conf.Debug)
env["BUILDKITE_AGENT_DEBUG_HTTP"] = fmt.Sprintf("%t", r.conf.DebugHTTP)
env["BUILDKITE_AGENT_DEBUG"] = fmt.Sprint(r.conf.Debug)
env["BUILDKITE_AGENT_DEBUG_HTTP"] = fmt.Sprint(r.conf.DebugHTTP)
env["BUILDKITE_AGENT_PID"] = strconv.Itoa(os.Getpid())

// We know the BUILDKITE_BIN_PATH dir, because it's the path to the
Expand All @@ -481,14 +508,14 @@ func (r *JobRunner) createEnvironment(ctx context.Context) ([]string, error) {
env["BUILDKITE_BUILD_PATH"] = r.conf.AgentConfiguration.BuildPath
env["BUILDKITE_SOCKETS_PATH"] = r.conf.AgentConfiguration.SocketsPath
env["BUILDKITE_GIT_MIRRORS_PATH"] = r.conf.AgentConfiguration.GitMirrorsPath
env["BUILDKITE_GIT_MIRRORS_SKIP_UPDATE"] = fmt.Sprintf("%t", r.conf.AgentConfiguration.GitMirrorsSkipUpdate)
env["BUILDKITE_GIT_MIRRORS_SKIP_UPDATE"] = fmt.Sprint(r.conf.AgentConfiguration.GitMirrorsSkipUpdate)
env["BUILDKITE_HOOKS_PATH"] = r.conf.AgentConfiguration.HooksPath
env["BUILDKITE_PLUGINS_PATH"] = r.conf.AgentConfiguration.PluginsPath
env["BUILDKITE_SSH_KEYSCAN"] = fmt.Sprintf("%t", r.conf.AgentConfiguration.SSHKeyscan)
env["BUILDKITE_GIT_SUBMODULES"] = fmt.Sprintf("%t", r.conf.AgentConfiguration.GitSubmodules)
env["BUILDKITE_COMMAND_EVAL"] = fmt.Sprintf("%t", r.conf.AgentConfiguration.CommandEval)
env["BUILDKITE_PLUGINS_ENABLED"] = fmt.Sprintf("%t", r.conf.AgentConfiguration.PluginsEnabled)
env["BUILDKITE_LOCAL_HOOKS_ENABLED"] = fmt.Sprintf("%t", r.conf.AgentConfiguration.LocalHooksEnabled)
env["BUILDKITE_SSH_KEYSCAN"] = fmt.Sprint(r.conf.AgentConfiguration.SSHKeyscan)
env["BUILDKITE_GIT_SUBMODULES"] = fmt.Sprint(r.conf.AgentConfiguration.GitSubmodules)
env["BUILDKITE_COMMAND_EVAL"] = fmt.Sprint(r.conf.AgentConfiguration.CommandEval)
env["BUILDKITE_PLUGINS_ENABLED"] = fmt.Sprint(r.conf.AgentConfiguration.PluginsEnabled)
env["BUILDKITE_LOCAL_HOOKS_ENABLED"] = fmt.Sprint(r.conf.AgentConfiguration.LocalHooksEnabled)
env["BUILDKITE_GIT_CHECKOUT_FLAGS"] = r.conf.AgentConfiguration.GitCheckoutFlags
env["BUILDKITE_GIT_CLONE_FLAGS"] = r.conf.AgentConfiguration.GitCloneFlags
env["BUILDKITE_GIT_FETCH_FLAGS"] = r.conf.AgentConfiguration.GitFetchFlags
Expand All @@ -498,7 +525,7 @@ func (r *JobRunner) createEnvironment(ctx context.Context) ([]string, error) {
env["BUILDKITE_SHELL"] = r.conf.AgentConfiguration.Shell
env["BUILDKITE_AGENT_EXPERIMENT"] = strings.Join(experiments.Enabled(ctx), ",")
env["BUILDKITE_REDACTED_VARS"] = strings.Join(r.conf.AgentConfiguration.RedactedVars, ",")
env["BUILDKITE_STRICT_SINGLE_HOOKS"] = fmt.Sprintf("%t", r.conf.AgentConfiguration.StrictSingleHooks)
env["BUILDKITE_STRICT_SINGLE_HOOKS"] = fmt.Sprint(r.conf.AgentConfiguration.StrictSingleHooks)
env["BUILDKITE_CANCEL_GRACE_PERIOD"] = strconv.Itoa(r.conf.AgentConfiguration.CancelGracePeriod)
env["BUILDKITE_SIGNAL_GRACE_PERIOD_SECONDS"] = strconv.Itoa(int(r.conf.AgentConfiguration.SignalGracePeriod / time.Second))
env["BUILDKITE_TRACE_CONTEXT_ENCODING"] = r.conf.AgentConfiguration.TraceContextEncoding
Expand Down Expand Up @@ -550,7 +577,7 @@ func (r *JobRunner) createEnvironment(ctx context.Context) ([]string, error) {
enablePluginValidation = true
}
}
env["BUILDKITE_PLUGIN_VALIDATION"] = fmt.Sprintf("%t", enablePluginValidation)
env["BUILDKITE_PLUGIN_VALIDATION"] = fmt.Sprint(enablePluginValidation)

if r.conf.AgentConfiguration.TracingBackend != "" {
env["BUILDKITE_TRACING_BACKEND"] = r.conf.AgentConfiguration.TracingBackend
Expand Down Expand Up @@ -613,7 +640,18 @@ func (r *JobRunner) executePreBootstrapHook(ctx context.Context, hook string) (b

// This (plus inherited) is the only ENV that should be exposed
// to the pre-bootstrap hook.
sh.Env.Set("BUILDKITE_ENV_FILE", r.envFile.Name())
// - Env files are designed to be validated by the pre-bootstrap hook
// - The pre-bootstrap hook may want to create annotations, so it can also
// have a few necessary and global args as env vars.
sh.Env.Set("BUILDKITE_ENV_FILE", r.envShellFile.Name())
sh.Env.Set("BUILDKITE_ENV_JSON_FILE", r.envJSONFile.Name())
apiConfig := r.apiClient.Config()
sh.Env.Set("BUILDKITE_JOB_ID", r.conf.Job.ID)
sh.Env.Set("BUILDKITE_AGENT_ACCESS_TOKEN", apiConfig.Token)
sh.Env.Set("BUILDKITE_AGENT_ENDPOINT", apiConfig.Endpoint)
sh.Env.Set("BUILDKITE_NO_HTTP2", fmt.Sprint(apiConfig.DisableHTTP2))
sh.Env.Set("BUILDKITE_AGENT_DEBUG", fmt.Sprint(r.conf.Debug))
sh.Env.Set("BUILDKITE_AGENT_DEBUG_HTTP", fmt.Sprint(r.conf.DebugHTTP))

sh.Writer = LogWriter{
l: r.agentLogger,
Expand Down
54 changes: 31 additions & 23 deletions agent/job_runner_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,43 +7,51 @@ import (
"testing"

"github.com/buildkite/agent/v3/logger"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

func TestTruncateEnv(t *testing.T) {
l := logger.NewBuffer()
env := map[string]string{"FOO": strings.Repeat("a", 100)}
err := truncateEnv(l, env, "FOO", 64)
require.NoError(t, err)
assert.Equal(t, "aaaaaaaaaaaaaaaaaaaaaaaaaa[value truncated 100 -> 59 bytes]", env["FOO"])
assert.Equal(t, 64, len(fmt.Sprintf("FOO=%s\000", env["FOO"])))
key := "FOO"
env := map[string]string{key: strings.Repeat("a", 100)}
limit := 64
if err := truncateEnv(l, env, key, limit); err != nil {
t.Fatalf("truncateEnv(logger, %v, %q, %d) = %v", env, key, limit, err)
}
if got, want := env["FOO"], "aaaaaaaaaaaaaaaaaaaaaaaaaa[value truncated 100 -> 59 bytes]"; got != want {
t.Errorf("after truncateEnv(logger, %v, %q, %d): env[%q] = %q, want %q", env, key, limit, key, got, want)
}
format := "FOO=%s\000"
if got, want := len(fmt.Sprintf(format, env["FOO"])), limit; got != want {
t.Errorf("after truncateEnv(logger, %v, %q, %d): len(fmt.Sprintf(%q, env[%q])) = %d, want %d", env, key, limit, format, key, got, want)
}
}

func TestValidateJobValue(t *testing.T) {
bkTarget := "github.com/buildkite/test"
bkTargetRe := regexp.MustCompile("^github.com/buildkite/.*")
ghTargetRe := regexp.MustCompile("^github.com/nope/.*")
bkTargetRE := regexp.MustCompile(`^github\.com/buildkite/.*`)
ghTargetRE := regexp.MustCompile(`^github\.com/nope/.*`)

tests := []struct {
name string
allowedTargets []*regexp.Regexp
pipelineTarget string
wantErr bool
}{{
name: "No error. Allowed targets no configured.",
allowedTargets: []*regexp.Regexp{},
pipelineTarget: bkTarget,
}, {
name: "No pipeline target match",
allowedTargets: []*regexp.Regexp{ghTargetRe},
pipelineTarget: bkTarget,
wantErr: true,
}, {
name: "Pipeline target match",
allowedTargets: []*regexp.Regexp{ghTargetRe, bkTargetRe},
pipelineTarget: bkTarget,
}}
}{
{
name: "No error. Allowed targets no configured.",
allowedTargets: []*regexp.Regexp{},
pipelineTarget: bkTarget,
}, {
name: "No pipeline target match",
allowedTargets: []*regexp.Regexp{ghTargetRE},
pipelineTarget: bkTarget,
wantErr: true,
}, {
name: "Pipeline target match",
allowedTargets: []*regexp.Regexp{ghTargetRE, bkTargetRE},
pipelineTarget: bkTarget,
},
}

for _, tc := range tests {
err := validateJobValue(tc.allowedTargets, tc.pipelineTarget)
Expand Down
10 changes: 7 additions & 3 deletions agent/run_job.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -349,11 +349,15 @@ func (r *JobRunner) cleanup(ctx context.Context, wg *sync.WaitGroup, exit core.P
wg.Wait()

// Remove the env file, if any
if r.envFile != nil {
if err := os.Remove(r.envFile.Name()); err != nil {
for _, f := range []*os.File{r.envShellFile, r.envJSONFile} {
if f == nil {
continue
}
if err := os.Remove(f.Name()); err != nil {
r.agentLogger.Warn("[JobRunner] Error cleaning up env file: %s", err)
continue
}
r.agentLogger.Debug("[JobRunner] Deleted env file: %s", r.envFile.Name())
r.agentLogger.Debug("[JobRunner] Deleted env file: %s", f.Name())
}

// Write some metrics about the job run
Expand Down
1 change: 1 addition & 0 deletions internal/mime/mime.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit eb40d04

Please sign in to comment.