fix conflict and modify Test_generateJobTree

.github/workflows/main.yml

@@ -1,23 +1,22 @@
 name: Test
-on: [push, pull_request]
+on: [push,pull_request]
 jobs:
   build:
     name: Test
     runs-on: ubuntu-latest
     steps:
 
-    - name: Set up Go 1.13
+    - name: Set up Go 1.14
       uses: actions/setup-go@v1
       with:
-        go-version: 1.13
+        go-version: 1.14
       id: go
 
     - name: Check out code into the Go module directory
-      uses: actions/checkout@v1
+      uses: actions/checkout@v2
 
     - name: Get dependencies
       run: go get -v -t -d ./...
 
     - name: Test without docker
-      run: go test -v -timeout 200s ./...
-
+      run: go test -v -timeout 200s ./...

.goreleaser.yml

@@ -99,34 +99,35 @@ builds:
     ldflags:
       - -s -w -X github.com/distribworks/dkron/v2/dkron.Version={{.Version}}
 
-nfpm:
-  vendor: Distributed Works
-  homepage: https://dkron.io
-  maintainer: Victor Castell <[email protected]>
-  description: Distributed, fault tolerant job scheduling system
-  license: LGPL 3.0
-
-  formats:
-    - deb
-    - rpm
-
-  # Override default /usr/local/bin destination for binaries
-  bindir: /usr/bin
-
-  empty_folders:
-    - /var/log/dkron
-
-  #files:
-  #  "builder/files/": "/etc/init.d"
-  #  "path/**/glob": "/var/foo/glob"
-  config_files:
-    "builder/files/dkron.yml": "/etc/dkron/dkron.yml"
-    "builder/files/dkron.service": "/lib/systemd/system/dkron.service"
-
-  overrides:
-    rpm:
-      replacements:
-        amd64: x86_64
+nfpms:
+  -
+    vendor: Distributed Works
+    homepage: https://dkron.io
+    maintainer: Victor Castell <[email protected]>
+    description: Distributed, fault tolerant job scheduling system
+    license: LGPL 3.0
+
+    formats:
+      - deb
+      - rpm
+
+    # Override default /usr/local/bin destination for binaries
+    bindir: /usr/bin
+
+    empty_folders:
+      - /var/log/dkron
+
+    #files:
+    #  "builder/files/": "/etc/init.d"
+    #  "path/**/glob": "/var/foo/glob"
+    config_files:
+      "builder/files/dkron.yml": "/etc/dkron/dkron.yml"
+      "builder/files/dkron.service": "/lib/systemd/system/dkron.service"
+
+    overrides:
+      rpm:
+        replacements:
+          amd64: x86_64
 
 snapshot:
   name_template: "{{ .Tag }}-next"

CHANGELOG.md

@@ -6,14 +6,52 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+### Fixes
+
+- fix: Graceful shutdown (#690) @andreygolev
+- fix: Fixes crash when plugin configuration is not defined in a job (#689) @andreygolev
+- fix: Defer panic fix in ExecutionDone gRPC call (#691) @andreygolev
+
+### Changes
+
+- chore: Default config will start and bootstrap a server
+- feat: isLeader handler added (#695)
+- Compile with go 1.14
+- Execution load balancing (#692) @andreygolev
+- Update Bootstrap and JQuery (#700)
+- chore: Upgrade all dependencies (#703)
+
+### Breaking Changes
+
+- refactor: Decrease plugin size by 75%, refactored plugin coding interface could affect new plugins development and require adaptations for existing plugin. (#696)
+- refactor: Use BuntDB for local storage, fixes [#687](https://github.com/distribworks/dkron/issues/687), require rolling upgrade. (#702) @andreygolev
+
+## [2.0.6] - 2020-02-14
+
+### Fixes
+
+- fix: Memory consumption on startup (#682)
+
+## [2.0.5] - 2020-02-12
+
+### Fixes
+
+- fix: Set the agent on dependent run (#675)
+- fix: Return the correct status code on leader endpoint (#671)
+
+### Changes
+
+- refactor: Check for missing agent (#675)
+- docs: Add code comment (#675)
+
 ## [2.0.4] - 2020-01-31
 
-* refactor: Remove dependency of the agent in store and reduce usage in Job (#669)
-* chore: Upgrade gin (#669)
-* chore: Add helper methods (#669)
-* refactor: Move directory creation to the Store instantiation (#669)
-* feat: Accept middlewares for API routes (#669)
-* doc: ACL
+- refactor: Remove dependency of the agent in store and reduce usage in Job (#669)
+- chore: Upgrade gin (#669)
+- chore: Add helper methods (#669)
+- refactor: Move directory creation to the Store instantiation (#669)
+- feat: Accept middlewares for API routes (#669)
+- doc: ACL
 
 ## [2.0.3] - 2020-01-04
 

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.13
+FROM golang:1.14
 LABEL maintainer="Victor Castell <[email protected]>"
 
 EXPOSE 8080 8946
@@ -12,6 +12,6 @@ COPY go.sum go.sum
 RUN go mod download
 
 COPY . .
-#RUN go install ./...
+RUN go install ./...
 
-#CMD ["dkron"]
+CMD ["dkron"]

builder/files/dkron.yml

@@ -1,5 +1,14 @@
 # Dkron example configuration file
-# server: false
+
+# This node is running in server mode
+server: true
+
+# Provides the number of expected servers in the datacenter.
+# Either this value should not be provided or the value must agree with other servers in the cluster.
+# When provided, Dkron waits until the specified number of servers are available and then bootstraps the cluster.
+# This allows an initial leader to be elected automatically. This flag requires server mode.
+bootstrap-expect: 1
+
 # bind-addr: "{{ GetPrivateIP }}:8946"
 # log-level: debug
 # tags:

builtin/bins/dkron-executor-http/http.go

@@ -16,7 +16,7 @@ import (
 	"time"
 
 	"github.com/armon/circbuf"
-	"github.com/distribworks/dkron/v2/dkron"
+	"github.com/distribworks/dkron/v2/plugin/types"
 )
 
 const (
@@ -44,17 +44,17 @@ type HTTP struct {
 //     "expectBody": "",            // Expect response body, support regexp, such as /success/
 //     "debug": "true"              // Debug option, will log everything when this option is not empty
 // }
-func (s *HTTP) Execute(args *dkron.ExecuteRequest) (*dkron.ExecuteResponse, error) {
+func (s *HTTP) Execute(args *types.ExecuteRequest) (*types.ExecuteResponse, error) {
 	out, err := s.ExecuteImpl(args)
-	resp := &dkron.ExecuteResponse{Output: out}
+	resp := &types.ExecuteResponse{Output: out}
 	if err != nil {
 		resp.Error = err.Error()
 	}
 	return resp, nil
 }
 
 // ExecuteImpl do http request
-func (s *HTTP) ExecuteImpl(args *dkron.ExecuteRequest) ([]byte, error) {
+func (s *HTTP) ExecuteImpl(args *types.ExecuteRequest) ([]byte, error) {
 	output, _ := circbuf.NewBuffer(maxBufSize)
 	var debug bool
 	if args.Config["debug"] != "" {

builtin/bins/dkron-executor-http/http_test.go

@@ -4,12 +4,12 @@ import (
 	"fmt"
 	"testing"
 
-	"github.com/distribworks/dkron/v2/dkron"
+	"github.com/distribworks/dkron/v2/plugin/types"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestExecute(t *testing.T) {
-	pa := &dkron.ExecuteRequest{
+	pa := &types.ExecuteRequest{
 		JobName: "testJob",
 		Config: map[string]string{
 			"method":     "GET",
@@ -28,7 +28,7 @@ func TestExecute(t *testing.T) {
 }
 
 func TestExecutePost(t *testing.T) {
-	pa := &dkron.ExecuteRequest{
+	pa := &types.ExecuteRequest{
 		JobName: "testJob",
 		Config: map[string]string{
 			"method":     "POST",
@@ -51,7 +51,7 @@ func TestExecutePost(t *testing.T) {
 
 // Note: badssl.com was meant for _manual_ testing. Maybe these tests should be disabled by default.
 func TestNoVerifyPeer(t *testing.T) {
-	pa := &dkron.ExecuteRequest{
+	pa := &types.ExecuteRequest{
 		JobName: "testJob",
 		Config: map[string]string{
 			"method":          "GET",
@@ -70,7 +70,7 @@ func TestNoVerifyPeer(t *testing.T) {
 
 func TestClientSSLCert(t *testing.T) {
 	// client certs: https://badssl.com/download/
-	pa := &dkron.ExecuteRequest{
+	pa := &types.ExecuteRequest{
 		JobName: "testJob",
 		Config: map[string]string{
 			"method":                "GET",
@@ -90,7 +90,7 @@ func TestClientSSLCert(t *testing.T) {
 
 func TestRootCA(t *testing.T) {
 	// untrusted root ca cert: https://badssl.com/certs/ca-untrusted-root.crt
-	pa := &dkron.ExecuteRequest{
+	pa := &types.ExecuteRequest{
 		JobName: "testJob",
 		Config: map[string]string{
 			"method":         "GET",

builtin/bins/dkron-executor-shell/shell.go

@@ -11,7 +11,7 @@ import (
 	"time"
 
 	"github.com/armon/circbuf"
-	"github.com/distribworks/dkron/v2/dkron"
+	dktypes "github.com/distribworks/dkron/v2/plugin/types"
 	"github.com/mattn/go-shellwords"
 )
 
@@ -28,17 +28,17 @@ const (
 type Shell struct{}
 
 // Execute method of the plugin
-func (s *Shell) Execute(args *dkron.ExecuteRequest) (*dkron.ExecuteResponse, error) {
+func (s *Shell) Execute(args *dktypes.ExecuteRequest) (*dktypes.ExecuteResponse, error) {
 	out, err := s.ExecuteImpl(args)
-	resp := &dkron.ExecuteResponse{Output: out}
+	resp := &dktypes.ExecuteResponse{Output: out}
 	if err != nil {
 		resp.Error = err.Error()
 	}
 	return resp, nil
 }
 
 // ExecuteImpl do execute command
-func (s *Shell) ExecuteImpl(args *dkron.ExecuteRequest) ([]byte, error) {
+func (s *Shell) ExecuteImpl(args *dktypes.ExecuteRequest) ([]byte, error) {
 	output, _ := circbuf.NewBuffer(maxBufSize)
 
 	shell, err := strconv.ParseBool(args.Config["shell"])

builtin/bins/dkron-executor-shell/shell_test.go

@@ -19,8 +19,7 @@ func Test_buildCmd(t *testing.T) {
 	// test not shell command
 	cmd, err = buildCmd("date && echo 'success'", false, []string{}, "")
 	assert.NoError(t, err)
-	out, err = cmd.CombinedOutput()
-	assert.Error(t, err)
+	assert.Len(t, cmd.Args, 1)
 }
 
 func Test_buildCmdWithCustomEnvironmentVariables(t *testing.T) {

builtin/bins/dkron-processor-files/files_output.go

@@ -6,7 +6,8 @@ import (
 	"os"
 	"strconv"
 
-	"github.com/distribworks/dkron/v2/dkron"
+	"github.com/distribworks/dkron/v2/plugin"
+	"github.com/distribworks/dkron/v2/plugin/types"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -19,8 +20,8 @@ type FilesOutput struct {
 	logDir  string
 }
 
-// Process method of the plugin
-func (l *FilesOutput) Process(args *dkron.ExecutionProcessorArgs) dkron.Execution {
+// Process method writes the execution output to a file
+func (l *FilesOutput) Process(args *plugin.ProcessorArgs) types.Execution {
 	log.SetFormatter(&log.TextFormatter{FullTimestamp: true})
 	l.parseConfig(args.Config)
 
@@ -39,7 +40,7 @@ func (l *FilesOutput) Process(args *dkron.ExecutionProcessorArgs) dkron.Executio
 	return args.Execution
 }
 
-func (l *FilesOutput) parseConfig(config dkron.PluginConfig) {
+func (l *FilesOutput) parseConfig(config plugin.Config) {
 	forward, err := strconv.ParseBool(config["forward"])
 	if err != nil {
 		l.forward = false

builtin/bins/dkron-processor-files/files_output_test.go

@@ -3,22 +3,23 @@ package main
 import (
 	"fmt"
 	"testing"
-	"time"
 
-	"github.com/distribworks/dkron/v2/dkron"
+	"github.com/distribworks/dkron/v2/plugin"
+	"github.com/distribworks/dkron/v2/plugin/types"
+	"github.com/golang/protobuf/ptypes"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestProcess(t *testing.T) {
-	now := time.Now()
+	now := ptypes.TimestampNow()
 
-	pa := &dkron.ExecutionProcessorArgs{
-		Execution: dkron.Execution{
+	pa := &plugin.ProcessorArgs{
+		Execution: types.Execution{
 			StartedAt: now,
 			NodeName:  "testNode",
 			Output:    []byte("test"),
 		},
-		Config: dkron.PluginConfig{
+		Config: plugin.Config{
 			"forward": "false",
 			"log_dir": "/tmp",
 		},

builtin/bins/dkron-processor-log/log_output.go

@@ -5,15 +5,18 @@ import (
 	"os"
 	"strconv"
 
-	"github.com/distribworks/dkron/v2/dkron"
+	"github.com/distribworks/dkron/v2/plugin"
+	"github.com/distribworks/dkron/v2/plugin/types"
 	log "github.com/sirupsen/logrus"
 )
 
+// LogOutput represent a LogOutputter
 type LogOutput struct {
 	forward bool
 }
 
-func (l *LogOutput) Process(args *dkron.ExecutionProcessorArgs) dkron.Execution {
+// Process method prints the execution output to the stdout
+func (l *LogOutput) Process(args *plugin.ProcessorArgs) types.Execution {
 	log.SetFormatter(&log.TextFormatter{FullTimestamp: true})
 
 	l.parseConfig(args.Config)
@@ -37,7 +40,7 @@ func (l *LogOutput) Process(args *dkron.ExecutionProcessorArgs) dkron.Execution
 	return args.Execution
 }
 
-func (l *LogOutput) parseConfig(config dkron.PluginConfig) {
+func (l *LogOutput) parseConfig(config plugin.Config) {
 	forward, err := strconv.ParseBool(config["forward"])
 	if err != nil {
 		l.forward = false