Merge branch 'main' into build-ui-esbuild-loader

argoproj · Feb 3, 2024 · 147fff6 · 147fff6
2 parents 9d49b09 + 13444e6
commit 147fff6
Show file tree

Hide file tree

Showing 70 changed files with 1,129 additions and 119 deletions.
diff --git a/.github/workflows/ci-build.yaml b/.github/workflows/ci-build.yaml
@@ -264,7 +264,13 @@ jobs:
         run: make cli STATIC_FILES=false
         if: ${{matrix.test == 'test-api' || matrix.test == 'test-cli' || matrix.test == 'test-java-sdk' || matrix.test == 'test-python-sdk'}}
       - name: Start controller/API
-        run: make start PROFILE=${{matrix.profile}} AUTH_MODE=client STATIC_FILES=false LOG_LEVEL=info API=${{matrix.test == 'test-api' || matrix.test == 'test-cli' || matrix.test == 'test-java-sdk' || matrix.test == 'test-python-sdk'}} UI=false > /tmp/argo.log 2>&1 &
+        run: |
+          make start PROFILE=${{matrix.profile}} \
+            AUTH_MODE=client STATIC_FILES=false \
+            LOG_LEVEL=info \
+            API=${{matrix.test == 'test-api' || matrix.test == 'test-cli' || matrix.test == 'test-java-sdk' || matrix.test == 'test-python-sdk'}} \
+            UI=false \
+            POD_STATUS_CAPTURE_FINALIZER=true > /tmp/argo.log 2>&1 &
       - name: Wait for controller to be up
         run: make wait API=${{matrix.test == 'test-api' || matrix.test == 'test-cli' || matrix.test == 'test-java-sdk' || matrix.test == 'test-python-sdk'}}
         timeout-minutes: 5

diff --git a/Makefile b/Makefile
@@ -96,6 +96,7 @@ RUN_MODE              := kubernetes
 endif
 
 ALWAYS_OFFLOAD_NODE_STATUS := false
+POD_STATUS_CAPTURE_FINALIZER ?= true
 
 $(info GIT_COMMIT=$(GIT_COMMIT) GIT_BRANCH=$(GIT_BRANCH) GIT_TAG=$(GIT_TAG) GIT_TREE_STATE=$(GIT_TREE_STATE) RELEASE_TAG=$(RELEASE_TAG) DEV_BRANCH=$(DEV_BRANCH) VERSION=$(VERSION))
 $(info KUBECTX=$(KUBECTX) DOCKER_DESKTOP=$(DOCKER_DESKTOP) K3D=$(K3D) DOCKER_PUSH=$(DOCKER_PUSH))
@@ -558,7 +559,7 @@ endif
 	grep '127.0.0.1.*postgres' /etc/hosts
 	grep '127.0.0.1.*mysql' /etc/hosts
 ifeq ($(RUN_MODE),local)
-	env DEFAULT_REQUEUE_TIME=$(DEFAULT_REQUEUE_TIME) ARGO_SECURE=$(SECURE) ALWAYS_OFFLOAD_NODE_STATUS=$(ALWAYS_OFFLOAD_NODE_STATUS) ARGO_LOGLEVEL=$(LOG_LEVEL) UPPERIO_DB_DEBUG=$(UPPERIO_DB_DEBUG) ARGO_AUTH_MODE=$(AUTH_MODE) ARGO_NAMESPACED=$(NAMESPACED) ARGO_NAMESPACE=$(KUBE_NAMESPACE) ARGO_MANAGED_NAMESPACE=$(MANAGED_NAMESPACE) ARGO_EXECUTOR_PLUGINS=$(PLUGINS) PROFILE=$(PROFILE) kit $(TASKS)
+	env DEFAULT_REQUEUE_TIME=$(DEFAULT_REQUEUE_TIME) ARGO_SECURE=$(SECURE) ALWAYS_OFFLOAD_NODE_STATUS=$(ALWAYS_OFFLOAD_NODE_STATUS) ARGO_LOGLEVEL=$(LOG_LEVEL) UPPERIO_DB_DEBUG=$(UPPERIO_DB_DEBUG) ARGO_AUTH_MODE=$(AUTH_MODE) ARGO_NAMESPACED=$(NAMESPACED) ARGO_NAMESPACE=$(KUBE_NAMESPACE) ARGO_MANAGED_NAMESPACE=$(MANAGED_NAMESPACE) ARGO_EXECUTOR_PLUGINS=$(PLUGINS) ARGO_POD_STATUS_CAPTURE_FINALIZER=$(POD_STATUS_CAPTURE_FINALIZER) PROFILE=$(PROFILE) kit $(TASKS)
 endif
 
 .PHONY: wait
@@ -581,7 +582,7 @@ mysql-cli:
 test-cli: ./dist/argo
 
 test-%:
-	go test -failfast -v -timeout $(E2E_SUITE_TIMEOUT) -count 1 --tags $* -parallel $(E2E_PARALLEL) ./test/e2e
+	E2E_WAIT_TIMEOUT=$(E2E_WAIT_TIMEOUT) go test -failfast -v -timeout $(E2E_SUITE_TIMEOUT) -count 1 --tags $* -parallel $(E2E_PARALLEL) ./test/e2e
 
 .PHONY: test-examples
 test-examples:
@@ -592,7 +593,7 @@ test-%-sdk:
 	make --directory sdks/$* install test -B
 
 Test%:
-	go test -failfast -v -timeout $(E2E_SUITE_TIMEOUT) -count 1 --tags api,cli,cron,executor,examples,corefunctional,functional,plugins -parallel $(E2E_PARALLEL) ./test/e2e  -run='.*/$*'
+	E2E_WAIT_TIMEOUT=$(E2E_WAIT_TIMEOUT) go test -failfast -v -timeout $(E2E_SUITE_TIMEOUT) -count 1 --tags api,cli,cron,executor,examples,corefunctional,functional,plugins -parallel $(E2E_PARALLEL) ./test/e2e  -run='.*/$*'
 
 
 # clean

diff --git a/README.md b/README.md
@@ -1,11 +1,14 @@
 <!-- markdownlint-disable-next-line MD041 -->
-[![slack](https://img.shields.io/badge/slack-argoproj-brightgreen.svg?logo=slack)](https://argoproj.github.io/community/join-slack)
 [![Security Status](https://github.com/argoproj/argo-workflows/workflows/Snyk/badge.svg)](https://github.com/argoproj/argo-workflows/actions/workflows/snyk.yml?query=branch%3Amain)
 [![OpenSSF Best Practices](https://bestpractices.coreinfrastructure.org/projects/3830/badge)](https://bestpractices.coreinfrastructure.org/projects/3830)
 [![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/argoproj/argo-workflows/badge)](https://api.securityscorecards.dev/projects/github.com/argoproj/argo-workflows)
 [![FOSSA License Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fargoproj%2Fargo-workflows.svg?type=shield)](https://app.fossa.com/projects/git%2Bgithub.com%2Fargoproj%2Fargo-workflows?ref=badge_shield)
-[![Artifact HUB](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/argo-workflows)](https://artifacthub.io/packages/helm/argo/argo-workflows)
+[![Slack](https://img.shields.io/badge/slack-argoproj-brightgreen.svg?logo=slack)](https://argoproj.github.io/community/join-slack)
 [![Twitter Follow](https://img.shields.io/twitter/follow/argoproj?style=social)](https://twitter.com/argoproj)
+[![LinkedIn](https://img.shields.io/badge/LinkedIn-argoproj-blue.svg?logo=linkedin)](https://www.linkedin.com/company/argoproj/)
+[![Release Version](https://img.shields.io/github/v/release/argoproj/argo-workflows?label=argo-workflows)](https://github.com/argoproj/argo-workflows/releases/latest)
+[![Artifact HUB](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/argo-workflows)](https://artifacthub.io/packages/helm/argo/argo-workflows)
+
 
 ## What is Argo Workflows?
 

diff --git a/api/jsonschema/schema.json b/api/jsonschema/schema.json
diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json
diff --git a/cmd/argoexec/commands/data.go b/cmd/argoexec/commands/data.go
@@ -3,18 +3,39 @@ package commands
 import (
 	"context"
 
+	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
 
 func NewDataCommand() *cobra.Command {
 	command := cobra.Command{
 		Use:   "data",
 		Short: "Process data",
-		RunE: func(cmd *cobra.Command, args []string) error {
-			ctx := context.Background()
-			wfExecutor := initExecutor()
-			return wfExecutor.Data(ctx)
+		Run: func(cmd *cobra.Command, args []string) {
+			ctx := cmd.Context()
+			err := execData(ctx)
+			if err != nil {
+				log.Fatalf("%+v", err)
+			}
 		},
 	}
 	return &command
 }
+
+func execData(ctx context.Context) error {
+	wfExecutor := initExecutor()
+
+	// Don't allow cancellation to impact capture of results, parameters, artifacts, or defers.
+	bgCtx := context.Background()
+	// Create a new empty (placeholder) task result with LabelKeyReportOutputsCompleted set to false.
+	wfExecutor.InitializeOutput(bgCtx)
+	defer wfExecutor.HandleError(bgCtx)
+	defer wfExecutor.FinalizeOutput(bgCtx) //Ensures the LabelKeyReportOutputsCompleted is set to true.
+
+	err := wfExecutor.Data(ctx)
+	if err != nil {
+		wfExecutor.AddError(err)
+		return err
+	}
+	return nil
+}
diff --git a/cmd/argoexec/commands/resource.go b/cmd/argoexec/commands/resource.go
@@ -33,12 +33,19 @@ func NewResourceCommand() *cobra.Command {
 
 func execResource(ctx context.Context, action string) error {
 	wfExecutor := initExecutor()
-	defer wfExecutor.HandleError(ctx)
+
+	// Don't allow cancellation to impact capture of results, parameters, artifacts, or defers.
+	bgCtx := context.Background()
+
+	wfExecutor.InitializeOutput(bgCtx)
+	defer wfExecutor.HandleError(bgCtx)
+	defer wfExecutor.FinalizeOutput(bgCtx) //Ensures the LabelKeyReportOutputsCompleted is set to true.
 	err := wfExecutor.StageFiles()
 	if err != nil {
 		wfExecutor.AddError(err)
 		return err
 	}
+
 	isDelete := action == "delete"
 	if isDelete && (wfExecutor.Template.Resource.SuccessCondition != "" || wfExecutor.Template.Resource.FailureCondition != "" || len(wfExecutor.Template.Outputs.Parameters) > 0) {
 		err = fmt.Errorf("successCondition, failureCondition and outputs are not supported for delete action")

diff --git a/community/sustainability_effort.md b/community/sustainability_effort.md
@@ -42,6 +42,9 @@ For general guidance regarding PR Reviews, see [the Contributing docs](../CONTRI
 ### Authoring PRs
 
 Participants can apply a `prioritized-review` label to any PRs they author.
+If you do not have permissions to add labels yet, you can request the label in the `#argo-wf-sustainability` channel.
+
+If a PR has reached a week without an Assignee, you can request an assignment in the channel.
 
 ## Participants
 

diff --git a/docs/argo-server-sso.md b/docs/argo-server-sso.md
@@ -83,7 +83,7 @@ metadata:
     # Must evaluate to a boolean.
     # If you want an account to be the default to use, this rule can be "true".
     # Details of the expression language are available in
-    # https://github.com/antonmedv/expr/blob/master/docs/language-definition.md.
+    # https://github.com/expr-lang/expr/blob/master/docs/language-definition.md.
     workflows.argoproj.io/rbac-rule: "'admin' in groups"
     # The precedence is used to determine which service account to use whe
     # Precedence is an integer. It may be negative. If omitted, it defaults to "0".

diff --git a/docs/data-sourcing-and-transformation.md b/docs/data-sourcing-and-transformation.md
@@ -53,6 +53,6 @@ A `data` template must always contain a `source`. Current available sources:
 
 A `data` template may contain any number of transformations (or zero). The transformations will be applied serially in order. Current available transformations:
 
-* `expression`: an [`expr`](https://github.com/antonmedv/expr) expression. See language definition [here](https://github.com/antonmedv/expr/blob/master/docs/Language-Definition.md). When defining `expr` expressions Argo will pass the available data to the environment as a variable called `data` (see example above).
+* `expression`: an [`expr`](https://github.com/expr-lang/expr) expression. See language definition [here](https://github.com/expr-lang/expr/blob/master/docs/Language-Definition.md). When defining `expr` expressions Argo will pass the available data to the environment as a variable called `data` (see example above).
 
     We understand that the `expression` transformation is limited. We intend to greatly expand the functionality of this template with our community's feedback. Please see the link at the top of this document to submit ideas or use cases for this feature.
diff --git a/docs/environment-variables.md b/docs/environment-variables.md
@@ -22,6 +22,7 @@ This document outlines environment variables that can be used to customize behav
 | `ARGO_AGENT_PATCH_RATE`                  | `time.Duration`     | `DEFAULT_REQUEUE_TIME`                                                                      | Rate that the Argo Agent will patch the workflow task-set.                                                                                                                                                                                                               |
 | `ARGO_AGENT_CPU_LIMIT`                   | `resource.Quantity` | `100m`                                                                                      | CPU resource limit for the agent.                                                                                                                                                                                                                                        |
 | `ARGO_AGENT_MEMORY_LIMIT`                | `resource.Quantity` | `256m`                                                                                      | Memory resource limit for the agent.                                                                                                                                                                                                                                     |
+| `ARGO_POD_STATUS_CAPTURE_FINALIZER`      | `bool`              | `false`                                                                                     | The finalizer blocks the deletion of pods until the controller captures their status.
 | `BUBBLE_ENTRY_TEMPLATE_ERR`              | `bool`              | `true`                                                                                      | Whether to bubble up template errors to workflow.                                                                                                                                                                                                                        |
 | `CACHE_GC_PERIOD`                        | `time.Duration`     | `0s`                                                                                        | How often to perform memoization cache GC, which is disabled by default and can be enabled by providing a non-zero duration.                                                                                                                                             |
 | `CACHE_GC_AFTER_NOT_HIT_DURATION`        | `time.Duration`     | `30s`                                                                                       | When a memoization cache has not been hit after this duration, it will be deleted.                                                                                                                                                                                       |

diff --git a/docs/events.md b/docs/events.md
@@ -156,7 +156,7 @@ requirements](https://kubernetes.io/docs/concepts/overview/working-with-objects/
 
 Because the endpoint accepts any JSON data, it is the user's responsibility to write a suitable expression to correctly filter the events they are interested in. Therefore, DO NOT assume the existence of any fields, and guard against them using a nil check.
 
-[Learn more about expression syntax](https://github.com/antonmedv/expr).
+[Learn more about expression syntax](https://github.com/expr-lang/expr).
 
 ### Expression Environment
 

diff --git a/docs/executor_swagger.md b/docs/executor_swagger.md
@@ -4837,7 +4837,7 @@ Cannot be updated.
 |------|------|---------|:--------:| ------- |-------------|---------|
 | configMapKeyRef | [ConfigMapKeySelector](#config-map-key-selector)| `ConfigMapKeySelector` |  | |  |  |
 | default | [AnyString](#any-string)| `AnyString` |  | |  |  |
-| event | string| `string` |  | | Selector (https://github.com/antonmedv/expr) that is evaluated against the event to get the value of the parameter. E.g. `payload.message` |  |
+| event | string| `string` |  | | Selector (https://github.com/expr-lang/expr) that is evaluated against the event to get the value of the parameter. E.g. `payload.message` |  |
 | expression | string| `string` |  | | Expression, if defined, is evaluated to specify the value for the parameter |  |
 | jqFilter | string| `string` |  | | JQFilter expression against the resource object in resource templates |  |
 | jsonPath | string| `string` |  | | JSONPath of a resource to retrieve an output parameter value from in resource templates |  |

diff --git a/docs/fields.md b/docs/fields.md
@@ -857,7 +857,7 @@ WorkflowStatus contains overall status information about a workflow
 |`storedTemplates`|[`Template`](#template)|StoredTemplates is a mapping between a template ref and the node's status.|
 |`storedWorkflowTemplateSpec`|[`WorkflowSpec`](#workflowspec)|StoredWorkflowSpec stores the WorkflowTemplate spec for future execution.|
 |`synchronization`|[`SynchronizationStatus`](#synchronizationstatus)|Synchronization stores the status of synchronization locks|
-|`taskResultsCompletionStatus`|`Map< boolean , string >`|TaskResultsCompletionStatus tracks task result completion status (mapped by pod name). Used to prevent premature archiving and garbage collection.|
+|`taskResultsCompletionStatus`|`Map< boolean , string >`|TaskResultsCompletionStatus tracks task result completion status (mapped by node ID). Used to prevent premature archiving and garbage collection.|
 
 ## CronWorkflowSpec
 
@@ -3552,7 +3552,7 @@ ValueFrom describes a location in which to obtain the value to a parameter
 |:----------:|:----------:|---------------|
 |`configMapKeyRef`|[`ConfigMapKeySelector`](#configmapkeyselector)|ConfigMapKeyRef is configmap selector for input parameter configuration|
 |`default`|`string`|Default specifies a value to be used if retrieving the value from the specified source fails|
-|`event`|`string`|Selector (https://github.com/antonmedv/expr) that is evaluated against the event to get the value of the parameter. E.g. `payload.message`|
+|`event`|`string`|Selector (https://github.com/expr-lang/expr) that is evaluated against the event to get the value of the parameter. E.g. `payload.message`|
 |`expression`|`string`|Expression, if defined, is evaluated to specify the value for the parameter|
 |`jqFilter`|`string`|JQFilter expression against the resource object in resource templates|
 |`jsonPath`|`string`|JSONPath of a resource to retrieve an output parameter value from in resource templates|

diff --git a/docs/retries.md b/docs/retries.md
@@ -86,7 +86,7 @@ spec:
 > v3.2 and after
 
 You can also use `expression` to control retries. The `expression` field
-accepts an [expr](https://github.com/antonmedv/expr) expression and has
+accepts an [expr](https://github.com/expr-lang/expr) expression and has
 access to the following variables:
 
 - `lastRetry.exitCode`: The exit code of the last retry, or "-1" if not available

diff --git a/docs/variables.md b/docs/variables.md
@@ -56,7 +56,7 @@ The tag is substituted with the result of evaluating the tag as an expression.
 Note that any hyphenated parameter names or step names will cause a parsing error. You can reference them by
 indexing into the parameter or step map, e.g. `inputs.parameters['my-param']` or `steps['my-step'].outputs.result`.
 
-[Learn about the expression syntax](https://github.com/antonmedv/expr/blob/master/docs/language-definition.md).
+[Learn about the expression syntax](https://github.com/expr-lang/expr/blob/master/docs/language-definition.md).
 
 #### Examples