Merge remote-tracking branch 'origin/master' into appset-use-db

Signed-off-by: Michael Crenshaw <[email protected]>

.github/workflows/ci-build.yaml

@@ -370,11 +370,11 @@ jobs:
           path: test-results
       - name: combine-go-coverage
         # We generate coverage reports for all Argo CD components, but only the applicationset-controller,
-        # app-controller, and repo-server report contain coverage data. The other components currently don't shut down
-        # gracefully, so no coverage data is produced. Once those components are fixed, we can add references to their
-        # coverage output directories.
+        # app-controller, repo-server, and commit-server report contain coverage data. The other components currently
+        # don't shut down gracefully, so no coverage data is produced. Once those components are fixed, we can add
+        # references to their coverage output directories.
         run: |
-          go tool covdata percent -i=test-results,e2e-code-coverage/applicationset-controller,e2e-code-coverage/repo-server,e2e-code-coverage/app-controller -o test-results/full-coverage.out
+          go tool covdata percent -i=test-results,e2e-code-coverage/applicationset-controller,e2e-code-coverage/repo-server,e2e-code-coverage/app-controller,e2e-code-coverage/commit-server -o test-results/full-coverage.out
       - name: Upload code coverage information to codecov.io
         uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0
         with:
@@ -429,6 +429,13 @@ jobs:
       GITHUB_TOKEN: ${{ secrets.E2E_TEST_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}
       GITLAB_TOKEN: ${{ secrets.E2E_TEST_GITLAB_TOKEN }}
     steps:
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be
+        with:
+          large-packages: false
+          docker-images: false
+          swap-storage: false
+          tool-cache: false
       - name: Checkout code
         uses: actions/checkout@8410ad0602e1e429cee44a835ae9f77f654a6694 # v4.0.0
       - name: Setup Golang

.github/workflows/image-reuse.yaml

@@ -17,11 +17,9 @@ on:
       platforms:
         required: true
         type: string
-        default: linux/amd64
       push:
         required: true
         type: boolean
-        default: false
       target:
         required: false
         type: string

.gitpod.Dockerfile

@@ -1,4 +1,4 @@
-FROM gitpod/workspace-full@sha256:230285e0b949e6d728d384b2029a4111db7b9c87c182f22f32a0be9e36b225df
+FROM gitpod/workspace-full@sha256:bec45ebdcc9b9c5ec28d5c61c16bf599200aa0d2dc1e69e2ed8ab0a424bae6db
 
 USER root
 

.mockery.yaml

@@ -26,6 +26,13 @@ packages:
   github.com/argoproj/argo-cd/v2/applicationset/utils:
     interfaces:
       Renderer:
+  github.com/argoproj/argo-cd/v2/commitserver/commit:
+    interfaces:
+      RepoClientFactory:
+  github.com/argoproj/argo-cd/v2/commitserver/apiclient:
+    interfaces:
+      CommitServiceClient:
+      Clientset:
   github.com/argoproj/argo-cd/v2/controller/cache:
     interfaces:
       LiveStateCache:

Dockerfile

@@ -140,7 +140,8 @@ RUN ln -s /usr/local/bin/argocd /usr/local/bin/argocd-server && \
     ln -s /usr/local/bin/argocd /usr/local/bin/argocd-dex && \
     ln -s /usr/local/bin/argocd /usr/local/bin/argocd-notifications && \
     ln -s /usr/local/bin/argocd /usr/local/bin/argocd-applicationset-controller && \
-    ln -s /usr/local/bin/argocd /usr/local/bin/argocd-k8s-auth
+    ln -s /usr/local/bin/argocd /usr/local/bin/argocd-k8s-auth && \
+    ln -s /usr/local/bin/argocd /usr/local/bin/argocd-commit-server
 
 USER $ARGOCD_USER_ID
 ENTRYPOINT ["/usr/bin/tini", "--"]

Makefile

@@ -472,6 +472,7 @@ start-e2e-local: mod-vendor-local dep-ui-local cli-local
 	mkdir -p /tmp/coverage/repo-server
 	mkdir -p /tmp/coverage/applicationset-controller
 	mkdir -p /tmp/coverage/notification
+	mkdir -p /tmp/coverage/commit-server
 	# set paths for locally managed ssh known hosts and tls certs data
 	ARGOCD_SSH_DATA_PATH=/tmp/argo-e2e/app/config/ssh \
 	ARGOCD_TLS_DATA_PATH=/tmp/argo-e2e/app/config/tls \
@@ -489,6 +490,7 @@ start-e2e-local: mod-vendor-local dep-ui-local cli-local
 	ARGOCD_APPLICATIONSET_CONTROLLER_TOKENREF_STRICT_MODE=true \
 	ARGOCD_APPLICATIONSET_CONTROLLER_ALLOWED_SCM_PROVIDERS=http://127.0.0.1:8341,http://127.0.0.1:8342,http://127.0.0.1:8343,http://127.0.0.1:8344 \
 	ARGOCD_E2E_TEST=true \
+	ARGOCD_HYDRATOR_ENABLED=true \
 		goreman -f $(ARGOCD_PROCFILE) start ${ARGOCD_START}
 	ls -lrt /tmp/coverage
 

Procfile

@@ -1,9 +1,10 @@
-controller: [ "$BIN_MODE" = 'true' ] && COMMAND=./dist/argocd || COMMAND='go run ./cmd/main.go' && sh -c "GOCOVERDIR=${ARGOCD_COVERAGE_DIR:-/tmp/coverage/app-controller} HOSTNAME=testappcontroller-1  FORCE_LOG_COLORS=1 ARGOCD_FAKE_IN_CLUSTER=true ARGOCD_TLS_DATA_PATH=${ARGOCD_TLS_DATA_PATH:-/tmp/argocd-local/tls} ARGOCD_SSH_DATA_PATH=${ARGOCD_SSH_DATA_PATH:-/tmp/argocd-local/ssh} ARGOCD_BINARY_NAME=argocd-application-controller $COMMAND --loglevel debug --redis localhost:${ARGOCD_E2E_REDIS_PORT:-6379} --repo-server localhost:${ARGOCD_E2E_REPOSERVER_PORT:-8081} --otlp-address=${ARGOCD_OTLP_ADDRESS} --application-namespaces=${ARGOCD_APPLICATION_NAMESPACES:-''} --server-side-diff-enabled=${ARGOCD_APPLICATION_CONTROLLER_SERVER_SIDE_DIFF:-'false'}"
-api-server: [ "$BIN_MODE" = 'true' ] && COMMAND=./dist/argocd || COMMAND='go run ./cmd/main.go' && sh -c "GOCOVERDIR=${ARGOCD_COVERAGE_DIR:-/tmp/coverage/api-server} FORCE_LOG_COLORS=1 ARGOCD_FAKE_IN_CLUSTER=true ARGOCD_TLS_DATA_PATH=${ARGOCD_TLS_DATA_PATH:-/tmp/argocd-local/tls} ARGOCD_SSH_DATA_PATH=${ARGOCD_SSH_DATA_PATH:-/tmp/argocd-local/ssh} ARGOCD_BINARY_NAME=argocd-server $COMMAND --loglevel debug --redis localhost:${ARGOCD_E2E_REDIS_PORT:-6379} --disable-auth=${ARGOCD_E2E_DISABLE_AUTH:-'true'} --insecure --dex-server http://localhost:${ARGOCD_E2E_DEX_PORT:-5556} --repo-server localhost:${ARGOCD_E2E_REPOSERVER_PORT:-8081} --port ${ARGOCD_E2E_APISERVER_PORT:-8080} --otlp-address=${ARGOCD_OTLP_ADDRESS} --application-namespaces=${ARGOCD_APPLICATION_NAMESPACES:-''}"
+controller: [ "$BIN_MODE" = 'true' ] && COMMAND=./dist/argocd || COMMAND='go run ./cmd/main.go' && sh -c "GOCOVERDIR=${ARGOCD_COVERAGE_DIR:-/tmp/coverage/app-controller} HOSTNAME=testappcontroller-1  FORCE_LOG_COLORS=1 ARGOCD_FAKE_IN_CLUSTER=true ARGOCD_TLS_DATA_PATH=${ARGOCD_TLS_DATA_PATH:-/tmp/argocd-local/tls} ARGOCD_SSH_DATA_PATH=${ARGOCD_SSH_DATA_PATH:-/tmp/argocd-local/ssh} ARGOCD_BINARY_NAME=argocd-application-controller $COMMAND --loglevel debug --redis localhost:${ARGOCD_E2E_REDIS_PORT:-6379} --repo-server localhost:${ARGOCD_E2E_REPOSERVER_PORT:-8081} --commit-server localhost:${ARGOCD_E2E_COMMITSERVER_PORT:-8086} --otlp-address=${ARGOCD_OTLP_ADDRESS} --application-namespaces=${ARGOCD_APPLICATION_NAMESPACES:-''} --server-side-diff-enabled=${ARGOCD_APPLICATION_CONTROLLER_SERVER_SIDE_DIFF:-'false'} --hydrator-enabled=${ARGOCD_HYDRATOR_ENABLED:='false'}"
+api-server: [ "$BIN_MODE" = 'true' ] && COMMAND=./dist/argocd || COMMAND='go run ./cmd/main.go' && sh -c "GOCOVERDIR=${ARGOCD_COVERAGE_DIR:-/tmp/coverage/api-server} FORCE_LOG_COLORS=1 ARGOCD_FAKE_IN_CLUSTER=true ARGOCD_TLS_DATA_PATH=${ARGOCD_TLS_DATA_PATH:-/tmp/argocd-local/tls} ARGOCD_SSH_DATA_PATH=${ARGOCD_SSH_DATA_PATH:-/tmp/argocd-local/ssh} ARGOCD_BINARY_NAME=argocd-server $COMMAND --loglevel debug --redis localhost:${ARGOCD_E2E_REDIS_PORT:-6379} --disable-auth=${ARGOCD_E2E_DISABLE_AUTH:-'true'} --insecure --dex-server http://localhost:${ARGOCD_E2E_DEX_PORT:-5556} --repo-server localhost:${ARGOCD_E2E_REPOSERVER_PORT:-8081} --port ${ARGOCD_E2E_APISERVER_PORT:-8080} --otlp-address=${ARGOCD_OTLP_ADDRESS} --application-namespaces=${ARGOCD_APPLICATION_NAMESPACES:-''} --hydrator-enabled=${ARGOCD_HYDRATOR_ENABLED:='false'}"
 dex: sh -c "ARGOCD_BINARY_NAME=argocd-dex go run github.com/argoproj/argo-cd/v2/cmd gendexcfg -o `pwd`/dist/dex.yaml && (test -f dist/dex.yaml || { echo 'Failed to generate dex configuration'; exit 1; }) && docker run --rm -p ${ARGOCD_E2E_DEX_PORT:-5556}:${ARGOCD_E2E_DEX_PORT:-5556} -v `pwd`/dist/dex.yaml:/dex.yaml ghcr.io/dexidp/dex:$(grep "image: ghcr.io/dexidp/dex" manifests/base/dex/argocd-dex-server-deployment.yaml | cut -d':' -f3) dex serve /dex.yaml"
 redis: hack/start-redis-with-password.sh
 repo-server: [ "$BIN_MODE" = 'true' ] && COMMAND=./dist/argocd || COMMAND='go run ./cmd/main.go' && sh -c "GOCOVERDIR=${ARGOCD_COVERAGE_DIR:-/tmp/coverage/repo-server} FORCE_LOG_COLORS=1 ARGOCD_FAKE_IN_CLUSTER=true ARGOCD_GNUPGHOME=${ARGOCD_GNUPGHOME:-/tmp/argocd-local/gpg/keys} ARGOCD_PLUGINSOCKFILEPATH=${ARGOCD_PLUGINSOCKFILEPATH:-./test/cmp}  ARGOCD_GPG_DATA_PATH=${ARGOCD_GPG_DATA_PATH:-/tmp/argocd-local/gpg/source} ARGOCD_TLS_DATA_PATH=${ARGOCD_TLS_DATA_PATH:-/tmp/argocd-local/tls} ARGOCD_SSH_DATA_PATH=${ARGOCD_SSH_DATA_PATH:-/tmp/argocd-local/ssh} ARGOCD_BINARY_NAME=argocd-repo-server ARGOCD_GPG_ENABLED=${ARGOCD_GPG_ENABLED:-false} $COMMAND --loglevel debug --port ${ARGOCD_E2E_REPOSERVER_PORT:-8081} --redis localhost:${ARGOCD_E2E_REDIS_PORT:-6379} --otlp-address=${ARGOCD_OTLP_ADDRESS}"
 cmp-server: [ "$ARGOCD_E2E_TEST" = 'true' ] && exit 0 || [ "$BIN_MODE" = 'true' ] && COMMAND=./dist/argocd || COMMAND='go run ./cmd/main.go' && sh -c "FORCE_LOG_COLORS=1 ARGOCD_FAKE_IN_CLUSTER=true ARGOCD_BINARY_NAME=argocd-cmp-server ARGOCD_PLUGINSOCKFILEPATH=${ARGOCD_PLUGINSOCKFILEPATH:-./test/cmp} $COMMAND --config-dir-path ./test/cmp --loglevel debug --otlp-address=${ARGOCD_OTLP_ADDRESS}"
+commit-server: [ "$BIN_MODE" = 'true' ] && COMMAND=./dist/argocd || COMMAND='go run ./cmd/main.go' && sh -c "GOCOVERDIR=${ARGOCD_COVERAGE_DIR:-/tmp/coverage/commit-server} FORCE_LOG_COLORS=1 ARGOCD_BINARY_NAME=argocd-commit-server $COMMAND --loglevel debug --port ${ARGOCD_E2E_COMMITSERVER_PORT:-8086}"
 ui: sh -c 'cd ui && ${ARGOCD_E2E_YARN_CMD:-yarn} start'
 git-server: test/fixture/testrepos/start-git.sh
 helm-registry: test/fixture/testrepos/start-helm-registry.sh

USERS.md

@@ -335,6 +335,7 @@ Currently, the following organizations are **officially** using Argo CD:
 1. [Swisscom](https://www.swisscom.ch)
 1. [Swissquote](https://github.com/swissquote)
 1. [Syncier](https://syncier.com/)
+1. [Synergy](https://synergy.net.au)
 1. [Syself](https://syself.com)
 1. [TableCheck](https://tablecheck.com/)
 1. [Tailor Brands](https://www.tailorbrands.com)

applicationset/services/internal/github_app/client.go

@@ -5,7 +5,7 @@ import (
 	"net/http"
 
 	"github.com/bradleyfalzon/ghinstallation/v2"
-	"github.com/google/go-github/v63/github"
+	"github.com/google/go-github/v66/github"
 
 	"github.com/argoproj/argo-cd/v2/applicationset/services/github_app_auth"
 )

applicationset/services/pull_request/github.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/google/go-github/v63/github"
+	"github.com/google/go-github/v66/github"
 	"golang.org/x/oauth2"
 )
 

applicationset/services/pull_request/github_test.go

@@ -3,7 +3,7 @@ package pull_request
 import (
 	"testing"
 
-	"github.com/google/go-github/v63/github"
+	"github.com/google/go-github/v66/github"
 	"github.com/stretchr/testify/require"
 )
 

applicationset/services/scm_provider/github.go

@@ -6,7 +6,7 @@ import (
 	"net/http"
 	"os"
 
-	"github.com/google/go-github/v63/github"
+	"github.com/google/go-github/v66/github"
 	"golang.org/x/oauth2"
 )
 

applicationset/utils/clusterUtils.go

@@ -2,22 +2,15 @@ package utils
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
-	"strconv"
-	"strings"
 	"sync"
-	"time"
-
-	log "github.com/sirupsen/logrus"
-	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/argoproj/argo-cd/v2/common"
 	appv1 "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1"
+	"github.com/argoproj/argo-cd/v2/util/db"
 
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
-	"k8s.io/utils/ptr"
 )
 
 var (
@@ -48,11 +41,15 @@ func ListClusters(ctx context.Context, clientset kubernetes.Interface, namespace
 	hasInClusterCredentials := false
 	for i, clusterSecret := range clusterSecrets {
 		// This line has changed from the original Argo CD code: now receives an error, and handles it
-		cluster, err := secretToCluster(&clusterSecret)
+		cluster, err := db.SecretToCluster(&clusterSecret)
 		if err != nil || cluster == nil {
 			return nil, fmt.Errorf("unable to convert cluster secret to cluster object '%s': %w", clusterSecret.Name, err)
 		}
 
+		// db.SecretToCluster populates these, but they're not meant to be available to the caller.
+		cluster.Labels = nil
+		cluster.Annotations = nil
+
 		clusterList.Items[i] = *cluster
 		if cluster.Server == appv1.KubernetesInternalAPIServerAddr {
 			hasInClusterCredentials = true
@@ -89,48 +86,3 @@ func getLocalCluster(clientset kubernetes.Interface) *appv1.Cluster {
 	cluster.ConnectionState.ModifiedAt = &now
 	return cluster
 }
-
-// secretToCluster converts a secret into a Cluster object
-func secretToCluster(s *corev1.Secret) (*appv1.Cluster, error) {
-	var config appv1.ClusterConfig
-	if len(s.Data["config"]) > 0 {
-		if err := json.Unmarshal(s.Data["config"], &config); err != nil {
-			// This line has changed from the original Argo CD: now returns an error rather than panicing.
-			return nil, err
-		}
-	}
-
-	var namespaces []string
-	for _, ns := range strings.Split(string(s.Data["namespaces"]), ",") {
-		if ns = strings.TrimSpace(ns); ns != "" {
-			namespaces = append(namespaces, ns)
-		}
-	}
-	var refreshRequestedAt *metav1.Time
-	if v, found := s.Annotations[appv1.AnnotationKeyRefresh]; found {
-		requestedAt, err := time.Parse(time.RFC3339, v)
-		if err != nil {
-			log.Warnf("Error while parsing date in cluster secret '%s': %v", s.Name, err)
-		} else {
-			refreshRequestedAt = &metav1.Time{Time: requestedAt}
-		}
-	}
-	var shard *int64
-	if shardStr := s.Data["shard"]; shardStr != nil {
-		if val, err := strconv.Atoi(string(shardStr)); err != nil {
-			log.Warnf("Error while parsing shard in cluster secret '%s': %v", s.Name, err)
-		} else {
-			shard = ptr.To(int64(val))
-		}
-	}
-	cluster := appv1.Cluster{
-		ID:                 string(s.UID),
-		Server:             strings.TrimRight(string(s.Data["server"]), "/"),
-		Name:               string(s.Data["name"]),
-		Namespaces:         namespaces,
-		Config:             config,
-		RefreshRequestedAt: refreshRequestedAt,
-		Shard:              shard,
-	}
-	return &cluster, nil
-}

assets/builtin-policy.csv

@@ -10,6 +10,7 @@ p, role:readonly, applications, get, */*, allow
 p, role:readonly, certificates, get, *, allow
 p, role:readonly, clusters, get, *, allow
 p, role:readonly, repositories, get, *, allow
+p, role:readonly, write-repositories, get, *, allow
 p, role:readonly, projects, get, *, allow
 p, role:readonly, accounts, get, *, allow
 p, role:readonly, gpgkeys, get, *, allow
@@ -34,6 +35,9 @@ p, role:admin, clusters, delete, *, allow
 p, role:admin, repositories, create, *, allow
 p, role:admin, repositories, update, *, allow
 p, role:admin, repositories, delete, *, allow
+p, role:admin, write-repositories, create, *, allow
+p, role:admin, write-repositories, update, *, allow
+p, role:admin, write-repositories, delete, *, allow
 p, role:admin, projects, create, *, allow
 p, role:admin, projects, update, *, allow
 p, role:admin, projects, delete, *, allow