Merge pull request #369 from pjbgf/libgit2-cacheless-conns

Dockerfile

@@ -72,6 +72,7 @@ WORKDIR /workspace
 COPY main.go main.go
 COPY controllers/ controllers/
 COPY pkg/ pkg/
+COPY internal/ internal/
 
 COPY --from=musl-tool-chain /workspace/build /workspace/build
 

api/go.mod

@@ -4,7 +4,7 @@ go 1.17
 
 require (
 	github.com/fluxcd/pkg/apis/meta v0.14.1
-	github.com/fluxcd/source-controller/api v0.24.4
+	github.com/fluxcd/source-controller/api v0.25.0
 	k8s.io/apimachinery v0.24.0
 	sigs.k8s.io/controller-runtime v0.11.2
 )

api/go.sum

@@ -21,8 +21,8 @@ github.com/fluxcd/pkg/apis/acl v0.0.3 h1:Lw0ZHdpnO4G7Zy9KjrzwwBmDZQuy4qEjaU/RvA6
 github.com/fluxcd/pkg/apis/acl v0.0.3/go.mod h1:XPts6lRJ9C9fIF9xVWofmQwftvhY25n1ps7W9xw0XLU=
 github.com/fluxcd/pkg/apis/meta v0.14.1 h1:lPDs9yV67DnwalHPb13bbnDkAatALfUiAMRHjUm4UBw=
 github.com/fluxcd/pkg/apis/meta v0.14.1/go.mod h1:1uJkTJGSZWrZxL5PFpx1IxGLrFmT1Cd0C2fFWrbv77I=
-github.com/fluxcd/source-controller/api v0.24.4 h1:m54sS1rJlgJf5j9qDRgKLhbPJAnJ9dY+VrstPKj0aQo=
-github.com/fluxcd/source-controller/api v0.24.4/go.mod h1:b0MmMPGE8gcpgSyGXe5m7see77tBW26eZrvGkkPstUs=
+github.com/fluxcd/source-controller/api v0.25.0 h1:+uL+hQb/6h2MHuE9/Iq054TrDWF70puAuWBcoBrZK5M=
+github.com/fluxcd/source-controller/api v0.25.0/go.mod h1:tuMrqHHpRt7mxdLeRXGIMtTKAMufLwLTm5uXkEOJWFw=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI=
@@ -231,7 +231,7 @@ gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA=
 gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-k8s.io/api v0.23.5 h1:zno3LUiMubxD/V1Zw3ijyKO3wxrhbUF1Ck+VjBvfaoA=
+k8s.io/api v0.24.0 h1:J0hann2hfxWr1hinZIDefw7Q96wmCBx6SSB8IY0MdDg=
 k8s.io/apimachinery v0.24.0 h1:ydFCyC/DjCvFCHK5OPMKBlxayQytB8pxy8YQInd5UyQ=
 k8s.io/apimachinery v0.24.0/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM=
 k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=

controllers/imageupdateautomation_controller.go

@@ -250,31 +250,20 @@ func (r *ImageUpdateAutomationReconciler) Reconcile(ctx context.Context, req ctr
 		return failWithError(err)
 	}
 
-	repositoryURL := origin.Spec.URL
+	// managed GIT transport only affects the libgit2 implementation
 	if managed.Enabled() {
-		// At present only HTTP connections have the ability to define remote options.
-		// Although this can be easily extended by ensuring that the fake URL below uses the
-		// target ssh scheme, and the libgit2/managed/ssh.go pulls that information accordingly.
-		//
-		// This is due to the fact the key libgit2 remote callbacks do not take place for HTTP
-		// whilst most still work for SSH.
-		if strings.HasPrefix(repositoryURL, "http") {
-			if access.auth != nil && len(access.auth.CAFile) > 0 {
-				// Due to the lack of the callback feature, a fake target URL is created to allow
-				// for the smart sub transport be able to pick the options specific for this
-				// GitRepository object.
-				// The URL should use unique information that do not collide in a multi tenant
-				// deployment.
-				repositoryURL = fmt.Sprintf("http://%s/%s/%d", auto.Name, auto.UID, auto.Generation)
-				managed.AddTransportOptions(repositoryURL,
-					managed.TransportOptions{
-						TargetURL: repositoryURL,
-						CABundle:  access.auth.CAFile,
-					})
-
-				// We remove the options from memory, to avoid accumulating unused options over time.
-				defer managed.RemoveTransportOptions(repositoryURL)
-			}
+		// We set the TransportOptionsURL of this set of authentication options here by constructing
+		// a unique URL that won't clash in a multi tenant environment. This unique URL is used by
+		// libgit2 managed transports. This enables us to bypass the inbuilt credentials callback in
+		// libgit2, which is inflexible and unstable.
+		// NB: The Transport Options URL must be unique, therefore it must use the object under
+		// reconciliation details, instead of the repository it depends on.
+		if strings.HasPrefix(origin.Spec.URL, "http") {
+			access.auth.TransportOptionsURL = fmt.Sprintf("http://%s/%s/%d", auto.Name, auto.UID, auto.Generation)
+		} else if strings.HasPrefix(origin.Spec.URL, "ssh") {
+			access.auth.TransportOptionsURL = fmt.Sprintf("ssh://%s/%s/%d", auto.Name, auto.UID, auto.Generation)
+		} else {
+			return failWithError(fmt.Errorf("git repository URL '%s' has invalid transport type, supported types are: http, https, ssh", origin.Spec.URL))
 		}
 	}
 
@@ -287,6 +276,20 @@ func (r *ImageUpdateAutomationReconciler) Reconcile(ctx context.Context, req ctr
 	}
 	defer repo.Free()
 
+	if managed.Enabled() {
+		// Checkout removes TransportOptions before returning, therefore this
+		// must happen after cloneInto.
+		// TODO(pjbgf): Git consolidation should improve the API workflow.
+		managed.AddTransportOptions(access.auth.TransportOptionsURL, managed.TransportOptions{
+			TargetURL:    origin.Spec.URL,
+			AuthOpts:     access.auth,
+			ProxyOptions: &libgit2.ProxyOptions{Type: libgit2.ProxyTypeAuto},
+			Context:      cloneCtx,
+		})
+
+		defer managed.RemoveTransportOptions(access.auth.TransportOptionsURL)
+	}
+
 	// When there's a push spec, the pushed-to branch is where commits
 	// shall be made
 
@@ -732,7 +735,28 @@ var errRemoteBranchMissing = errors.New("remote branch missing")
 // switchToBranch switches to a branch after fetching latest from upstream.
 // If the branch does not exist, it is created using the head as the starting point.
 func switchToBranch(repo *libgit2.Repository, ctx context.Context, branch string, access repoAccess) error {
+	origin, err := repo.Remotes.Lookup(originRemote)
+	if err != nil {
+		return fmt.Errorf("cannot lookup remote: %w", err)
+	}
+	defer origin.Free()
+
+	callbacks := access.remoteCallbacks(ctx)
+	if managed.Enabled() {
+		// Override callbacks with dummy ones as they are not needed within Managed Transport.
+		// However, not setting them may lead to git2go panicing.
+		callbacks = managed.RemoteCallbacks()
+	}
+
 	branchRef := fmt.Sprintf("origin/%s", branch)
+	// Force the fetching of the remote branch.
+	err = origin.Fetch([]string{branch}, &libgit2.FetchOptions{
+		RemoteCallbacks: callbacks,
+	}, "")
+	if err != nil {
+		return fmt.Errorf("cannot fetch remote branch: %w", err)
+	}
+
 	remoteBranch, err := repo.LookupBranch(branchRef, libgit2.BranchRemote)
 	if err != nil && !libgit2.IsErrorCode(err, libgit2.ErrorCodeNotFound) {
 		return err
@@ -806,6 +830,11 @@ func push(ctx context.Context, path, branch string, access repoAccess) error {
 	defer origin.Free()
 
 	callbacks := access.remoteCallbacks(ctx)
+	if managed.Enabled() {
+		// Override callbacks with dummy ones as they are not needed within Managed Transport.
+		// However, not setting them may lead to git2go panicing.
+		callbacks = managed.RemoteCallbacks()
+	}
 
 	// calling repo.Push will succeed even if a reference update is
 	// rejected; to detect this case, this callback is supplied.

controllers/update_test.go

@@ -1593,7 +1593,7 @@ func createSSHIdentitySecret(kClient client.Client, name, namespace, repoURL str
 	if err != nil {
 		return err
 	}
-	knownhosts, err := ssh.ScanHostKey(url.Host, 5*time.Second)
+	knownhosts, err := ssh.ScanHostKey(url.Host, 5*time.Second, []string{}, false)
 	if err != nil {
 		return err
 	}

go.mod

@@ -12,11 +12,11 @@ require (
 	github.com/fluxcd/image-reflector-controller/api v0.18.0
 	github.com/fluxcd/pkg/apis/acl v0.0.3
 	github.com/fluxcd/pkg/apis/meta v0.14.1
-	github.com/fluxcd/pkg/gittestserver v0.5.2
+	github.com/fluxcd/pkg/gittestserver v0.5.3
 	github.com/fluxcd/pkg/runtime v0.16.1
-	github.com/fluxcd/pkg/ssh v0.3.2
-	github.com/fluxcd/source-controller v0.24.4
-	github.com/fluxcd/source-controller/api v0.24.4
+	github.com/fluxcd/pkg/ssh v0.4.1
+	github.com/fluxcd/source-controller v0.25.0
+	github.com/fluxcd/source-controller/api v0.25.0
 	github.com/go-logr/logr v1.2.3
 	github.com/google/go-containerregistry v0.9.0
 	github.com/libgit2/git2go/v33 v33.0.9
@@ -31,10 +31,6 @@ require (
 	sigs.k8s.io/kustomize/kyaml v0.13.7
 )
 
-require github.com/sosedoff/gitkit v0.3.0 // indirect
-
-replace github.com/sosedoff/gitkit => github.com/fluxcd/gitkit v0.5.1
-
 // Fix CVE-2022-28948
 replace gopkg.in/yaml.v3 => gopkg.in/yaml.v3 v3.0.0
 
@@ -51,8 +47,9 @@ require (
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/emicklei/go-restful v2.9.5+incompatible // indirect
-	github.com/emirpasic/gods v1.12.0 // indirect
+	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
+	github.com/fluxcd/gitkit v0.5.1 // indirect
 	github.com/fluxcd/pkg/gitutil v0.1.0 // indirect
 	github.com/fluxcd/pkg/version v0.1.0 // indirect
 	github.com/fsnotify/fsnotify v1.5.1 // indirect
@@ -73,14 +70,17 @@ require (
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-hclog v0.12.0 // indirect
 	github.com/hashicorp/go-retryablehttp v0.7.1 // indirect
 	github.com/huandu/xstrings v1.3.2 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/kevinburke/ssh_config v1.1.0 // indirect
+	github.com/kevinburke/ssh_config v1.2.0 // indirect
 	github.com/mailru/easyjson v0.7.6 // indirect
+	github.com/matryer/is v1.4.0 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
@@ -91,22 +91,23 @@ require (
 	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/prometheus/client_golang v1.12.1 // indirect
+	github.com/prometheus/client_golang v1.12.2 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
 	github.com/prometheus/common v0.32.1 // indirect
 	github.com/prometheus/procfs v0.7.3 // indirect
 	github.com/sergi/go-diff v1.2.0 // indirect
 	github.com/shopspring/decimal v1.2.0 // indirect
 	github.com/spf13/cast v1.4.1 // indirect
+	github.com/spf13/cobra v1.4.0 // indirect
 	github.com/xanzy/ssh-agent v0.3.1 // indirect
 	github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
 	go.uber.org/zap v1.21.0 // indirect
 	golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e
-	golang.org/x/net v0.0.0-20220516155154-20f960328961 // indirect
+	golang.org/x/net v0.0.0-20220524220425-1d687d428aca // indirect
 	golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect
-	golang.org/x/sys v0.0.0-20220513210249-45d2b4557a2a // indirect
+	golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
 	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
@@ -117,11 +118,11 @@ require (
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
-	k8s.io/apiextensions-apiserver v0.23.5 // indirect
+	k8s.io/apiextensions-apiserver v0.24.0 // indirect
 	k8s.io/component-base v0.24.0 // indirect
 	k8s.io/klog/v2 v2.60.1 // indirect
 	k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 // indirect
-	sigs.k8s.io/cli-utils v0.30.0 // indirect
+	sigs.k8s.io/cli-utils v0.31.1 // indirect
 	sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect