Merge pull request #312 from somtochiama/sourceignore

Add `sourceignore` pkg to `pkg/oci`

oci/client/build.go

@@ -23,14 +23,16 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/fluxcd/pkg/oci/client/internal/fs"
+	"github.com/fluxcd/pkg/sourceignore"
 )
 
 // Build archives the given directory as a tarball to the given local path.
 // While archiving, any environment specific data (for example, the user and group name) is stripped from file headers.
-func (c *Client) Build(artifactPath, sourceDir string) (err error) {
+func (c *Client) Build(artifactPath, sourceDir string, ignorePaths []string) (err error) {
 	if f, err := os.Stat(sourceDir); os.IsNotExist(err) || !f.IsDir() {
 		return fmt.Errorf("invalid source dir path: %s", sourceDir)
 	}
@@ -46,6 +48,14 @@ func (c *Client) Build(artifactPath, sourceDir string) (err error) {
 		}
 	}()
 
+	ignore := strings.Join(ignorePaths, "\n")
+	domain := strings.Split(filepath.Clean(sourceDir), string(filepath.Separator))
+	ps := sourceignore.ReadPatterns(strings.NewReader(ignore), domain)
+	matcher := sourceignore.NewMatcher(ps)
+	filter := func(p string, fi os.FileInfo) bool {
+		return matcher.Match(strings.Split(p, string(filepath.Separator)), fi.IsDir())
+	}
+
 	sz := &writeCounter{}
 	mw := io.MultiWriter(tf, sz)
 
@@ -61,6 +71,10 @@ func (c *Client) Build(artifactPath, sourceDir string) (err error) {
 			return nil
 		}
 
+		if len(ignorePaths) > 0 && filter(p, fi) {
+			return nil
+		}
+
 		header, err := tar.FileInfoHeader(fi, p)
 		if err != nil {
 			return err

oci/client/build_test.go

@@ -17,11 +17,15 @@ limitations under the License.
 package client
 
 import (
+	"bytes"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 
 	. "github.com/onsi/gomega"
+
+	"github.com/fluxcd/pkg/untar"
 )
 
 func TestBuild(t *testing.T) {
@@ -33,13 +37,37 @@ func TestBuild(t *testing.T) {
 	artifactPath := filepath.Join(tmpDir, "files.tar.gz")
 
 	// test with non-existent path
-	err := c.Build(artifactPath, "testdata/non-existent")
+	err := c.Build(artifactPath, "testdata/non-existent", nil)
 	g.Expect(err).To(HaveOccurred())
 
-	err = c.Build(artifactPath, testDir)
+	ignorePaths := []string{"ignore.txt", "ignore-dir/", "!/deploy"}
+	err = c.Build(artifactPath, testDir, ignorePaths)
+	g.Expect(err).ToNot(HaveOccurred())
+
+	_, err = os.Stat(artifactPath)
 	g.Expect(err).ToNot(HaveOccurred())
 
-	if _, err := os.Stat(artifactPath); err != nil {
-		g.Expect(err).ToNot(HaveOccurred())
+	b, err := os.ReadFile(artifactPath)
+	g.Expect(err).ToNot(HaveOccurred())
+
+	untarDir := t.TempDir()
+	_, err = untar.Untar(bytes.NewReader(b), untarDir)
+	g.Expect(err).To(BeNil())
+
+	for _, path := range ignorePaths {
+		var shouldExist bool
+		if strings.HasPrefix(path, "!") {
+			shouldExist = true
+			path = path[1:]
+		}
+
+		fullPath := filepath.Join(untarDir, testDir, path)
+		_, err := os.Stat(fullPath)
+		if shouldExist {
+			g.Expect(err).To(BeNil())
+			return
+		}
+		g.Expect(err).ToNot(BeNil())
+		g.Expect(os.IsNotExist(err)).To(BeTrue())
 	}
 }

oci/client/push.go

@@ -32,7 +32,7 @@ import (
 
 // Push creates an artifact from the given directory, uploads the artifact
 // to the given OCI repository and returns the digest.
-func (c *Client) Push(ctx context.Context, url, sourceDir string, meta Metadata) (string, error) {
+func (c *Client) Push(ctx context.Context, url, sourceDir string, meta Metadata, ignorePaths []string) (string, error) {
 	ref, err := name.ParseReference(url)
 	if err != nil {
 		return "", fmt.Errorf("invalid URL: %w", err)
@@ -46,7 +46,7 @@ func (c *Client) Push(ctx context.Context, url, sourceDir string, meta Metadata)
 
 	tmpFile := filepath.Join(tmpDir, "artifact.tgz")
 
-	if err := c.Build(tmpFile, sourceDir); err != nil {
+	if err := c.Build(tmpFile, sourceDir, ignorePaths); err != nil {
 		return "", err
 	}
 

oci/client/push_pull_test.go

@@ -44,7 +44,7 @@ func Test_Push_Pull(t *testing.T) {
 	}
 
 	testDir := "testdata/artifact"
-	_, err := c.Push(ctx, url, testDir, metadata)
+	_, err := c.Push(ctx, url, testDir, metadata, nil)
 	g.Expect(err).ToNot(HaveOccurred())
 
 	tags, err := crane.ListTags(fmt.Sprintf("%s/%s", dockerReg, repo))

oci/client/testdata/artifact/deploy/repo.yaml

@@ -0,0 +1,8 @@
+apiVersion: source.toolkit.fluxcd.io/v1beta2
+kind: HelmRepository
+metadata:
+  name: podinfo
+  namespace: default
+spec:
+  interval: 1m
+  url: https://stefanprodan.github.io/podinfo

oci/client/testdata/artifact/ignore-dir/deployment.yaml

@@ -0,0 +1,21 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+  labels:
+    app: nginx
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: nginx
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      containers:
+        - name: nginx
+          image: nginx:1.14.2
+          ports:
+            - containerPort: 80

oci/client/testdata/artifact/ignore.txt

@@ -0,0 +1 @@
+excluded file

oci/go.mod

@@ -2,13 +2,17 @@ module github.com/fluxcd/pkg/oci
 
 go 1.18
 
-replace github.com/fluxcd/pkg/untar => ../untar
+replace (
+	github.com/fluxcd/pkg/sourceignore => ../sourceignore
+	github.com/fluxcd/pkg/untar => ../untar
+)
 
 require (
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.0
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0
 	github.com/aws/aws-sdk-go v1.44.53
 	github.com/distribution/distribution/v3 v3.0.0-20220729163034-26163d82560f
+	github.com/fluxcd/pkg/sourceignore v0.0.0-00010101000000-000000000000
 	github.com/fluxcd/pkg/untar v0.1.0
 	github.com/google/go-containerregistry v0.10.0
 	github.com/onsi/gomega v1.19.0
@@ -20,6 +24,7 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1 // indirect
 	github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d // indirect
+	github.com/acomagu/bufpipe v1.0.3 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bshuster-repo/logrus-logstash-hook v1.0.0 // indirect
 	github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd // indirect
@@ -39,6 +44,9 @@ require (
 	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
 	github.com/felixge/httpsnoop v1.0.1 // indirect
 	github.com/fsnotify/fsnotify v1.5.1 // indirect
+	github.com/go-git/gcfg v1.5.0 // indirect
+	github.com/go-git/go-billy/v5 v5.3.1 // indirect
+	github.com/go-git/go-git/v5 v5.4.2 // indirect
 	github.com/go-logr/logr v1.2.3 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
 	github.com/go-openapi/jsonreference v0.20.0 // indirect
@@ -57,6 +65,7 @@ require (
 	github.com/gorilla/mux v1.8.0 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
@@ -96,6 +105,7 @@ require (
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.28.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/api v0.24.2 // indirect