Implement Size field on archived artifacts

This adds a Size field to Artifacts, which reflects the number of bytes written to the artifact when it's being archived. Signed-off-by: Kevin McDermott <[email protected]>
fluxcd · Feb 28, 2022 · f7105ea · f7105ea
1 parent 25e6e16
commit f7105ea
Show file tree

Hide file tree

Showing 14 changed files with 95 additions and 5 deletions.
diff --git a/api/v1beta2/artifact_types.go b/api/v1beta2/artifact_types.go
@@ -51,6 +51,10 @@ type Artifact struct {
 	// artifact.
 	// +required
 	LastUpdateTime metav1.Time `json:"lastUpdateTime,omitempty"`
+
+	// Size is the number of bytes in the file.
+	// +optional
+	Size *int64 `json:"size,omitempty"`
 }
 
 // HasRevision returns true if the given revision matches the current Revision

diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go
diff --git a/config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml b/config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml
@@ -391,6 +391,10 @@ spec:
                       in the origin source system. It can be a Git commit SHA, Git
                       tag, a Helm index timestamp, a Helm chart version, etc.
                     type: string
+                  size:
+                    description: Size is the number of bytes in the file.
+                    format: int64
+                    type: integer
                   url:
                     description: URL is the HTTP address of this artifact. It is used
                       by the consumers of the artifacts to fetch and use the artifacts.

diff --git a/config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml b/config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml
@@ -559,6 +559,10 @@ spec:
                       in the origin source system. It can be a Git commit SHA, Git
                       tag, a Helm index timestamp, a Helm chart version, etc.
                     type: string
+                  size:
+                    description: Size is the number of bytes in the file.
+                    format: int64
+                    type: integer
                   url:
                     description: URL is the HTTP address of this artifact. It is used
                       by the consumers of the artifacts to fetch and use the artifacts.
@@ -663,6 +667,10 @@ spec:
                         in the origin source system. It can be a Git commit SHA, Git
                         tag, a Helm index timestamp, a Helm chart version, etc.
                       type: string
+                    size:
+                      description: Size is the number of bytes in the file.
+                      format: int64
+                      type: integer
                     url:
                       description: URL is the HTTP address of this artifact. It is
                         used by the consumers of the artifacts to fetch and use the

diff --git a/config/crd/bases/source.toolkit.fluxcd.io_helmcharts.yaml b/config/crd/bases/source.toolkit.fluxcd.io_helmcharts.yaml
@@ -438,6 +438,10 @@ spec:
                       in the origin source system. It can be a Git commit SHA, Git
                       tag, a Helm index timestamp, a Helm chart version, etc.
                     type: string
+                  size:
+                    description: Size is the number of bytes in the file.
+                    format: int64
+                    type: integer
                   url:
                     description: URL is the HTTP address of this artifact. It is used
                       by the consumers of the artifacts to fetch and use the artifacts.

diff --git a/config/crd/bases/source.toolkit.fluxcd.io_helmrepositories.yaml b/config/crd/bases/source.toolkit.fluxcd.io_helmrepositories.yaml
@@ -364,6 +364,10 @@ spec:
                       in the origin source system. It can be a Git commit SHA, Git
                       tag, a Helm index timestamp, a Helm chart version, etc.
                     type: string
+                  size:
+                    description: Size is the number of bytes in the file.
+                    format: int64
+                    type: integer
                   url:
                     description: URL is the HTTP address of this artifact. It is used
                       by the consumers of the artifacts to fetch and use the artifacts.

diff --git a/controllers/artifact_matchers_test.go b/controllers/artifact_matchers_test.go
@@ -54,6 +54,9 @@ func (m matchArtifact) Match(actual interface{}) (success bool, err error) {
 	if ok, err = Equal(m.expected.Checksum).Match(actualArtifact.Checksum); !ok {
 		return ok, err
 	}
+	if ok, err = Equal(m.expected.Size).Match(actualArtifact.Size); !ok {
+		return ok, err
+	}
 
 	return ok, err
 }

diff --git a/controllers/bucket_controller_test.go b/controllers/bucket_controller_test.go
@@ -200,6 +200,7 @@ func TestBucketReconciler_reconcileStorage(t *testing.T) {
 				Revision: "c",
 				Checksum: "2e7d2c03a9507ae265ecf5b5356885a53393a2029d241394997265a1a25aefc6",
 				URL:      testStorage.Hostname + "/reconcile-storage/c.txt",
+				Size:     int64p(int64(len("c"))),
 			},
 			assertPaths: []string{
 				"/reconcile-storage/c.txt",
@@ -251,6 +252,7 @@ func TestBucketReconciler_reconcileStorage(t *testing.T) {
 				Revision: "f",
 				Checksum: "3b9c358f36f0a31b6ad3e14f309c7cf198ac9246e8316f9ce543d5b19ac02b80",
 				URL:      testStorage.Hostname + "/reconcile-storage/hostname.txt",
+				Size:     int64p(int64(len("file"))),
 			},
 		},
 	}

diff --git a/controllers/gitrepository_controller_test.go b/controllers/gitrepository_controller_test.go
@@ -19,6 +19,7 @@ package controllers
 import (
 	"context"
 	"fmt"
+	"net/http"
 	"net/url"
 	"os"
 	"path/filepath"
@@ -818,6 +819,16 @@ func TestGitRepositoryReconciler_reconcileArtifact(t *testing.T) {
 			wantErr: true,
 		},
 	}
+	artifactSize := func(g *WithT, artifactURL string) *int64 {
+		if artifactURL == "" {
+			return nil
+		}
+		res, err := http.Get(artifactURL)
+		g.Expect(err).NotTo(HaveOccurred())
+		g.Expect(res.StatusCode).To(Equal(http.StatusOK))
+		defer res.Body.Close()
+		return &res.ContentLength
+	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -850,6 +861,10 @@ func TestGitRepositoryReconciler_reconcileArtifact(t *testing.T) {
 			g.Expect(err != nil).To(Equal(tt.wantErr))
 			g.Expect(got).To(Equal(tt.want))
 
+			if obj.Status.Artifact != nil {
+				g.Expect(obj.Status.Artifact.Size).To(Equal(artifactSize(g, obj.Status.Artifact.URL)))
+			}
+
 			if tt.afterFunc != nil {
 				tt.afterFunc(g, obj)
 			}

diff --git a/controllers/helmchart_controller_test.go b/controllers/helmchart_controller_test.go
@@ -198,6 +198,7 @@ func TestHelmChartReconciler_reconcileStorage(t *testing.T) {
 				Revision: "c",
 				Checksum: "2e7d2c03a9507ae265ecf5b5356885a53393a2029d241394997265a1a25aefc6",
 				URL:      testStorage.Hostname + "/reconcile-storage/c.txt",
+				Size:     int64p(int64(len("c"))),
 			},
 			assertPaths: []string{
 				"/reconcile-storage/c.txt",
@@ -250,6 +251,7 @@ func TestHelmChartReconciler_reconcileStorage(t *testing.T) {
 				Revision: "f",
 				Checksum: "3b9c358f36f0a31b6ad3e14f309c7cf198ac9246e8316f9ce543d5b19ac02b80",
 				URL:      testStorage.Hostname + "/reconcile-storage/hostname.txt",
+				Size:     int64p(int64(len("file"))),
 			},
 		},
 	}

diff --git a/controllers/helmrepository_controller_test.go b/controllers/helmrepository_controller_test.go
@@ -166,6 +166,7 @@ func TestHelmRepositoryReconciler_reconcileStorage(t *testing.T) {
 				Revision: "c",
 				Checksum: "2e7d2c03a9507ae265ecf5b5356885a53393a2029d241394997265a1a25aefc6",
 				URL:      testStorage.Hostname + "/reconcile-storage/c.txt",
+				Size:     int64p(int64(len("c"))),
 			},
 			assertPaths: []string{
 				"/reconcile-storage/c.txt",
@@ -218,6 +219,7 @@ func TestHelmRepositoryReconciler_reconcileStorage(t *testing.T) {
 				Revision: "f",
 				Checksum: "3b9c358f36f0a31b6ad3e14f309c7cf198ac9246e8316f9ce543d5b19ac02b80",
 				URL:      testStorage.Hostname + "/reconcile-storage/hostname.txt",
+				Size:     int64p(int64(len("file"))),
 			},
 		},
 	}

diff --git a/controllers/storage.go b/controllers/storage.go
@@ -194,7 +194,8 @@ func (s *Storage) Archive(artifact *sourcev1.Artifact, dir string, filter Archiv
 	}()
 
 	h := newHash()
-	mw := io.MultiWriter(h, tf)
+	sz := &writeCounter{}
+	mw := io.MultiWriter(h, tf, sz)
 
 	gw := gzip.NewWriter(mw)
 	tw := tar.NewWriter(gw)
@@ -286,6 +287,8 @@ func (s *Storage) Archive(artifact *sourcev1.Artifact, dir string, filter Archiv
 
 	artifact.Checksum = fmt.Sprintf("%x", h.Sum(nil))
 	artifact.LastUpdateTime = metav1.Now()
+	artifact.Size = &sz.written
+
 	return nil
 }
 
@@ -305,7 +308,8 @@ func (s *Storage) AtomicWriteFile(artifact *sourcev1.Artifact, reader io.Reader,
 	}()
 
 	h := newHash()
-	mw := io.MultiWriter(h, tf)
+	sz := &writeCounter{}
+	mw := io.MultiWriter(h, tf, sz)
 
 	if _, err := io.Copy(mw, reader); err != nil {
 		tf.Close()
@@ -325,6 +329,8 @@ func (s *Storage) AtomicWriteFile(artifact *sourcev1.Artifact, reader io.Reader,
 
 	artifact.Checksum = fmt.Sprintf("%x", h.Sum(nil))
 	artifact.LastUpdateTime = metav1.Now()
+	artifact.Size = &sz.written
+
 	return nil
 }
 
@@ -344,7 +350,8 @@ func (s *Storage) Copy(artifact *sourcev1.Artifact, reader io.Reader) (err error
 	}()
 
 	h := newHash()
-	mw := io.MultiWriter(h, tf)
+	sz := &writeCounter{}
+	mw := io.MultiWriter(h, tf, sz)
 
 	if _, err := io.Copy(mw, reader); err != nil {
 		tf.Close()
@@ -360,6 +367,8 @@ func (s *Storage) Copy(artifact *sourcev1.Artifact, reader io.Reader) (err error
 
 	artifact.Checksum = fmt.Sprintf("%x", h.Sum(nil))
 	artifact.LastUpdateTime = metav1.Now()
+	artifact.Size = &sz.written
+
 	return nil
 }
 
@@ -471,3 +480,15 @@ func (s *Storage) LocalPath(artifact sourcev1.Artifact) string {
 func newHash() hash.Hash {
 	return sha256.New()
 }
+
+// writecounter is an implementation of io.Writer that only records the number
+// of bytes written.
+type writeCounter struct {
+	written int64
+}
+
+func (wc *writeCounter) Write(p []byte) (int, error) {
+	n := len(p)
+	wc.written += int64(n)
+	return n, nil
+}
diff --git a/controllers/suite_test.go b/controllers/suite_test.go
@@ -193,3 +193,7 @@ func randStringRunes(n int) string {
 	}
 	return string(b)
 }
+
+func int64p(i int64) *int64 {
+	return &i
+}
diff --git a/docs/api/source.md b/docs/api/source.md
@@ -931,6 +931,18 @@ Kubernetes meta/v1.Time
 artifact.</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>size</code><br>
+<em>
+int64
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>Size is the number of bytes in the file.</p>
+</td>
+</tr>
 </tbody>
 </table>
 </div>
@@ -1568,8 +1580,8 @@ Artifact
 <td>
 <code>includedArtifacts</code><br>
 <em>
-<a href="#source.toolkit.fluxcd.io/v1beta2.*github.com/fluxcd/source-controller/api/v1beta2.Artifact">
-[]*github.com/fluxcd/source-controller/api/v1beta2.Artifact
+<a href="#source.toolkit.fluxcd.io/v1beta2.*./api/v1beta2.Artifact">
+[]*./api/v1beta2.Artifact
 </a>
 </em>
 </td>