From f7105ea736638f21f2c3fe637483ee5254f6792c Mon Sep 17 00:00:00 2001 From: Kevin McDermott Date: Tue, 22 Feb 2022 10:30:58 +0000 Subject: [PATCH] Implement Size field on archived artifacts This adds a Size field to Artifacts, which reflects the number of bytes written to the artifact when it's being archived. Signed-off-by: Kevin McDermott --- api/v1beta2/artifact_types.go | 4 +++ api/v1beta2/zz_generated.deepcopy.go | 5 ++++ .../source.toolkit.fluxcd.io_buckets.yaml | 4 +++ ...rce.toolkit.fluxcd.io_gitrepositories.yaml | 8 ++++++ .../source.toolkit.fluxcd.io_helmcharts.yaml | 4 +++ ...ce.toolkit.fluxcd.io_helmrepositories.yaml | 4 +++ controllers/artifact_matchers_test.go | 3 +++ controllers/bucket_controller_test.go | 2 ++ controllers/gitrepository_controller_test.go | 15 +++++++++++ controllers/helmchart_controller_test.go | 2 ++ controllers/helmrepository_controller_test.go | 2 ++ controllers/storage.go | 27 ++++++++++++++++--- controllers/suite_test.go | 4 +++ docs/api/source.md | 16 +++++++++-- 14 files changed, 95 insertions(+), 5 deletions(-) diff --git a/api/v1beta2/artifact_types.go b/api/v1beta2/artifact_types.go index 363f79b1a..64829b6ba 100644 --- a/api/v1beta2/artifact_types.go +++ b/api/v1beta2/artifact_types.go @@ -51,6 +51,10 @@ type Artifact struct { // artifact. // +required LastUpdateTime metav1.Time `json:"lastUpdateTime,omitempty"` + + // Size is the number of bytes in the file. + // +optional + Size *int64 `json:"size,omitempty"` } // HasRevision returns true if the given revision matches the current Revision diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go index 53c86a93a..b789d81da 100644 --- a/api/v1beta2/zz_generated.deepcopy.go +++ b/api/v1beta2/zz_generated.deepcopy.go @@ -32,6 +32,11 @@ import ( func (in *Artifact) DeepCopyInto(out *Artifact) { *out = *in in.LastUpdateTime.DeepCopyInto(&out.LastUpdateTime) + if in.Size != nil { + in, out := &in.Size, &out.Size + *out = new(int64) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Artifact. diff --git a/config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml b/config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml index 31f7f7e27..9607665e4 100644 --- a/config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml +++ b/config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml @@ -391,6 +391,10 @@ spec: in the origin source system. It can be a Git commit SHA, Git tag, a Helm index timestamp, a Helm chart version, etc. type: string + size: + description: Size is the number of bytes in the file. + format: int64 + type: integer url: description: URL is the HTTP address of this artifact. It is used by the consumers of the artifacts to fetch and use the artifacts. diff --git a/config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml b/config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml index 4436b5137..7d445f7cb 100644 --- a/config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml +++ b/config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml @@ -559,6 +559,10 @@ spec: in the origin source system. It can be a Git commit SHA, Git tag, a Helm index timestamp, a Helm chart version, etc. type: string + size: + description: Size is the number of bytes in the file. + format: int64 + type: integer url: description: URL is the HTTP address of this artifact. It is used by the consumers of the artifacts to fetch and use the artifacts. @@ -663,6 +667,10 @@ spec: in the origin source system. It can be a Git commit SHA, Git tag, a Helm index timestamp, a Helm chart version, etc. type: string + size: + description: Size is the number of bytes in the file. + format: int64 + type: integer url: description: URL is the HTTP address of this artifact. It is used by the consumers of the artifacts to fetch and use the diff --git a/config/crd/bases/source.toolkit.fluxcd.io_helmcharts.yaml b/config/crd/bases/source.toolkit.fluxcd.io_helmcharts.yaml index 45dfb71bd..75b6bfee6 100644 --- a/config/crd/bases/source.toolkit.fluxcd.io_helmcharts.yaml +++ b/config/crd/bases/source.toolkit.fluxcd.io_helmcharts.yaml @@ -438,6 +438,10 @@ spec: in the origin source system. It can be a Git commit SHA, Git tag, a Helm index timestamp, a Helm chart version, etc. type: string + size: + description: Size is the number of bytes in the file. + format: int64 + type: integer url: description: URL is the HTTP address of this artifact. It is used by the consumers of the artifacts to fetch and use the artifacts. diff --git a/config/crd/bases/source.toolkit.fluxcd.io_helmrepositories.yaml b/config/crd/bases/source.toolkit.fluxcd.io_helmrepositories.yaml index 002718627..cd687f6bb 100644 --- a/config/crd/bases/source.toolkit.fluxcd.io_helmrepositories.yaml +++ b/config/crd/bases/source.toolkit.fluxcd.io_helmrepositories.yaml @@ -364,6 +364,10 @@ spec: in the origin source system. It can be a Git commit SHA, Git tag, a Helm index timestamp, a Helm chart version, etc. type: string + size: + description: Size is the number of bytes in the file. + format: int64 + type: integer url: description: URL is the HTTP address of this artifact. It is used by the consumers of the artifacts to fetch and use the artifacts. diff --git a/controllers/artifact_matchers_test.go b/controllers/artifact_matchers_test.go index 06ab529de..5007cc6dd 100644 --- a/controllers/artifact_matchers_test.go +++ b/controllers/artifact_matchers_test.go @@ -54,6 +54,9 @@ func (m matchArtifact) Match(actual interface{}) (success bool, err error) { if ok, err = Equal(m.expected.Checksum).Match(actualArtifact.Checksum); !ok { return ok, err } + if ok, err = Equal(m.expected.Size).Match(actualArtifact.Size); !ok { + return ok, err + } return ok, err } diff --git a/controllers/bucket_controller_test.go b/controllers/bucket_controller_test.go index 17429fb92..3ff729f3b 100644 --- a/controllers/bucket_controller_test.go +++ b/controllers/bucket_controller_test.go @@ -200,6 +200,7 @@ func TestBucketReconciler_reconcileStorage(t *testing.T) { Revision: "c", Checksum: "2e7d2c03a9507ae265ecf5b5356885a53393a2029d241394997265a1a25aefc6", URL: testStorage.Hostname + "/reconcile-storage/c.txt", + Size: int64p(int64(len("c"))), }, assertPaths: []string{ "/reconcile-storage/c.txt", @@ -251,6 +252,7 @@ func TestBucketReconciler_reconcileStorage(t *testing.T) { Revision: "f", Checksum: "3b9c358f36f0a31b6ad3e14f309c7cf198ac9246e8316f9ce543d5b19ac02b80", URL: testStorage.Hostname + "/reconcile-storage/hostname.txt", + Size: int64p(int64(len("file"))), }, }, } diff --git a/controllers/gitrepository_controller_test.go b/controllers/gitrepository_controller_test.go index a91a0f624..5f20e18ae 100644 --- a/controllers/gitrepository_controller_test.go +++ b/controllers/gitrepository_controller_test.go @@ -19,6 +19,7 @@ package controllers import ( "context" "fmt" + "net/http" "net/url" "os" "path/filepath" @@ -818,6 +819,16 @@ func TestGitRepositoryReconciler_reconcileArtifact(t *testing.T) { wantErr: true, }, } + artifactSize := func(g *WithT, artifactURL string) *int64 { + if artifactURL == "" { + return nil + } + res, err := http.Get(artifactURL) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(res.StatusCode).To(Equal(http.StatusOK)) + defer res.Body.Close() + return &res.ContentLength + } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -850,6 +861,10 @@ func TestGitRepositoryReconciler_reconcileArtifact(t *testing.T) { g.Expect(err != nil).To(Equal(tt.wantErr)) g.Expect(got).To(Equal(tt.want)) + if obj.Status.Artifact != nil { + g.Expect(obj.Status.Artifact.Size).To(Equal(artifactSize(g, obj.Status.Artifact.URL))) + } + if tt.afterFunc != nil { tt.afterFunc(g, obj) } diff --git a/controllers/helmchart_controller_test.go b/controllers/helmchart_controller_test.go index b031e9d50..154eed083 100644 --- a/controllers/helmchart_controller_test.go +++ b/controllers/helmchart_controller_test.go @@ -198,6 +198,7 @@ func TestHelmChartReconciler_reconcileStorage(t *testing.T) { Revision: "c", Checksum: "2e7d2c03a9507ae265ecf5b5356885a53393a2029d241394997265a1a25aefc6", URL: testStorage.Hostname + "/reconcile-storage/c.txt", + Size: int64p(int64(len("c"))), }, assertPaths: []string{ "/reconcile-storage/c.txt", @@ -250,6 +251,7 @@ func TestHelmChartReconciler_reconcileStorage(t *testing.T) { Revision: "f", Checksum: "3b9c358f36f0a31b6ad3e14f309c7cf198ac9246e8316f9ce543d5b19ac02b80", URL: testStorage.Hostname + "/reconcile-storage/hostname.txt", + Size: int64p(int64(len("file"))), }, }, } diff --git a/controllers/helmrepository_controller_test.go b/controllers/helmrepository_controller_test.go index a337d04bb..c1b08bcb3 100644 --- a/controllers/helmrepository_controller_test.go +++ b/controllers/helmrepository_controller_test.go @@ -166,6 +166,7 @@ func TestHelmRepositoryReconciler_reconcileStorage(t *testing.T) { Revision: "c", Checksum: "2e7d2c03a9507ae265ecf5b5356885a53393a2029d241394997265a1a25aefc6", URL: testStorage.Hostname + "/reconcile-storage/c.txt", + Size: int64p(int64(len("c"))), }, assertPaths: []string{ "/reconcile-storage/c.txt", @@ -218,6 +219,7 @@ func TestHelmRepositoryReconciler_reconcileStorage(t *testing.T) { Revision: "f", Checksum: "3b9c358f36f0a31b6ad3e14f309c7cf198ac9246e8316f9ce543d5b19ac02b80", URL: testStorage.Hostname + "/reconcile-storage/hostname.txt", + Size: int64p(int64(len("file"))), }, }, } diff --git a/controllers/storage.go b/controllers/storage.go index 0e9e5fe8b..55f9a077c 100644 --- a/controllers/storage.go +++ b/controllers/storage.go @@ -194,7 +194,8 @@ func (s *Storage) Archive(artifact *sourcev1.Artifact, dir string, filter Archiv }() h := newHash() - mw := io.MultiWriter(h, tf) + sz := &writeCounter{} + mw := io.MultiWriter(h, tf, sz) gw := gzip.NewWriter(mw) tw := tar.NewWriter(gw) @@ -286,6 +287,8 @@ func (s *Storage) Archive(artifact *sourcev1.Artifact, dir string, filter Archiv artifact.Checksum = fmt.Sprintf("%x", h.Sum(nil)) artifact.LastUpdateTime = metav1.Now() + artifact.Size = &sz.written + return nil } @@ -305,7 +308,8 @@ func (s *Storage) AtomicWriteFile(artifact *sourcev1.Artifact, reader io.Reader, }() h := newHash() - mw := io.MultiWriter(h, tf) + sz := &writeCounter{} + mw := io.MultiWriter(h, tf, sz) if _, err := io.Copy(mw, reader); err != nil { tf.Close() @@ -325,6 +329,8 @@ func (s *Storage) AtomicWriteFile(artifact *sourcev1.Artifact, reader io.Reader, artifact.Checksum = fmt.Sprintf("%x", h.Sum(nil)) artifact.LastUpdateTime = metav1.Now() + artifact.Size = &sz.written + return nil } @@ -344,7 +350,8 @@ func (s *Storage) Copy(artifact *sourcev1.Artifact, reader io.Reader) (err error }() h := newHash() - mw := io.MultiWriter(h, tf) + sz := &writeCounter{} + mw := io.MultiWriter(h, tf, sz) if _, err := io.Copy(mw, reader); err != nil { tf.Close() @@ -360,6 +367,8 @@ func (s *Storage) Copy(artifact *sourcev1.Artifact, reader io.Reader) (err error artifact.Checksum = fmt.Sprintf("%x", h.Sum(nil)) artifact.LastUpdateTime = metav1.Now() + artifact.Size = &sz.written + return nil } @@ -471,3 +480,15 @@ func (s *Storage) LocalPath(artifact sourcev1.Artifact) string { func newHash() hash.Hash { return sha256.New() } + +// writecounter is an implementation of io.Writer that only records the number +// of bytes written. +type writeCounter struct { + written int64 +} + +func (wc *writeCounter) Write(p []byte) (int, error) { + n := len(p) + wc.written += int64(n) + return n, nil +} diff --git a/controllers/suite_test.go b/controllers/suite_test.go index b4a6ca69d..a585eeddd 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -193,3 +193,7 @@ func randStringRunes(n int) string { } return string(b) } + +func int64p(i int64) *int64 { + return &i +} diff --git a/docs/api/source.md b/docs/api/source.md index 91ac4e946..83392ee9b 100644 --- a/docs/api/source.md +++ b/docs/api/source.md @@ -931,6 +931,18 @@ Kubernetes meta/v1.Time artifact.

+ + +size
+ +int64 + + + +(Optional) +

Size is the number of bytes in the file.

+ + @@ -1568,8 +1580,8 @@ Artifact includedArtifacts
- -[]*github.com/fluxcd/source-controller/api/v1beta2.Artifact + +[]*./api/v1beta2.Artifact