Skip to content

Commit

Permalink
[wip] migrate DB definitions from grype
Browse files Browse the repository at this point in the history
Signed-off-by: Alex Goodman <[email protected]>
  • Loading branch information
wagoodman committed Jan 19, 2024
1 parent 1a04db7 commit 5731d0e
Show file tree
Hide file tree
Showing 219 changed files with 17,374 additions and 253 deletions.
2 changes: 1 addition & 1 deletion cmd/grype-db/application/build_info.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import (
"runtime"
"runtime/debug"

grypeDB "github.com/anchore/grype/grype/db/v3"
grypeDB "github.com/anchore/grype-db/pkg/db/v3"
)

const valueNotProvided = "[not provided]"
Expand Down
8 changes: 4 additions & 4 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -9,19 +9,23 @@ require (
github.com/adrg/xdg v0.4.0
github.com/anchore/go-logger v0.0.0-20230725134548-c21dafa1ec5a
github.com/anchore/grype v0.74.0
github.com/anchore/packageurl-go v0.1.1-0.20230104203445-02e0a6721501
github.com/anchore/syft v0.100.0
github.com/dustin/go-humanize v1.0.1
github.com/glebarez/sqlite v1.10.0
github.com/go-test/deep v1.1.0
github.com/google/go-cmp v0.6.0
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
github.com/google/uuid v1.5.0
github.com/gookit/color v1.5.4
github.com/hako/durafmt v0.0.0-20210608085754-5c1018a4e16b
github.com/hashicorp/go-cleanhttp v0.5.2
github.com/hashicorp/go-getter v1.7.3
github.com/hashicorp/go-multierror v1.1.1
github.com/iancoleman/strcase v0.3.0
github.com/jinzhu/copier v0.4.0
github.com/klauspost/compress v1.17.4
github.com/mholt/archiver/v3 v3.5.1
github.com/mitchellh/go-homedir v1.1.0
github.com/mitchellh/mapstructure v1.5.0
github.com/pkg/profile v1.7.0
Expand Down Expand Up @@ -62,7 +66,6 @@ require (
github.com/anchore/go-macholibre v0.0.0-20220308212642-53e6d0aaf6fb // indirect
github.com/anchore/go-struct-converter v0.0.0-20221118182256-c68fdcfa2092 // indirect
github.com/anchore/go-version v1.2.2-0.20210903204242-51efa5b487c4 // indirect
github.com/anchore/packageurl-go v0.1.1-0.20230104203445-02e0a6721501 // indirect
github.com/anchore/stereoscope v0.0.0-20231220161148-590920dabc54 // indirect
github.com/andybalholm/brotli v1.0.4 // indirect
github.com/aquasecurity/go-pep440-version v0.0.0-20210121094942-22b2f8951d46 // indirect
Expand Down Expand Up @@ -116,10 +119,8 @@ require (
github.com/google/licensecheck v0.3.1 // indirect
github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 // indirect
github.com/google/s2a-go v0.1.7 // indirect
github.com/google/uuid v1.5.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
github.com/googleapis/gax-go/v2 v2.12.0 // indirect
github.com/hako/durafmt v0.0.0-20210608085754-5c1018a4e16b // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-safetemp v1.0.0 // indirect
github.com/hashicorp/go-version v1.6.0 // indirect
Expand All @@ -140,7 +141,6 @@ require (
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.18 // indirect
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
github.com/mholt/archiver/v3 v3.5.1 // indirect
github.com/microsoft/go-rustaudit v0.0.0-20220730194248-4b17361d90a5 // indirect
github.com/mitchellh/go-testing-interface v1.14.1 // indirect
github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect
Expand Down
68 changes: 68 additions & 0 deletions internal/file/copy.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
package file

import (
"fmt"
"io"
"os"
"path"

"github.com/spf13/afero"
)

func CopyDir(fs afero.Fs, src string, dst string) error {
var err error
var fds []os.DirEntry
var srcinfo os.FileInfo

if srcinfo, err = fs.Stat(src); err != nil {
return err
}

if err = fs.MkdirAll(dst, srcinfo.Mode()); err != nil {
return err
}

if fds, err = os.ReadDir(src); err != nil {
return err
}
for _, fd := range fds {
srcPath := path.Join(src, fd.Name())
dstPath := path.Join(dst, fd.Name())

if fd.IsDir() {
if err = CopyDir(fs, srcPath, dstPath); err != nil {
return fmt.Errorf("could not copy dir (%s -> %s): %w", srcPath, dstPath, err)
}
} else {
if err = CopyFile(fs, srcPath, dstPath); err != nil {
return fmt.Errorf("could not copy file (%s -> %s): %w", srcPath, dstPath, err)
}
}
}
return nil
}

func CopyFile(fs afero.Fs, src, dst string) error {
var err error
var srcFd afero.File
var dstFd afero.File
var srcinfo os.FileInfo

if srcFd, err = fs.Open(src); err != nil {
return err
}
defer srcFd.Close()

if dstFd, err = fs.Create(dst); err != nil {
return err
}
defer dstFd.Close()

if _, err = io.Copy(dstFd, srcFd); err != nil {
return err
}
if srcinfo, err = fs.Stat(src); err != nil {
return err
}
return fs.Chmod(dst, srcinfo.Mode())
}
9 changes: 6 additions & 3 deletions internal/file/exists.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,13 @@ import (
"github.com/spf13/afero"
)

func Exists(fs afero.Fs, path string) bool {
func Exists(fs afero.Fs, path string) (bool, error) {
info, err := fs.Stat(path)
if os.IsNotExist(err) {
return false
return false, nil
} else if err != nil {
return false, err
}
return !info.IsDir()

return !info.IsDir(), nil
}
130 changes: 10 additions & 120 deletions internal/file/getter.go
Original file line number Diff line number Diff line change
@@ -1,23 +1,15 @@
package file

import (
"crypto/tls"
"crypto/x509"
"fmt"
"io"
"io/fs"
"math"
"net/http"
"strings"
"time"

"github.com/hashicorp/go-cleanhttp"
"github.com/hashicorp/go-getter"
"github.com/hashicorp/go-getter/helper/url"
"github.com/wagoodman/go-progress"

"github.com/anchore/go-logger"
"github.com/anchore/grype-db/internal/log"
"github.com/anchore/grype-db/internal/stringutil"
)

var (
Expand All @@ -34,52 +26,29 @@ type Getter interface {
GetToDir(dst, src string, monitor ...*progress.Manual) error
}

type hashiGoGetter struct {
type HashiGoGetter struct {
httpGetter getter.HttpGetter
}

// NewGetter creates and returns a new Getter. Providing an http.Client is optional. If one is provided,
// it will be used for all HTTP(S) getting; otherwise, go-getter's default getters will be used.
func NewGetter(httpClient *http.Client) Getter {
return &hashiGoGetter{
func NewGetter(httpClient *http.Client) *HashiGoGetter {
return &HashiGoGetter{
httpGetter: getter.HttpGetter{
Client: httpClient,
},
}
}

func NewDefaultGetter() Getter {
return NewGetter(cleanhttp.DefaultClient())
}

func HTTPClientWithCerts(fileSystem fs.FS, caCertPath string) (*http.Client, error) {
httpClient := cleanhttp.DefaultClient()
if caCertPath != "" {
rootCAs := x509.NewCertPool()

pemBytes, err := fs.ReadFile(fileSystem, caCertPath)
if err != nil {
return nil, fmt.Errorf("unable to configure root CAs for curator: %w", err)
}
rootCAs.AppendCertsFromPEM(pemBytes)

httpClient.Transport.(*http.Transport).TLSClientConfig = &tls.Config{
MinVersion: tls.VersionTLS12,
RootCAs: rootCAs,
}
}
return httpClient, nil
}

func (g hashiGoGetter) GetFile(dst, src string, monitors ...*progress.Manual) error {
func (g HashiGoGetter) GetFile(dst, src string, monitors ...*progress.Manual) error {
if len(monitors) > 1 {
return fmt.Errorf("multiple monitors provided, which is not allowed")
}

return getWithRetry(getterClient(dst, src, false, g.httpGetter, monitors))
return getterClient(dst, src, false, g.httpGetter, monitors).Get()
}

func (g hashiGoGetter) GetToDir(dst, src string, monitors ...*progress.Manual) error {
func (g HashiGoGetter) GetToDir(dst, src string, monitors ...*progress.Manual) error {
// though there are multiple getters, only the http/https getter requires extra validation
if err := validateHTTPSource(src); err != nil {
return err
Expand All @@ -88,69 +57,12 @@ func (g hashiGoGetter) GetToDir(dst, src string, monitors ...*progress.Manual) e
return fmt.Errorf("multiple monitors provided, which is not allowed")
}

return getWithRetry(getterClient(dst, src, true, g.httpGetter, monitors))
}

func getWithRetry(client *getter.Client) error {
var err error
attempt := 1
for interval := range retryIntervals() {
fields := logger.Fields{
"url": client.Src,
"to": client.Dst,
}

if attempt > 1 {
fields["attempt"] = attempt
}

log.WithFields(fields).Info("downloading file")

err = client.Get()
if err == nil {
break
}

time.Sleep(interval)
attempt++
}
return err
}

func retryIntervals() <-chan time.Duration {
return exponentialBackoffDurations(250*time.Millisecond, 5*time.Second, 2)
}

func exponentialBackoffDurations(minDuration, maxDuration time.Duration, step float64) <-chan time.Duration {
sleepDurations := make(chan time.Duration)
go func() {
defer close(sleepDurations)
for attempt := 0; ; attempt++ {
duration := exponentialBackoffDuration(minDuration, maxDuration, step, attempt)

sleepDurations <- duration

if duration == maxDuration {
break
}
}
}()
return sleepDurations
}

func exponentialBackoffDuration(minDuration, maxDuration time.Duration, step float64, attempt int) time.Duration {
duration := time.Duration(float64(minDuration) * math.Pow(step, float64(attempt)))
if duration < minDuration {
return minDuration
} else if duration > maxDuration {
return maxDuration
}
return duration
return getterClient(dst, src, true, g.httpGetter, monitors).Get()
}

func validateHTTPSource(src string) error {
// we are ignoring any sources that are not destined to use the http getter object
if !hasAnyOfPrefixes(src, "http://", "https://") {
if !stringutil.HasAnyOfPrefixes(src, "http://", "https://") {
return nil
}

Expand All @@ -159,7 +71,7 @@ func validateHTTPSource(src string) error {
return fmt.Errorf("bad URL provided %q: %w", src, err)
}
// only allow for sources with archive extensions
if !hasAnyOfSuffixes(u.Path, archiveExtensions...) {
if !stringutil.HasAnyOfSuffixes(u.Path, archiveExtensions...) {
return ErrNonArchiveSource
}
return nil
Expand Down Expand Up @@ -230,25 +142,3 @@ func getterDecompressorNames() (names []string) {
}
return names
}

// hasAnyOfSuffixes returns an indication if the given string has any of the given suffixes.
func hasAnyOfSuffixes(input string, suffixes ...string) bool {
for _, suffix := range suffixes {
if strings.HasSuffix(input, suffix) {
return true
}
}

return false
}

// hasAnyOfPrefixes returns an indication if the given string has any of the given prefixes.
func hasAnyOfPrefixes(input string, prefixes ...string) bool {
for _, prefix := range prefixes {
if strings.HasPrefix(input, prefix) {
return true
}
}

return false
}
Loading

0 comments on commit 5731d0e

Please sign in to comment.