feat: add Arch Linux Tracker

arch-linux/archlinux.go

@@ -0,0 +1,115 @@
+package arch
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+
+	"github.com/aquasecurity/vuln-list-update/utils"
+
+	"github.com/cheggaaa/pb/v3"
+	"golang.org/x/xerrors"
+)
+
+const (
+	archLinuxDir       = "arch-linux"
+	securityTrackerURL = "https://security.archlinux.org/json"
+	retry              = 3
+)
+
+type securityGroups []struct {
+	Name       string   `json:"name"`
+	Packages   []string `json:"packages"`
+	Status     string   `json:"status"`
+	Severity   string   `json:"severity"`
+	Type       string   `json:"type"`
+	Affected   string   `json:"affected"`
+	Fixed      string   `json:"fixed"`
+	Issues     []string `json:"issues"`
+	Advisories []string `json:"advisories"`
+}
+
+type options struct {
+	url   string
+	dir   string
+	retry int
+}
+
+type option func(*options)
+
+func WithURL(url string) option {
+	return func(opts *options) { opts.url = url }
+}
+
+func WithDir(dir string) option {
+	return func(opts *options) { opts.dir = dir }
+}
+
+func WithRetry(retry int) option {
+	return func(opts *options) { opts.retry = retry }
+}
+
+type ArchLinux struct {
+	*options
+}
+
+func NewArchLinux(opts ...option) ArchLinux {
+	o := &options{
+		url:   securityTrackerURL,
+		dir:   filepath.Join(utils.VulnListDir(), archLinuxDir),
+		retry: retry,
+	}
+
+	for _, opt := range opts {
+		opt(o)
+	}
+
+	return ArchLinux{
+		options: o,
+	}
+}
+
+func (al ArchLinux) Update() error {
+	log.Println("Fetching Arch Linux data...")
+	asgs, err := al.retrieveSecurityGroups()
+	if err != nil {
+		return xerrors.Errorf("failed to retrieve Arch Linux Security Groups: %w", err)
+	}
+
+	log.Printf("Removing old dir (%s)...", al.dir)
+	if err = os.RemoveAll(al.dir); err != nil {
+		return xerrors.Errorf("failed to remove Arch Linux dir: %w", err)
+	}
+
+	// Save all JSON files
+	log.Println("Saving new data...")
+	bar := pb.StartNew(len(asgs))
+	if err = os.MkdirAll(al.dir, os.ModePerm); err != nil {
+		return xerrors.Errorf("failed to create the directory: %w", err)
+	}
+	for _, asg := range asgs {
+		filePath := filepath.Join(al.dir, fmt.Sprintf("%s.json", asg.Name))
+		if err = utils.Write(filePath, asg); err != nil {
+			return xerrors.Errorf("failed to write Debian CVE details: %w", err)
+		}
+		bar.Increment()
+	}
+	bar.Finish()
+
+	return nil
+}
+
+func (al ArchLinux) retrieveSecurityGroups() (securityGroups, error) {
+	secJSON, err := utils.FetchURL(al.url, "", al.retry)
+	if err != nil {
+		return nil, xerrors.Errorf("failed to fetch cve data from Arch Linux. err: %w", err)
+	}
+
+	var asgs securityGroups
+	if err = json.Unmarshal(secJSON, &asgs); err != nil {
+		return nil, xerrors.Errorf("json unmarshal error: %w", err)
+	}
+	return asgs, nil
+}

arch-linux/archlinux_test.go

@@ -0,0 +1,81 @@
+package arch_test
+
+import (
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/aquasecurity/vuln-list-update/arch-linux"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestUpdate(t *testing.T) {
+	tests := []struct {
+		name          string
+		inputJSONFile string
+		wantErr       string
+	}{
+		{
+			name:          "happy path",
+			inputJSONFile: "testdata/archlinux.json",
+		},
+		{
+			name:    "sad path, unreachable Arch Linux service",
+			wantErr: "connection refused",
+		},
+		{
+			name:          "sad path, invalid json",
+			inputJSONFile: "testdata/invalid.json",
+			wantErr:       "json unmarshal error",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				b, err := os.ReadFile(tt.inputJSONFile)
+				require.NoError(t, err)
+
+				_, _ = io.WriteString(w, string(b))
+			}))
+			defer ts.Close()
+
+			// Intentionally close to induce network errors
+			if tt.inputJSONFile == "" {
+				ts.Close()
+			}
+
+			dir := t.TempDir()
+			c := arch.NewArchLinux(arch.WithURL(ts.URL), arch.WithDir(filepath.Join(dir)), arch.WithRetry(0))
+			err := c.Update()
+			if tt.wantErr != "" {
+				require.Error(t, err)
+				assert.Contains(t, err.Error(), tt.wantErr)
+				return
+			}
+
+			entries, err := os.ReadDir(dir)
+			require.NoError(t, err)
+
+			for _, e := range entries {
+				if e.IsDir() {
+					continue
+				}
+
+				filePath := e.Name()
+				gotJSON, err := os.ReadFile(filepath.Join(dir, filePath))
+				require.NoError(t, err)
+
+				wantJSON, err := os.ReadFile(filepath.Join("testdata", "golden", filepath.Base(filePath)))
+				require.NoError(t, err)
+
+				assert.JSONEq(t, string(wantJSON), string(gotJSON))
+			}
+		})
+	}
+}

arch-linux/testdata/archlinux.json

@@ -0,0 +1,38 @@
+[
+  {
+    "name":"AVG-20",
+    "packages":[
+      "curl"
+    ],
+    "status":"Fixed",
+    "severity":"Low",
+    "type":"denial of service",
+    "affected":"7.50.2-1",
+    "fixed":"7.50.3-1",
+    "ticket":null,
+    "issues":[
+      "CVE-2016-7167"
+    ],
+    "advisories":[
+      "ASA-201609-19"
+    ]
+  },
+  {
+    "name":"AVG-4",
+    "packages":[
+      "bzip2"
+    ],
+    "status":"Fixed",
+    "severity":"Low",
+    "type":"denial of service",
+    "affected":"1.0.6-5",
+    "fixed":"1.0.6-6",
+    "ticket":null,
+    "issues":[
+      "CVE-2016-3189"
+    ],
+    "advisories":[
+      "ASA-201702-19"
+    ]
+  }
+]

arch-linux/testdata/golden/AVG-20.json

@@ -0,0 +1,17 @@
+{
+  "name":"AVG-20",
+  "packages":[
+    "curl"
+  ],
+  "status":"Fixed",
+  "severity":"Low",
+  "type":"denial of service",
+  "affected":"7.50.2-1",
+  "fixed":"7.50.3-1",
+  "issues":[
+    "CVE-2016-7167"
+  ],
+  "advisories":[
+    "ASA-201609-19"
+  ]
+}

arch-linux/testdata/golden/AVG-4.json

@@ -0,0 +1,17 @@
+{
+  "name":"AVG-4",
+  "packages":[
+    "bzip2"
+  ],
+  "status":"Fixed",
+  "severity":"Low",
+  "type":"denial of service",
+  "affected":"1.0.6-5",
+  "fixed":"1.0.6-6",
+  "issues":[
+    "CVE-2016-3189"
+  ],
+  "advisories":[
+    "ASA-201702-19"
+  ]
+}

arch-linux/testdata/invalid.json

@@ -0,0 +1 @@
+invalid json

go.mod

@@ -6,13 +6,12 @@ require (
 	github.com/PuerkitoBio/goquery v1.6.0
 	github.com/araddon/dateparse v0.0.0-20190426192744-0d74ffceef83
 	github.com/cheggaaa/pb v2.0.7+incompatible
+	github.com/cheggaaa/pb/v3 v3.0.8
 	github.com/elazarl/goproxy v0.0.0-20190421051319-9d40249d3c2f // indirect
 	github.com/elazarl/goproxy/ext v0.0.0-20190421051319-9d40249d3c2f // indirect
-	github.com/fatih/color v1.10.0 // indirect
 	github.com/kr/pretty v0.1.0 // indirect
 	github.com/kylelemons/godebug v1.1.0
 	github.com/mattn/go-jsonpointer v0.0.0-20180225143300-37667080efed
-	github.com/mattn/go-runewidth v0.0.7 // indirect
 	github.com/parnurzeal/gorequest v0.2.16
 	github.com/pkg/errors v0.8.0 // indirect
 	github.com/shurcooL/githubv4 v0.0.0-20191127044304-8f68eb5628d0

go.sum

@@ -1,12 +1,16 @@
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/PuerkitoBio/goquery v1.6.0 h1:j7taAbelrdcsOlGeMenZxc2AWXD5fieT1/znArdnx94=
 github.com/PuerkitoBio/goquery v1.6.0/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
+github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM=
+github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
 github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
 github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
 github.com/araddon/dateparse v0.0.0-20190426192744-0d74ffceef83 h1:ukTLOeMC0aVxbJWVg6hOsVJ0VPIo8w++PbNsze/pqF8=
 github.com/araddon/dateparse v0.0.0-20190426192744-0d74ffceef83/go.mod h1:SLqhdZcd+dF3TEVL2RMoob5bBP5R1P1qkox+HtCBgGI=
 github.com/cheggaaa/pb v2.0.7+incompatible h1:gLKifR1UkZ/kLkda5gC0K6c8g+jU2sINPtBeOiNlMhU=
 github.com/cheggaaa/pb v2.0.7+incompatible/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s=
+github.com/cheggaaa/pb/v3 v3.0.8 h1:bC8oemdChbke2FHIIGy9mn4DPJ2caZYQnfbRqwmdCoA=
+github.com/cheggaaa/pb/v3 v3.0.8/go.mod h1:UICbiLec/XO6Hw6k+BHEtHeQFzzBH4i2/qk/ow1EJTA=
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/elazarl/goproxy v0.0.0-20190421051319-9d40249d3c2f h1:8GDPb0tCY8LQ+OJ3dbHb5sA6YZWXFORQYZx5sdsTlMs=
@@ -34,14 +38,17 @@ github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHX
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-jsonpointer v0.0.0-20180225143300-37667080efed h1:fCWISZq4YN4ulCJx7x0KB15rqxLEe3mtNJL8cSOGKZU=
 github.com/mattn/go-jsonpointer v0.0.0-20180225143300-37667080efed/go.mod h1:SDJ4hurDYyQ9/7nc+eCYtXqdufgK4Cq9TJlwPklqEYA=
-github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54=
-github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/mattn/go-runewidth v0.0.12 h1:Y41i/hVW3Pgwr8gV+J23B9YEY0zxjptBuCWEaxmAOow=
+github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
 github.com/parnurzeal/gorequest v0.2.16 h1:T/5x+/4BT+nj+3eSknXmCTnEVGSzFzPGdpqmUVVZXHQ=
 github.com/parnurzeal/gorequest v0.2.16/go.mod h1:3Kh2QUMJoqw3icWAecsyzkpY7UzRfDhbRdTjtNwNiUE=
 github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc=
 github.com/shurcooL/githubv4 v0.0.0-20191127044304-8f68eb5628d0 h1:T9uus1QvcPgeLShS30YOnnzk3r9Vvygp45muhlrufgY=
 github.com/shurcooL/githubv4 v0.0.0-20191127044304-8f68eb5628d0/go.mod h1:hAF0iLZy4td2EX+/8Tw+4nodhlMrwN3HupfaXj3zkGo=
@@ -55,16 +62,13 @@ github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2
 github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
-github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -73,11 +77,10 @@ golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4Iltr
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20200116001909-b77594299b42 h1:vEOn+mP2zCOVzKckCZy6YsCtDblrpj/w7B9nxGNELpg=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57 h1:F5Gozwx4I1xtr/sr/8CFbb57iKi3297KFs0QDbGN60A=
+golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=

main.go

@@ -16,6 +16,7 @@ import (
 
 	"github.com/aquasecurity/vuln-list-update/alpine"
 	"github.com/aquasecurity/vuln-list-update/amazon"
+	arch_linux "github.com/aquasecurity/vuln-list-update/arch-linux"
 	"github.com/aquasecurity/vuln-list-update/cwe"
 	debianoval "github.com/aquasecurity/vuln-list-update/debian/oval"
 	"github.com/aquasecurity/vuln-list-update/debian/tracker"
@@ -40,7 +41,7 @@ const (
 
 var (
 	target = flag.String("target", "", "update target (nvd, alpine, redhat, redhat-oval, "+
-		"debian, debian-oval, ubuntu, amazon, oracle-oval, suse-cvrf, photon, ghsa, glad, cwe)")
+		"debian, debian-oval, ubuntu, amazon, oracle-oval, suse-cvrf, photon, arch-linux, ghsa, glad, cwe)")
 	years = flag.String("years", "", "update years (only redhat)")
 )
 
@@ -178,6 +179,12 @@ func run() error {
 			return xerrors.Errorf("error in CWE update: %w", err)
 		}
 		commitMsg = "CWE Advisories"
+	case "arch-linux":
+		al := arch_linux.NewArchLinux()
+		if err := al.Update(); err != nil {
+			return xerrors.Errorf("error in CWE update: %w", err)
+		}
+		commitMsg = "Arch Linux Security Tracker"
 	default:
 		return xerrors.New("unknown target")
 	}