feat(fetch/redhat): fetch including-unpatched (#388)

commands/fetch-redhat.go

@@ -85,7 +85,7 @@ func fetchRedHat(_ *cobra.Command, args []string) (err error) {
 		}
 
 		roots := make([]redhat.Root, 0, len(m))
-		for _, k := range []string{fmt.Sprintf("rhel-%s.oval.xml.bz2", v), fmt.Sprintf("rhel-%s-extras.oval.xml.bz2", v), fmt.Sprintf("rhel-%s-supplementary.oval.xml.bz2", v), fmt.Sprintf("rhel-%s-els.oval.xml.bz2", v), fmt.Sprintf("com.redhat.rhsa-RHEL%s.xml", v), fmt.Sprintf("com.redhat.rhsa-RHEL%s-ELS.xml", v)} {
+		for _, k := range []string{fmt.Sprintf("rhel-%s-including-unpatched.oval.xml.bz2", v), fmt.Sprintf("rhel-%s-extras-including-unpatched.oval.xml.bz2", v), fmt.Sprintf("rhel-%s-supplementary.oval.xml.bz2", v), fmt.Sprintf("rhel-%s-els.oval.xml.bz2", v), fmt.Sprintf("com.redhat.rhsa-RHEL%s.xml", v), fmt.Sprintf("com.redhat.rhsa-RHEL%s-ELS.xml", v)} {
 			roots = append(roots, m[k])
 		}
 

db/db.go

@@ -154,8 +154,8 @@ func chunkSlice(length int, chunkSize int) <-chan IndexChunk {
 
 func filterByRedHatMajor(packs []models.Package, majorVer string) (filtered []models.Package) {
 	for _, p := range packs {
-		if strings.Contains(p.Version, ".el"+majorVer) ||
-			strings.Contains(p.Version, ".module+el"+majorVer) {
+		if p.NotFixedYet ||
+			strings.Contains(p.Version, ".el"+majorVer) || strings.Contains(p.Version, ".module+el"+majorVer) {
 			filtered = append(filtered, p)
 		}
 	}

db/db_test.go

@@ -373,6 +373,35 @@ func Test_filterByRedHatMajor(t *testing.T) {
 				},
 			},
 		},
+		{
+			in: args{
+				packs: []models.Package{
+					{
+						Name:    "name-el7",
+						Version: "0:0.0.1-0.0.1.el7",
+					},
+					{
+						Name:    "name-el8",
+						Version: "0:0.0.1-0.0.1.el8",
+					},
+					{
+						Name:        "notfixedyet",
+						NotFixedYet: true,
+					},
+				},
+				majorVer: "8",
+			},
+			expected: []models.Package{
+				{
+					Name:    "name-el8",
+					Version: "0:0.0.1-0.0.1.el8",
+				},
+				{
+					Name:        "notfixedyet",
+					NotFixedYet: true,
+				},
+			},
+		},
 	}
 
 	for i, tt := range tests {

db/rdb.go

@@ -132,6 +132,8 @@ func (r *RDBDriver) MigrateDB() error {
 		&models.Advisory{},
 		&models.Cve{},
 		&models.Bugzilla{},
+		&models.Resolution{},
+		&models.Component{},
 		&models.Cpe{},
 		&models.Debian{},
 	); err != nil {
@@ -156,9 +158,7 @@ func (r *RDBDriver) MigrateDB() error {
 				}
 			}
 		case dialectMysql, dialectPostgreSQL:
-			if err != nil {
-				return xerrors.Errorf("Failed to migrate. err: %w", err)
-			}
+			return xerrors.Errorf("Failed to migrate. err: %w", err)
 		default:
 			return xerrors.Errorf("Not Supported DB dialects. r.name: %s", r.name)
 		}
@@ -196,6 +196,8 @@ func (r *RDBDriver) GetByPackName(family, osVer, packName, arch string) ([]model
 		Preload("Advisory").
 		Preload("Advisory.Cves").
 		Preload("Advisory.Bugzillas").
+		Preload("Advisory.AffectedResolution").
+		Preload("Advisory.AffectedResolution.Components").
 		Preload("Advisory.AffectedCPEList").
 		Preload("References")
 
@@ -253,6 +255,8 @@ func (r *RDBDriver) GetByCveID(family, osVer, cveID, arch string) ([]models.Defi
 		Preload("Advisory").
 		Preload("Advisory.Cves").
 		Preload("Advisory.Bugzillas").
+		Preload("Advisory.AffectedResolution").
+		Preload("Advisory.AffectedResolution.Components").
 		Preload("Advisory.AffectedCPEList").
 		Preload("References")
 

fetcher/redhat/redhat.go

@@ -42,7 +42,7 @@ func FetchFiles(versions []string) (map[string][]util.FetchResult, error) {
 			}
 			results[v] = rs
 
-			rs, err = fetchOVALv2([]string{v, fmt.Sprintf("%s-extras", v), fmt.Sprintf("%s-supplementary", v), fmt.Sprintf("%s-els", v)})
+			rs, err = fetchOVALv2([]string{fmt.Sprintf("%s-including-unpatched", v), fmt.Sprintf("%s-extras-including-unpatched", v), fmt.Sprintf("%s-supplementary", v), fmt.Sprintf("%s-els", v)})
 			if err != nil {
 				return nil, xerrors.Errorf("Failed to fetch OVALv2. err: %w", err)
 			}
@@ -54,7 +54,7 @@ func FetchFiles(versions []string) (map[string][]util.FetchResult, error) {
 			}
 			results[v] = rs
 
-			rs, err = fetchOVALv2([]string{v, fmt.Sprintf("%s-extras", v), fmt.Sprintf("%s-supplementary", v)})
+			rs, err = fetchOVALv2([]string{fmt.Sprintf("%s-including-unpatched", v), fmt.Sprintf("%s-extras-including-unpatched", v), fmt.Sprintf("%s-supplementary", v)})
 			if err != nil {
 				return nil, xerrors.Errorf("Failed to fetch OVALv2. err: %w", err)
 			}
@@ -66,7 +66,7 @@ func FetchFiles(versions []string) (map[string][]util.FetchResult, error) {
 			}
 			results[v] = rs
 
-			rs, err = fetchOVALv2([]string{v})
+			rs, err = fetchOVALv2([]string{fmt.Sprintf("%s-including-unpatched", v)})
 			if err != nil {
 				return nil, xerrors.Errorf("Failed to fetch OVALv2. err: %w", err)
 			}
@@ -77,7 +77,7 @@ func FetchFiles(versions []string) (map[string][]util.FetchResult, error) {
 				break
 			}
 
-			rs, err := fetchOVALv2([]string{v})
+			rs, err := fetchOVALv2([]string{fmt.Sprintf("%s-including-unpatched", v)})
 			if err != nil {
 				return nil, xerrors.Errorf("Failed to fetch OVALv2. err: %w", err)
 			}

models/models.go

@@ -53,7 +53,7 @@ type Package struct {
 	Name            string `gorm:"index:idx_packages_name"` // If the type:text, varchar(255) is specified, MySQL overflows and gives an error. No problem in GORMv2. (https://github.com/go-gorm/mysql/tree/15e2cbc6fd072be99215a82292e025dab25e2e16#configuration)
 	Version         string `gorm:"type:varchar(255)"`       // affected earlier than this version
 	Arch            string `gorm:"type:varchar(255)"`       // Used for Amazon Linux, Oracle Linux and Fedora
-	NotFixedYet     bool   // Ubuntu Only
+	NotFixedYet     bool   // Used for RedHat, Ubuntu
 	ModularityLabel string `gorm:"type:varchar(255)"` // RHEL 8 or later only
 }
 
@@ -75,6 +75,7 @@ type Advisory struct {
 	Severity           string `gorm:"type:varchar(255)"`
 	Cves               []Cve
 	Bugzillas          []Bugzilla
+	AffectedResolution []Resolution
 	AffectedCPEList    []Cpe
 	AffectedRepository string `gorm:"type:varchar(255)"` // Amazon Linux 2 Only
 	Issued             time.Time
@@ -105,6 +106,23 @@ type Bugzilla struct {
 	Title      string `gorm:"type:varchar(255)"`
 }
 
+// Resolution : >definitions>definition>metadata>advisory>affected>resolution
+type Resolution struct {
+	ID         uint `gorm:"primary_key" json:"-"`
+	AdvisoryID uint `gorm:"index:idx_resolution_advisory_id" json:"-" xml:"-"`
+
+	State      string `gorm:"type:varchar(255)"`
+	Components []Component
+}
+
+// Component : >definitions>definition>metadata>advisory>affected>resolution>component
+type Component struct {
+	ID           uint `gorm:"primary_key" json:"-"`
+	ResolutionID uint `gorm:"index:idx_component_resolution_id" json:"-" xml:"-"`
+
+	Component string `gorm:"type:varchar(255)"`
+}
+
 // Cpe : >definitions>definition>metadata>advisory>affected_cpe_list
 type Cpe struct {
 	ID         uint `gorm:"primary_key" json:"-"`

models/redhat/redhat.go

@@ -18,11 +18,11 @@ func ConvertToModel(v string, roots []Root) []models.Definition {
 	defs := map[string]models.Definition{}
 	for _, root := range roots {
 		for _, d := range root.Definitions.Definitions {
-			if strings.Contains(d.Description, "** REJECT **") {
+			if strings.HasPrefix(d.ID, "oval:com.redhat.unaffected:def:") || strings.Contains(d.Description, "** REJECT **") {
 				continue
 			}
 
-			cves := []models.Cve{}
+			var cves = make([]models.Cve, 0, len(d.Advisory.Cves))
 			for _, c := range d.Advisory.Cves {
 				cves = append(cves, models.Cve{
 					CveID:  c.CveID,
@@ -35,7 +35,7 @@ func ConvertToModel(v string, roots []Root) []models.Definition {
 				})
 			}
 
-			rs := []models.Reference{}
+			var rs = make([]models.Reference, 0, len(d.References))
 			for _, r := range d.References {
 				rs = append(rs, models.Reference{
 					Source: r.Source,
@@ -44,14 +44,14 @@ func ConvertToModel(v string, roots []Root) []models.Definition {
 				})
 			}
 
-			cl := []models.Cpe{}
+			var cpes = make([]models.Cpe, 0, len(d.Advisory.AffectedCPEList))
 			for _, cpe := range d.Advisory.AffectedCPEList {
-				cl = append(cl, models.Cpe{
+				cpes = append(cpes, models.Cpe{
 					Cpe: cpe,
 				})
 			}
 
-			bs := []models.Bugzilla{}
+			var bs = make([]models.Bugzilla, 0, len(d.Advisory.Bugzillas))
 			for _, b := range d.Advisory.Bugzillas {
 				bs = append(bs, models.Bugzilla{
 					BugzillaID: b.ID,
@@ -60,6 +60,22 @@ func ConvertToModel(v string, roots []Root) []models.Definition {
 				})
 			}
 
+			var ress = make([]models.Resolution, 0, len(d.Advisory.Affected.Resolution))
+			for _, r := range d.Advisory.Affected.Resolution {
+				ress = append(ress, models.Resolution{
+					State: r.State,
+					Components: func() []models.Component {
+						var comps = make([]models.Component, 0, len(r.Component))
+						for _, c := range r.Component {
+							comps = append(comps, models.Component{
+								Component: c,
+							})
+						}
+						return comps
+					}(),
+				})
+			}
+
 			issued := util.ParsedOrDefaultTime([]string{"2006-01-02"}, d.Advisory.Issued.Date)
 			updated := util.ParsedOrDefaultTime([]string{"2006-01-02"}, d.Advisory.Updated.Date)
 
@@ -68,14 +84,14 @@ func ConvertToModel(v string, roots []Root) []models.Definition {
 				Title:        d.Title,
 				Description:  d.Description,
 				Advisory: models.Advisory{
-					Severity:        d.Advisory.Severity,
-					Cves:            cves,
-					Bugzillas:       bs,
-					AffectedCPEList: cl,
-					Issued:          issued,
-					Updated:         updated,
+					Severity:           d.Advisory.Severity,
+					Cves:               cves,
+					Bugzillas:          bs,
+					AffectedResolution: ress,
+					AffectedCPEList:    cpes,
+					Issued:             issued,
+					Updated:            updated,
 				},
-				Debian:        nil,
 				AffectedPacks: collectRedHatPacks(v, d.Criteria),
 				References:    rs,
 			}
@@ -100,19 +116,23 @@ func ConvertToModel(v string, roots []Root) []models.Definition {
 }
 
 func collectRedHatPacks(v string, cri Criteria) []models.Package {
-	ps := walkRedHat(cri, []models.Package{}, "")
 	pkgs := map[string]models.Package{}
-	for _, p := range ps {
-		// OVALv1 includes definitions other than the target RHEL version
-		if !strings.Contains(p.Version, ".el"+v) && !strings.Contains(p.Version, ".module+el"+v) {
-			continue
-		}
-
+	for _, p := range walkRedHat(cri, []models.Package{}, "") {
 		n := p.Name
 		if p.ModularityLabel != "" {
 			n = fmt.Sprintf("%s::%s", p.ModularityLabel, p.Name)
 		}
 
+		if p.NotFixedYet {
+			pkgs[n] = p
+			continue
+		}
+
+		// OVALv1 includes definitions other than the target RHEL version
+		if !strings.Contains(p.Version, ".el"+v) && !strings.Contains(p.Version, ".module+el"+v) {
+			continue
+		}
+
 		// since different versions are defined for the same package, the newer version is adopted
 		// example: OVALv2: oval:com.redhat.rhsa:def:20111349, oval:com.redhat.rhsa:def:20120451
 		if base, ok := pkgs[n]; ok {
@@ -130,19 +150,26 @@ func collectRedHatPacks(v string, cri Criteria) []models.Package {
 
 func walkRedHat(cri Criteria, acc []models.Package, label string) []models.Package {
 	for _, c := range cri.Criterions {
-		if strings.HasPrefix(c.Comment, "Module ") && strings.HasSuffix(c.Comment, " is enabled") {
+		switch {
+		case strings.HasPrefix(c.Comment, "Module ") && strings.HasSuffix(c.Comment, " is enabled"):
 			label = strings.TrimSuffix(strings.TrimPrefix(c.Comment, "Module "), " is enabled")
+		case strings.Contains(c.Comment, " is earlier than "):
+			ss := strings.Split(c.Comment, " is earlier than ")
+			if len(ss) != 2 {
+				continue
+			}
+			acc = append(acc, models.Package{
+				Name:            ss[0],
+				Version:         strings.Split(ss[1], " ")[0],
+				ModularityLabel: label,
+			})
+		case !strings.HasPrefix(c.Comment, "Red Hat Enterprise Linux") && !strings.HasPrefix(c.Comment, "Red Hat CoreOS") && strings.HasSuffix(c.Comment, " is installed"):
+			acc = append(acc, models.Package{
+				Name:            strings.TrimSuffix(c.Comment, " is installed"),
+				ModularityLabel: label,
+				NotFixedYet:     true,
+			})
 		}
-
-		ss := strings.Split(c.Comment, " is earlier than ")
-		if len(ss) != 2 {
-			continue
-		}
-		acc = append(acc, models.Package{
-			Name:            ss[0],
-			Version:         strings.Split(ss[1], " ")[0],
-			ModularityLabel: label,
-		})
 	}
 
 	if len(cri.Criterias) == 0 {

models/redhat/redhat_test.go

@@ -253,6 +253,48 @@ func TestWalkRedHat(t *testing.T) {
 				},
 			},
 		},
+		{
+			version: "8",
+			cri: Criteria{
+				Criterias: []Criteria{
+					{
+						Criterias: []Criteria{
+							{
+								Criterias: []Criteria{
+									{
+										Criterias: []Criteria{
+											{
+												Criterions: []Criterion{
+													{Comment: "python2 is installed"},
+													{Comment: "python2 is signed with Red Hat redhatrelease2 key"},
+												},
+											},
+										},
+									},
+								},
+								Criterions: []Criterion{
+									{Comment: "Module inkscape:flatpak is enabled"},
+								},
+							},
+						},
+						Criterions: []Criterion{
+							{Comment: "Red Hat Enterprise Linux 8 is installed"},
+							{Comment: "Red Hat CoreOS 4 is installed"},
+						},
+					},
+				},
+				Criterions: []Criterion{
+					{Comment: "Red Hat Enterprise Linux must be installed"},
+				},
+			},
+			expected: []models.Package{
+				{
+					Name:            "python2",
+					ModularityLabel: "inkscape:flatpak",
+					NotFixedYet:     true,
+				},
+			},
+		},
 	}
 
 	for i, tt := range tests {

models/redhat/types.go

@@ -110,7 +110,7 @@ type Bugzilla struct {
 
 // AffectedPkgs : >definitions>definition>metadata>advisory>affected
 type AffectedPkgs struct {
-	Resolution struct {
+	Resolution []struct {
 		State     string   `xml:"state,attr"`
 		Component []string `xml:"component"`
 	} `xml:"resolution"`