Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
    Cache eviction was trying to evict the records until the new blob fits w/o checking if any records are aactually cached.

    This could happen when a record does not fit into cache:

```
    dvl@dvl-mbp cpe2cve % echo "cpe:/a:foo:bar:*" | go run . -cpe 1 -e 1 -cve 1 -cache_size 1 ~/feeds/cve_20200424000000_nvd.json.gz
    panic: attempted to evict non-existent record

    goroutine 14 [running]:
    github.com/facebookincubator/nvdtools/cvefeed.(*Cache).evict(0xc05df0b9f0, 0x0)
            /Users/dvl/go/src/github.com/facebookincubator/nvdtools/cvefeed/cvecache.go:238 +0x140
    github.com/facebookincubator/nvdtools/cvefeed.(*Cache).Get(0xc05df0b9f0, 0xc00008e008, 0x1, 0x1, 0x1, 0xc07bb2bd70, 0x10)
            /Users/dvl/go/src/github.com/facebookincubator/nvdtools/cvefeed/cvecache.go:171 +0x425
    main.processAll(0xc07bb45b60, 0xc07bb45bc0, 0xc05e3e20f0, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
            /Users/dvl/go/src/github.com/facebookincubator/nvdtools/cmd/cpe2cve/cpe2cve.go:73 +0xd6a
    main.processInput.func1(0xc07bb45b60, 0xc07bb45bc0, 0xc05e3e20f0, 0xc0001340f0, 0xc07bb2bd38, 0xc07bb2bd40)
            /Users/dvl/go/src/github.com/facebookincubator/nvdtools/cmd/cpe2cve/cpe2cve.go:139 +0xcc
    created by main.processInput
            /Users/dvl/go/src/github.com/facebookincubator/nvdtools/cmd/cpe2cve/cpe2cve.go:138 +0x309
    exit status 2
```

    After the fix:
```
    dvl@dvl-mbp cpe2cve % echo "cpe:/a:foo:bar:*" | go run . -cpe 1 -e 1 -cve 1 -cache_size 1 ~/feeds/cve_20200424000000_nvd.json.gz
    dvl@dvl-mbp cpe2cve %
```
  • Loading branch information
skogtwin committed Jul 22, 2020
1 parent aed862a commit 3f12633
Showing 1 changed file with 3 additions and 3 deletions.
6 changes: 3 additions & 3 deletions cvefeed/cvecache.go
Original file line number Diff line number Diff line change
Expand Up @@ -166,10 +166,10 @@ func (c *Cache) Get(cpes []*wfn.Attributes) []MatchResult {
cves.res = c.match(cpes)
cves.updateResSize(key)
c.mu.Lock()
c.size += cves.size
if c.MaxSize != 0 && c.size > c.MaxSize {
if c.MaxSize != 0 && c.size+cves.size > c.MaxSize {
c.evict(int64(cacheEvictPercentage * float64(c.MaxSize)))
}
c.size += cves.size
cves.evictionIndex = c.evictionQ.push(key)
c.mu.Unlock()
close(cves.ready)
Expand Down Expand Up @@ -231,7 +231,7 @@ func (c *Cache) matchDict(cpes []*wfn.Attributes, dict Dictionary) (results []Ma
// evict the least recently used records untile nbytes of capacity is achieved or no more records left.
// It is not concurrency-safe, c.mu should be locked before calling it.
func (c *Cache) evict(nbytes int64) {
for c.size+nbytes > c.MaxSize {
for c.size > 0 && c.size+nbytes > c.MaxSize {
key := c.evictionQ.pop()
cd, ok := c.data[key]
if !ok { // should not happen
Expand Down

0 comments on commit 3f12633

Please sign in to comment.