feat(redhat): support affected CPE list in OVAL v2 (#101)

aquasecurity · Jan 28, 2022 · 5c53ef8 · 5c53ef8
1 parent d8e373e
commit 5c53ef8
Show file tree

Hide file tree

Showing 97 changed files with 3,182 additions and 1,286 deletions.
diff --git a/pkg/db/advisory.go b/pkg/db/advisory.go
@@ -8,22 +8,21 @@ import (
 	"golang.org/x/xerrors"
 )
 
-func (dbc Config) PutAdvisory(tx *bolt.Tx, source, pkgName, cveID string, advisory interface{}) error {
-	root, err := tx.CreateBucketIfNotExists([]byte(source))
-	if err != nil {
-		return xerrors.Errorf("failed to create a bucket: %w", err)
+func (dbc Config) PutAdvisory(tx *bolt.Tx, bktNames []string, key string, advisory interface{}) error {
+	if err := dbc.put(tx, bktNames, key, advisory); err != nil {
+		return xerrors.Errorf("failed to put advisory: %w", err)
 	}
-	return dbc.put(root, pkgName, cveID, advisory)
+	return nil
 }
 
-func (dbc Config) ForEachAdvisory(source, pkgName string) (value map[string]Value, err error) {
-	return dbc.forEach(source, pkgName)
+func (dbc Config) ForEachAdvisory(sources []string, pkgName string) (map[string]Value, error) {
+	return dbc.forEach(append(sources, pkgName))
 }
 
 func (dbc Config) GetAdvisories(source, pkgName string) ([]types.Advisory, error) {
-	advisories, err := dbc.ForEachAdvisory(source, pkgName)
+	advisories, err := dbc.ForEachAdvisory([]string{source}, pkgName)
 	if err != nil {
-		return nil, xerrors.Errorf("error in advisory foreach: %w", err)
+		return nil, xerrors.Errorf("advisory foreach error: %w", err)
 	}
 	if len(advisories) == 0 {
 		return nil, nil

diff --git a/pkg/db/advisory_detail.go b/pkg/db/advisory_detail.go
@@ -6,66 +6,72 @@ import (
 	bolt "go.etcd.io/bbolt"
 
 	"golang.org/x/xerrors"
-
-	"github.com/aquasecurity/trivy-db/pkg/types"
 )
 
 const (
 	advisoryDetailBucket = "advisory-detail"
 )
 
-func (dbc Config) PutAdvisoryDetail(tx *bolt.Tx, vulnerabilityID string, source string, pkgName string,
-	advisory interface{}) (err error) {
-	root, err := tx.CreateBucketIfNotExists([]byte(advisoryDetailBucket))
-	if err != nil {
-		return err
+func (dbc Config) PutAdvisoryDetail(tx *bolt.Tx, vulnID, pkgName string, nestedBktNames []string, advisory interface{}) error {
+	bktNames := append([]string{advisoryDetailBucket, vulnID}, nestedBktNames...)
+	if err := dbc.put(tx, bktNames, pkgName, advisory); err != nil {
+		return xerrors.Errorf("failed to put advisory detail: %w", err)
 	}
-	nested, err := root.CreateBucketIfNotExists([]byte(vulnerabilityID))
-	if err != nil {
-		return err
+	return nil
+}
+
+// SaveAdvisoryDetails Extract advisories from 'advisory-detail' bucket and copy them in each
+func (dbc Config) SaveAdvisoryDetails(tx *bolt.Tx, vulnID string) error {
+	root := tx.Bucket([]byte(advisoryDetailBucket))
+	if root == nil {
+		return nil
+	}
+
+	cveBucket := root.Bucket([]byte(vulnID))
+	if cveBucket == nil {
+		return nil
+	}
+
+	if err := dbc.saveAdvisories(tx, cveBucket, []string{}, vulnID); err != nil {
+		return xerrors.Errorf("walk advisories error: %w", err)
 	}
-	return dbc.put(nested, source, pkgName, advisory)
+
+	return nil
 }
 
-func (dbc Config) GetAdvisoryDetails(cveID string) ([]types.AdvisoryDetail, error) {
-	var advisories []types.AdvisoryDetail
-	err := db.View(func(tx *bolt.Tx) error {
-		root := tx.Bucket([]byte(advisoryDetailBucket))
-		if root == nil {
-			return nil
-		}
-		cveBucket := root.Bucket([]byte(cveID))
-		if cveBucket == nil {
-			return nil
-		}
-		err := cveBucket.ForEach(func(platform, v []byte) error {
-			packageBucket := cveBucket.Bucket(platform)
-			if packageBucket == nil {
-				return nil
+// saveAdvisories walks all key-values under the 'advisory-detail' bucket and copy them in each vendor's bucket.
+func (dbc Config) saveAdvisories(tx *bolt.Tx, bkt *bolt.Bucket, bktNames []string, vulnID string) error {
+	if bkt == nil {
+		return nil
+	}
+
+	err := bkt.ForEach(func(k, v []byte) error {
+		// When the key is a bucket, it walks recursively.
+		if v == nil {
+			bkts := append(bktNames, string(k))
+			if err := dbc.saveAdvisories(tx, bkt.Bucket(k), bkts, vulnID); err != nil {
+				return xerrors.Errorf("walk advisories error: %w", err)
+			}
+		} else {
+			detail := map[string]interface{}{}
+			if err := json.Unmarshal(v, &detail); err != nil {
+				return xerrors.Errorf("failed to unmarshall the advisory detail: %w", err)
+			}
+
+			// Put the advisory in vendor's bucket such as Debian and Ubuntu
+			bkts := append(bktNames, string(k))
+			if err := dbc.put(tx, bkts, vulnID, detail); err != nil {
+				return xerrors.Errorf("database put error: %w", err)
 			}
-			err := packageBucket.ForEach(func(packageName, v []byte) error {
-				detail := map[string]interface{}{}
-				if err := json.Unmarshal(v, &detail); err != nil {
-					return xerrors.Errorf("failed to unmarshall advisory_detail: %w", err)
-				}
-				advisories = append(advisories, types.AdvisoryDetail{
-					PlatformName: string(platform),
-					PackageName:  string(packageName),
-					AdvisoryItem: detail,
-				})
-				return nil
-			})
-			return err
-		})
-		if err != nil {
-			return xerrors.Errorf("error in db foreach: %w", err)
 		}
+
 		return nil
 	})
 	if err != nil {
-		return nil, xerrors.Errorf("failed to get all key/value in the specified bucket: %w", err)
+		return xerrors.Errorf("foreach error: %w", err)
 	}
-	return advisories, nil
+
+	return nil
 }
 
 func (dbc Config) DeleteAdvisoryDetailBucket() error {

diff --git a/pkg/db/advisory_detail_test.go b/pkg/db/advisory_detail_test.go
@@ -0,0 +1,85 @@
+package db_test
+
+import (
+	"testing"
+
+	bolt "go.etcd.io/bbolt"
+
+	"github.com/aquasecurity/trivy-db/pkg/types"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/aquasecurity/trivy-db/pkg/db"
+	"github.com/aquasecurity/trivy-db/pkg/dbtest"
+)
+
+func TestConfig_SaveAdvisoryDetails(t *testing.T) {
+	type want struct {
+		key   []string
+		value types.Advisory
+	}
+	tests := []struct {
+		name     string
+		fixtures []string
+		vulnID   string
+		want     []want
+		wantErr  string
+	}{
+		{
+			name:     "happy path",
+			fixtures: []string{"testdata/fixtures/advisory-detail.yaml"},
+			vulnID:   "CVE-2019-14904",
+			want: []want{
+				{
+					key: []string{"alpine 3.14", "ansible", "CVE-2019-14904"},
+					value: types.Advisory{
+						FixedVersion: "2.9.3-r0",
+					},
+				},
+				{
+					key: []string{"debian 10", "ansible", "CVE-2019-14904"},
+					value: types.Advisory{
+						FixedVersion: "2.3.4",
+					},
+				},
+				{
+					key: []string{"Red Hat", "cpe:/o:redhat:enterprise_linux:6::server", "ansible", "CVE-2019-14904"},
+					value: types.Advisory{
+						FixedVersion: "3.4.5",
+					},
+				},
+			},
+		},
+		{
+			name:     "missing ID",
+			fixtures: []string{"testdata/fixtures/advisory-detail.yaml"},
+			vulnID:   "CVE-2019-9999",
+			want:     nil,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			// Initialize DB for testing
+			tmpDir := dbtest.InitDB(t, tt.fixtures)
+			defer db.Close()
+
+			dbc := db.Config{}
+			err := dbc.BatchUpdate(func(tx *bolt.Tx) error {
+				return dbc.SaveAdvisoryDetails(tx, tt.vulnID)
+			})
+
+			if tt.wantErr != "" {
+				require.NotNil(t, err)
+				assert.Contains(t, err.Error(), tt.wantErr)
+				return
+			}
+
+			require.NoError(t, err)
+			require.NoError(t, db.Close()) // Need to close before dbtest.JSONEq is called
+			for _, w := range tt.want {
+				dbtest.JSONEq(t, db.Path(tmpDir), w.key, w.value)
+			}
+		})
+	}
+}
diff --git a/pkg/db/advisory_test.go b/pkg/db/advisory_test.go
@@ -81,11 +81,11 @@ func TestConfig_ForEachAdvisory(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			// Initialize DB
-			dbtest.InitTestDB(t, tt.fixtures)
+			dbtest.InitDB(t, tt.fixtures)
 			defer db.Close()
 
 			dbc := db.Config{}
-			got, err := dbc.ForEachAdvisory(tt.args.source, tt.args.pkgName)
+			got, err := dbc.ForEachAdvisory([]string{tt.args.source}, tt.args.pkgName)
 
 			if tt.wantErr != "" {
 				require.NotNil(t, err)
@@ -202,7 +202,7 @@ func TestConfig_GetAdvisories(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			// Initialize DB
-			dbtest.InitTestDB(t, tt.fixtures)
+			dbtest.InitDB(t, tt.fixtures)
 			defer db.Close()
 
 			dbc := db.Config{}

diff --git a/pkg/db/db.go b/pkg/db/db.go
@@ -32,9 +32,7 @@ type Operation interface {
 		vulnerability types.VulnerabilityDetail) (err error)
 	DeleteVulnerabilityDetailBucket() (err error)
 
-	PutAdvisory(tx *bolt.Tx, source string, pkgName string, vulnerabilityID string,
-		advisory interface{}) (err error)
-	ForEachAdvisory(source string, pkgName string) (value map[string]Value, err error)
+	ForEachAdvisory(sources []string, pkgName string) (value map[string]Value, err error)
 	GetAdvisories(source string, pkgName string) (advisories []types.Advisory, err error)
 
 	PutVulnerabilityID(tx *bolt.Tx, vulnerabilityID string) (err error)
@@ -43,12 +41,18 @@ type Operation interface {
 	PutVulnerability(tx *bolt.Tx, vulnerabilityID string, vulnerability types.Vulnerability) (err error)
 	GetVulnerability(vulnerabilityID string) (vulnerability types.Vulnerability, err error)
 
-	GetAdvisoryDetails(cveID string) ([]types.AdvisoryDetail, error)
-	PutAdvisoryDetail(tx *bolt.Tx, vulnerabilityID string, source string, pkgName string,
-		advisory interface{}) (err error)
+	SaveAdvisoryDetails(tx *bolt.Tx, cveID string) (err error)
+	PutAdvisoryDetail(tx *bolt.Tx, vulnerabilityID, pkgName string, nestedBktNames []string, advisory interface{}) (err error)
 	DeleteAdvisoryDetailBucket() error
 
 	PutDataSource(tx *bolt.Tx, bktName string, source types.DataSource) (err error)
+
+	// For Red Hat
+	PutRedHatRepositories(tx *bolt.Tx, repository string, cpeIndices []int) (err error)
+	PutRedHatNVRs(tx *bolt.Tx, nvr string, cpeIndices []int) (err error)
+	PutRedHatCPEs(tx *bolt.Tx, cpeIndex int, cpe string) (err error)
+	RedHatRepoToCPEs(repository string) (cpeIndices []int, err error)
+	RedHatNVRToCPEs(nvr string) (cpeIndices []int, err error)
 }
 
 type Config struct {
@@ -109,24 +113,66 @@ func (dbc Config) BatchUpdate(fn func(tx *bolt.Tx) error) error {
 	return nil
 }
 
-func (dbc Config) put(root *bolt.Bucket, nestedBucket, key string, value interface{}) error {
-	nested, err := root.CreateBucketIfNotExists([]byte(nestedBucket))
+func (dbc Config) put(tx *bolt.Tx, bktNames []string, key string, value interface{}) error {
+	if len(bktNames) == 0 {
+		return xerrors.Errorf("empty bucket name")
+	}
+
+	bkt, err := tx.CreateBucketIfNotExists([]byte(bktNames[0]))
 	if err != nil {
-		return xerrors.Errorf("failed to create a bucket: %w", err)
+		return xerrors.Errorf("failed to create '%s' bucket: %w", bktNames[0], err)
+	}
+
+	for _, bktName := range bktNames[1:] {
+		bkt, err = bkt.CreateBucketIfNotExists([]byte(bktName))
+		if err != nil {
+			return xerrors.Errorf("failed to create a bucket: %w", err)
+		}
 	}
 	v, err := json.Marshal(value)
 	if err != nil {
 		return xerrors.Errorf("failed to unmarshal JSON: %w", err)
 	}
-	return nested.Put([]byte(key), v)
+
+	return bkt.Put([]byte(key), v)
+}
+
+func (dbc Config) get(bktNames []string, key string) (value []byte, err error) {
+	err = db.View(func(tx *bolt.Tx) error {
+		if len(bktNames) == 0 {
+			return xerrors.Errorf("empty bucket name")
+		}
+
+		bkt := tx.Bucket([]byte(bktNames[0]))
+		if bkt == nil {
+			return nil
+		}
+		for _, bktName := range bktNames[1:] {
+			bkt = bkt.Bucket([]byte(bktName))
+			if bkt == nil {
+				return nil
+			}
+		}
+		value = bkt.Get([]byte(key))
+		return nil
+	})
+	if err != nil {
+		return nil, xerrors.Errorf("failed to get data from db: %w", err)
+	}
+	return value, nil
 }
 
 type Value struct {
 	Source  types.DataSource
 	Content []byte
 }
 
-func (dbc Config) forEach(rootBucket, nestedBucket string) (map[string]Value, error) {
+func (dbc Config) forEach(bktNames []string) (map[string]Value, error) {
+	if len(bktNames) < 2 {
+		return nil, xerrors.Errorf("bucket must be nested: %v", bktNames)
+	}
+	rootBucket, nestedBuckets := bktNames[0], bktNames[1:]
+
 	values := map[string]Value{}
 	err := db.View(func(tx *bolt.Tx) error {
 		var rootBuckets []string
@@ -154,20 +200,26 @@ func (dbc Config) forEach(rootBucket, nestedBucket string) (map[string]Value, er
 				log.Logger.Debugf("Data source error: %s", err)
 			}
 
-			nested := root.Bucket([]byte(nestedBucket))
-			if nested == nil {
+			bkt := root
+			for _, nestedBkt := range nestedBuckets {
+				bkt = bkt.Bucket([]byte(nestedBkt))
+				if bkt == nil {
+					break
+				}
+			}
+			if bkt == nil {
 				continue
 			}
 
-			err = nested.ForEach(func(k, v []byte) error {
+			err = bkt.ForEach(func(k, v []byte) error {
 				values[string(k)] = Value{
 					Source:  source,
 					Content: v,
 				}
 				return nil
 			})
 			if err != nil {
-				return xerrors.Errorf("error in db foreach: %w", err)
+				return xerrors.Errorf("db foreach error: %w", err)
 			}
 		}
 		return nil