feat(cache): based on JSON (aquasecurity#398)

* refactor(docker_conf): rename and remove unnecessary options * feat(rpc): define new API * fix(cli): change default timeout * fix(import): fix package names * refactor(vulnerability): remove old mock * refactor(utils): remove un-needed functions * feat(cache): implement cache communicating with a server * refactor(scan): separate scan function as local scanner * test(scanner): add tests for ScanImage * refactor(scan): remove unused options * test(vulnerability): generate mock * refactor(server): split a file * feat(server): implement new RPC server * feat(client): implement new RPC client * fix(cache): use new cache interface * fix(standalone): use new scanner * fix(client): use new scanner * fix(server): pass cache * test(integration): make sure an error is not nil before calling the method * fix(mod): update dependencies * test(integration): ensure the image load finishes * feat(docker): support DOCKER_HOST and DOCKER_CERT_PATH * chore(mod): update dependencies * refactor(rpc): remove old client * feat(server): support old API for backward compatibility * fix(server): check a schema version of JSON cache * fix(rpc): add a version to packages * feat(rpc): add PutImage * test: rename expectations * refactor(cache): rename LayerCache to ImageCache * refactor: rename ImageInfo to ImageReference * fix(applier): pass image_id to ApplyLayer * feat(cache): handle image cache * chore(mod): update dependencies * refactor(server): pass only config * feat(cli): add -removed-pkgs option * refactor(err): wrap errors
GuaoGuao · Feb 27, 2020 · 9b2fd97 · 9b2fd97
1 parent 6d724c5
commit 9b2fd97
Show file tree

Hide file tree

Showing 81 changed files with 7,570 additions and 2,360 deletions.
diff --git a/Makefile b/Makefile
@@ -47,7 +47,7 @@ build:
 
 .PHONY: protoc
 protoc:
-	protoc --proto_path=$(GOSRC):. --twirp_out=. --go_out=. ./rpc/detector/service.proto
+	find ./rpc/ -name "*.proto" -type f -exec protoc --proto_path=$(GOSRC):. --twirp_out=. --go_out=. {} \;
 
 .PHONY: install
 install:

diff --git a/go.mod b/go.mod
@@ -3,8 +3,7 @@ module github.com/aquasecurity/trivy
 go 1.13
 
 require (
-	github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5 // indirect
-	github.com/aquasecurity/fanal v0.0.0-20200112144021-9a35ce3bd793
+	github.com/aquasecurity/fanal v0.0.0-20200221125056-947c2a5bb130
 	github.com/aquasecurity/go-dep-parser v0.0.0-20190819075924-ea223f0ef24b
 	github.com/aquasecurity/trivy-db v0.0.0-20191226181755-d6cabf5bc5d1
 	github.com/caarlos0/env/v6 v6.0.0
@@ -13,27 +12,24 @@ require (
 	github.com/docker/docker v0.0.0-20180924202107-a9c061deec0f
 	github.com/genuinetools/reg v0.16.0
 	github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d // indirect
-	github.com/golang/protobuf v1.3.2
+	github.com/golang/protobuf v1.3.3
 	github.com/google/go-github/v28 v28.1.1
 	github.com/google/wire v0.3.0
 	github.com/knqyf263/go-deb-version v0.0.0-20190517075300-09fca494f03d
 	github.com/knqyf263/go-rpm-version v0.0.0-20170716094938-74609b86c936
 	github.com/knqyf263/go-version v1.1.1
 	github.com/kylelemons/godebug v1.1.0
 	github.com/olekukonko/tablewriter v0.0.2-0.20190607075207-195002e6e56a
-	github.com/prometheus/procfs v0.0.5 // indirect
+	github.com/opencontainers/go-digest v1.0.0-rc1
 	github.com/stretchr/testify v1.4.0
 	github.com/twitchtv/twirp v5.10.1+incompatible
-	github.com/urfave/cli v1.20.0
+	github.com/urfave/cli v1.22.1
 	go.uber.org/atomic v1.5.1 // indirect
 	go.uber.org/multierr v1.4.0 // indirect
 	go.uber.org/zap v1.13.0
-	golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529
 	golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421
 	golang.org/x/tools v0.0.0-20191121040551-947d4aa89328 // indirect
 	golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898
 	gopkg.in/yaml.v2 v2.2.4 // indirect
 	k8s.io/utils v0.0.0-20191010214722-8d271d903fe4
 )
-
-replace github.com/genuinetools/reg => github.com/tomoyamachi/reg v0.16.1-0.20190706172545-2a2250fd7c00
diff --git a/go.sum b/go.sum
diff --git a/integration/docker_engine_test.go b/integration/docker_engine_test.go
@@ -4,6 +4,7 @@ package integration
 
 import (
 	"context"
+	"io"
 	"io/ioutil"
 	"os"
 	"strings"
@@ -229,7 +230,7 @@ func TestRun_WithDockerEngine(t *testing.T) {
 			name:          "sad path, invalid image",
 			invalidImage:  true,
 			testfile:      "badimage:latest",
-			expectedError: "error in image scan: failed to analyze image: failed to extract files: failed to get the v2 manifest: Get https://registry-1.docker.io/v2/library/badimage/manifests/latest: http: non-successful response (status=401 body=\"{\\\"errors\\\":[{\\\"code\\\":\\\"UNAUTHORIZED\\\",\\\"message\\\":\\\"authentication required\\\",\\\"detail\\\":[{\\\"Type\\\":\\\"repository\\\",\\\"Class\\\":\\\"\\\",\\\"Name\\\":\\\"library/badimage\\\",\\\"Action\\\":\\\"pull\\\"}]}]}\\n\")",
+			expectedError: "unable to initialize a image struct: Error reading manifest latest in docker.io/library/badimage",
 		},
 	}
 
@@ -256,8 +257,9 @@ func TestRun_WithDockerEngine(t *testing.T) {
 				})
 
 				// load image into docker engine
-				_, err = cli.ImageLoad(ctx, testfile, true)
+				res, err := cli.ImageLoad(ctx, testfile, true)
 				require.NoError(t, err, tc.name)
+				io.Copy(ioutil.Discard, res.Body)
 
 				// tag our image to something unique
 				err = cli.ImageTag(ctx, tc.imageTag, tc.testfile)
@@ -293,7 +295,8 @@ func TestRun_WithDockerEngine(t *testing.T) {
 			err = app.Run(trivyArgs)
 			switch {
 			case tc.expectedError != "":
-				assert.Equal(t, tc.expectedError, err.Error(), tc.name)
+				require.NotNil(t, err)
+				assert.Contains(t, err.Error(), tc.expectedError, tc.name)
 			default:
 				assert.NoError(t, err, tc.name)
 			}
@@ -311,6 +314,10 @@ func TestRun_WithDockerEngine(t *testing.T) {
 					Force:         true,
 					PruneChildren: true,
 				})
+				_, err = cli.ImageRemove(ctx, tc.imageTag, types.ImageRemoveOptions{
+					Force:         true,
+					PruneChildren: true,
+				})
 				assert.NoError(t, err, tc.name)
 			}
 		})

diff --git a/internal/app.go b/internal/app.go
@@ -104,6 +104,12 @@ var (
 		EnvVar: "TRIVY_DEBUG",
 	}
 
+	removedPkgsFlag = cli.BoolFlag{
+		Name:   "removed-pkgs",
+		Usage:  "detect vulnerabilities of removed packages (only for Alpine)",
+		EnvVar: "TRIVY_REMOVED_PKGS",
+	}
+
 	vulnTypeFlag = cli.StringFlag{
 		Name:   "vuln-type",
 		Value:  "os,library",
@@ -127,7 +133,7 @@ var (
 
 	timeoutFlag = cli.DurationFlag{
 		Name:   "timeout",
-		Value:  time.Second * 60,
+		Value:  time.Second * 120,
 		Usage:  "docker timeout",
 		EnvVar: "TRIVY_TIMEOUT",
 	}
@@ -192,6 +198,7 @@ OPTIONS:
 		noProgressFlag,
 		ignoreUnfixedFlag,
 		debugFlag,
+		removedPkgsFlag,
 		vulnTypeFlag,
 		cacheDirFlag,
 		ignoreFileFlag,
@@ -242,6 +249,7 @@ func NewClientCommand() cli.Command {
 			quietFlag,
 			ignoreUnfixedFlag,
 			debugFlag,
+			removedPkgsFlag,
 			vulnTypeFlag,
 			ignoreFileFlag,
 			cacheDirFlag,

diff --git a/internal/client/config/config.go b/internal/client/config/config.go
@@ -30,12 +30,13 @@ type Config struct {
 	Format   string
 	Template string
 
-	Timeout       time.Duration
-	vulnType      string
-	severities    string
-	IgnoreFile    string
-	IgnoreUnfixed bool
-	ExitCode      int
+	Timeout         time.Duration
+	ScanRemovedPkgs bool
+	vulnType        string
+	severities      string
+	IgnoreFile      string
+	IgnoreUnfixed   bool
+	ExitCode        int
 
 	RemoteAddr    string
 	token         string
@@ -73,12 +74,13 @@ func New(c *cli.Context) (Config, error) {
 		Format:   c.String("format"),
 		Template: c.String("template"),
 
-		Timeout:       c.Duration("timeout"),
-		vulnType:      c.String("vuln-type"),
-		severities:    c.String("severity"),
-		IgnoreFile:    c.String("ignorefile"),
-		IgnoreUnfixed: c.Bool("ignore-unfixed"),
-		ExitCode:      c.Int("exit-code"),
+		Timeout:         c.Duration("timeout"),
+		ScanRemovedPkgs: c.Bool("removed-pkgs"),
+		vulnType:        c.String("vuln-type"),
+		severities:      c.String("severity"),
+		IgnoreFile:      c.String("ignorefile"),
+		IgnoreUnfixed:   c.Bool("ignore-unfixed"),
+		ExitCode:        c.Int("exit-code"),
 
 		RemoteAddr:    c.String("remote"),
 		token:         c.String("token"),

diff --git a/internal/client/inject.go b/internal/client/inject.go
@@ -3,18 +3,26 @@
 package client
 
 import (
+	"context"
+	"time"
+
 	"github.com/aquasecurity/fanal/cache"
-	"github.com/aquasecurity/trivy/pkg/rpc/client/library"
-	"github.com/aquasecurity/trivy/pkg/rpc/client/ospkg"
+	"github.com/aquasecurity/trivy/pkg/rpc/client"
 	"github.com/aquasecurity/trivy/pkg/scanner"
 	"github.com/aquasecurity/trivy/pkg/vulnerability"
 	"github.com/google/wire"
 )
 
-func initializeScanner(cacheClient cache.Cache, ospkgCustomHeaders ospkg.CustomHeaders, libraryCustomHeaders library.CustomHeaders,
-	ospkgURL ospkg.RemoteURL, libURL library.RemoteURL) scanner.Scanner {
-	wire.Build(scanner.ClientSet)
-	return scanner.Scanner{}
+func initializeDockerScanner(ctx context.Context, imageName string, layerCache cache.ImageCache, customHeaders client.CustomHeaders,
+	url client.RemoteURL, timeout time.Duration) (scanner.Scanner, error) {
+	wire.Build(scanner.RemoteDockerSet)
+	return scanner.Scanner{}, nil
+}
+
+func initializeArchiveScanner(ctx context.Context, filePath string, layerCache cache.ImageCache, customHeaders client.CustomHeaders,
+	url client.RemoteURL, timeout time.Duration) (scanner.Scanner, error) {
+	wire.Build(scanner.RemoteArchiveSet)
+	return scanner.Scanner{}, nil
 }
 
 func initializeVulnerabilityClient() vulnerability.Client {

diff --git a/internal/client/run.go b/internal/client/run.go
@@ -1,20 +1,19 @@
 package client
 
 import (
+	"context"
 	"os"
 
-	"github.com/urfave/cli"
-	"golang.org/x/xerrors"
-
-	"github.com/aquasecurity/fanal/cache"
 	"github.com/aquasecurity/trivy/internal/client/config"
-	"github.com/aquasecurity/trivy/internal/operation"
+	"github.com/aquasecurity/trivy/pkg/cache"
 	"github.com/aquasecurity/trivy/pkg/log"
 	"github.com/aquasecurity/trivy/pkg/report"
-	"github.com/aquasecurity/trivy/pkg/rpc/client/library"
-	"github.com/aquasecurity/trivy/pkg/rpc/client/ospkg"
+	"github.com/aquasecurity/trivy/pkg/rpc/client"
+	"github.com/aquasecurity/trivy/pkg/scanner"
 	"github.com/aquasecurity/trivy/pkg/types"
 	"github.com/aquasecurity/trivy/pkg/utils"
+	"github.com/urfave/cli"
+	"golang.org/x/xerrors"
 )
 
 func Run(cliCtx *cli.Context) error {
@@ -37,31 +36,40 @@ func run(c config.Config) (err error) {
 
 	// configure cache dir
 	utils.SetCacheDir(c.CacheDir)
-	cacheClient := cache.Initialize(c.CacheDir)
-	cacheOperation := operation.NewCache(cacheClient)
 	log.Logger.Debugf("cache dir:  %s", utils.CacheDir())
 
 	if c.ClearCache {
-		return cacheOperation.ClearImages()
+		log.Logger.Warn("A client doesn't have image cache")
+		return nil
 	}
 
-	scanOptions := types.ScanOptions{
-		VulnType:  c.VulnType,
-		RemoteURL: c.RemoteAddr,
-	}
-	log.Logger.Debugf("Vulnerability type:  %s", scanOptions.VulnType)
+	var scanner scanner.Scanner
+	ctx := context.Background()
+	remoteCache := cache.NewRemoteCache(cache.RemoteURL(c.RemoteAddr), c.CustomHeaders)
 
-	dockerOption, err := types.GetDockerOption()
-	if err != nil {
-		return xerrors.Errorf("failed to get docker option: %w", err)
+	if c.Input != "" {
+		// scan tar file
+		scanner, err = initializeArchiveScanner(ctx, c.Input, remoteCache,
+			client.CustomHeaders(c.CustomHeaders), client.RemoteURL(c.RemoteAddr), c.Timeout)
+		if err != nil {
+			return xerrors.Errorf("unable to initialize the archive scanner: %w", err)
+		}
+	} else {
+		// scan an image in Docker Engine or Docker Registry
+		scanner, err = initializeDockerScanner(ctx, c.ImageName, remoteCache,
+			client.CustomHeaders(c.CustomHeaders), client.RemoteURL(c.RemoteAddr), c.Timeout)
+		if err != nil {
+			return xerrors.Errorf("unable to initialize the docker scanner: %w", err)
+		}
 	}
-	dockerOption.Timeout = c.Timeout
 
-	scanner := initializeScanner(cacheClient,
-		ospkg.CustomHeaders(c.CustomHeaders), library.CustomHeaders(c.CustomHeaders),
-		ospkg.RemoteURL(c.RemoteAddr), library.RemoteURL(c.RemoteAddr))
+	scanOptions := types.ScanOptions{
+		VulnType:            c.VulnType,
+		ScanRemovedPackages: c.ScanRemovedPkgs,
+	}
+	log.Logger.Debugf("Vulnerability type:  %s", scanOptions.VulnType)
 
-	results, err := scanner.ScanImage(c.ImageName, c.Input, scanOptions, dockerOption)
+	results, err := scanner.ScanImage(scanOptions)
 	if err != nil {
 		return xerrors.Errorf("error in image scan: %w", err)
 	}

diff --git a/internal/client/wire_gen.go b/internal/client/wire_gen.go
diff --git a/internal/operation/operation.go b/internal/operation/operation.go
@@ -14,15 +14,16 @@ import (
 )
 
 var SuperSet = wire.NewSet(
-	cache.Initialize,
+	cache.NewFSCache,
+	wire.Bind(new(cache.LocalImageCache), new(cache.FSCache)),
 	NewCache,
 )
 
 type Cache struct {
-	client cache.Cache
+	client cache.LocalImageCache
 }
 
-func NewCache(client cache.Cache) Cache {
+func NewCache(client cache.LocalImageCache) Cache {
 	return Cache{client: client}
 }