refactor: remove parallel walk

Signed-off-by: knqyf263 <knqyf263@gmail.com>
aquasecurity · knqyf263 · Apr 17, 2024 · Sep 13, 2023 · Oct 19, 2023 · Oct 19, 2023
commit 544953b3558cd37c3119afabf5cfc4955a36241e
@@ -76,7 +76,6 @@ require (
 	github.com/owenrumney/go-sarif/v2 v2.2.0
 	github.com/package-url/packageurl-go v0.1.2-0.20230812223828-f8bb31c1f10b
 	github.com/samber/lo v1.38.1
-	github.com/saracen/walker v0.1.3
 	github.com/secure-systems-lab/go-securesystemslib v0.7.0
 	github.com/sigstore/rekor v1.2.1
 	github.com/sirupsen/logrus v1.9.3

@@ -1558,8 +1558,6 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb
 github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4=
 github.com/samber/lo v1.38.1 h1:j2XEAqXKb09Am4ebOg31SpvzUTTs6EN3VfgeLUhPdXM=
 github.com/samber/lo v1.38.1/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA=
-github.com/saracen/walker v0.1.3 h1:YtcKKmpRPy6XJTHJ75J2QYXXZYWnZNQxPCVqZSHVV/g=
-github.com/saracen/walker v0.1.3/go.mod h1:FU+7qU8DeQQgSZDmmThMJi93kPkLFgy0oVAcLxurjIk=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo=

@@ -56,7 +56,7 @@ func NewArtifact(rootPath string, c cache.ArtifactCache, opt artifact.Option) (a
 		rootPath: filepath.ToSlash(filepath.Clean(rootPath)),
 		cache:    c,
 		walker: walker.NewFS(buildPathsToSkip(rootPath, opt.SkipFiles), buildPathsToSkip(rootPath, opt.SkipDirs),
-			opt.Slow, opt.WalkOption.ErrorCallback),
+			opt.WalkOption.ErrorCallback),
 		analyzer:       a,
 		handlerManager: handlerManager,
 

@@ -158,7 +158,7 @@ func TestArtifact_Inspect(t *testing.T) {
 			fields: fields{
 				dir: "./testdata/unknown",
 			},
-			wantErr: "walk error",
+			wantErr: "walk dir error",
 		},
 		{
 			name: "happy path with single file",

@@ -5,11 +5,9 @@ import (
 	"os"
 	"path/filepath"
 
-	swalker "github.com/saracen/walker"
 	"golang.org/x/xerrors"
 
 	dio "github.com/aquasecurity/go-dep-parser/pkg/io"
-	"github.com/aquasecurity/trivy/pkg/log"
 )
 
 type ErrorCallback func(pathname string, err error) error
@@ -19,7 +17,7 @@ type FS struct {
 	errCallback ErrorCallback
 }
 
-func NewFS(skipFiles, skipDirs []string, slow bool, errCallback ErrorCallback) FS {
+func NewFS(skipFiles, skipDirs []string, errCallback ErrorCallback) FS {
 	if errCallback == nil {
 		errCallback = func(pathname string, err error) error {
 			// ignore permission errors
@@ -32,78 +30,50 @@ func NewFS(skipFiles, skipDirs []string, slow bool, errCallback ErrorCallback) F
 	}
 
 	return FS{
-		walker:      newWalker(skipFiles, skipDirs, slow),
+		walker:      newWalker(skipFiles, skipDirs, false),
 		errCallback: errCallback,
 	}
 }
 
-// Walk walks the file tree rooted at root, calling WalkFunc for each file or
+// Walk walks the file tree rooted at root, calling WalkDirFunc for each file or
 // directory in the tree, including root, but a directory to be ignored will be skipped.
 func (w FS) Walk(root string, fn WalkFunc) error {
-	// walk function called for every path found
-	walkFn := func(pathname string, fi os.FileInfo) error {
-		pathname = filepath.Clean(pathname)
+	err := filepath.WalkDir(root, func(filePath string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return w.errCallback(filePath, err)
+		}
+
+		filePath = filepath.Clean(filePath)
 
 		// For exported rootfs (e.g. images/alpine/etc/alpine-release)
-		relPath, err := filepath.Rel(root, pathname)
+		relPath, err := filepath.Rel(root, filePath)
 		if err != nil {
 			return xerrors.Errorf("filepath rel (%s): %w", relPath, err)
 		}
 		relPath = filepath.ToSlash(relPath)
 
-		if fi.IsDir() {
+		info, err := d.Info()
+		if err != nil {
+			return xerrors.Errorf("file info error: %w", err)
+		}
+
+		if info.IsDir() {
 			if w.shouldSkipDir(relPath) {
 				return filepath.SkipDir
 			}
 			return nil
-		} else if !fi.Mode().IsRegular() {
+		} else if !info.Mode().IsRegular() {
 			return nil
 		} else if w.shouldSkipFile(relPath) {
 			return nil
 		}
 
-		if err = fn(relPath, fi, w.fileOpener(pathname)); err != nil {
+		if err = fn(relPath, info, w.fileOpener(filePath)); err != nil {
 			return xerrors.Errorf("failed to analyze file: %w", err)
 		}
 		return nil
-	}
-
-	if w.slow {
-		// In series: fast, with higher CPU/memory
-		return w.walkSlow(root, walkFn)
-	}
-
-	// In parallel: slow, with lower CPU/memory
-	return w.walkFast(root, walkFn)
-}
-
-type fastWalkFunc func(pathname string, fi os.FileInfo) error
-
-func (w FS) walkFast(root string, walkFn fastWalkFunc) error {
-	// error function called for every error encountered
-	errorCallbackOption := swalker.WithErrorCallback(w.errCallback)
-
-	// Multiple goroutines stat the filesystem concurrently. The provided
-	// walkFn must be safe for concurrent use.
-	log.Logger.Debugf("Walk the file tree rooted at '%s' in parallel", root)
-	if err := swalker.Walk(root, walkFn, errorCallbackOption); err != nil {
-		return xerrors.Errorf("walk error: %w", err)
-	}
-	return nil
-}
-
-func (w FS) walkSlow(root string, walkFn fastWalkFunc) error {
-	log.Logger.Debugf("Walk the file tree rooted at '%s' in series", root)
-	err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
-		if err != nil {
-			return w.errCallback(path, err)
-		}
-		info, err := d.Info()
-		if err != nil {
-			return xerrors.Errorf("file info error: %w", err)
-		}
-		return walkFn(path, info)
 	})
+
 	if err != nil {
 		return xerrors.Errorf("walk dir error: %w", err)
 	}

@@ -93,7 +93,7 @@ func TestDir_Walk(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			w := walker.NewFS(tt.fields.skipFiles, tt.fields.skipDirs, true, tt.fields.errCallback)
+			w := walker.NewFS(tt.fields.skipFiles, tt.fields.skipDirs, tt.fields.errCallback)
 
 			err := w.Walk(tt.rootDir, tt.analyzeFn)
 			if tt.wantErr != "" {