Whitelist repo (#378)

* fixing commit latest bug * bumping go-git and adding repo whitelist * adding tests
gitleaks · May 9, 2020 · 473d0d5 · 473d0d5
1 parent 8bd4649
commit 473d0d5
Show file tree

Hide file tree

Showing 12 changed files with 282 additions and 65 deletions.
diff --git a/audit/audit.go b/audit/audit.go
@@ -33,6 +33,15 @@ func Run(m *manager.Manager) error {
 }
 
 func runHelper(r *Repo) error {
+	// Ignore whitelisted repos
+	for _, wlRepo := range r.Manager.Config.Whitelist.Repos {
+		if RegexMatched(r.Manager.Opts.RepoPath, wlRepo) {
+			return nil
+		}
+		if RegexMatched(r.Manager.Opts.Repo, wlRepo) {
+			return nil
+		}
+	}
 	if r.Manager.Opts.OpenLocal() {
 		r.Name = path.Base(r.Manager.Opts.RepoPath)
 		if err := r.Open(); err != nil {

diff --git a/audit/audit_test.go b/audit/audit_test.go
@@ -173,6 +173,16 @@ func TestAudit(t *testing.T) {
 			},
 			wantPath: "../test_data/test_local_owner_aws_leak.json",
 		},
+		{
+			description: "test owner path whitelist repo",
+			opts: options.Options{
+				OwnerPath:    "../test_data/test_repos/",
+				Report:       "../test_data/test_local_owner_aws_leak_whitelist_repo.json.got",
+				ReportFormat: "json",
+				Config:       "../test_data/test_configs/aws_key_local_owner_whitelist_repo.toml",
+			},
+			wantPath: "../test_data/test_local_owner_aws_leak_whitelist_repo.json",
+		},
 		{
 			description: "test entropy and regex",
 			opts: options.Options{

diff --git a/audit/repo.go b/audit/repo.go
@@ -5,6 +5,7 @@ import (
 	"context"
 	"crypto/md5"
 	"fmt"
+	"github.com/go-git/go-git/v5"
 	"io"
 	"os"
 	"path"
@@ -16,14 +17,13 @@ import (
 	"github.com/zricethezav/gitleaks/v4/manager"
 
 	"github.com/BurntSushi/toml"
+	"github.com/go-git/go-billy/v5"
+	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/go-git/go-git/v5/plumbing/object"
+	"github.com/go-git/go-git/v5/plumbing/storer"
+	"github.com/go-git/go-git/v5/storage/memory"
 	"github.com/sergi/go-diff/diffmatchpatch"
 	log "github.com/sirupsen/logrus"
-	"gopkg.in/src-d/go-billy.v4"
-	"gopkg.in/src-d/go-git.v4"
-	"gopkg.in/src-d/go-git.v4/plumbing"
-	"gopkg.in/src-d/go-git.v4/plumbing/object"
-	"gopkg.in/src-d/go-git.v4/plumbing/storer"
-	"gopkg.in/src-d/go-git.v4/storage/memory"
 )
 
 // Repo wraps a *git.Repository object in addition to a manager object and the name of the repo.
@@ -259,7 +259,6 @@ func (repo *Repo) Audit() error {
 	auditTimeStart := time.Now()
 
 	// audit commit patches OR all files at commit. See https://github.com/zricethezav/gitleaks/issues/326
-	// TODO having --commit= and --fites-at-commit= set should probably be guarded against
 	if repo.Manager.Opts.Commit != "" {
 		return inspectCommit(repo.Manager.Opts.Commit, repo, inspectCommitPatches)
 	} else if repo.Manager.Opts.FilesAtCommit != "" {

diff --git a/audit/util.go b/audit/util.go
@@ -13,10 +13,10 @@ import (
 	"github.com/zricethezav/gitleaks/v4/manager"
 
 	log "github.com/sirupsen/logrus"
-	"gopkg.in/src-d/go-git.v4"
-	"gopkg.in/src-d/go-git.v4/plumbing"
-	fdiff "gopkg.in/src-d/go-git.v4/plumbing/format/diff"
-	"gopkg.in/src-d/go-git.v4/plumbing/object"
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing"
+	fdiff "github.com/go-git/go-git/v5/plumbing/format/diff"
+	"github.com/go-git/go-git/v5/plumbing/object"
 )
 
 // Inspect patch accepts a patch, commit, and repo. If the patches contains files that are
@@ -164,7 +164,7 @@ func InspectFile(content string, fullpath string, c *object.Commit, repo *Repo)
 	// We want to check if there is a whitelist for this file
 	if len(repo.config.Whitelist.Files) != 0 {
 		for _, reFileName := range repo.config.Whitelist.Files {
-			if fileMatched(filename, reFileName) {
+			if RegexMatched(filename, reFileName) {
 				log.Debugf("whitelisted file found, skipping audit of file: %s", filename)
 				return
 			}
@@ -174,7 +174,7 @@ func InspectFile(content string, fullpath string, c *object.Commit, repo *Repo)
 	// We want to check if there is a whitelist for this path
 	if len(repo.config.Whitelist.Paths) != 0 {
 		for _, reFilePath := range repo.config.Whitelist.Paths {
-			if fileMatched(path, reFilePath) {
+			if RegexMatched(path, reFilePath) {
 				log.Debugf("file in whitelisted path found, skipping audit of file: %s", filename)
 				return
 			}
@@ -190,12 +190,12 @@ func InspectFile(content string, fullpath string, c *object.Commit, repo *Repo)
 		}
 
 		// If it has fileNameRegex and it doesnt match we continue to next rule
-		if ruleContainFileNameRegex(rule) && !fileMatched(filename, rule.FileNameRegex) {
+		if ruleContainFileNameRegex(rule) && !RegexMatched(filename, rule.FileNameRegex) {
 			continue
 		}
 
 		// If it has filePathRegex and it doesnt match we continue to next rule
-		if ruleContainFilePathRegex(rule) && !fileMatched(path, rule.FilePathRegex) {
+		if ruleContainFilePathRegex(rule) && !RegexMatched(path, rule.FilePathRegex) {
 			continue
 		}
 
@@ -376,7 +376,7 @@ func isOffenderWhiteListed(offender string, whitelist []config.Whitelist) bool {
 func isFileNameWhiteListed(filename string, whitelist []config.Whitelist) bool {
 	if len(whitelist) != 0 {
 		for _, wl := range whitelist {
-			if fileMatched(filename, wl.File) {
+			if RegexMatched(filename, wl.File) {
 				return true
 			}
 		}
@@ -387,15 +387,17 @@ func isFileNameWhiteListed(filename string, whitelist []config.Whitelist) bool {
 func isFilePathWhiteListed(filepath string, whitelist []config.Whitelist) bool {
 	if len(whitelist) != 0 {
 		for _, wl := range whitelist {
-			if fileMatched(filepath, wl.Path) {
+			if RegexMatched(filepath, wl.Path) {
 				return true
 			}
 		}
 	}
 	return false
 }
 
-func fileMatched(f interface{}, re *regexp.Regexp) bool {
+// RegexMatched matched an interface to a regular expression. The interface f can 
+// be a string type or go-git *object.File type.
+func RegexMatched(f interface{}, re *regexp.Regexp) bool {
 	if re == nil {
 		return false
 	}

diff --git a/config/config.go b/config/config.go
@@ -51,6 +51,7 @@ type Config struct {
 		Commits     []string
 		Files       []*regexp.Regexp
 		Paths       []*regexp.Regexp
+		Repos       []*regexp.Regexp
 	}
 }
 
@@ -63,6 +64,7 @@ type TomlLoader struct {
 		Commits     []string
 		Files       []string
 		Paths       []string
+		Repos       []string
 	}
 	Rules []struct {
 		Description   string
@@ -210,6 +212,16 @@ func (tomlLoader TomlLoader) Parse() (Config, error) {
 		}
 		cfg.Whitelist.Paths = append(cfg.Whitelist.Paths, re)
 	}
+
+	// global repo whitelists
+	for _, wlRepo := range tomlLoader.Whitelist.Repos {
+		re, err := regexp.Compile(wlRepo)
+		if err != nil {
+			return cfg, fmt.Errorf("problem loading config: %v", err)
+		}
+		cfg.Whitelist.Repos = append(cfg.Whitelist.Repos, re)
+	}
+
 	cfg.Whitelist.Commits = tomlLoader.Whitelist.Commits
 	cfg.Whitelist.Description = tomlLoader.Whitelist.Description
 

diff --git a/go.mod b/go.mod
@@ -4,16 +4,15 @@ go 1.14
 
 require (
 	github.com/BurntSushi/toml v0.3.1
+	github.com/go-git/go-billy/v5 v5.0.0
+	github.com/go-git/go-git/v5 v5.0.0
 	github.com/google/go-cmp v0.4.0 // indirect
-	github.com/google/go-github/v29 v29.0.3
+	github.com/google/go-github/v31 v31.0.0
 	github.com/hako/durafmt v0.0.0-20191009132224-3f39dc1ed9f4
 	github.com/jessevdk/go-flags v1.4.0
 	github.com/mattn/go-colorable v0.1.2
-	github.com/sergi/go-diff v1.0.0
+	github.com/sergi/go-diff v1.1.0
 	github.com/sirupsen/logrus v1.4.2
 	github.com/xanzy/go-gitlab v0.21.0
-	golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 // indirect
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
-	gopkg.in/src-d/go-billy.v4 v4.3.2
-	gopkg.in/src-d/go-git.v4 v4.13.1
 )