update to V7 (#469)

gitleaks · Dec 2, 2020 · c509063 · c509063
1 parent 40545fb
commit c509063
Show file tree

Hide file tree

Showing 88 changed files with 2,608 additions and 2,673 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.14.1 AS build
+FROM golang:1.15.5 AS build
 WORKDIR /go/src/github.com/zricethezav/gitleaks
 ARG ldflags
 COPY . .

diff --git a/Makefile b/Makefile
@@ -2,8 +2,8 @@
 
 VERSION := `git fetch --tags && git tag | sort -V | tail -1`
 PKG=github.com/zricethezav/gitleaks
-LDFLAGS=-ldflags "-X=github.com/zricethezav/gitleaks/v6/version.Version=$(VERSION)"
-_LDFLAGS="github.com/zricethezav/gitleaks/v6/version.Version=$(VERSION)"
+LDFLAGS=-ldflags "-X=github.com/zricethezav/gitleaks/v7/version.Version=$(VERSION)"
+_LDFLAGS="github.com/zricethezav/gitleaks/v7/version.Version=$(VERSION)"
 COVER=--cover --coverprofile=cover.out
 
 test-cover:
@@ -17,9 +17,6 @@ test:
 	golint ./...
 	go test ./... --race $(PKG) -v
 
-test-integration:
-	go test github.com/zricethezav/gitleaks/hosts -v -integration
-
 build:
 	go fmt ./...
 	golint ./...

diff --git a/README.md b/README.md
@@ -8,18 +8,79 @@
 Gitleaks is a SAST tool for detecting hardcoded secrets like passwords, api keys, and tokens in git repos. Gitleaks aims to be the **easy-to-use, all-in-one solution** for finding secrets, past or present, in your code. 
 
 ### Features:
-- Scans for [commited](https://github.com/zricethezav/gitleaks/wiki/Scanning) secrets
-- Scans for [uncommitted](https://github.com/zricethezav/gitleaks/wiki/Scanning#uncommitted-changes-scan) secrets as part of shifting security left
+- Scan for [commited](https://github.com/zricethezav/gitleaks/wiki/Scanning) secrets
+- Scan for [uncommitted](https://github.com/zricethezav/gitleaks/wiki/Scanning#uncommitted-changes-scan) secrets as part of shifting security left
+- Scan for entire directories and files
 - Available [Github Action](https://github.com/marketplace/actions/gitleaks)
-- Gitlab and Github API support which allows scans of whole organizations, users, and pull/merge requests
 - [Custom rules](https://github.com/zricethezav/gitleaks/wiki/Configuration) via toml configuration
 - High performance using [go-git](https://github.com/go-git/go-git)
-- JSON and CSV reporting
+- JSON, SARIF, and CSV reporting
 - Private repo scans using key or password based authentication
 
 
-## Installation, Documentation and Examples
-This project is documented [here](https://github.com/zricethezav/gitleaks/wiki)
+### Installation
+Written in Go, gitleaks is available in binary form for many popular platforms and OS types from the [releases page](https://github.com/zricethezav/gitleaks/releases). Alternatively, executed via Docker or it can be installed using Go directly.
+
+##### MacOS
+
+```
+brew install gitleaks
+```
+
+##### Docker
+
+```bash
+docker pull zricethezav/gitleaks
+```
+
+##### Go
+```bash
+GO111MODULE=on go get github.com/zricethezav/gitleaks/v6
+```
+
+### Usage
+```
+Usage:
+  gitleaks [OPTIONS]
+
+Application Options:
+  -v, --verbose           Show verbose output from scan
+  -r, --repo-url=         Repository URL
+  -p, --path=             Path to directory (repo if contains .git) or file
+  -c, --config-path=      Path to config
+      --repo-config-path= Path to gitleaks config relative to repo root
+      --clone-path=       Path to clone repo to disk
+      --clone-cleanup=    Deletes cloned repo after scan
+      --version           Version number
+      --username=         Username for git repo
+      --password=         Password for git repo
+      --access-token=     Access token for git repo
+      --threads=          Maximum number of threads gitleaks spawns
+      --ssh-key=          Path to ssh key used for auth
+      --unstaged          Run gitleaks on unstaged code
+      --branch=           Branch to scan
+      --redact            Redact secrets from log messages and leaks
+      --debug             Log debug messages
+      --no-git            Treat git repos as plain directories and scan those
+                          files
+  -o, --report=           Report output path
+  -f, --format=           JSON, CSV, SARIF (default: json)
+      --files-at-commit=  Sha of commit to scan all files at commit
+      --commit=           Sha of commit to scan or "latest" to scan the last
+                          commit of the repository
+      --commits=          Comma separated list of a commits to scan
+      --commits-file=     Path to file of line separated list of commits to scan
+      --commit-from=      Commit to start scan from
+      --commit-to=        Commit to stop scan
+      --commit-since=     Scan commits more recent than a specific date. Ex:
+                          '2006-01-02' or '2006-01-02T15:04:05-0700' format.
+      --commit-until=     Scan commits older than a specific date. Ex:
+                          '2006-01-02' or '2006-01-02T15:04:05-0700' format.
+      --depth=            Number of commits to scan
+
+Help Options:
+  -h, --help              Show this help message
+```
 
 
 ###  Sponsors ❤️
@@ -34,6 +95,8 @@ These users are [sponsors](https://github.com/sponsors/zricethezav) of gitleaks:
 [![Adam Shannon](https://github.com/adamdecaf.png?size=50)](https://github.com/adamdecaf) | 
 ---|
 ----
+
+
 #### Logo Attribution
 The Gitleaks logo uses the Git Logo created <a href="https://twitter.com/jasonlong">Jason Long</a> is licensed under the <a href="https://creativecommons.org/licenses/by/3.0/">Creative Commons Attribution 3.0 Unported License</a>.
 
diff --git a/config/config.go b/config/config.go
@@ -6,9 +6,10 @@ import (
 	"regexp"
 	"strconv"
 
-	"github.com/zricethezav/gitleaks/v6/options"
+	"github.com/zricethezav/gitleaks/v7/options"
 
 	"github.com/BurntSushi/toml"
+	"github.com/go-git/go-git/v5"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -94,10 +95,10 @@ func NewConfig(options options.Options) (Config, error) {
 	tomlLoader := TomlLoader{}
 
 	var err error
-	if options.Config != "" {
-		_, err = toml.DecodeFile(options.Config, &tomlLoader)
+	if options.ConfigPath != "" {
+		_, err = toml.DecodeFile(options.ConfigPath, &tomlLoader)
 		// append a allowlist rule for allowlisting the config
-		tomlLoader.AllowList.Files = append(tomlLoader.AllowList.Files, path.Base(options.Config))
+		tomlLoader.AllowList.Files = append(tomlLoader.AllowList.Files, path.Base(options.ConfigPath))
 	} else {
 		_, err = toml.Decode(DefaultConfig, &tomlLoader)
 	}
@@ -139,6 +140,8 @@ func (tomlLoader TomlLoader) Parse() (Config, error) {
 		// rule specific allowlists
 		var allowList AllowList
 
+		allowList.Description = rule.AllowList.Description
+
 		// rule specific regexes
 		for _, re := range rule.AllowList.Regexes {
 			allowListedRegex, err := regexp.Compile(re)
@@ -166,6 +169,9 @@ func (tomlLoader TomlLoader) Parse() (Config, error) {
 			allowList.Paths = append(allowList.Paths, allowListedRegex)
 		}
 
+		// rule specific commits
+		allowList.Commits = rule.AllowList.Commits
+
 		var entropies []Entropy
 		for _, e := range rule.Entropies {
 			min, err := strconv.ParseFloat(e.Min, 64)
@@ -250,3 +256,60 @@ func (tomlLoader TomlLoader) Parse() (Config, error) {
 
 	return cfg, nil
 }
+
+// LoadRepoConfig accepts a repo and config path related to the target repo's root.
+func LoadRepoConfig(repo *git.Repository, repoConfig string) (Config, error) {
+	gitRepoConfig, err := repo.Config()
+	if err != nil {
+		return Config{}, err
+	}
+	if !gitRepoConfig.Core.IsBare {
+		wt, err := repo.Worktree()
+		if err != nil {
+			return Config{}, err
+		}
+		_, err = wt.Filesystem.Stat(repoConfig)
+		if err != nil {
+			return Config{}, err
+		}
+		r, err := wt.Filesystem.Open(repoConfig)
+		if err != nil {
+			return Config{}, err
+		}
+		var tomlLoader TomlLoader
+		_, err = toml.DecodeReader(r, &tomlLoader)
+		if err != nil {
+			return Config{}, err
+		}
+
+		return tomlLoader.Parse()
+	}
+
+	log.Debug("attempting to load repo config from bare worktree, this may use an old config")
+	ref, err := repo.Head()
+	if err != nil {
+		return Config{}, err
+	}
+
+	c, err := repo.CommitObject(ref.Hash())
+	if err != nil {
+		return Config{}, err
+	}
+
+	f, err := c.File(repoConfig)
+	if err != nil {
+		return Config{}, err
+	}
+
+	var tomlLoader TomlLoader
+	r, err := f.Reader()
+	if err != nil {
+		return Config{}, err
+	}
+	_, err = toml.DecodeReader(r, &tomlLoader)
+	if err != nil {
+		return Config{}, err
+	}
+
+	return tomlLoader.Parse()
+}
diff --git a/config/config_test.go b/config/config_test.go
@@ -7,7 +7,7 @@ import (
 	"regexp"
 	"testing"
 
-	"github.com/zricethezav/gitleaks/v6/options"
+	"github.com/zricethezav/gitleaks/v7/options"
 )
 
 func TestParse(t *testing.T) {
@@ -26,95 +26,95 @@ func TestParse(t *testing.T) {
 		{
 			description: "test successful load",
 			opts: options.Options{
-				Config: "../test_data/test_configs/aws_key.toml",
+				ConfigPath: "../test_data/test_configs/aws_key.toml",
 			},
 		},
 		{
 			description: "test bad toml",
 			opts: options.Options{
-				Config: "../test_data/test_configs/bad_aws_key.toml",
+				ConfigPath: "../test_data/test_configs/bad_aws_key.toml",
 			},
 			wantErr: fmt.Errorf("Near line 7 (last key parsed 'rules.description'): expected value but found \"AWS\" instead"),
 		},
 		{
 			description: "test bad regex",
 			opts: options.Options{
-				Config: "../test_data/test_configs/bad_regex_aws_key.toml",
+				ConfigPath: "../test_data/test_configs/bad_regex_aws_key.toml",
 			},
 			wantErr: fmt.Errorf("problem loading config: error parsing regexp: invalid nested repetition operator: `???`"),
 		},
 		{
 			description: "test bad global allowlist file regex",
 			opts: options.Options{
-				Config: "../test_data/test_configs/bad_aws_key_global_allowlist_file.toml",
+				ConfigPath: "../test_data/test_configs/bad_aws_key_global_allowlist_file.toml",
 			},
 			wantErr: fmt.Errorf("problem loading config: error parsing regexp: missing argument to repetition operator: `??`"),
 		},
 		{
 			description: "test bad global file regex",
 			opts: options.Options{
-				Config: "../test_data/test_configs/bad_aws_key_file_regex.toml",
+				ConfigPath: "../test_data/test_configs/bad_aws_key_file_regex.toml",
 			},
 			wantErr: fmt.Errorf("problem loading config: error parsing regexp: missing argument to repetition operator: `??`"),
 		},
 		{
 			description: "test successful load big ol thing",
 			opts: options.Options{
-				Config: "../test_data/test_configs/large.toml",
+				ConfigPath: "../test_data/test_configs/large.toml",
 			},
 		},
 		{
 			description: "test load entropy",
 			opts: options.Options{
-				Config: "../test_data/test_configs/entropy.toml",
+				ConfigPath: "../test_data/test_configs/entropy.toml",
 			},
 		},
 		{
 			description: "test entropy bad range",
 			opts: options.Options{
-				Config: "../test_data/test_configs/bad_entropy_1.toml",
+				ConfigPath: "../test_data/test_configs/bad_entropy_1.toml",
 			},
 			wantErr: fmt.Errorf("problem loading config: entropy Min value cannot be higher than Max value"),
 		},
 		{
 			description: "test entropy value max",
 			opts: options.Options{
-				Config: "../test_data/test_configs/bad_entropy_2.toml",
+				ConfigPath: "../test_data/test_configs/bad_entropy_2.toml",
 			},
 			wantErr: fmt.Errorf("strconv.ParseFloat: parsing \"x\": invalid syntax"),
 		},
 		{
 			description: "test entropy value min",
 			opts: options.Options{
-				Config: "../test_data/test_configs/bad_entropy_3.toml",
+				ConfigPath: "../test_data/test_configs/bad_entropy_3.toml",
 			},
 			wantErr: fmt.Errorf("strconv.ParseFloat: parsing \"x\": invalid syntax"),
 		},
 		{
 			description: "test entropy value group",
 			opts: options.Options{
-				Config: "../test_data/test_configs/bad_entropy_4.toml",
+				ConfigPath: "../test_data/test_configs/bad_entropy_4.toml",
 			},
 			wantErr: fmt.Errorf("strconv.ParseInt: parsing \"x\": invalid syntax"),
 		},
 		{
 			description: "test entropy value group",
 			opts: options.Options{
-				Config: "../test_data/test_configs/bad_entropy_5.toml",
+				ConfigPath: "../test_data/test_configs/bad_entropy_5.toml",
 			},
 			wantErr: fmt.Errorf("problem loading config: group cannot be lower than 0"),
 		},
 		{
 			description: "test entropy value group",
 			opts: options.Options{
-				Config: "../test_data/test_configs/bad_entropy_6.toml",
+				ConfigPath: "../test_data/test_configs/bad_entropy_6.toml",
 			},
 			wantErr: fmt.Errorf("problem loading config: group cannot be higher than number of groups in regexp"),
 		},
 		{
 			description: "test entropy range limits",
 			opts: options.Options{
-				Config: "../test_data/test_configs/bad_entropy_7.toml",
+				ConfigPath: "../test_data/test_configs/bad_entropy_7.toml",
 			},
 			wantErr: fmt.Errorf("problem loading config: invalid entropy ranges, must be within 0.0-8.0"),
 		},
@@ -151,7 +151,7 @@ func TestParseFields(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	config, err := NewConfig(options.Options{Config: configPath})
+	config, err := NewConfig(options.Options{ConfigPath: configPath})
 	if err != nil {
 		t.Fatalf("Couldn't parse config: %v", err)
 	}

diff --git a/config/default.go b/config/default.go
@@ -7,7 +7,7 @@ const DefaultConfig = `
 title = "gitleaks config"
 
 [[rules]]
-	description = "AWS Manager ID"
+	description = "AWS Access Key"
 	regex = '''(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}'''
 	tags = ["key", "AWS"]
 
@@ -131,6 +131,31 @@ title = "gitleaks config"
 	regex = '''(?i)twilio(.{0,20})?SK[0-9a-f]{32}'''
 	tags = ["key", "twilio"]
 
+[[rules]]
+	description = "Dynatrace ttoken"
+	regex = '''dt0[a-zA-Z]{1}[0-9]{2}\.[A-Z0-9]{24}\.[A-Z0-9]{64}'''
+	tags = ["key", "Dynatrace"]
+
+[[rules]]
+	description = "Shopify shared secret"
+	regex = '''shpss_[a-fA-F0-9]{32}'''
+	tags = ["key", "Shopify"]
+
+[[rules]]
+	description = "Shopify access token"
+	regex = '''shpat_[a-fA-F0-9]{32}'''
+	tags = ["key", "Shopify"]
+
+[[rules]]
+	description = "Shopify custom app access token"
+	regex = '''shpca_[a-fA-F0-9]{32}'''
+	tags = ["key", "Shopify"]
+
+[[rules]]
+	description = "Shopify private app access token"
+	regex = '''shppa_[a-fA-F0-9]{32}'''
+	tags = ["key", "Shopify"]
+
 [allowlist]
 	description = "Allowlisted files"
 	files = ['''^\.?gitleaks.toml$''',