Merge branch 'master' into entropy_trace

gitleaks · Dec 23, 2024 · 22d78ab · 22d78ab
2 parents 80bc6ee + 7edfc6b
commit 22d78ab
Show file tree

Hide file tree

Showing 86 changed files with 1,418 additions and 706 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.22 AS build
+FROM golang:1.23 AS build
 WORKDIR /go/src/github.com/zricethezav/gitleaks
 COPY . .
 RUN VERSION=$(git describe --tags --abbrev=0) && \

diff --git a/README.md b/README.md
@@ -162,8 +162,9 @@ Flags:
       --no-banner                     suppress banner
       --no-color                      turn off color for verbose output
       --redact uint[=100]             redact secrets from logs and stdout. To redact only parts of the secret just apply a percent value from 0..100. For example --redact=20 (default 100%)
-  -f, --report-format string          output format (json, jsonextra, csv, junit, sarif) (default "json")
+  -f, --report-format string          output format (json, csv, junit, sarif) (default "json")
   -r, --report-path string            report file
+      --report-template string        template file used to generate the report (implies --report-format=template)
   -v, --verbose                       show verbose output from scan
       --version                       version for gitleaks
 
@@ -393,6 +394,47 @@ Currently supported encodings:
 
 - `base64` (both standard and base64url)
 
+#### Reporting
+
+Gitleaks has built-in support for several report formats: [`json`](https://github.com/gitleaks/gitleaks/blob/master/testdata/expected/report/json_simple.json), [`csv`](https://github.com/gitleaks/gitleaks/blob/master/testdata/expected/report/csv_simple.csv?plain=1), [`junit`](https://github.com/gitleaks/gitleaks/blob/master/testdata/expected/report/junit_simple.xml), and [`sarif`](https://github.com/gitleaks/gitleaks/blob/master/testdata/expected/report/sarif_simple.sarif).
+
+If none of these formats fit your need, you can create your own report format with a [Go `text/template` .tmpl file](https://www.digitalocean.com/community/tutorials/how-to-use-templates-in-go#step-4-writing-a-template) and the `--report-template` flag. The template can use [extended functionality from the `Masterminds/sprig` template library](https://masterminds.github.io/sprig/).
+
+For example, the following template provides a custom JSON output:
+```gotemplate
+# jsonextra.tmpl
+[{{ $lastFinding := (sub (len . ) 1) }}
+{{- range $i, $finding := . }}{{with $finding}}
+    {
+        "Description": {{ quote .Description }},
+        "StartLine": {{ .StartLine }},
+        "EndLine": {{ .EndLine }},
+        "StartColumn": {{ .StartColumn }},
+        "EndColumn": {{ .EndColumn }},
+        "Line": {{ quote .Line }},
+        "Match": {{ quote .Match }},
+        "Secret": {{ quote .Secret }},
+        "File": "{{ .File }}",
+        "SymlinkFile": {{ quote .SymlinkFile }},
+        "Commit": {{ quote .Commit }},
+        "Entropy": {{ .Entropy }},
+        "Author": {{ quote .Author }},
+        "Email": {{ quote .Email }},
+        "Date": {{ quote .Date }},
+        "Message": {{ quote .Message }},
+        "Tags": [{{ $lastTag := (sub (len .Tags ) 1) }}{{ range $j, $tag := .Tags }}{{ quote . }}{{ if ne $j $lastTag }},{{ end }}{{ end }}],
+        "RuleID": {{ quote .RuleID }},
+        "Fingerprint": {{ quote .Fingerprint }}
+    }{{ if ne $i $lastFinding }},{{ end }}
+{{- end}}{{ end }}
+]
+```
+
+Usage:
+```sh
+$ gitleaks dir ~/leaky-repo/ --report-path "report.json" --report-format template --report-template testdata/report/jsonextra.tmpl
+```
+
 ## Sponsorships
 
 <p align="left">

diff --git a/cmd/detect.go b/cmd/detect.go
@@ -62,6 +62,10 @@ func runDetect(cmd *cobra.Command, args []string) {
 
 	detector := Detector(cmd, cfg, source)
 
+	// set follow symlinks flag
+	if detector.FollowSymlinks, err = cmd.Flags().GetBool("follow-symlinks"); err != nil {
+		log.Fatal().Err(err).Msg("")
+	}
 	// set exit code
 	exitCode, err := cmd.Flags().GetInt("exit-code")
 	if err != nil {
@@ -83,7 +87,12 @@ func runDetect(cmd *cobra.Command, args []string) {
 	// start the detector scan
 	if noGit {
 		var paths <-chan sources.ScanTarget
-		paths, err = sources.DirectoryTargets(source, detector.Sema, detector.FollowSymlinks)
+		paths, err = sources.DirectoryTargets(
+			source,
+			detector.Sema,
+			detector.FollowSymlinks,
+			detector.Config.Allowlist.PathAllowed,
+		)
 		if err != nil {
 			log.Fatal().Err(err)
 		}
@@ -120,10 +129,5 @@ func runDetect(cmd *cobra.Command, args []string) {
 		}
 	}
 
-	// set follow symlinks flag
-	if detector.FollowSymlinks, err = cmd.Flags().GetBool("follow-symlinks"); err != nil {
-		log.Fatal().Err(err).Msg("")
-	}
-
-	findingSummaryAndExit(findings, cmd, cfg, exitCode, start, err)
+	findingSummaryAndExit(detector, findings, exitCode, start, err)
 }
diff --git a/cmd/directory.go b/cmd/directory.go
@@ -45,14 +45,23 @@ func runDirectory(cmd *cobra.Command, args []string) {
 
 	detector := Detector(cmd, cfg, source)
 
+	// set follow symlinks flag
+	if detector.FollowSymlinks, err = cmd.Flags().GetBool("follow-symlinks"); err != nil {
+		log.Fatal().Err(err).Msg("")
+	}
 	// set exit code
 	exitCode, err := cmd.Flags().GetInt("exit-code")
 	if err != nil {
 		log.Fatal().Err(err).Msg("could not get exit code")
 	}
 
 	var paths <-chan sources.ScanTarget
-	paths, err = sources.DirectoryTargets(source, detector.Sema, detector.FollowSymlinks)
+	paths, err = sources.DirectoryTargets(
+		source,
+		detector.Sema,
+		detector.FollowSymlinks,
+		detector.Config.Allowlist.PathAllowed,
+	)
 	if err != nil {
 		log.Fatal().Err(err)
 	}
@@ -63,5 +72,5 @@ func runDirectory(cmd *cobra.Command, args []string) {
 		log.Error().Err(err).Msg("failed scan directory")
 	}
 
-	findingSummaryAndExit(findings, cmd, cfg, exitCode, start, err)
+	findingSummaryAndExit(detector, findings, exitCode, start, err)
 }
diff --git a/cmd/generate/config/base/config.go b/cmd/generate/config/base/config.go
@@ -2,9 +2,10 @@ package base
 
 import (
 	"fmt"
-	"github.com/zricethezav/gitleaks/v8/config"
-	"regexp"
 	"strings"
+
+	regexp "github.com/wasilibs/go-re2"
+	"github.com/zricethezav/gitleaks/v8/config"
 )
 
 func CreateGlobalConfig() config.Config {
@@ -64,7 +65,7 @@ func CreateGlobalConfig() config.Config {
 				// ----------- Golang files -----------
 				regexp.MustCompile(`go\.(mod|sum|work(\.sum)?)$`),
 				regexp.MustCompile(`(^|/)vendor/modules\.txt$`),
-				regexp.MustCompile(`(^|/)vendor/(github\.com|golang\.org/x|google\.golang\.org|gopkg\.in|istio\.io|k8s\.io|sigs\.k8s\.io)/.*$`),
+				regexp.MustCompile(`(^|/)vendor/(github\.com|golang\.org/x|google\.golang\.org|gopkg\.in|istio\.io|k8s\.io|sigs\.k8s\.io)(/.*)?$`),
 
 				// ----------- Java files -----------
 				// Gradle
@@ -75,30 +76,27 @@ func CreateGlobalConfig() config.Config {
 
 				// ----------- JavaScript files -----------
 				// Dependencies and lock files.
-				regexp.MustCompile(`(^|/)node_modules/.*?$`),
-				regexp.MustCompile(`(^|/)package-lock\.json$`),
-				regexp.MustCompile(`(^|/)yarn\.lock$`),
-				regexp.MustCompile(`(^|/)pnpm-lock\.yaml$`),
-				regexp.MustCompile(`(^|/)npm-shrinkwrap\.json$`),
-				regexp.MustCompile(`(^|/)bower_components/.*?$`),
+				regexp.MustCompile(`(^|/)node_modules(/.*)?$`),
+				regexp.MustCompile(`(^|/)(npm-shrinkwrap\.json|package-lock\.json|pnpm-lock\.yaml|yarn\.lock)$`),
+				regexp.MustCompile(`(^|/)bower_components(/.*)?$`),
 				// TODO: Add more common static assets, such as swagger-ui.
 				regexp.MustCompile(`(^|/)(angular|jquery(-?ui)?|plotly|swagger-?ui)[a-zA-Z0-9.-]*(\.min)?\.js(\.map)?$`),
 
 				// ----------- Python files -----------
 				// Dependencies and lock files.
 				regexp.MustCompile(`(^|/)(Pipfile|poetry)\.lock$`),
 				// Virtual environments
-				regexp.MustCompile(`(?i)/?(v?env|virtualenv)/lib(64)?/.+$`),
-				regexp.MustCompile(`(?i)(^|/)(lib(64)?/python[23](\.\d{1,2})+/|python/[23](\.\d{1,2})+/lib(64)?/).+$`),
+				regexp.MustCompile(`(?i)/?(v?env|virtualenv)/lib(64)?(/.*)?$`),
+				regexp.MustCompile(`(?i)(^|/)(lib(64)?/python[23](\.\d{1,2})+|python/[23](\.\d{1,2})+/lib(64)?)(/.*)?$`),
 				// dist-info directory (https://py-pkgs.org/04-package-structure.html#building-sdists-and-wheels)
-				regexp.MustCompile(`(?i)(^|/)[a-z0-9_.]+-[0-9.]+\.dist-info/.+$`),
+				regexp.MustCompile(`(?i)(^|/)[a-z0-9_.]+-[0-9.]+\.dist-info(/.+)?$`),
 
 				// ----------- Ruby files -----------
-				regexp.MustCompile(`(^|/)vendor/(bundle|ruby)/.*?$`),
+				regexp.MustCompile(`(^|/)vendor/(bundle|ruby)(/.*?)?$`),
 				regexp.MustCompile(`\.gem$`), // tar archive
 
 				// Misc
-				regexp.MustCompile(`verification-metadata.xml`),
+				regexp.MustCompile(`verification-metadata\.xml`),
 				regexp.MustCompile(`Database.refactorlog`),
 				//regexp.MustCompile(`vendor`),
 			},

diff --git a/cmd/generate/config/main.go b/cmd/generate/config/main.go
@@ -4,10 +4,9 @@ import (
 	"os"
 	"text/template"
 
-	"github.com/zricethezav/gitleaks/v8/cmd/generate/config/base"
-
 	"github.com/rs/zerolog/log"
 
+	"github.com/zricethezav/gitleaks/v8/cmd/generate/config/base"
 	"github.com/zricethezav/gitleaks/v8/cmd/generate/config/rules"
 	"github.com/zricethezav/gitleaks/v8/config"
 )
@@ -90,6 +89,7 @@ func main() {
 		rules.FlutterwaveEncKey(),
 		rules.FlyIOAccessToken(),
 		rules.FrameIO(),
+		rules.Freemius(),
 		rules.FreshbooksAccessToken(),
 		rules.GoCardless(),
 		// TODO figure out what makes sense for GCP
@@ -108,6 +108,7 @@ func main() {
 		rules.GitlabKubernetesAgentToken(),
 		rules.GitlabOauthAppSecret(),
 		rules.GitlabPat(),
+		rules.GitlabPatRoutable(),
 		rules.GitlabPipelineTriggerToken(),
 		rules.GitlabRunnerRegistrationToken(),
 		rules.GitlabRunnerAuthenticationToken(),
@@ -184,6 +185,9 @@ func main() {
 		rules.SentryAccessToken(),
 		rules.SentryOrgToken(),
 		rules.SentryUserToken(),
+		rules.SettlemintApplicationAccessToken(),
+		rules.SettlemintPersonalAccessToken(),
+		rules.SettlemintServiceAccessToken(),
 		rules.ShippoAPIToken(),
 		rules.ShopifyAccessToken(),
 		rules.ShopifyCustomAccessToken(),

diff --git a/cmd/generate/config/rules/1password.go b/cmd/generate/config/rules/1password.go
@@ -1,10 +1,11 @@
 package rules
 
 import (
+	regexp "github.com/wasilibs/go-re2"
+
 	"github.com/zricethezav/gitleaks/v8/cmd/generate/config/utils"
 	"github.com/zricethezav/gitleaks/v8/cmd/generate/secrets"
 	"github.com/zricethezav/gitleaks/v8/config"
-	"regexp"
 )
 
 // https://developer.1password.com/docs/service-accounts/security/?token-example=encoded

diff --git a/cmd/generate/config/rules/age.go b/cmd/generate/config/rules/age.go
@@ -1,9 +1,9 @@
 package rules
 
 import (
-	"github.com/zricethezav/gitleaks/v8/cmd/generate/config/utils"
-	"regexp"
+	regexp "github.com/wasilibs/go-re2"
 
+	"github.com/zricethezav/gitleaks/v8/cmd/generate/config/utils"
 	"github.com/zricethezav/gitleaks/v8/config"
 )
 

diff --git a/cmd/generate/config/rules/authress.go b/cmd/generate/config/rules/authress.go
@@ -2,8 +2,8 @@ package rules
 
 import (
 	"fmt"
-	"github.com/zricethezav/gitleaks/v8/cmd/generate/config/utils"
 
+	"github.com/zricethezav/gitleaks/v8/cmd/generate/config/utils"
 	"github.com/zricethezav/gitleaks/v8/cmd/generate/secrets"
 	"github.com/zricethezav/gitleaks/v8/config"
 )

diff --git a/cmd/generate/config/rules/aws.go b/cmd/generate/config/rules/aws.go
@@ -1,10 +1,10 @@
 package rules
 
 import (
+	regexp "github.com/wasilibs/go-re2"
+
 	"github.com/zricethezav/gitleaks/v8/cmd/generate/config/utils"
 	"github.com/zricethezav/gitleaks/v8/cmd/generate/secrets"
-	"regexp"
-
 	"github.com/zricethezav/gitleaks/v8/config"
 )
 

diff --git a/cmd/generate/config/rules/azure.go b/cmd/generate/config/rules/azure.go
@@ -2,11 +2,13 @@ package rules
 
 import (
 	"fmt"
+
+	regexp "github.com/wasilibs/go-re2"
+
 	"github.com/zricethezav/gitleaks/v8/cmd/generate/config/utils"
-	"regexp"
+	"github.com/zricethezav/gitleaks/v8/config"
 
 	"github.com/zricethezav/gitleaks/v8/cmd/generate/secrets"
-	"github.com/zricethezav/gitleaks/v8/config"
 )
 
 // References:

diff --git a/cmd/generate/config/rules/clojars.go b/cmd/generate/config/rules/clojars.go
@@ -1,11 +1,12 @@
 package rules
 
 import (
+	regexp "github.com/wasilibs/go-re2"
+
 	"github.com/zricethezav/gitleaks/v8/cmd/generate/config/utils"
-	"regexp"
+	"github.com/zricethezav/gitleaks/v8/config"
 
 	"github.com/zricethezav/gitleaks/v8/cmd/generate/secrets"
-	"github.com/zricethezav/gitleaks/v8/config"
 )
 
 func Clojars() *config.Rule {

diff --git a/cmd/generate/config/rules/curl.go b/cmd/generate/config/rules/curl.go
@@ -2,9 +2,9 @@ package rules
 
 import (
 	"fmt"
-	"github.com/zricethezav/gitleaks/v8/cmd/generate/config/utils"
-	"regexp"
+	regexp "github.com/wasilibs/go-re2"
 
+	"github.com/zricethezav/gitleaks/v8/cmd/generate/config/utils"
 	"github.com/zricethezav/gitleaks/v8/config"
 )
 

diff --git a/cmd/generate/config/rules/doppler.go b/cmd/generate/config/rules/doppler.go
@@ -1,11 +1,12 @@
 package rules
 
 import (
+	regexp "github.com/wasilibs/go-re2"
+
 	"github.com/zricethezav/gitleaks/v8/cmd/generate/config/utils"
-	"regexp"
+	"github.com/zricethezav/gitleaks/v8/config"
 
 	"github.com/zricethezav/gitleaks/v8/cmd/generate/secrets"
-	"github.com/zricethezav/gitleaks/v8/config"
 )
 
 func Doppler() *config.Rule {

diff --git a/cmd/generate/config/rules/duffel.go b/cmd/generate/config/rules/duffel.go
@@ -1,11 +1,12 @@
 package rules
 
 import (
+	regexp "github.com/wasilibs/go-re2"
+
 	"github.com/zricethezav/gitleaks/v8/cmd/generate/config/utils"
-	"regexp"
+	"github.com/zricethezav/gitleaks/v8/config"
 
 	"github.com/zricethezav/gitleaks/v8/cmd/generate/secrets"
-	"github.com/zricethezav/gitleaks/v8/config"
 )
 
 func Duffel() *config.Rule {

diff --git a/cmd/generate/config/rules/dynatrace.go b/cmd/generate/config/rules/dynatrace.go
@@ -1,11 +1,12 @@
 package rules
 
 import (
+	regexp "github.com/wasilibs/go-re2"
+
 	"github.com/zricethezav/gitleaks/v8/cmd/generate/config/utils"
-	"regexp"
+	"github.com/zricethezav/gitleaks/v8/config"
 
 	"github.com/zricethezav/gitleaks/v8/cmd/generate/secrets"
-	"github.com/zricethezav/gitleaks/v8/config"
 )
 
 func Dynatrace() *config.Rule {

diff --git a/cmd/generate/config/rules/easypost.go b/cmd/generate/config/rules/easypost.go
@@ -1,11 +1,12 @@
 package rules
 
 import (
+	regexp "github.com/wasilibs/go-re2"
+
 	"github.com/zricethezav/gitleaks/v8/cmd/generate/config/utils"
-	"regexp"
+	"github.com/zricethezav/gitleaks/v8/config"
 
 	"github.com/zricethezav/gitleaks/v8/cmd/generate/secrets"
-	"github.com/zricethezav/gitleaks/v8/config"
 )
 
 func EasyPost() *config.Rule {

diff --git a/cmd/generate/config/rules/etsy.go b/cmd/generate/config/rules/etsy.go
@@ -2,6 +2,7 @@ package rules
 
 import (
 	"fmt"
+
 	"github.com/zricethezav/gitleaks/v8/cmd/generate/config/utils"
 	"github.com/zricethezav/gitleaks/v8/cmd/generate/secrets"
 	"github.com/zricethezav/gitleaks/v8/config"