Skip to content

Commit

Permalink
Refactor detect, add entropy to all findings (#804)
Browse files Browse the repository at this point in the history 
Refactor `detect`, add `entropy` to all findings
  • Loading branch information
@zricethezav
zricethezav authored Mar 12, 2022
1 parent 9326f35 commit 6e72472
Show file tree
Hide file tree
Showing 24 changed files with 1,086 additions and 913 deletions.
89 changes: 62 additions & 27 deletions cmd/detect.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ import (

"github.com/zricethezav/gitleaks/v8/config"
"github.com/zricethezav/gitleaks/v8/detect"
"github.com/zricethezav/gitleaks/v8/git"
"github.com/zricethezav/gitleaks/v8/report"
)

Expand All @@ -35,56 +34,92 @@ func runDetect(cmd *cobra.Command, args []string) {
err error
)

viper.Unmarshal(&vc)
// Load config
if err = viper.Unmarshal(&vc); err != nil {
log.Fatal().Err(err).Msg("Failed to load config")
}
cfg, err := vc.Translate()
if err != nil {
log.Fatal().Err(err).Msg("Failed to load config")
}

cfg.Path, _ = cmd.Flags().GetString("config")
source, _ := cmd.Flags().GetString("source")
logOpts, _ := cmd.Flags().GetString("log-opts")
verbose, _ := cmd.Flags().GetBool("verbose")
redact, _ := cmd.Flags().GetBool("redact")
noGit, _ := cmd.Flags().GetBool("no-git")
exitCode, _ := cmd.Flags().GetInt("exit-code")
if cfg.Path == "" {
cfg.Path = filepath.Join(source, ".gitleaks.toml")
}

// start timer
start := time.Now()

// Setup detector
detector := detect.NewDetector(cfg)
detector.Config.Path, err = cmd.Flags().GetString("config")
if err != nil {
log.Fatal().Err(err)
}
source, err := cmd.Flags().GetString("source")
if err != nil {
log.Fatal().Err(err)
}
// if config path is not set, then use the {source}/.gitleaks.toml path.
// note that there may not be a `{source}/.gitleaks.toml` file, this is ok.
if detector.Config.Path == "" {
detector.Config.Path = filepath.Join(source, ".gitleaks.toml")
}
// set verbose flag
if detector.Verbose, err = cmd.Flags().GetBool("verbose"); err != nil {
log.Fatal().Err(err)
}
// set redact flag
if detector.Redact, err = cmd.Flags().GetBool("redact"); err != nil {
log.Fatal().Err(err)
}

// set exit code
exitCode, err := cmd.Flags().GetInt("exit-code")
if err != nil {
log.Fatal().Err(err)
}

// determine what type of scan:
// - git: scan the history of the repo
// - no-git: scan files by treating the repo as a plain directory
noGit, err := cmd.Flags().GetBool("no-git")
if err != nil {
log.Fatal().Err(err)
}

// start the detector scan
if noGit {
if logOpts != "" {
log.Fatal().Err(err).Msg("--log-opts cannot be used with --no-git")
}
findings, err = detect.FromFiles(source, cfg, detect.Options{
Verbose: verbose,
Redact: redact,
})
findings, err = detector.DetectFiles(source)
if err != nil {
log.Fatal().Err(err).Msg("Failed to scan files")
// don't exit on error, just log it
log.Error().Err(err)
}

} else {
files, err := git.GitLog(source, logOpts)
logOpts, err := cmd.Flags().GetString("log-opts")
if err != nil {
log.Fatal().Err(err).Msg("Failed to get git log")
log.Fatal().Err(err)
}
findings, err = detector.DetectGit(source, logOpts, detect.DetectType)
if err != nil {
// don't exit on error, just log it
log.Error().Err(err)
}

findings = detect.FromGit(files, cfg, detect.Options{Verbose: verbose, Redact: redact})
}

// log info about the scan
log.Info().Msgf("scan completed in %s", time.Since(start))
if len(findings) != 0 {
log.Warn().Msgf("leaks found: %d", len(findings))
} else {
log.Info().Msg("no leaks found")
}

log.Info().Msgf("scan completed in %s", time.Since(start))

// write report if desired
reportPath, _ := cmd.Flags().GetString("report-path")
ext, _ := cmd.Flags().GetString("report-format")
if reportPath != "" {
report.Write(findings, cfg, ext, reportPath)
if err = report.Write(findings, cfg, ext, reportPath); err != nil {
log.Fatal().Err(err)
}
}

if len(findings) != 0 {
Expand Down
61 changes: 47 additions & 14 deletions cmd/protect.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ import (

"github.com/zricethezav/gitleaks/v8/config"
"github.com/zricethezav/gitleaks/v8/detect"
"github.com/zricethezav/gitleaks/v8/git"
"github.com/zricethezav/gitleaks/v8/report"
)

Expand All @@ -30,41 +29,75 @@ func runProtect(cmd *cobra.Command, args []string) {
initConfig()
var vc config.ViperConfig

viper.Unmarshal(&vc)
if err := viper.Unmarshal(&vc); err != nil {
log.Fatal().Err(err).Msg("Failed to load config")
}
cfg, err := vc.Translate()
if err != nil {
log.Fatal().Err(err).Msg("Failed to load config")
}

cfg.Path, _ = cmd.Flags().GetString("config")
source, _ := cmd.Flags().GetString("source")
verbose, _ := cmd.Flags().GetBool("verbose")
redact, _ := cmd.Flags().GetBool("redact")
exitCode, _ := cmd.Flags().GetInt("exit-code")
staged, _ := cmd.Flags().GetBool("staged")
if cfg.Path == "" {
cfg.Path = filepath.Join(source, ".gitleaks.toml")
}
start := time.Now()

files, err := git.GitDiff(source, staged)
// Setup detector
detector := detect.NewDetector(cfg)
detector.Config.Path, err = cmd.Flags().GetString("config")
if err != nil {
log.Fatal().Err(err)
}
source, err := cmd.Flags().GetString("source")
if err != nil {
log.Fatal().Err(err).Msg("Failed to get git log")
log.Fatal().Err(err)
}
// if config path is not set, then use the {source}/.gitleaks.toml path.
// note that there may not be a `{source}/.gitleaks.toml` file, this is ok.
if detector.Config.Path == "" {
detector.Config.Path = filepath.Join(source, ".gitleaks.toml")
}
// set verbose flag
if detector.Verbose, err = cmd.Flags().GetBool("verbose"); err != nil {
log.Fatal().Err(err)
}
// set redact flag
if detector.Redact, err = cmd.Flags().GetBool("redact"); err != nil {
log.Fatal().Err(err)
}

findings := detect.FromGit(files, cfg, detect.Options{Verbose: verbose, Redact: redact})
// get log options for git scan
logOpts, err := cmd.Flags().GetString("log-opts")
if err != nil {
log.Fatal().Err(err)
}

// start git scan
var findings []report.Finding
if staged {
findings, err = detector.DetectGit(source, logOpts, detect.ProtectStagedType)
} else {
findings, err = detector.DetectGit(source, logOpts, detect.ProtectType)
}
if err != nil {
// don't exit on error, just log it
log.Error().Err(err)
}

// log info about the scan
log.Info().Msgf("scan completed in %s", time.Since(start))
if len(findings) != 0 {
log.Warn().Msgf("leaks found: %d", len(findings))
} else {
log.Info().Msg("no leaks found")
}

log.Info().Msgf("scan duration: %s", time.Since(start))

reportPath, _ := cmd.Flags().GetString("report-path")
ext, _ := cmd.Flags().GetString("report-format")
if reportPath != "" {
report.Write(findings, cfg, ext, reportPath)
if err = report.Write(findings, cfg, ext, reportPath); err != nil {
log.Fatal().Err(err)
}
}
if len(findings) != 0 {
os.Exit(exitCode)
Expand Down
15 changes: 11 additions & 4 deletions cmd/root.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,10 @@ func init() {
rootCmd.PersistentFlags().StringP("log-level", "l", "info", "log level (debug, info, warn, error, fatal)")
rootCmd.PersistentFlags().BoolP("verbose", "v", false, "show verbose output from scan")
rootCmd.PersistentFlags().Bool("redact", false, "redact secrets from logs and stdout")
viper.BindPFlag("config", rootCmd.PersistentFlags().Lookup("config"))
err := viper.BindPFlag("config", rootCmd.PersistentFlags().Lookup("config"))
if err != nil {
log.Fatal().Msgf("err binding config %s", err.Error())
}
}

func initLog() {
Expand All @@ -71,7 +74,7 @@ func initLog() {
}

func initConfig() {
fmt.Fprintf(os.Stderr, banner)
fmt.Fprint(os.Stderr, banner)
cfgPath, err := rootCmd.Flags().GetString("config")
if err != nil {
log.Fatal().Msg(err.Error())
Expand All @@ -97,14 +100,18 @@ func initConfig() {
log.Debug().Msgf("Unable to load gitleaks config from %s since --source=%s is a file, using default config",
filepath.Join(source, ".gitleaks.toml"), source)
viper.SetConfigType("toml")
viper.ReadConfig(strings.NewReader(config.DefaultConfig))
if err = viper.ReadConfig(strings.NewReader(config.DefaultConfig)); err != nil {
log.Fatal().Msgf("err reading toml %s", err.Error())
}
return
}

if _, err := os.Stat(filepath.Join(source, ".gitleaks.toml")); os.IsNotExist(err) {
log.Debug().Msgf("No gitleaks config found in path %s, using default gitleaks config", filepath.Join(source, ".gitleaks.toml"))
viper.SetConfigType("toml")
viper.ReadConfig(strings.NewReader(config.DefaultConfig))
if err = viper.ReadConfig(strings.NewReader(config.DefaultConfig)); err != nil {
log.Fatal().Msgf("err reading default config toml %s", err.Error())
}
return
} else {
log.Debug().Msgf("Using existing gitleaks config %s from `(--source)/.gitleaks.toml`", filepath.Join(source, ".gitleaks.toml"))
Expand Down
28 changes: 17 additions & 11 deletions config/allowlist.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,23 @@ package config

import "regexp"

// Allowlist allows a rule to be ignored for specific
// regexes, paths, and/or commits
type Allowlist struct {
// Short human readable description of the allowlist.
Description string
Regexes []*regexp.Regexp
Paths []*regexp.Regexp
Commits []string

// Regexes is slice of content regular expressions that are allowed to be ignored.
Regexes []*regexp.Regexp

// Paths is a slice of path regular expressions that are allowed to be ignored.
Paths []*regexp.Regexp

// Commits is a slice of commit SHAs that are allowed to be ignored.
Commits []string
}

// CommitAllowed returns true if the commit is allowed to be ignored.
func (a *Allowlist) CommitAllowed(c string) bool {
if c == "" {
return false
Expand All @@ -21,16 +31,12 @@ func (a *Allowlist) CommitAllowed(c string) bool {
return false
}

// PathAllowed returns true if the path is allowed to be ignored.
func (a *Allowlist) PathAllowed(path string) bool {
if anyRegexMatch(path, a.Paths) {
return true
}
return false
return anyRegexMatch(path, a.Paths)
}

// RegexAllowed returns true if the regex is allowed to be ignored.
func (a *Allowlist) RegexAllowed(s string) bool {
if anyRegexMatch(s, a.Regexes) {
return true
}
return false
return anyRegexMatch(s, a.Regexes)
}
55 changes: 4 additions & 51 deletions config/config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,10 @@ func TestTranslate(t *testing.T) {
}

var vc ViperConfig
viper.Unmarshal(&vc)
err = viper.Unmarshal(&vc)
if err != nil {
t.Error(err)
}
cfg, err := vc.Translate()
if tt.wantError != nil {
if err == nil {
Expand All @@ -115,53 +118,3 @@ func TestTranslate(t *testing.T) {
assert.Equal(t, cfg.Rules, tt.cfg.Rules)
}
}

func TestIncludeEntropy(t *testing.T) {
tests := []struct {
rule Rule
secret string
entropy float32
include bool
}{
{
rule: Rule{
RuleID: "generic-api-key",
SecretGroup: 4,
Entropy: 3.5,
Regex: regexp.MustCompile(`(?i)((key|api|token|secret|password)[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([0-9a-zA-Z\-_=]{8,64})['\"]`),
},
secret: `e7322523fb86ed64c836a979cf8465fbd436378c653c1db38f9ae87bc62a6fd5`,
entropy: 3.7906235872459746,
include: true,
},
{
rule: Rule{
RuleID: "generic-api-key",
SecretGroup: 4,
Entropy: 4,
Regex: regexp.MustCompile(`(?i)((key|api|token|secret|password)[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([0-9a-zA-Z\-_=]{8,64})['\"]`),
},
secret: `e7322523fb86ed64c836a979cf8465fbd436378c653c1db38f9ae87bc62a6fd5`,
entropy: 3.7906235872459746,
include: false,
},
{
rule: Rule{
RuleID: "generic-api-key",
SecretGroup: 4,
Entropy: 3.0,
Regex: regexp.MustCompile(`(?i)((key|api|token|secret|password)[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([0-9a-zA-Z\-_=]{8,64})['\"]`),
},
secret: `ssh-keyboard-interactive`,
entropy: 0,
include: false,
},
}

for _, tt := range tests {
include, entropy := tt.rule.IncludeEntropy(tt.secret)
assert.Equal(t, true, tt.rule.EntropySet())
assert.Equal(t, tt.entropy, float32(entropy))
assert.Equal(t, tt.include, include)
}
}
Loading

0 comments on commit 6e72472

Please sign in to comment.