Skip to content

Commit

Permalink
refactor(config): keyword map (#1538)
Browse files Browse the repository at this point in the history
  • Loading branch information
@rgmz
rgmz authored Sep 30, 2024
1 parent a971a32 commit c5b15c9
Show file tree
Hide file tree
Showing 5 changed files with 57 additions and 42 deletions.
62 changes: 33 additions & 29 deletions cmd/generate/config/utils/validate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,20 +11,9 @@ import (
"strings"
)

func Validate(r config.Rule, truePositives []string, falsePositives []string) *config.Rule {
// normalize keywords like in the config package
var keywords []string
for _, k := range r.Keywords {
keywords = append(keywords, strings.ToLower(k))
}
r.Keywords = keywords

rules := make(map[string]config.Rule)
rules[r.RuleID] = r
d := detect.NewDetector(config.Config{
Rules: rules,
Keywords: keywords,
})
func Validate(rule config.Rule, truePositives []string, falsePositives []string) *config.Rule {
r := &rule
d := createSingleRuleDetector(r)
for _, tp := range truePositives {
if len(d.DetectString(tp)) != 1 {
log.Fatal().
Expand All @@ -43,22 +32,12 @@ func Validate(r config.Rule, truePositives []string, falsePositives []string) *c
Msg("Failed to Validate. False positive was detected by regex.")
}
}
return &r
return r
}

func ValidateWithPaths(r config.Rule, truePositives map[string]string, falsePositives map[string]string) *config.Rule {
var keywords []string
for _, k := range r.Keywords {
keywords = append(keywords, strings.ToLower(k))
}
r.Keywords = keywords

rules := make(map[string]config.Rule)
rules[r.RuleID] = r
d := detect.NewDetector(config.Config{
Rules: rules,
Keywords: keywords,
})
func ValidateWithPaths(rule config.Rule, truePositives map[string]string, falsePositives map[string]string) *config.Rule {
r := &rule
d := createSingleRuleDetector(r)
for path, tp := range truePositives {
f := detect.Fragment{Raw: tp, FilePath: path}
if len(d.Detect(f)) != 1 {
Expand All @@ -81,5 +60,30 @@ func ValidateWithPaths(r config.Rule, truePositives map[string]string, falsePosi
Msg("Failed to Validate. False positive was detected by regex and/or path.")
}
}
return &r
return r
}

func createSingleRuleDetector(r *config.Rule) *detect.Detector {
// normalize keywords like in the config package
var (
uniqueKeywords = make(map[string]struct{})
keywords []string
)
for _, keyword := range r.Keywords {
k := strings.ToLower(keyword)
if _, ok := uniqueKeywords[k]; ok {
continue
}
keywords = append(keywords, k)
uniqueKeywords[k] = struct{}{}
}
r.Keywords = keywords

rules := map[string]config.Rule{
r.RuleID: *r,
}
return detect.NewDetector(config.Config{
Rules: rules,
Keywords: uniqueKeywords,
})
}
22 changes: 13 additions & 9 deletions config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ type Config struct {
Description string
Rules map[string]Rule
Allowlist Allowlist
Keywords []string
Keywords map[string]struct{}

// used to keep sarif results consistent
OrderedRules []string
Expand All @@ -75,10 +75,10 @@ type Extend struct {

func (vc *ViperConfig) Translate() (Config, error) {
var (
keywords []string
keywords = make(map[string]struct{})
orderedRules []string
rulesMap = make(map[string]Rule)
)
rulesMap := make(map[string]Rule)

for _, r := range vc.Rules {
var allowlistRegexes []*regexp.Regexp
Expand All @@ -94,7 +94,7 @@ func (vc *ViperConfig) Translate() (Config, error) {
r.Keywords = []string{}
} else {
for _, k := range r.Keywords {
keywords = append(keywords, strings.ToLower(k))
keywords[strings.ToLower(k)] = struct{}{}
}
}

Expand Down Expand Up @@ -245,7 +245,9 @@ func (c *Config) extend(extensionConfig Config) {
if !ok {
// Rule doesn't exist, add it to the config.
c.Rules[ruleID] = baseRule
c.Keywords = append(c.Keywords, baseRule.Keywords...)
for _, k := range baseRule.Keywords {
c.Keywords[k] = struct{}{}
}
c.OrderedRules = append(c.OrderedRules, ruleID)
} else {
// Rule exists, merge our changes into the base.
Expand All @@ -255,10 +257,12 @@ func (c *Config) extend(extensionConfig Config) {
baseRule.Allowlist.RegexTarget = currentRule.Allowlist.RegexTarget
baseRule.Allowlist.StopWords = append(baseRule.Allowlist.StopWords, currentRule.Allowlist.StopWords...)
// The keywords from the base rule and the extended rule must be merged into the global keywords list
c.Keywords = append(c.Keywords, baseRule.Keywords...)
c.Keywords = append(c.Keywords, currentRule.Keywords...)

delete(c.Rules, ruleID)
for _, k := range baseRule.Keywords {
c.Keywords[k] = struct{}{}
}
for _, k := range currentRule.Keywords {
c.Keywords[k] = struct{}{}
}
c.Rules[ruleID] = baseRule
}
}
Expand Down
3 changes: 2 additions & 1 deletion detect/detect.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (

"github.com/rs/zerolog/log"
"github.com/spf13/viper"
"golang.org/x/exp/maps"
)

const (
Expand Down Expand Up @@ -108,7 +109,7 @@ func NewDetector(cfg config.Config) *Detector {
findingMutex: &sync.Mutex{},
findings: make([]report.Finding, 0),
Config: cfg,
prefilter: *ahocorasick.NewTrieBuilder().AddStrings(cfg.Keywords).Build(),
prefilter: *ahocorasick.NewTrieBuilder().AddStrings(maps.Keys(cfg.Keywords)).Build(),
Sema: semgroup.NewGroup(context.Background(), 40),
}
}
Expand Down
7 changes: 5 additions & 2 deletions go.mod
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
module github.com/zricethezav/gitleaks/v8

go 1.22
go 1.22.0

toolchain go1.22.5

require (
github.com/BobuSumisu/aho-corasick v1.0.3
Expand All @@ -13,6 +15,7 @@ require (
github.com/spf13/cobra v1.2.1
github.com/spf13/viper v1.8.1
github.com/stretchr/testify v1.7.0
golang.org/x/exp v0.0.0-20240909161429-701f63a606c0
)

require (
Expand Down Expand Up @@ -40,7 +43,7 @@ require (
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/subosito/gotenv v1.2.0 // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
golang.org/x/sync v0.8.0 // indirect
golang.org/x/sys v0.6.0 // indirect
golang.org/x/text v0.3.8 // indirect
gopkg.in/ini.v1 v1.62.0 // indirect
Expand Down
5 changes: 4 additions & 1 deletion go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -309,6 +309,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk=
golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
Expand Down Expand Up @@ -393,8 +395,9 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
Expand Down

2 comments on commit c5b15c9

@matiasmoyano
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

hello, i have a comment on this commit,
I have a pipeline that crashes on gitleaks
`
go fmt ./...

33 | go: errors parsing go.mod:
34 | /codebuild/output/src2754723846/src/gitleaks/go.mod:3: invalid go version '1.22.0': must match format 1.23
35 | /codebuild/output/src2754723846/src/gitleaks/go.mod:5: unknown directive: toolchain
36 | make: *** [Makefile:13: format] Error 1
`

what I believe is happening is that for some reason go is not compatible with writing the version as 1.22.0, it should be 1.22 or 1.23
I don't know why

@rgmz
Copy link
Contributor Author

@rgmz rgmz commented on c5b15c9 Oct 1, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's weird. I think this was automatically added from go mod tidy, but it should be easy to revert.

-go 1.22.0
+go 1.22

Please sign in to comment.