Adding support for nmhe (#3219)

* adding support for nmhe * updating docs
projectdiscovery · Jan 22, 2023 · 6c56a20 · 6c56a20
1 parent 6ebf5a7
commit 6c56a20
Show file tree

Hide file tree

Showing 9 changed files with 34 additions and 15 deletions.
diff --git a/README.md b/README.md
@@ -221,6 +221,7 @@ OPTIMIZATIONS:
    -retries int                        number of times to retry a failed request (default 1)
    -ldp, -leave-default-ports          leave default HTTP/HTTPS ports (eg. host:80,host:443)
    -mhe, -max-host-error int           max errors for a host before skipping from scan (default 30)
+   -nmhe, -no-mhe                      disable skipping host from scan based on errors
    -project                            use a project folder to avoid sending same request multiple times
    -project-path string                set a specific project path
    -spm, -stop-at-first-match          stop processing HTTP requests after the first match (may break template/workflow logic)

diff --git a/README_CN.md b/README_CN.md
@@ -189,6 +189,7 @@ Nuclei是一款注重于可配置性、可扩展性和易用性的基于模板
    -retries int                          重试次数（默认：1）
    -ldp, -leave-default-ports            指定HTTP/HTTPS默认端口（例如：host:80，host:443）
    -mhe, -max-host-error int             某主机扫描失败次数，跳过该主机（默认：30）
+   -nmhe, -no-mhe                        disable skipping host from scan based on errors
    -project                              使用项目文件夹避免多次发送同一请求
    -project-path string                  设置特定的项目文件夹
    -spm, -stop-at-first-path             得到一个结果后停止（或许会中断模板和工作流的逻辑）

diff --git a/README_ID.md b/README_ID.md
@@ -188,6 +188,7 @@ OPTIMIZATIONS:
    -retries int                        number of times to retry a failed request (default 1)
    -ldp, -leave-default-ports          leave default HTTP/HTTPS ports (eg. host:80,host:443
    -mhe, -max-host-error int           max errors for a host before skipping from scan (default 30)
+   -nmhe, -no-mhe                      disable skipping host from scan based on errors
    -project                            use a project folder to avoid sending same request multiple times
    -project-path string                set a specific project path
    -spm, -stop-at-first-path           stop processing HTTP requests after the first match (may break template/workflow logic)

diff --git a/README_KR.md b/README_KR.md
@@ -178,6 +178,7 @@ OPTIMIZATIONS:
    -retries int                실패한 요청을 재시도하는 횟수 (기본 1)
    -ldp, -leave-default-ports  leave default HTTP/HTTPS ports (eg. host:80,host:443
    -mhe, -max-host-error int   스캔을 건너뛰기 전에 호스트에 대한 최대 오류 수 (기본 30)
+   -nmhe, -no-mhe                      disable skipping host from scan based on errors
    -project                    프로젝트 폴더를 사용하여 동일한 요청을 여러 번 보내지 않음
    -project-path string        특정 프로젝트 경로 설정
    -spm, -stop-at-first-match  첫 번째 일치 후 HTTP 요청 처리 중지 (template/workflow 로직이 중단될 수 있음)

diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go
@@ -243,6 +243,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.IntVar(&options.Retries, "retries", 1, "number of times to retry a failed request"),
 		flagSet.BoolVarP(&options.LeaveDefaultPorts, "leave-default-ports", "ldp", false, "leave default HTTP/HTTPS ports (eg. host:80,host:443)"),
 		flagSet.IntVarP(&options.MaxHostError, "max-host-error", "mhe", 30, "max errors for a host before skipping from scan"),
+		flagSet.BoolVarP(&options.NoHostErrors, "no-mhe", "nmhe", false, "disable skipping host from scan based on errors"),
 		flagSet.BoolVar(&options.Project, "project", false, "use a project folder to avoid sending same request multiple times"),
 		flagSet.StringVar(&options.ProjectPath, "project-path", os.TempDir(), "set a specific project path"),
 		flagSet.BoolVarP(&options.StopAtFirstMatch, "stop-at-first-match", "spm", false, "stop processing HTTP requests after the first match (may break template/workflow logic)"),

diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go
@@ -3,8 +3,6 @@ package runner
 import (
 	"bufio"
 	"fmt"
-	"io"
-	"log"
 	"os"
 	"path/filepath"
 	"strings"
@@ -24,6 +22,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 	"github.com/projectdiscovery/stringsutil"
 	fileutil "github.com/projectdiscovery/utils/file"
+	logutil "github.com/projectdiscovery/utils/log"
 )
 
 func ConfigureOptions() error {
@@ -260,8 +259,7 @@ func configureOutput(options *types.Options) {
 	}
 
 	// disable standard logger (ref: https://github.com/golang/go/issues/19895)
-	log.SetFlags(0)
-	log.SetOutput(io.Discard)
+	logutil.DisableDefaultLogger()
 }
 
 // loadResolvers loads resolvers from both user provided flag and file

diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go
@@ -392,12 +392,6 @@ func (r *Runner) RunEnumeration() error {
 		r.options.ExcludeTags = append(r.options.ExcludeTags, ignoreFile.Tags...)
 		r.options.ExcludedTemplates = append(r.options.ExcludedTemplates, ignoreFile.Files...)
 	}
-	var cache *hosterrorscache.Cache
-	if r.options.MaxHostError > 0 {
-		cache = hosterrorscache.New(r.options.MaxHostError, hosterrorscache.DefaultMaxHostsCount)
-		cache.SetVerbose(r.options.Verbose)
-	}
-	r.hostErrors = cache
 
 	// Create the executer options which will be used throughout the execution
 	// stage by the nuclei engine modules.
@@ -411,12 +405,19 @@ func (r *Runner) RunEnumeration() error {
 		Interactsh:      r.interactsh,
 		ProjectFile:     r.projectFile,
 		Browser:         r.browser,
-		HostErrorsCache: cache,
 		Colorizer:       r.colorizer,
 		ResumeCfg:       r.resumeCfg,
 		ExcludeMatchers: excludematchers.New(r.options.ExcludeMatchers),
 		InputHelper:     input.NewHelper(),
 	}
+
+	if r.options.ShouldUseHostError() {
+		cache := hosterrorscache.New(r.options.MaxHostError, hosterrorscache.DefaultMaxHostsCount)
+		cache.SetVerbose(r.options.Verbose)
+		r.hostErrors = cache
+		executerOpts.HostErrorsCache = cache
+	}
+
 	engine := core.New(r.options)
 	engine.SetExecuterOptions(executerOpts)
 

diff --git a/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go b/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go
@@ -89,9 +89,6 @@ func (c *Cache) normalizeCacheValue(value string) string {
 //   - host type
 func (c *Cache) Check(value string) bool {
 	finalValue := c.normalizeCacheValue(value)
-	if !c.failedTargets.Has(finalValue) {
-		return false
-	}
 
 	existingCacheItem, err := c.failedTargets.GetIFPresent(finalValue)
 	if err != nil {

diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go
@@ -141,6 +141,8 @@ type Options struct {
 	MetricsPort int
 	// MaxHostError is the maximum number of errors allowed for a host
 	MaxHostError int
+	// NoHostErrors disables host skipping after maximum number of errors
+	NoHostErrors bool
 	// BulkSize is the of targets analyzed in parallel for each template
 	BulkSize int
 	// TemplateThreads is the number of templates executed in parallel
@@ -389,5 +391,21 @@ func DefaultOptions() *Options {
 
 // HasCloudOptions returns true if cloud options have been specified
 func (options *Options) HasCloudOptions() bool {
-	return options.ScanList || options.DeleteScan != "" || options.ScanOutput != "" || options.ListDatasources || options.ListTargets || options.ListTemplates || options.RemoveDatasource != "" || options.AddTarget != "" || options.AddTemplate != "" || options.RemoveTarget != "" || options.RemoveTemplate != "" || options.GetTarget != "" || options.GetTemplate != ""
+	return options.ScanList ||
+		options.DeleteScan != "" ||
+		options.ScanOutput != "" ||
+		options.ListDatasources ||
+		options.ListTargets ||
+		options.ListTemplates ||
+		options.RemoveDatasource != "" ||
+		options.AddTarget != "" ||
+		options.AddTemplate != "" ||
+		options.RemoveTarget != "" ||
+		options.RemoveTemplate != "" ||
+		options.GetTarget != "" ||
+		options.GetTemplate != ""
+}
+
+func (options *Options) ShouldUseHostError() bool {
+	return options.MaxHostError > 0 && !options.NoHostErrors
 }