From 23f07cb671ce56b76c6d0f063825ea1e85c47cb1 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Tue, 20 Aug 2024 10:29:51 +0700 Subject: [PATCH 1/6] docs: update example usage Signed-off-by: Dwi Siswanto --- README.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index cf7a192..490ec4f 100644 --- a/README.md +++ b/README.md @@ -42,11 +42,16 @@ import ( ) func main() { - // it requires a list of resolvers - resolvers := []string{"8.8.8.8:53", "8.8.4.4:53"} + // It requires a list of resolvers. + // Valid protocols are "udp", "tcp", "doh", "dot". Default are "udp". + resolvers := []string{"8.8.8.8:53", "8.8.4.4:53", "tcp:1.1.1.1"} retries := 2 hostname := "hackerone.com" - dnsClient := retryabledns.New(resolvers, retries) + + dnsClient, err := retryabledns.New(resolvers, retries) + if err != nil { + log.Fatal(err) + } ips, err := dnsClient.Resolve(hostname) if err != nil { From 2e48f41c183d3431fd5c83d21c87c338ee72262c Mon Sep 17 00:00:00 2001 From: Tom Palarz Date: Sat, 31 Aug 2024 03:51:37 -0500 Subject: [PATCH 2/6] Cleaned up error logic in QueryMultiple() and added ErrRetriesExceeded --- README.md | 3 ++- client.go | 12 ++++++++++-- client_test.go | 24 ++++++++++++++++++++++++ 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index cf7a192..2d72239 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,7 @@ func main() { // Query Types: dns.TypeA, dns.TypeNS, dns.TypeCNAME, dns.TypeSOA, dns.TypePTR, dns.TypeMX, dns.TypeANY // dns.TypeTXT, dns.TypeAAAA, dns.TypeSRV (from github.com/miekg/dns) + // retryabledns.ErrRetriesExceeded will be returned if a result isn't returned in max retries dnsResponses, err := dnsClient.Query(hostname, dns.TypeA) if err != nil { log.Fatal(err) @@ -69,4 +70,4 @@ func main() { Credits: - `https://github.com/lixiangzhong/dnsutil` -- `https://github.com/rs/dnstrace` \ No newline at end of file +- `https://github.com/rs/dnstrace` diff --git a/client.go b/client.go index 5d74157..6f90dac 100644 --- a/client.go +++ b/client.go @@ -22,6 +22,8 @@ import ( sliceutil "github.com/projectdiscovery/utils/slice" ) +var ErrRetriesExceeded = errors.New("could not resolve, max retries exceeded") + var internalRangeCheckerInstance *internalRangeChecker func init() { @@ -187,7 +189,7 @@ func (c *Client) Do(msg *dns.Msg) (*dns.Msg, error) { // In case we get a non empty answer stop retrying return resp, nil } - return resp, errors.New("could not resolve, max retries exceeded") + return resp, ErrRetriesExceeded } // Query sends a provided dns request and return enriched response @@ -322,8 +324,9 @@ func (c *Client) queryMultiple(host string, requestTypes []uint16, resolver Reso var ( resp *dns.Msg trResp chan *dns.Envelope + i int ) - for i := 0; i < c.options.MaxRetries; i++ { + for i = 0; i < c.options.MaxRetries; i++ { index := atomic.AddUint32(&c.serversIndex, 1) if !hasResolver { resolver = c.resolvers[index%uint32(len(c.resolvers))] @@ -421,6 +424,11 @@ func (c *Client) queryMultiple(host string, requestTypes []uint16, resolver Reso break } } + // Finished retry loop at limit, bail out + if i == c.options.MaxRetries { + err = ErrRetriesExceeded + break + } } return &dnsdata, err diff --git a/client_test.go b/client_test.go index 2bed4c3..7b47075 100644 --- a/client_test.go +++ b/client_test.go @@ -123,6 +123,30 @@ func TestQueryMultiple(t *testing.T) { require.NotZero(t, d.TTL) } +func TestRetries(t *testing.T) { + client, _ := New([]string{"127.0.0.1"}, 5) + + // Test that error is returned on max retries, should conn refused 5 times then err + _, err := client.QueryMultiple("scanme.sh", []uint16{dns.TypeA}) + require.True(t, err == ErrRetriesExceeded) + + msg := &dns.Msg{} + msg.Id = dns.Id() + msg.SetEdns0(4096, false) + msg.Question = make([]dns.Question, 1) + msg.RecursionDesired = true + question := dns.Question{ + Name: "scanme.sh", + Qtype: dns.TypeA, + Qclass: dns.ClassINET, + } + msg.Question[0] = question + + // Test with raw Do() interface as well + _, err = client.Do(msg) + require.True(t, err == ErrRetriesExceeded) +} + func TestTrace(t *testing.T) { client, _ := New([]string{"8.8.8.8:53", "1.1.1.1:53"}, 5) From d932d2aa6d7db7afbb732e1a9d5fd8638465a240 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Sep 2024 14:35:00 +0000 Subject: [PATCH 3/6] chore(deps): bump github.com/projectdiscovery/utils from 0.2.7 to 0.2.8 Bumps [github.com/projectdiscovery/utils](https://github.com/projectdiscovery/utils) from 0.2.7 to 0.2.8. - [Release notes](https://github.com/projectdiscovery/utils/releases) - [Changelog](https://github.com/projectdiscovery/utils/blob/main/CHANGELOG.md) - [Commits](https://github.com/projectdiscovery/utils/compare/v0.2.7...v0.2.8) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/utils dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 28bf732..01f05f3 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/microcosm-cc/bluemonday v1.0.25 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/projectdiscovery/utils v0.2.7 + github.com/projectdiscovery/utils v0.2.8 github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/mod v0.12.0 // indirect diff --git a/go.sum b/go.sum index 3a83fdc..f76aecc 100644 --- a/go.sum +++ b/go.sum @@ -18,8 +18,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= -github.com/projectdiscovery/utils v0.2.7 h1:XWdz7SscL++jqsnQ9ecHzSZE0RK33tyPcnqcXw+vmKs= -github.com/projectdiscovery/utils v0.2.7/go.mod h1:N0N7tbdNFPegd9NpJ3onCPClaBrERcOIB88yww6UCF8= +github.com/projectdiscovery/utils v0.2.8 h1:++NcCJ+lXEfNBHKBs6q+cWa8JrVS8cYdGcW9jOgZebI= +github.com/projectdiscovery/utils v0.2.8/go.mod h1:UYJ8GKbZaezPFRT/cOk01LreQZ1QK0fko1EUnSUiSGU= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= From 509afcbc09c512af956789b09e6e7c677158637a Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Mon, 2 Sep 2024 17:33:39 +0200 Subject: [PATCH 4/6] limit cname recursion --- client.go | 13 +++++++++++++ options.go | 1 + 2 files changed, 14 insertions(+) diff --git a/client.go b/client.go index 5d74157..a9bda45 100644 --- a/client.go +++ b/client.go @@ -22,6 +22,10 @@ import ( sliceutil "github.com/projectdiscovery/utils/slice" ) +var ( + DefaultMaxPerCNAMEFollows = 32 +) + var internalRangeCheckerInstance *internalRangeChecker func init() { @@ -62,6 +66,10 @@ func NewWithOptions(options Options) (*Client, error) { knownHosts, _ = hostsfile.ParseDefault() } + if options.MaxPerCNAMEFollows == 0 { + options.MaxPerCNAMEFollows = DefaultMaxPerCNAMEFollows + } + httpClient := doh.NewHttpClientWithTimeout(options.Timeout) client := Client{ @@ -472,6 +480,7 @@ func (c *Client) Trace(host string, requestType uint16, maxrecursion int) (*Trac msg.SetQuestion(host, requestType) servers := RootDNSServersIPv4 seenNS := make(map[string]struct{}) + seenCName := make(map[string]int) for i := 1; i < maxrecursion; i++ { msg.SetQuestion(host, requestType) dnsdatas, err := c.QueryParallel(host, requestType, servers) @@ -534,6 +543,10 @@ func (c *Client) Trace(host string, requestType uint16, maxrecursion int) (*Trac // follow cname if any if nextCname != "" { + seenCName[nextCname]++ + if seenCName[nextCname] > c.options.MaxPerCNAMEFollows { + break + } host = nextCname } } diff --git a/options.go b/options.go index df94050..7e0967e 100644 --- a/options.go +++ b/options.go @@ -20,6 +20,7 @@ type Options struct { LocalAddrIP net.IP LocalAddrPort uint16 ConnectionPoolThreads int + MaxPerCNAMEFollows int } // Returns a net.Addr of a UDP or TCP type depending on whats required From b2c6d23db52988a168f2f084aad5363f8b891bed Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Tue, 3 Sep 2024 05:33:26 +0700 Subject: [PATCH 5/6] docs(README): update key-features Signed-off-by: Dwi Siswanto --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 490ec4f..1504233 100644 --- a/README.md +++ b/README.md @@ -4,10 +4,11 @@ Based on `miekg/dns` and freely inspired by `bogdanovich/dns_resolver`. ## Features -- Supports system default resolvers along with user supplied ones -- Retries dns requests in case of I/O, Time, Network failures +- Supports both system default DNS resolvers and user-provided ones +- Retries DNS requests in case of I/O errors, timeouts, or network failures - Allows arbitrary query types - Resolution with random resolvers +- Compatible with various DNS resolver protocols (TCP, UDP, DoH, and DoT) ### Using *go get* From 0b895e3c7cc6ef4ed44173d564bae44601838446 Mon Sep 17 00:00:00 2001 From: mzack Date: Fri, 6 Sep 2024 02:31:06 +0200 Subject: [PATCH 6/6] adding support for proxy --- client.go | 124 ++++++++++++++++++++++++++++++++++------------ doh/doh_client.go | 5 +- doh/util.go | 62 +++++++++++++++++++---- go.mod | 2 +- options.go | 1 + 5 files changed, 151 insertions(+), 43 deletions(-) diff --git a/client.go b/client.go index e98797f..658bc43 100644 --- a/client.go +++ b/client.go @@ -9,6 +9,7 @@ import ( "fmt" "math/rand" "net" + "net/url" "strings" "sync" "sync/atomic" @@ -20,10 +21,9 @@ import ( iputil "github.com/projectdiscovery/utils/ip" mapsutil "github.com/projectdiscovery/utils/maps" sliceutil "github.com/projectdiscovery/utils/slice" + "golang.org/x/net/proxy" ) -var () - var ( // DefaultMaxPerCNAMEFollows is the default number of times a CNAME can be followed within a trace DefaultMaxPerCNAMEFollows = 32 @@ -53,6 +53,9 @@ type Client struct { tcpClient *dns.Client dohClient *doh.Client dotClient *dns.Client + udpProxy proxy.Dialer + tcpProxy proxy.Dialer + dotProxy proxy.Dialer knownHosts map[string][]string } @@ -76,39 +79,70 @@ func NewWithOptions(options Options) (*Client, error) { options.MaxPerCNAMEFollows = DefaultMaxPerCNAMEFollows } - httpClient := doh.NewHttpClientWithTimeout(options.Timeout) + httpClient := doh.NewHttpClient( + doh.WithTimeout(options.Timeout), + doh.WithInsecureSkipVerify(), + doh.WithProxy(options.Proxy), // no-op if empty + ) - client := Client{ - options: options, - resolvers: parsedBaseResolvers, - udpClient: &dns.Client{ - Net: "", - Timeout: options.Timeout, - Dialer: &net.Dialer{ - LocalAddr: options.GetLocalAddr(UDP), - }, - }, - tcpClient: &dns.Client{ - Net: TCP.String(), - Timeout: options.Timeout, - Dialer: &net.Dialer{ - LocalAddr: options.GetLocalAddr(TCP), - }, - }, - dohClient: doh.NewWithOptions( - doh.Options{ - HttpClient: httpClient, - }, - ), - dotClient: &dns.Client{ - Net: "tcp-tls", - Timeout: options.Timeout, - Dialer: &net.Dialer{ - LocalAddr: options.GetLocalAddr(TCP), - }, + udpDialer := &net.Dialer{LocalAddr: options.GetLocalAddr(UDP)} + tcpDialer := &net.Dialer{LocalAddr: options.GetLocalAddr(TCP)} + dotDialer := &net.Dialer{LocalAddr: options.GetLocalAddr(TCP)} + + udpClient := &dns.Client{ + Net: "", + Timeout: options.Timeout, + Dialer: udpDialer, + } + tcpClient := &dns.Client{ + Net: TCP.String(), + Timeout: options.Timeout, + Dialer: tcpDialer, + } + dohClient := doh.NewWithOptions( + doh.Options{ + HttpClient: httpClient, }, + ) + dotClient := &dns.Client{ + Net: "tcp-tls", + Timeout: options.Timeout, + Dialer: dotDialer, + } + + client := Client{ + options: options, + resolvers: parsedBaseResolvers, + udpClient: udpClient, + tcpClient: tcpClient, + dohClient: dohClient, + dotClient: dotClient, knownHosts: knownHosts, } + + if options.Proxy != "" { + proxyURL, err := url.Parse(options.Proxy) + if err != nil { + return nil, fmt.Errorf("invalid proxy URL: %v", err) + } + proxyDialer, err := proxy.FromURL(proxyURL, udpDialer) + if err != nil { + return nil, fmt.Errorf("error creating proxy dialer: %v", err) + } + tcpProxyDialer, err := proxy.FromURL(proxyURL, tcpDialer) + if err != nil { + return nil, fmt.Errorf("error creating proxy dialer: %v", err) + } + dotProxyDialer, err := proxy.FromURL(proxyURL, dotDialer) + if err != nil { + return nil, fmt.Errorf("error creating proxy dialer: %v", err) + } + + client.udpProxy = proxyDialer + client.tcpProxy = tcpProxyDialer + client.dotProxy = dotProxyDialer + } + if options.ConnectionPoolThreads > 1 { client.udpConnPool = mapsutil.SyncLockMap[string, *ConnPool]{ Map: make(mapsutil.Map[string, *ConnPool]), @@ -170,12 +204,30 @@ func (c *Client) Do(msg *dns.Msg) (*dns.Msg, error) { case *NetworkResolver: switch r.Protocol { case TCP: - resp, _, err = c.tcpClient.Exchange(msg, resolver.String()) + if c.tcpProxy != nil { + var tcpConn *dns.Conn + tcpConn, err = c.dialWithProxy(c.tcpProxy, "tcp", resolver.String()) + if err != nil { + break + } + defer tcpConn.Close() + resp, _, err = c.tcpClient.ExchangeWithConn(msg, tcpConn) + } else { + resp, _, err = c.tcpClient.Exchange(msg, resolver.String()) + } case UDP: if c.options.ConnectionPoolThreads > 1 { if udpConnPool, ok := c.udpConnPool.Get(resolver.String()); ok { resp, _, err = udpConnPool.Exchange(context.TODO(), c.udpClient, msg) } + } else if c.udpProxy != nil { + var udpConn *dns.Conn + udpConn, err = c.dialWithProxy(c.udpProxy, "udp", resolver.String()) + if err != nil { + break + } + defer udpConn.Close() + resp, _, err = c.udpClient.ExchangeWithConn(msg, udpConn) } else { resp, _, err = c.udpClient.Exchange(msg, resolver.String()) } @@ -204,6 +256,14 @@ func (c *Client) Do(msg *dns.Msg) (*dns.Msg, error) { return resp, ErrRetriesExceeded } +func (c *Client) dialWithProxy(dialer proxy.Dialer, network, addr string) (*dns.Conn, error) { + conn, err := dialer.Dial(network, addr) + if err != nil { + return nil, err + } + return &dns.Conn{Conn: conn}, nil +} + // Query sends a provided dns request and return enriched response func (c *Client) Query(host string, requestType uint16) (*DNSData, error) { return c.QueryMultiple(host, []uint16{requestType}) diff --git a/doh/doh_client.go b/doh/doh_client.go index 58a61e2..8f47b12 100644 --- a/doh/doh_client.go +++ b/doh/doh_client.go @@ -21,7 +21,10 @@ func NewWithOptions(options Options) *Client { } func New() *Client { - httpClient := NewHttpClientWithTimeout(DefaultTimeout) + httpClient := NewHttpClient( + WithTimeout(DefaultTimeout), + WithInsecureSkipVerify(), + ) return NewWithOptions(Options{DefaultResolver: Cloudflare, HttpClient: httpClient}) } diff --git a/doh/util.go b/doh/util.go index e23b2c4..db742f6 100644 --- a/doh/util.go +++ b/doh/util.go @@ -3,17 +3,61 @@ package doh import ( "crypto/tls" "net/http" + "net/url" "time" ) -func NewHttpClientWithTimeout(timeout time.Duration) *http.Client { - httpClient := &http.Client{ - Timeout: timeout, - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - }, +// ClientOption defines a function type for configuring an http.Client +type ClientOption func(*http.Client) + +// WithTimeout sets the timeout for the http.Client +func WithTimeout(timeout time.Duration) ClientOption { + return func(c *http.Client) { + c.Timeout = timeout + } +} + +// WithInsecureSkipVerify sets the InsecureSkipVerify option for the TLS config +func WithInsecureSkipVerify() ClientOption { + return func(c *http.Client) { + transport, ok := c.Transport.(*http.Transport) + if !ok { + transport = &http.Transport{} + c.Transport = transport + } + if transport.TLSClientConfig == nil { + transport.TLSClientConfig = &tls.Config{} + } + transport.TLSClientConfig.InsecureSkipVerify = true + } +} + +// WithProxy sets a proxy for the http.Client +func WithProxy(proxyURL string) ClientOption { + return func(c *http.Client) { + if proxyURL == "" { + return + } + proxyURL, err := url.Parse(proxyURL) + if err != nil { + return + } + + transport, ok := c.Transport.(*http.Transport) + if !ok { + transport = &http.Transport{} + c.Transport = transport + } + + transport.Proxy = http.ProxyURL(proxyURL) + } +} + +// NewHttpClient creates a new http.Client with the given options +func NewHttpClient(opts ...ClientOption) *http.Client { + client := &http.Client{} + for _, opt := range opts { + opt(client) } - return httpClient + return client } diff --git a/go.mod b/go.mod index 01f05f3..32a94c6 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/mod v0.12.0 // indirect - golang.org/x/net v0.23.0 // indirect + golang.org/x/net v0.23.0 golang.org/x/sys v0.18.0 // indirect golang.org/x/tools v0.13.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/options.go b/options.go index 7e0967e..5a56fa8 100644 --- a/options.go +++ b/options.go @@ -21,6 +21,7 @@ type Options struct { LocalAddrPort uint16 ConnectionPoolThreads int MaxPerCNAMEFollows int + Proxy string } // Returns a net.Addr of a UDP or TCP type depending on whats required