Parallelize DNS record lookups

Support using TCP or TCP with TLS for DNS queries
Fix incorrect logger injection for the scanner
Increase default DNS buffer from `1024` to `4096`
Strip unneeded information from release builds

Author: wolveix

.github/workflows/release.yml

@@ -19,7 +19,7 @@ jobs:
           mkdir -p builds/compressed
           go install github.com/mitchellh/gox@latest
           cd cmd/dss
-          gox --output "../../builds/dss-{{.OS}}-{{.Arch}}" -osarch 'darwin/amd64 darwin/arm64 linux/amd64 linux/arm freebsd/amd64 windows/amd64'
+          gox --output "../../builds/dss-{{.OS}}-{{.Arch}}" -ldflags '-s -w' -osarch 'darwin/amd64 darwin/arm64 linux/amd64 linux/arm freebsd/amd64 windows/amd64'
           cd ../../builds
           find . -maxdepth 1 -type f -execdir zip 'compressed/{}.zip' '{}' \;
       - name: Upload Binaries

README.md

@@ -207,7 +207,8 @@ You can then email this inbox from any address, and you'll receive an email back
 | `--concurrent`   | `-c`  | The number of domains to scan concurrently (defaults to your number of CPU threads)                             |
 | `--debug`        | `-d`  | Print debug logs                                                                                                |
 | `--dkimSelector` |       | Specify a comma seperated list of DKIM selectors (default "")                                                   |
-| `--dnsBuffer`    |       | Specify the allocated buffer for DNS responses (default 1024)                                                   |
+| `--dnsBuffer`    |       | Specify the allocated buffer for DNS responses (default 4096)                                                   |
+| `--dnsProtocol`  |       | Use udp, tcp, or tcp-tls for DNS queries (default udp)                                                          |
 | `--format`       | `-f`  | Format to print results in (yaml, json, csv) (default "yaml")                                                   |
 | `--nameservers`  | `-n`  | Use specific nameservers, in host[:port] format; may be specified multiple times                                |
 | `--outputFile`   | `-o`  | Output the results to a specified file (creates a file with the current unix timestamp if no file is specified) |

cmd/dss/main.go

@@ -25,7 +25,7 @@ var (
 		Use:     "dss",
 		Short:   "Scan a domain's DNS records.",
 		Long:    "Scan a domain's DNS records.\nhttps://github.com/GlobalCyberAlliance/domain-security-scanner",
-		Version: "3.0.2",
+		Version: "3.0.3",
 		PersistentPreRun: func(cmd *cobra.Command, args []string) {
 			if debug {
 				log = zerolog.New(zerolog.ConsoleWriter{Out: os.Stdout, TimeFormat: time.RFC3339}).With().Timestamp().Logger().Level(zerolog.DebugLevel)
@@ -58,7 +58,7 @@ var (
 	cfg                               *Config
 	log                               zerolog.Logger
 	writeToFileCounter                int
-	format, outputFile                string
+	dnsProtocol, format, outputFile   string
 	dkimSelector, nameservers         []string
 	advise, debug, checkTLS, zoneFile bool
 	dnsBuffer                         uint16
@@ -73,7 +73,8 @@ func main() {
 	cmd.PersistentFlags().Uint16VarP(&concurrent, "concurrent", "c", uint16(runtime.NumCPU()), "The number of domains to scan concurrently")
 	cmd.PersistentFlags().BoolVarP(&debug, "debug", "d", false, "Print debug logs")
 	cmd.PersistentFlags().StringSliceVar(&dkimSelector, "dkimSelector", []string{}, "Specify a DKIM selector")
-	cmd.PersistentFlags().Uint16Var(&dnsBuffer, "dnsBuffer", 1024, "Specify the allocated buffer for DNS responses")
+	cmd.PersistentFlags().Uint16Var(&dnsBuffer, "dnsBuffer", 4096, "Specify the allocated buffer for DNS responses")
+	cmd.PersistentFlags().StringVar(&dnsProtocol, "dnsProtocol", "udp", "Use udp, tcp, or tcp-tls for DNS queries")
 	cmd.PersistentFlags().StringVarP(&format, "format", "f", "yaml", "Format to print results in (yaml, json)")
 	cmd.PersistentFlags().StringSliceVarP(&nameservers, "nameservers", "n", nil, "Use specific nameservers, in `host[:port]` format; may be specified multiple times")
 	cmd.PersistentFlags().StringVarP(&outputFile, "outputFile", "o", "", "Output the results to a specified file (creates a file with the current unix timestamp if no file is specified)")

cmd/dss/scan.go

@@ -7,7 +7,6 @@ import (
 	"github.com/GlobalCyberAlliance/domain-security-scanner/pkg/advisor"
 	"github.com/GlobalCyberAlliance/domain-security-scanner/pkg/model"
 	"github.com/GlobalCyberAlliance/domain-security-scanner/pkg/scanner"
-	"github.com/rs/zerolog"
 	"github.com/spf13/cobra"
 )
 
@@ -25,14 +24,15 @@ var cmdScan = &cobra.Command{
 			scanner.WithCacheDuration(cache),
 			scanner.WithConcurrentScans(concurrent),
 			scanner.WithDNSBuffer(dnsBuffer),
+			scanner.WithDNSProtocol(dnsProtocol),
 			scanner.WithNameservers(nameservers),
 		}
 
 		if len(dkimSelector) > 0 {
 			opts = append(opts, scanner.WithDKIMSelectors(dkimSelector...))
 		}
 
-		sc, err := scanner.New(zerolog.Logger{}, timeout, opts...)
+		sc, err := scanner.New(log, timeout, opts...)
 		if err != nil {
 			log.Fatal().Err(err).Msg("An unexpected error occurred.")
 		}

cmd/dss/serve.go

@@ -7,7 +7,6 @@ import (
 	"github.com/GlobalCyberAlliance/domain-security-scanner/pkg/http"
 	"github.com/GlobalCyberAlliance/domain-security-scanner/pkg/mail"
 	"github.com/GlobalCyberAlliance/domain-security-scanner/pkg/scanner"
-	"github.com/rs/zerolog"
 	"github.com/spf13/cobra"
 )
 
@@ -52,14 +51,15 @@ var (
 				scanner.WithCacheDuration(cache),
 				scanner.WithConcurrentScans(concurrent),
 				scanner.WithDNSBuffer(dnsBuffer),
+				scanner.WithDNSProtocol(dnsProtocol),
 				scanner.WithNameservers(nameservers),
 			}
 
 			if len(dkimSelector) > 0 {
 				opts = append(opts, scanner.WithDKIMSelectors(dkimSelector...))
 			}
 
-			sc, err := scanner.New(zerolog.Logger{}, timeout, opts...)
+			sc, err := scanner.New(log, timeout, opts...)
 			if err != nil {
 				log.Fatal().Err(err).Msg("could not create domain scanner")
 			}
@@ -83,14 +83,15 @@ var (
 				scanner.WithCacheDuration(cache),
 				scanner.WithConcurrentScans(concurrent),
 				scanner.WithDNSBuffer(dnsBuffer),
+				scanner.WithDNSProtocol(dnsProtocol),
 				scanner.WithNameservers(nameservers),
 			}
 
 			if len(dkimSelector) > 0 {
 				opts = append(opts, scanner.WithDKIMSelectors(dkimSelector...))
 			}
 
-			sc, err := scanner.New(zerolog.Logger{}, timeout, opts...)
+			sc, err := scanner.New(log, timeout, opts...)
 			if err != nil {
 				log.Fatal().Err(err).Msg("could not create domain scanner")
 			}

pkg/advisor/advisor.go

@@ -69,53 +69,44 @@ func NewAdvisor(timeout time.Duration, cacheLifetime time.Duration, checkTLS boo
 	return &advisor
 }
 
-func (a *Advisor) CheckAll(domain string, bimi string, dkim string, dmarc string, mx []string, spf string) *Advice {
-	var adviceDomain, adviceBIMI, adviceDKIM, adviceDMARC, adviceMX, adviceSPF []string
+func (a *Advisor) CheckAll(domain, bimi, dkim, dmarc string, mx []string, spf string) *Advice {
+	advice := &Advice{}
 	var wg sync.WaitGroup
 
 	wg.Add(6)
 	go func() {
-		adviceDomain = a.CheckDomain(domain)
+		advice.Domain = a.CheckDomain(domain)
 		wg.Done()
 	}()
 
 	go func() {
-		adviceBIMI = a.CheckBIMI(bimi)
+		advice.BIMI = a.CheckBIMI(bimi)
 		wg.Done()
 	}()
 
 	go func() {
-		adviceDKIM = a.CheckDKIM(dkim)
+		advice.DKIM = a.CheckDKIM(dkim)
 		wg.Done()
 	}()
 
 	go func() {
-		adviceDMARC = a.CheckDMARC(dmarc)
+		advice.DMARC = a.CheckDMARC(dmarc)
 		wg.Done()
 	}()
 
 	go func() {
-		adviceMX = a.CheckMX(mx)
+		advice.MX = a.CheckMX(mx)
 		wg.Done()
 	}()
 
 	go func() {
-		adviceSPF = a.CheckSPF(spf)
+		advice.SPF = a.CheckSPF(spf)
 		wg.Done()
 	}()
 
 	wg.Wait()
 
-	advice := Advice{
-		Domain: adviceDomain,
-		BIMI:   adviceBIMI,
-		DKIM:   adviceDKIM,
-		DMARC:  adviceDMARC,
-		MX:     adviceMX,
-		SPF:    adviceSPF,
-	}
-
-	return &advice
+	return advice
 }
 
 func (a *Advisor) CheckBIMI(bimi string) (advice []string) {

pkg/scanner/options.go

@@ -5,6 +5,7 @@ import (
 	"net"
 	"net/netip"
 	"runtime"
+	"strings"
 	"time"
 
 	"github.com/miekg/dns"
@@ -68,8 +69,8 @@ func WithDKIMSelectors(selectors ...string) Option {
 // WithDNSBuffer increases the allocated buffer for DNS responses
 func WithDNSBuffer(bufferSize uint16) Option {
 	return func(s *Scanner) error {
-		if bufferSize > 4096 {
-			s.logger.Warn().Msg("buffer size should not be larger than 4096")
+		if bufferSize <= 0 {
+			return fmt.Errorf("invalid DNS buffer size: %d", bufferSize)
 		}
 
 		s.dnsBuffer = bufferSize
@@ -78,6 +79,22 @@ func WithDNSBuffer(bufferSize uint16) Option {
 	}
 }
 
+// WithDNSProtocol sets the DNS protocol to use for queries.
+func WithDNSProtocol(protocol string) Option {
+	return func(s *Scanner) error {
+		protocol = strings.ToLower(protocol)
+
+		switch protocol {
+		case "udp", "tcp", "tcp-tls":
+			s.dnsClient.Net = protocol
+		default:
+			return fmt.Errorf("invalid DNS protocol: %s, valid options: udp, tcp, tcp-tls", protocol)
+		}
+
+		return nil
+	}
+}
+
 // WithNameservers allows the caller to provide a custom set of nameservers for
 // a *Scanner to use. If ns is nil, or zero-length, the *Scanner will use
 // the nameservers specified in /etc/resolv.conf.

pkg/scanner/options_test.go

@@ -128,6 +128,34 @@ func TestOptionWithDNSBuffer(t *testing.T) {
 	})
 }
 
+func TestOptionWithDNSProtocol(t *testing.T) {
+	logger := zerolog.Nop()
+	timeout := time.Second * 5
+
+	t.Run("InvalidProtocol", func(t *testing.T) {
+		_, err := New(logger, timeout, WithDNSProtocol("invalid_protocol"))
+		assert.Error(t, err)
+	})
+
+	t.Run("ValidProtocolTCP", func(t *testing.T) {
+		scanner, err := New(logger, timeout, WithDNSProtocol("TCP"))
+		assert.NoError(t, err)
+		assert.Equal(t, "tcp", scanner.dnsClient.Net)
+	})
+
+	t.Run("ValidProtocolTCPWithTLS", func(t *testing.T) {
+		scanner, err := New(logger, timeout, WithDNSProtocol("TCP-tls"))
+		assert.NoError(t, err)
+		assert.Equal(t, "tcp-tls", scanner.dnsClient.Net)
+	})
+
+	t.Run("ValidProtocolUDP", func(t *testing.T) {
+		scanner, err := New(logger, timeout, WithDNSProtocol("UDP"))
+		assert.NoError(t, err)
+		assert.Equal(t, "udp", scanner.dnsClient.Net)
+	})
+}
+
 func TestOptionWithNameservers(t *testing.T) {
 	logger := zerolog.Nop()
 	timeout := time.Second * 5

pkg/scanner/requests.go

@@ -5,7 +5,6 @@ import (
 	"strings"
 
 	"github.com/miekg/dns"
-	"github.com/pkg/errors"
 )
 
 const (
@@ -60,17 +59,17 @@ func (s *Scanner) getDNSRecords(domain string, recordType uint16) (records []str
 			answer.Header().Rrtype = recordType
 		}
 
-		switch t := answer.(type) {
+		switch dnsRec := answer.(type) {
 		case *dns.A:
-			records = append(records, t.A.String())
+			records = append(records, dnsRec.A.String())
 		case *dns.AAAA:
-			records = append(records, t.AAAA.String())
+			records = append(records, dnsRec.AAAA.String())
 		case *dns.MX:
-			records = append(records, t.Mx)
+			records = append(records, dnsRec.Mx)
 		case *dns.NS:
-			records = append(records, t.Ns)
+			records = append(records, dnsRec.Ns)
 		case *dns.TXT:
-			records = append(records, t.Txt...)
+			records = append(records, dnsRec.Txt...)
 		}
 	}
 
@@ -81,14 +80,20 @@ func (s *Scanner) getDNSRecords(domain string, recordType uint16) (records []str
 // It returns a slice of dns.RR (DNS resource records) and an error if any occurred.
 func (s *Scanner) getDNSAnswers(domain string, recordType uint16) ([]dns.RR, error) {
 	req := &dns.Msg{}
-	req.SetQuestion(dns.Fqdn(domain), recordType)
+	req.Id = dns.Id()
+	req.RecursionDesired = true
 	req.SetEdns0(s.dnsBuffer, true) // increases the response buffer size
+	req.SetQuestion(dns.Fqdn(domain), recordType)
 
 	in, _, err := s.dnsClient.Exchange(req, s.getNS())
 	if err != nil {
 		return nil, err
 	}
 
+	if in.Rcode != dns.RcodeSuccess {
+		return nil, fmt.Errorf("DNS query failed with rcode %v", in.Rcode)
+	}
+
 	if in.MsgHdr.Truncated && s.dnsBuffer < 4096 {
 		s.logger.Warn().Msg(fmt.Sprintf("DNS buffer %v was too small for %v, retrying with larger buffer (4096)", s.dnsBuffer, domain))
 
@@ -103,35 +108,6 @@ func (s *Scanner) getDNSAnswers(domain string, recordType uint16) ([]dns.RR, err
 	return in.Answer, nil
 }
 
-// GetDNSRecords is a convenience wrapper which will scan all provided DNS record types
-// and fill the pointered ScanResult. It returns an error if any occurred.
-func (s *Scanner) GetDNSRecords(scanResult *Result, recordTypes ...string) (err error) {
-	for _, recordType := range recordTypes {
-		switch strings.ToUpper(recordType) {
-		case "BIMI":
-			scanResult.BIMI, err = s.getTypeBIMI(scanResult.Domain)
-		case "DKIM":
-			scanResult.DKIM, err = s.getTypeDKIM(scanResult.Domain)
-		case "DMARC":
-			scanResult.DMARC, err = s.getTypeDMARC(scanResult.Domain)
-		case "MX":
-			scanResult.MX, err = s.getDNSRecords(scanResult.Domain, dns.TypeMX)
-		case "NS":
-			scanResult.NS, err = s.getDNSRecords(scanResult.Domain, dns.TypeNS)
-		case "SPF":
-			scanResult.SPF, err = s.getTypeSPF(scanResult.Domain)
-		default:
-			return errors.New("invalid dns record type")
-		}
-
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
 func (s *Scanner) getTypeBIMI(domain string) (string, error) {
 	for _, dname := range []string{
 		"default._bimi." + domain,